global: PostgreSQL Text type compatibility fix

* Changes Text in models because from SQLAlchemy>=1.0 it's arised a
  exception if the length is specified. (closes inveniosoftware#3037)

* Refactores SbmFORMATEXTENSION and sbmGFILERESULT.

* Removes Invenio hacks on MySQL Index and Primary Key creation.

* PEP8/257 code style improvements.

Signed-off-by: Leonardo Rossi <[email protected]>

invenio/ext/sqlalchemy/engines/mysql.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2011, 2012, 2013, 2014 CERN.
+# Copyright (C) 2011, 2012, 2013, 2014, 2015 CERN.
 #
 # Invenio is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
@@ -21,86 +21,11 @@
 
 import base64
 
-# SQLAlchemy
 import sqlalchemy
 
-import sqlalchemy.types as types
+from sqlalchemy import types as types
 from sqlalchemy.ext.compiler import compiles
-
-from sqlalchemy.schema import CreateIndex, PrimaryKeyConstraint
-
-
-@compiles(CreateIndex, 'mysql')
-def visit_create_index(element, compiler, **kw):
-    """Return create index statement with defined length for text field.
-
-    example:
-    CREATE TABLE tableA
-        ...
-        description TEXT(40)
-        ...
-        INDEX ix_tableA_description ON (description(40))
-    """
-    index = element.element
-    preparer = compiler.preparer
-    table = preparer.format_table(index.table)
-    name = preparer.quote(index.name, index.name.quote)
-
-    text = "ALTER TABLE %s ADD " % (table, )
-    if index.unique:
-        text += "UNIQUE "
-    text += "INDEX %s" % (name, )
-
-    lst = index.kwargs.get('mysql_length', None)
-
-    columns = []
-    for i, c in enumerate(index.columns):
-        cname = c.name
-        suffix = ''
-        if isinstance(lst, (list, tuple)) and len(lst) > i \
-                and lst[i] is not None:
-            suffix = '(%d)' % lst[i]
-        elif c.type != types.NULLTYPE \
-                and str(c.type).startswith('TEXT') \
-                and (c.type.length is not None):
-            suffix = '(%d)' % c.type.length
-        columns.append(cname + suffix)
-
-    text += '(' + ', '.join(columns) + ')'
-
-    if 'mysql_using' in index.kwargs:
-        using = index.kwargs['mysql_using']
-        if using is not None:
-            text += " USING %s" % (preparer.quote(using, index.quote))
-
-    return text
-
-
-@compiles(PrimaryKeyConstraint, 'mysql')
-def visit_primary_key_constraint(*element):
-    """
-    Return create primary key constrains.
-
-    Return create primary key constrains with defined length
-    for text field.
-    """
-    constraint, compiler = element
-    if len(constraint) == 0:
-        return ''
-    text = ""
-    if constraint.name is not None:
-        text += "CONSTRAINT %s " % \
-                compiler.preparer.format_constraint(constraint)
-    text += "PRIMARY KEY "
-    text += "(%s)" % ', '.join(c.name +
-                               (c.type != types.NULLTYPE
-                                and (str(c.type).startswith('TEXT')
-                                     and (c.type.length is not None))
-                                and '(%d)' % c.type.length
-                                or '')
-                               for c in constraint)
-    text += compiler.define_constraint_deferrability(constraint)
-    return text
+from sqlalchemy.types import TypeDecorator
 
 
 @compiles(types.Text, 'sqlite')
@@ -140,8 +65,6 @@ def compile_largebinary(element, compiler, **kw):
     """Redefine LargeBinary filed type for MySQL."""
     return 'LONGBLOB'
 
-from sqlalchemy.types import TypeDecorator
-
 
 class iBinary(TypeDecorator):
 

invenio/modules/accounts/models.py

@@ -253,7 +253,7 @@ def __str__(self):
 
     # FIXME Unique(login_method(70), name)
     __table_args__ = (db.Index('login_method_name', login_method, name,
-                               mysql_length=[60, None]),
+                               mysql_length=60),
                       db.Model.__table_args__)
 
     @classmethod

invenio/modules/baskets/models.py

@@ -125,7 +125,7 @@ class BskRECORDCOMMENT(db.Model):
     user = db.relationship(User)
 
     __table_args__ = (db.Index('bskRECORDCOMMENT_reply_order_cached_data',
-                               reply_order_cached_data, mysql_length=[40]),
+                               reply_order_cached_data, mysql_length=40),
                       db.Model.__table_args__)
 
 

invenio/modules/collections/models.py

@@ -38,9 +38,10 @@
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.orderinglist import ordering_list
 from sqlalchemy.orm.collections import attribute_mapped_collection
+from sqlalchemy.schema import Index
 
 from werkzeug.utils import cached_property
-# Create your models here.
+
 
 external_collection_mapper = attribute_multi_dict_collection(
     creator=lambda k, v: CollectionExternalcollection(type=k,
@@ -59,19 +60,22 @@ def __repr__(self):
                'nbrecs: {0.nbrecs}>'.format(self)
 
     def __unicode__(self):
+        """Unicode."""
         suffix = ' ({0})'.format(_('default')) if self.id == 1 else ''
         return u"{0.id}. {0.name}{1}".format(self, suffix)
 
     def __str__(self):
+        """Str."""
+        # TODO it's compatible with python 3?
         return unicode(self).encode('utf-8')
 
     __tablename__ = 'collection'
     id = db.Column(db.MediumInteger(9, unsigned=True),
                    primary_key=True)
-    name = db.Column(db.String(255), unique=True, index=True,
-                     nullable=False)
-    dbquery = db.Column(db.Text(20), nullable=True,
-                        index=True)
+    name = db.Column(db.String(255), unique=True, index=True, nullable=False)
+    dbquery = db.Column(
+        db.Text().with_variant(db.Text(20), 'mysql'),
+        nullable=True)
 
     @property
     def nbrecs(self):
@@ -136,6 +140,7 @@ def examples(self):
 
     @property
     def name_ln(self):
+        """Name ln."""
         from invenio.legacy.search_engine import get_coll_i18nname
         return get_coll_i18nname(self.name,
                                  getattr(g, 'ln', cfg['CFG_SITE_LANG']))
@@ -151,6 +156,7 @@ def name_ln(self):
     @property
     # @cache.memoize(make_name=lambda fname: fname + '::' + g.ln)
     def portalboxes_ln(self):
+        """Get Portalboxes ln."""
         return db.object_session(self).query(CollectionPortalbox).\
             with_parent(self).\
             options(db.joinedload_all(CollectionPortalbox.portalbox)).\
@@ -159,6 +165,7 @@ def portalboxes_ln(self):
 
     @property
     def most_specific_dad(self):
+        """Most specific dad."""
         results = sorted(
             db.object_session(self).query(Collection).join(
                 Collection.sons
@@ -262,9 +269,7 @@ def _make_field_fieldvalue(type_):
     @property
     # @cache.memoize(make_name=lambda fname: fname + '::' + g.ln)
     def search_within(self):
-        """
-        Collect search within options.
-        """
+        """Collect search within options."""
         default = [('', _('any field'))]
         found = [(o.field.code, o.field.name_ln) for o in self._search_within]
         if not found:
@@ -423,6 +428,9 @@ def breadcrumbs(self, builder=None, ln=None):
         return breadcrumbs
 
 
+Index('collection_dbquery_idx', Collection.dbquery, mysql_length=20)
+
+
 class Collectionname(db.Model):
 
     """Represent a Collectionname record."""
@@ -440,10 +448,12 @@ class Collectionname(db.Model):
 
     @db.hybrid_property
     def ln_type(self):
+        """Get ln type."""
         return (self.ln, self.type)
 
     @ln_type.setter
     def set_ln_type(self, value):
+        """Set ln type."""
         (self.ln, self.type) = value
 
 
@@ -480,6 +490,7 @@ def set_ln_type(self, value):
 class Collectiondetailedrecordpagetabs(db.Model):
 
     """Represent a Collectiondetailedrecordpagetabs record."""
+
     __tablename__ = 'collectiondetailedrecordpagetabs'
     id_collection = db.Column(db.MediumInteger(9, unsigned=True),
                               db.ForeignKey(Collection.id),
@@ -493,6 +504,7 @@ class Collectiondetailedrecordpagetabs(db.Model):
 class CollectionCollection(db.Model):
 
     """Represent a CollectionCollection record."""
+
     __tablename__ = 'collection_collection'
     id_dad = db.Column(db.MediumInteger(9, unsigned=True),
                        db.ForeignKey(Collection.id), primary_key=True)
@@ -514,6 +526,7 @@ class CollectionCollection(db.Model):
 class Example(db.Model):
 
     """Represent a Example record."""
+
     __tablename__ = 'example'
     id = db.Column(db.MediumInteger(9, unsigned=True), primary_key=True,
                    autoincrement=True)
@@ -524,6 +537,7 @@ class Example(db.Model):
 class CollectionExample(db.Model):
 
     """Represent a CollectionExample record."""
+
     __tablename__ = 'collection_example'
     id_collection = db.Column(db.MediumInteger(9, unsigned=True),
                               db.ForeignKey(Collection.id), primary_key=True)
@@ -539,6 +553,7 @@ class CollectionExample(db.Model):
 class Portalbox(db.Model):
 
     """Represent a Portalbox record."""
+
     __tablename__ = 'portalbox'
     id = db.Column(db.MediumInteger(9, unsigned=True), autoincrement=True,
                    primary_key=True)
@@ -547,8 +562,7 @@ class Portalbox(db.Model):
 
 
 def get_pbx_pos():
-    """Returns a list of all the positions for a portalbox"""
-
+    """Return a list of all the positions for a portalbox."""
     position = {}
     position["rt"] = "Right Top"
     position["lt"] = "Left Top"
@@ -562,6 +576,7 @@ def get_pbx_pos():
 class CollectionPortalbox(db.Model):
 
     """Represent a CollectionPortalbox record."""
+
     __tablename__ = 'collection_portalbox'
     id_collection = db.Column(db.MediumInteger(9, unsigned=True),
                               db.ForeignKey(Collection.id), primary_key=True)
@@ -583,6 +598,7 @@ class CollectionPortalbox(db.Model):
 class Externalcollection(db.Model):
 
     """Represent a Externalcollection record."""
+
     __tablename__ = 'externalcollection'
     id = db.Column(db.MediumInteger(9, unsigned=True),
                    primary_key=True)
@@ -591,6 +607,7 @@ class Externalcollection(db.Model):
 
     @property
     def engine(self):
+        """Engine."""
         from invenio.legacy.websearch_external_collections.searcher import (
             external_collections_dictionary
         )
@@ -601,6 +618,7 @@ def engine(self):
 class CollectionExternalcollection(db.Model):
 
     """Represent a CollectionExternalcollection record."""
+
     __tablename__ = 'collection_externalcollection'
     id_collection = db.Column(db.MediumInteger(9,
                                                unsigned=True),
@@ -634,6 +652,7 @@ def _collection_type(type_):
 class CollectionFormat(db.Model):
 
     """Represent a CollectionFormat record."""
+
     __tablename__ = 'collection_format'
     id_collection = db.Column(db.MediumInteger(9, unsigned=True),
                               db.ForeignKey(Collection.id), primary_key=True)

invenio/modules/comments/models.py

@@ -96,7 +96,7 @@ def expand(self, id_user):
             CmtCOLLAPSED.id_user == id_user)).delete(synchronize_session=False)
 
     __table_args__ = (db.Index('cmtRECORDCOMMENT_reply_order_cached_data',
-                               reply_order_cached_data, mysql_length=[40]),
+                               reply_order_cached_data, mysql_length=40),
                       db.Model.__table_args__)
 
 

invenio/modules/knowledge/models.py

@@ -27,9 +27,10 @@
 from invenio.modules.collections.models import Collection
 from invenio.utils.text import slugify
 
+from sqlalchemy.dialects import mysql
 from sqlalchemy.event import listens_for
-
 from sqlalchemy.orm.collections import attribute_mapped_collection
+from sqlalchemy.schema import Index
 
 
 class KnwKB(db.Model):
@@ -281,7 +282,9 @@ class KnwKBRVAL(db.Model):
     __tablename__ = 'knwKBRVAL'
     m_key = db.Column(db.String(255), nullable=False, primary_key=True,
                       index=True)
-    m_value = db.Column(db.Text(30), nullable=False, index=True)
+    m_value = db.Column(
+        db.Text().with_variant(mysql.TEXT(30), 'mysql'),
+        nullable=False)
     id_knwKB = db.Column(db.MediumInteger(8), db.ForeignKey(KnwKB.id),
                          nullable=False, server_default='0',
                          primary_key=True)
@@ -343,4 +346,7 @@ def to_dict(self):
                 'kbid': self.kb.id if self.kb else None,
                 'kbname': self.kb.name if self.kb else None}
 
+
+Index('ix_knwKBRVAL_m_value', KnwKBRVAL.m_value, mysql_length=30)
+
 __all__ = ('KnwKB', 'KnwKBDDEF', 'KnwKBRVAL')

invenio/modules/pages/models.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2014 CERN.
+# Copyright (C) 2014, 2015 CERN.
 #
 # Invenio is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
@@ -17,27 +17,27 @@
 # along with Invenio; if not, write to the Free Software Foundation, Inc.,
 # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
 
-"""
-    invenio.modules.pages.models
-    ----------------------------
+"""Contains page model."""
 
-    Contains page model.
-"""
+from datetime import datetime
 
 from invenio.ext.sqlalchemy import db
-from datetime import datetime
 
 
 class Page(db.Model):
+
     """Represents a page."""
+
     __tablename__ = 'pages'
 
     id = db.Column(db.Integer(15, unsigned=True), nullable=False,
                    primary_key=True,
                    autoincrement=True)
     url = db.Column(db.String(100), unique=True, nullable=False)
     title = db.Column(db.String(200), nullable=True)
-    content = db.Column(db.TEXT(length=2**32-2), nullable=True)
+    content = db.Column(
+        db.Text().with_variant(db.Text(length=2**32-2), 'mysql'),
+        nullable=True)
     # Default is pages/templates/default.html
     template_name = db.Column(db.String(70), nullable=True)
     created = db.Column(db.DateTime(), nullable=False, default=datetime.now)