Support for Oracle: auto-shorten long table names, use UnicodeText() instead of Unicode(4000) because of size limitations for the mapped datatypes in Oracle

pokedex/db/__init__.py

@@ -38,6 +38,22 @@ def connect(uri=None, session_args={}, engine_args={}, engine_prefix=''):
             table.kwargs['mysql_engine'] = 'InnoDB'
             table.kwargs['mysql_charset'] = 'utf8'
 
+    ### Do some fixery for Oracle
+    if uri.startswith('oracle:') or uri.startswith('oracle+cx_oracle:'):
+        # Oracle requires auto_setinputsizes=False (or at least a special
+        # set of exclusions from it, which I don't know)
+        if 'auto_setinputsizes' not in uri:
+            uri += '?auto_setinputsizes=FALSE'
+
+        # Shorten table names, Oracle limits table and column names to 30 chars
+        # Easy solution : drop the vowels, differents words are unlikely to
+        # end up the same after the vowels are gone
+        for table in metadata.tables.values():
+            table.description = table.name[:]
+            if len(table.name) > 30:
+                for letter in ['a', 'e', 'i', 'o', 'u', 'y']:
+                    table.name=table.name.replace(letter,'')
+
     ### Connect
     engine_args[engine_prefix + 'url'] = uri
     engine = engine_from_config(engine_args, prefix=engine_prefix)

pokedex/db/load.py

@@ -138,6 +138,22 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s
     table_names = _get_table_names(metadata, tables)
     table_objs = [metadata.tables[name] for name in table_names]
 
+    # Oracle fixery again, load doesn't know we modified the schema
+    # flag for oracle stuff
+    oranames = (session.connection().dialect.name == 'oracle')
+    if oranames:
+        # Prepare a dictionary to match old<->new names
+        oradict = {}
+
+        # Shorten table names, Oracle limits table and column names to 30 chars
+        for table in table_objs:
+            tname = table.name[:]
+            oradict[tname]=table.description
+            if len(tname) > 30:
+                for letter in ['a', 'e', 'i', 'o', 'u', 'y']:
+                    table.name=table.name.replace(letter,'')
+                oradict[tname]=table.description
+                
     if recursive:
         table_objs.extend(find_dependent_tables(table_objs))
 
@@ -181,9 +197,11 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s
         insert_stmt = table_obj.insert()
 
         print_start(table_name)
-
         try:
             csvpath = "%s/%s.csv" % (directory, table_name)
+            # In oracle mode, use the original names instead of current
+            if oranames:
+                csvpath = "%s/%s.csv" % (directory, oradict[table_name])    
             csvfile = open(csvpath, 'rb')
         except IOError:
             # File doesn't exist; don't load anything!
@@ -250,6 +268,8 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s
 
             for column_name, value in zip(column_names, csvs):
                 column = table_obj.c[column_name]
+                if not column.nullable and value == '':
+                    value = ' '
                 if column.nullable and value == '':
                     # Empty string in a nullable column really means NULL
                     value = None

pokedex/db/tables.py

@@ -38,7 +38,7 @@ from sqlalchemy.orm.session import Session
 from sqlalchemy.orm.interfaces import AttributeExtension
 from sqlalchemy.sql import and_, or_
 from sqlalchemy.schema import ColumnDefault
-from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode
+from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode, UnicodeText
 
 from pokedex.db import markdown, multilang
 
@@ -130,7 +130,7 @@ create_translation_table('ability_names', Ability, 'names',
         info=dict(description="The name", format='plaintext', official=True, ripped=True)),
 )
 create_translation_table('ability_prose', Ability, 'prose',
-    effect = Column(Unicode(4000), nullable=True,
+    effect = Column(UnicodeText(), nullable=True,
         info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
     short_effect = Column(Unicode(512), nullable=True,
         info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
@@ -920,7 +920,7 @@ create_translation_table('item_names', Item, 'names',
 create_translation_table('item_prose', Item, 'prose',
     short_effect = Column(Unicode(256), nullable=True,
         info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)),
-    effect = Column(Unicode(4000), nullable=True,
+    effect = Column(UnicodeText(), nullable=True,
         info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)),
 )
 create_translation_table('item_flavor_summaries', Item, 'flavor_summaries',
@@ -1219,7 +1219,7 @@ class MoveEffect(TableBase):
 create_translation_table('move_effect_prose', MoveEffect, 'prose',
     short_effect = Column(Unicode(256), nullable=True,
         info=dict(description="A short summary of the effect", format='markdown')),
-    effect = Column(Unicode(4000), nullable=True,
+    effect = Column(UnicodeText(), nullable=True,
         info=dict(description="A detailed description of the effect", format='markdown')),
 )
 

pokedex/doc/tabledoc.py

@@ -56,6 +56,8 @@ def column_type_str(column):
         return 'bool'
     if type(type_) == types.Unicode:
         return u'unicode – %s' % column.info['format']
+    if type(type_) == types.UnicodeText:
+        return u'unicode – %s' % column.info['format']
     if type(type_) == types.Enum:
         return 'enum: [%s]' % ', '.join(type_.enums)
     if type(type_) == markdown.MarkdownColumn: