From 5501be34a82774773cb74ceaeda961827ff9aab1 Mon Sep 17 00:00:00 2001 From: Epithumia Date: Wed, 18 Dec 2013 14:32:13 +0100 Subject: [PATCH] Support for Oracle: auto-shorten long table names, use UnicodeText() instead of Unicode(4000) because of size limitations for the mapped datatypes in Oracle --- pokedex/db/__init__.py | 16 ++++++++++++++++ pokedex/db/load.py | 22 +++++++++++++++++++++- pokedex/db/tables.py | 8 ++++---- pokedex/doc/tabledoc.py | 2 ++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/pokedex/db/__init__.py b/pokedex/db/__init__.py index 464629d..37b3746 100644 --- a/pokedex/db/__init__.py +++ b/pokedex/db/__init__.py @@ -38,6 +38,22 @@ def connect(uri=None, session_args={}, engine_args={}, engine_prefix=''): table.kwargs['mysql_engine'] = 'InnoDB' table.kwargs['mysql_charset'] = 'utf8' + ### Do some fixery for Oracle + if uri.startswith('oracle:') or uri.startswith('oracle+cx_oracle:'): + # Oracle requires auto_setinputsizes=False (or at least a special + # set of exclusions from it, which I don't know) + if 'auto_setinputsizes' not in uri: + uri += '?auto_setinputsizes=FALSE' + + # Shorten table names, Oracle limits table and column names to 30 chars + # Easy solution : drop the vowels, differents words are unlikely to + # end up the same after the vowels are gone + for table in metadata.tables.values(): + table.description = table.name[:] + if len(table.name) > 30: + for letter in ['a', 'e', 'i', 'o', 'u', 'y']: + table.name=table.name.replace(letter,'') + ### Connect engine_args[engine_prefix + 'url'] = uri engine = engine_from_config(engine_args, prefix=engine_prefix) diff --git a/pokedex/db/load.py b/pokedex/db/load.py index d241519..8471902 100644 --- a/pokedex/db/load.py +++ b/pokedex/db/load.py @@ -138,6 +138,22 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s table_names = _get_table_names(metadata, tables) table_objs = [metadata.tables[name] for name in table_names] + # Oracle fixery again, load doesn't know we modified the schema + # flag for oracle stuff + oranames = (session.connection().dialect.name == 'oracle') + if oranames: + # Prepare a dictionary to match old<->new names + oradict = {} + + # Shorten table names, Oracle limits table and column names to 30 chars + for table in table_objs: + tname = table.name[:] + oradict[tname]=table.description + if len(tname) > 30: + for letter in ['a', 'e', 'i', 'o', 'u', 'y']: + table.name=table.name.replace(letter,'') + oradict[tname]=table.description + if recursive: table_objs.extend(find_dependent_tables(table_objs)) @@ -181,9 +197,11 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s insert_stmt = table_obj.insert() print_start(table_name) - try: csvpath = "%s/%s.csv" % (directory, table_name) + # In oracle mode, use the original names instead of current + if oranames: + csvpath = "%s/%s.csv" % (directory, oradict[table_name]) csvfile = open(csvpath, 'rb') except IOError: # File doesn't exist; don't load anything! @@ -250,6 +268,8 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s for column_name, value in zip(column_names, csvs): column = table_obj.c[column_name] + if not column.nullable and value == '': + value = ' ' if column.nullable and value == '': # Empty string in a nullable column really means NULL value = None diff --git a/pokedex/db/tables.py b/pokedex/db/tables.py index b24ea86..5f42079 100644 --- a/pokedex/db/tables.py +++ b/pokedex/db/tables.py @@ -38,7 +38,7 @@ from sqlalchemy.orm.session import Session from sqlalchemy.orm.interfaces import AttributeExtension from sqlalchemy.sql import and_, or_ from sqlalchemy.schema import ColumnDefault -from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode +from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode, UnicodeText from pokedex.db import markdown, multilang @@ -130,7 +130,7 @@ create_translation_table('ability_names', Ability, 'names', info=dict(description="The name", format='plaintext', official=True, ripped=True)), ) create_translation_table('ability_prose', Ability, 'prose', - effect = Column(Unicode(4000), nullable=True, + effect = Column(UnicodeText(), nullable=True, info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), short_effect = Column(Unicode(512), nullable=True, info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), @@ -920,7 +920,7 @@ create_translation_table('item_names', Item, 'names', create_translation_table('item_prose', Item, 'prose', short_effect = Column(Unicode(256), nullable=True, info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)), - effect = Column(Unicode(4000), nullable=True, + effect = Column(UnicodeText(), nullable=True, info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)), ) create_translation_table('item_flavor_summaries', Item, 'flavor_summaries', @@ -1219,7 +1219,7 @@ class MoveEffect(TableBase): create_translation_table('move_effect_prose', MoveEffect, 'prose', short_effect = Column(Unicode(256), nullable=True, info=dict(description="A short summary of the effect", format='markdown')), - effect = Column(Unicode(4000), nullable=True, + effect = Column(UnicodeText(), nullable=True, info=dict(description="A detailed description of the effect", format='markdown')), ) diff --git a/pokedex/doc/tabledoc.py b/pokedex/doc/tabledoc.py index 172a454..755303b 100644 --- a/pokedex/doc/tabledoc.py +++ b/pokedex/doc/tabledoc.py @@ -56,6 +56,8 @@ def column_type_str(column): return 'bool' if type(type_) == types.Unicode: return u'unicode – %s' % column.info['format'] + if type(type_) == types.UnicodeText: + return u'unicode – %s' % column.info['format'] if type(type_) == types.Enum: return 'enum: [%s]' % ', '.join(type_.enums) if type(type_) == markdown.MarkdownColumn: