diff --git a/pokedex/compatibility.py b/pokedex/compatibility.py index 67eff35..4eea75b 100644 --- a/pokedex/compatibility.py +++ b/pokedex/compatibility.py @@ -2,6 +2,7 @@ Currently these are functions missing from Python 2.5. """ +from __future__ import print_function try: from itertools import permutations @@ -127,14 +128,15 @@ except ImportError: for i, name in enumerate(field_names): template += ' %s = _property(_itemgetter(%d))\n' % (name, i) if verbose: - print template + print(template) # Execute the template string in a temporary namespace namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, _property=property, _tuple=tuple) try: - exec template in namespace - except SyntaxError, e: + exec ("exec template in namespace") + except SyntaxError: + e = _sys.exc_info()[1] raise SyntaxError(e.message + ':\n' + template) result = namespace[typename] diff --git a/pokedex/db/load.py b/pokedex/db/load.py index 37d7529..da3130f 100644 --- a/pokedex/db/load.py +++ b/pokedex/db/load.py @@ -1,4 +1,6 @@ """CSV to database or vice versa.""" +from __future__ import print_function + import csv import fnmatch import os.path @@ -58,7 +60,7 @@ def _get_verbose_prints(verbose): # Also, space-pad to keep the cursor in a known column num_spaces = 66 - len(truncated_thing) - print "%s...%s" % (truncated_thing, ' ' * num_spaces), + print("%s...%s" % (truncated_thing, ' ' * num_spaces), end='') sys.stdout.flush() if sys.stdout.isatty(): @@ -91,7 +93,7 @@ def _get_verbose_prints(verbose): pass def print_done(msg='ok'): - print msg + print(msg) return print_start, print_status, print_done diff --git a/pokedex/db/markdown.py b/pokedex/db/markdown.py index f6c10a3..8bbc0d0 100644 --- a/pokedex/db/markdown.py +++ b/pokedex/db/markdown.py @@ -14,6 +14,7 @@ from __future__ import absolute_import import re import markdown +import six from sqlalchemy.orm.session import object_session try: # Markdown 2.1+ @@ -22,6 +23,7 @@ except ImportError: # Old Markdown from markdown import etree, AtomicString +@six.python_2_unicode_compatible class MarkdownString(object): """Wraps a Markdown string. @@ -44,11 +46,8 @@ class MarkdownString(object): self.session = session self.language = language - def __unicode__(self): - return self.as_text() - def __str__(self): - return self.as_text().encode() + return self.as_text() def __html__(self): return self.as_html() @@ -98,7 +97,7 @@ def _markdownify_effect_text(move, effect_text, language=None): return effect_text effect_text = effect_text.replace( u'$effect_chance', - unicode(move.effect_chance), + str(move.effect_chance), ) # "The target" vs "each target"; for Conquest, but hopefully main series @@ -165,7 +164,7 @@ class PokedexLinkPattern(markdown.inlinepatterns.Pattern): Handles matches using factory """ - regex = ur'(?x) \[ ([^]]*) \] \{ ([-a-z0-9]+) : ([-a-z0-9 ]+) \}' + regex = u'(?x) \\[ ([^]]*) \\] \\{ ([-a-z0-9]+) : ([-a-z0-9 ]+) \\}' def __init__(self, factory, session, string_language=None, game_language=None): markdown.inlinepatterns.Pattern.__init__(self, self.regex) diff --git a/pokedex/db/multilang.py b/pokedex/db/multilang.py index c0c1965..06a5ab8 100644 --- a/pokedex/db/multilang.py +++ b/pokedex/db/multilang.py @@ -136,7 +136,7 @@ def create_translation_table(_table_name, foreign_class, relation_name, # Add ye columns # Column objects have a _creation_order attribute in ascending order; use # this to get the (unordered) kwargs sorted correctly - kwitems = kwargs.items() + kwitems = list(kwargs.items()) kwitems.sort(key=lambda kv: kv[1]._creation_order) for name, column in kwitems: column.name = name diff --git a/pokedex/db/translations.py b/pokedex/db/translations.py index 5d1bda6..39dc07c 100755 --- a/pokedex/db/translations.py +++ b/pokedex/db/translations.py @@ -20,18 +20,17 @@ put it in flavor_summary tables. Routes names and other repetitive numeric things are replaced by e.g. "Route {num}" so translators only have to work on each set once. - """ +from __future__ import print_function import binascii import csv -import heapq -import itertools import os import re -import sys from collections import defaultdict +from six.moves import zip + from pokedex.db import tables from pokedex.defaults import get_default_csv_dir @@ -253,7 +252,7 @@ class Translations(object): ) if len(warning) > 79: warning = warning[:76] + u'...' - print warning.encode('utf-8') + print(warning) def reader_for_class(self, cls, reader_class=csv.reader): tablename = cls.__table__.name @@ -366,7 +365,7 @@ def group_by_object(stream): Yields ((class name, object ID), (list of messages)) pairs. """ stream = iter(stream) - current = stream.next() + current = next(stream) current_key = current.cls, current.id group = [current] for message in stream: @@ -395,27 +394,37 @@ class Merge(object): def add_iterator(self, iterator): iterator = iter(iterator) try: - value = iterator.next() + value = next(iterator) except StopIteration: return - else: - heapq.heappush(self.next_values, (value, iterator)) + + self.next_values.append((value, iterator)) def __iter__(self): return self - def next(self): - if self.next_values: - value, iterator = heapq.heappop(self.next_values) - self.add_iterator(iterator) - return value - else: + def __next__(self): + if not self.next_values: raise StopIteration + min_idx = min(range(len(self.next_values)), key=lambda i: self.next_values[i][0]) + value, iterator = self.next_values[min_idx] + + try: + next_value = next(iterator) + except StopIteration: + del self.next_values[min_idx] + else: + self.next_values[min_idx] = next_value, iterator + + return value + + next = __next__ + def merge_adjacent(gen): """Merge adjacent messages that compare equal""" gen = iter(gen) - last = gen.next() + last = next(gen) for this in gen: if this.merge_key == last.merge_key: last.merge(this) @@ -441,16 +450,16 @@ def leftjoin(left_stream, right_stream, key=lambda x: x, unused=None): left_stream = iter(left_stream) right_stream = iter(right_stream) try: - right = right_stream.next() + right = next(right_stream) for left in left_stream: while right and key(left) > key(right): if unused is not None: unused(right) - right = right_stream.next() + right = next(right_stream) if key(left) == key(right): yield left, right del left - right = right_stream.next() + right = next(right_stream) else: yield left, None except StopIteration: @@ -478,7 +487,7 @@ def yield_source_csv_messages(cls, foreign_cls, csvreader, force_column=None): """Yield all messages from one source CSV file. """ columns = list(cls.__table__.c) - column_names = csvreader.next() + column_names = next(csvreader) # Assumptions: rows are in lexicographic order # (taking numeric values as numbers of course) # Assumptions about the order of columns: @@ -503,11 +512,13 @@ def _yield_csv_messages(foreign_cls, columns, first_string_index, csvreader, ori id = int(values[0]) messages = [] for string, column in zip(values[first_string_index:], string_columns): + if isinstance(string, bytes): + string = string.decode('utf-8') message = Message( foreign_cls.__name__, id, column.name, - string.decode('utf-8'), + string, column.type.length, pot=pot_for_column(cls, column, force_column is not None), origin=origin, @@ -524,7 +535,7 @@ def yield_guessed_csv_messages(file): """Yield messages from a CSV file, using the header to figure out what the data means. """ csvreader = csv.reader(file, lineterminator='\n') - column_names = csvreader.next() + column_names = next(csvreader) if column_names == 'language_id,table,id,column,source_crc,string'.split(','): # A translation CSV return yield_translation_csv_messages(file, True) @@ -553,14 +564,16 @@ def yield_translation_csv_messages(file, no_header=False): """ csvreader = csv.reader(file, lineterminator='\n') if not no_header: - columns = csvreader.next() + columns = next(csvreader) assert columns == 'language_id,table,id,column,source_crc,string'.split(',') for language_id, table, id, column, source_crc, string in csvreader: + if isinstance(string, bytes): + string = string.decode('utf-8') yield Message( table, int(id), column, - string.decode('utf-8'), + string, origin='target CSV', source_crc=source_crc, language_id=int(language_id), @@ -591,7 +604,7 @@ def pot_for_column(cls, column, summary=False): def number_replace(source, string): numbers_iter = iter(number_re.findall(source)) - next_number = lambda match: numbers_iter.next() + next_number = lambda match: next(numbers_iter) return re.sub(r'\{num\}', next_number, string) def match_to_source(source, *translations): @@ -617,7 +630,7 @@ def match_to_source(source, *translations): current_source = number_replace(source.string, translation.source) current_crc = crc(current_source) elif '{num}' in translation.string: - print (u'Warning: {num} appears in %s, but not marked for number replacement. Discarding!' % translation).encode('utf-8') + print(u'Warning: {num} appears in %s, but not marked for number replacement. Discarding!' % translation) continue else: current_string = translation.string @@ -655,5 +668,5 @@ def merge_translations(source_stream, *translation_streams, **kwargs): synchronize(source, t, key=lambda m: m.merge_key, unused=kwargs.get('unused')) for t in translation_streams ] - for messages in itertools.izip(source, *streams): + for messages in zip(source, *streams): yield match_to_source(*messages) diff --git a/pokedex/lookup.py b/pokedex/lookup.py index 9c8f80b..15fc0c0 100644 --- a/pokedex/lookup.py +++ b/pokedex/lookup.py @@ -4,6 +4,7 @@ import random import re import unicodedata +from six import text_type import whoosh import whoosh.index import whoosh.query @@ -196,8 +197,8 @@ class PokedexLookup(object): q = self.session.query(cls).order_by(cls.id) for row in q: - row_key = dict(table=unicode(cls.__tablename__), - row_id=unicode(row.id)) + row_key = dict(table=text_type(cls.__tablename__), + row_id=text_type(row.id)) def add(name, language, iso639, iso3166): normalized_name = self.normalize_name(name) @@ -242,7 +243,7 @@ class PokedexLookup(object): # decompose! But the results are considered letters, not combining # characters, so testing for Mn works well, and combining them again # makes them look right. - nkfd_form = unicodedata.normalize('NFKD', unicode(name)) + nkfd_form = unicodedata.normalize('NFKD', text_type(name)) name = u"".join(c for c in nkfd_form if unicodedata.category(c) != 'Mn') name = unicodedata.normalize('NFC', name) @@ -292,8 +293,8 @@ class PokedexLookup(object): # And, just to complicate matters: "type" and language need to be # considered separately. def merge_requirements(func): - user = filter(func, user_valid_types) - system = filter(func, valid_types) + user = list(filter(func, user_valid_types)) + system = list(filter(func, valid_types)) if user and system: merged = list(set(user) & set(system)) @@ -463,7 +464,7 @@ class PokedexLookup(object): elif name_as_number is not None: # Don't spell-check numbers! exact_only = True - query = whoosh.query.Term(u'row_id', unicode(name_as_number)) + query = whoosh.query.Term(u'row_id', text_type(name_as_number)) else: # Not an integer query = whoosh.query.Term(u'name', name) @@ -547,7 +548,7 @@ class PokedexLookup(object): # n.b.: It's possible we got a list of valid_types and none of them # were valid, but this function is guaranteed to return # *something*, so it politely selects from the entire index instead - table_names = self.indexed_tables.keys() + table_names = list(self.indexed_tables) table_names.remove('pokemon_forms') # Pick a random table, then pick a random item from it. Small tables @@ -561,7 +562,7 @@ class PokedexLookup(object): .offset(random.randint(0, count - 1)) \ .first() - return self.lookup(unicode(id), valid_types=[table_name]) + return self.lookup(text_type(id), valid_types=[table_name]) def prefix_lookup(self, prefix, valid_types=[]): """Returns terms starting with the given exact prefix. diff --git a/pokedex/main.py b/pokedex/main.py index fd29593..180248a 100644 --- a/pokedex/main.py +++ b/pokedex/main.py @@ -1,4 +1,6 @@ # encoding: utf8 +from __future__ import print_function + from optparse import OptionParser import os import sys @@ -58,8 +60,8 @@ def get_session(options): session = pokedex.db.connect(engine_uri) if options.verbose: - print "Connected to database %(engine)s (from %(got_from)s)" \ - % dict(engine=session.bind.url, got_from=got_from) + print("Connected to database %(engine)s (from %(got_from)s)" + % dict(engine=session.bind.url, got_from=got_from)) return session @@ -78,8 +80,8 @@ def get_lookup(options, session=None, recreate=False): index_dir, got_from = defaults.get_default_index_dir_with_origin() if options.verbose: - print "Opened lookup index %(index_dir)s (from %(got_from)s)" \ - % dict(index_dir=index_dir, got_from=got_from) + print("Opened lookup index %(index_dir)s (from %(got_from)s)" + % dict(index_dir=index_dir, got_from=got_from)) lookup = pokedex.lookup.PokedexLookup(index_dir, session=session) @@ -100,8 +102,8 @@ def get_csv_directory(options): if csvdir is None: csvdir, got_from = defaults.get_default_csv_dir_with_origin() - print "Using CSV directory %(csvdir)s (from %(got_from)s)" \ - % dict(csvdir=csvdir, got_from=got_from) + print("Using CSV directory %(csvdir)s (from %(got_from)s)" + % dict(csvdir=csvdir, got_from=got_from)) return csvdir @@ -142,11 +144,11 @@ def command_load(*args): options, tables = parser.parse_args(list(args)) if not options.engine_uri: - print "WARNING: You're reloading the default database, but not the lookup index. They" - print " might get out of sync, and pokedex commands may not work correctly!" - print "To fix this, run `pokedex reindex` when this command finishes. Or, just use" - print "`pokedex setup` to do both at once." - print + print("WARNING: You're reloading the default database, but not the lookup index. They") + print(" might get out of sync, and pokedex commands may not work correctly!") + print("To fix this, run `pokedex reindex` when this command finishes. Or, just use") + print("`pokedex setup` to do both at once.") + print() if options.langs == 'none': langs = [] @@ -173,7 +175,7 @@ def command_reindex(*args): session = get_session(options) lookup = get_lookup(options, session=session, recreate=True) - print "Recreated lookup index." + print("Recreated lookup index.") def command_setup(*args): @@ -190,7 +192,7 @@ def command_setup(*args): lookup = get_lookup(options, session=session, recreate=True) - print "Recreated lookup index." + print("Recreated lookup index.") def command_status(*args): @@ -201,37 +203,37 @@ def command_status(*args): # Database, and a lame check for whether it's been inited at least once session = get_session(options) - print " - OK! Connected successfully." + print(" - OK! Connected successfully.") if pokedex.db.tables.Pokemon.__table__.exists(session.bind): - print " - OK! Database seems to contain some data." + print(" - OK! Database seems to contain some data.") else: - print " - WARNING: Database appears to be empty." + print(" - WARNING: Database appears to be empty.") # CSV; simple checks that the dir exists csvdir = get_csv_directory(options) if not os.path.exists(csvdir): - print " - ERROR: No such directory!" + print(" - ERROR: No such directory!") elif not os.path.isdir(csvdir): - print " - ERROR: Not a directory!" + print(" - ERROR: Not a directory!") else: - print " - OK! Directory exists." + print(" - OK! Directory exists.") if os.access(csvdir, os.R_OK): - print " - OK! Can read from directory." + print(" - OK! Can read from directory.") else: - print " - ERROR: Can't read from directory!" + print(" - ERROR: Can't read from directory!") if os.access(csvdir, os.W_OK): - print " - OK! Can write to directory." + print(" - OK! Can write to directory.") else: - print " - WARNING: Can't write to directory! " \ - "`dump` will not work. You may need to sudo." + print(" - WARNING: Can't write to directory! " + "`dump` will not work. You may need to sudo.") # Index; the PokedexLookup constructor covers most tests and will # cheerfully bomb if they fail lookup = get_lookup(options, recreate=False) - print " - OK! Opened successfully." + print(" - OK! Opened successfully.") ### User-facing commands @@ -247,11 +249,11 @@ def command_lookup(*args): results = lookup.lookup(name) if not results: - print "No matches." + print("No matches.") elif results[0].exact: - print "Matched:" + print("Matched:") else: - print "Fuzzy-matched:" + print("Fuzzy-matched:") for result in results: if hasattr(result.object, 'full_name'): @@ -259,15 +261,15 @@ def command_lookup(*args): else: name = result.object.name - print "%s: %s" % (result.object.__tablename__, name), + print("%s: %s" % (result.object.__tablename__, name), end='') if result.language: - print "(%s in %s)" % (result.name, result.language) + print("(%s in %s)" % (result.name, result.language)) else: - print + print() def command_help(): - print u"""pokedex -- a command-line Pokédex interface + print(u"""pokedex -- a command-line Pokédex interface usage: pokedex {command} [options...] Run `pokedex setup` first, or nothing will work! See https://github.com/veekun/pokedex/wiki/CLI for more documentation. @@ -319,7 +321,7 @@ Dump options: Additionally, load and dump accept a list of table names (possibly with wildcards) and/or csv fileames as an argument list. -""".encode(sys.getdefaultencoding(), 'replace') +""".encode(sys.getdefaultencoding(), 'replace')) sys.exit(0) diff --git a/pokedex/roomaji.py b/pokedex/roomaji.py index aa378ac..5844a36 100644 --- a/pokedex/roomaji.py +++ b/pokedex/roomaji.py @@ -128,7 +128,7 @@ class Romanizer(object): if last_kana == 'sokuon': raise ValueError("Sokuon cannot be the last character.") - return unicode(''.join(characters)) + return u''.join(characters) romanizers = dict() diff --git a/pokedex/struct/_pokemon_struct.py b/pokedex/struct/_pokemon_struct.py index 79ff0e0..ba67eff 100644 --- a/pokedex/struct/_pokemon_struct.py +++ b/pokedex/struct/_pokemon_struct.py @@ -467,7 +467,7 @@ character_table = { # And the reverse dict, used with str.translate() inverse_character_table = dict() -for in_, out in character_table.iteritems(): +for in_, out in character_table.items(): inverse_character_table[ord(out)] = in_ diff --git a/pokedex/tests/test_database_sanity.py b/pokedex/tests/test_database_sanity.py index 8614bc2..276491b 100644 --- a/pokedex/tests/test_database_sanity.py +++ b/pokedex/tests/test_database_sanity.py @@ -53,7 +53,6 @@ def test_unique_form_order(session): query = query.options(joinedload('pokemon.species')) for form in query: - print form.name try: previous_species = species_by_form_order[form.order] except KeyError: diff --git a/pokedex/tests/test_media.py b/pokedex/tests/test_media.py index 3015585..5c7a2fe 100644 --- a/pokedex/tests/test_media.py +++ b/pokedex/tests/test_media.py @@ -4,6 +4,7 @@ If run directly from the command line, also tests the accessors and the names of all the media by getting just about everything in a naive brute-force way. This, of course, takes a lot of time to run. """ +from __future__ import print_function import pytest @@ -49,7 +50,6 @@ def test_strict_castform(session, media_root): with pytest.raises(ValueError): castform = session.query(tables.PokemonSpecies).filter_by(identifier=u'castform').first() rainy_castform = [f for f in castform.forms if f.form_identifier == 'rainy'][0] - print rainy_castform rainy_castform = media.PokemonFormMedia(media_root, rainy_castform) rainy_castform.overworld('up', strict=True) @@ -81,13 +81,11 @@ def hit(filenames, method, *args, **kwargs): """ try: medium = method(*args, **kwargs) - #print 'Hit', medium.relative_path assert medium.exists - except ValueError, e: - #print 'DNF', e + except ValueError: return False except: - print 'Error while processing', method, args, kwargs + print('Error while processing', method, args, kwargs) raise try: filenames.remove(medium.path) @@ -242,9 +240,9 @@ def test_get_everything(session, media_root): unaccessed_filenames.remove(filename) if unaccessed_filenames: - print 'Unaccessed files:' + print('Unaccessed files:') for filename in unaccessed_filenames: - print filename + print(filename) assert unaccessed_filenames == set() diff --git a/pokedex/tests/test_schema.py b/pokedex/tests/test_schema.py index 2288ac9..736e960 100644 --- a/pokedex/tests/test_schema.py +++ b/pokedex/tests/test_schema.py @@ -38,7 +38,6 @@ def test_class_order(): class_names = [table.__name__ for table in tables.mapped_classes] def key(name): return name != 'Language', name - print [(a,b) for (a,b) in zip(class_names, sorted(class_names, key=key)) if a!=b] assert class_names == sorted(class_names, key=key) def test_i18n_table_creation(): diff --git a/pokedex/tests/test_strings.py b/pokedex/tests/test_strings.py index 4d14818..0517460 100644 --- a/pokedex/tests/test_strings.py +++ b/pokedex/tests/test_strings.py @@ -88,15 +88,15 @@ def test_markdown(session): assert '10%' in move.effect_map[language].as_text() assert '10%' in move.effect.as_html() assert '10%' in move.effect_map[language].as_html() - assert '10%' in unicode(move.effect) - assert '10%' in unicode(move.effect_map[language]) + assert '10%' in str(move.effect) + assert '10%' in str(move.effect_map[language]) assert '10%' in move.effect.__html__() assert '10%' in move.effect_map[language].__html__() def test_markdown_string(session): en = util.get(session, tables.Language, 'en') md = markdown.MarkdownString('[]{move:thunderbolt} [paralyzes]{mechanic:paralysis} []{form:sky shaymin}. []{pokemon:mewthree} does not exist.', session, en) - assert unicode(md) == 'Thunderbolt paralyzes Sky Shaymin. mewthree does not exist.' + assert str(md) == 'Thunderbolt paralyzes Sky Shaymin. mewthree does not exist.' assert md.as_html() == '

Thunderbolt paralyzes Sky Shaymin. mewthree does not exist.

' class ObjectTestExtension(markdown.PokedexLinkExtension): diff --git a/pokedex/tests/test_translations.py b/pokedex/tests/test_translations.py index 78c9ec2..19c6356 100644 --- a/pokedex/tests/test_translations.py +++ b/pokedex/tests/test_translations.py @@ -112,7 +112,6 @@ def test_merge_translations(): for result, expected in zip(result_stream, expected_list): res_src, res_crc, res_str, res_match = result exp_src, exp_match, exp_str = expected - print result, expected assert res_src.string == exp_src assert res_str == exp_str, (res_str, exp_str) if exp_match is None: @@ -122,7 +121,6 @@ def test_merge_translations(): elif exp_match is False: assert res_crc == translations.crc('----') assert res_match == exp_match - print 'unused:', unused for message in unused: assert message.string == 'unused' assert message.id == 100 diff --git a/setup.py b/setup.py index 896a9f1..4373e13 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ setup( 'whoosh>=2.5,<2.7', 'markdown', 'construct', + 'six>=1.9.0', ], entry_points = {