1.5.4dev: fix message extraction failing with Python 3, Babel and Jin…

…ja2 3+ (closes #13479)

git-svn-id: http://trac.edgewall.org/intertrac/log:/trunk@17594 af82e41b-90c4-0310-8c96-b1721e28e2e2

messages.cfg

@@ -6,7 +6,6 @@
 [python: **.py]
 
 [html: **/templates/**.html]
-extensions = jinja2.ext.do, jinja2.ext.with_
 variable_start_string = ${
 variable_end_string = }
 line_statement_prefix = #
@@ -17,7 +16,6 @@ newstyle_gettext = yes
 silent = no
 
 [text: **/templates/**.txt]
-extensions = jinja2.ext.do, jinja2.ext.with_
 variable_start_string = ${
 variable_end_string = }
 line_statement_prefix = #

trac/dist.py

@@ -24,9 +24,11 @@
 from html.parser import HTMLParser
 import io
 import os
+import pkg_resources
 import re
 from tokenize import generate_tokens, COMMENT, NAME, OP, STRING
 
+import jinja2
 from jinja2.ext import babel_extract as jinja2_extractor
 
 from distutils import log as distlog
@@ -36,6 +38,10 @@
 from setuptools.command.install_lib import install_lib as _install_lib
 
 
+_jinja2_ext_with = pkg_resources.parse_version(jinja2.__version__) < \
+                   pkg_resources.parse_version('3')
+
+
 def simplify_message(message):
     """Transforms an extracted messsage (string or tuple) into one in
     which the repeated white-space has been simplified to a single
@@ -143,7 +149,7 @@ def extract_python(fileobj, keywords, comment_tags, options):
         if 'cleandoc_keywords' in options:
             cleandoc_keywords.update(options['cleandoc_keywords'])
 
-        tokens = generate_tokens(fileobj.readline)
+        tokens = generate_tokens(lambda: fileobj.readline().decode(encoding))
         tok = value = None
         for _ in tokens:
             prev_tok, prev_value = tok, value
@@ -166,7 +172,7 @@ def extract_python(fileobj, keywords, comment_tags, options):
                 continue
             elif call_stack == -1 and tok == COMMENT:
                 # Strip the comment token from the line
-                value = value.decode(encoding)[1:].strip()
+                value = value[1:].strip()
                 if in_translator_comments and \
                         translator_comments[-1][0] == lineno - 1:
                     # We're already inside a translator comment, continue
@@ -274,7 +280,8 @@ def extract_javascript_script(fileobj, keywords, comment_tags, options):
         extractor = ScriptExtractor(out)
         extractor.feed(str(fileobj.read(), 'utf-8'))
         extractor.close()
-        out.seek(0)
+        # extract_javascript expects a binary file object
+        out = io.BytesIO(out.getvalue().encode('utf-8'))
         return extract_javascript(out, keywords, comment_tags, options)
 
 
@@ -292,6 +299,9 @@ def extract_html(fileobj, keywords, comment_tags, options):
         """
         if fileobj:
             extractor = jinja2_extractor
+            options.setdefault('extensions', 'jinja2.ext.do, jinja2.ext.with_'
+                                             if _jinja2_ext_with else
+                                             'jinja2.ext.do')
             fileobj.seek(0)
             for m in extractor(fileobj, keywords, comment_tags, options):
                 # lineno, func, message, comments = m