From 3583d4e3184eed6f6026949c51795c156220a093 Mon Sep 17 00:00:00 2001 From: Yuval Langer Date: Tue, 18 Apr 2017 18:26:28 +0300 Subject: [PATCH 1/2] Standard library imports goes before other imports --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 651c086..25881e8 100755 --- a/setup.py +++ b/setup.py @@ -1,13 +1,13 @@ #!/usr/bin/env python +import os + from distutils.core import setup from distutils.extension import Extension from distutils.command import build_ext from my_build_ext import build_ext as _link_build_ext from distutils.util import strtobool -import os - EMBED = strtobool(os.environ.get("EMBED", "1")) if not os.path.exists("fribidi-src"): From d143a3b5557e0a4b3ecd2edd163df8c126ec7f2f Mon Sep 17 00:00:00 2001 From: Yuval Langer Date: Wed, 19 Apr 2017 04:24:56 +0300 Subject: [PATCH 2/2] Port to Py2/3. Test passes all but 2 in Py3 and all but 1 in Py2. --- pyfribidi.c | 66 +++++++++++++++++++++++++++++++++++++++------ pyfribidi.py | 5 ++-- requirements-27.txt | 2 ++ requirements-36.txt | 2 ++ setup.py | 5 ++-- test_pyfribidi.py | 16 +++++++---- time_pyfribidi.py | 20 +++++++------- 7 files changed, 90 insertions(+), 26 deletions(-) create mode 100644 requirements-27.txt create mode 100644 requirements-36.txt diff --git a/pyfribidi.c b/pyfribidi.c index 82c4092..67f0913 100644 --- a/pyfribidi.c +++ b/pyfribidi.c @@ -21,17 +21,26 @@ #include #include +#if PY_MAJOR_VERSION >= 3 +#define PYFRIBIDI_UNICODE_OBJECT PyObject +#else +#define PYFRIBIDI_UNICODE_OBJECT PyUnicodeObject +#endif static PyObject * -unicode_log2vis (PyUnicodeObject* string, +unicode_log2vis (PYFRIBIDI_UNICODE_OBJECT * string, FriBidiParType base_direction, int clean, int reordernsm) { int i; +#if PY_MAJOR_VERSION >= 3 + Py_ssize_t length = PyUnicode_GetLength(string); +#else int length = string->length; +#endif FriBidiChar *logical = NULL; /* input fribidi unicode buffer */ FriBidiChar *visual = NULL; /* output fribidi unicode buffer */ - FriBidiStrIndex new_len = 0; /* length of the UTF-8 buffer */ - PyUnicodeObject *result = NULL; + // FriBidiStrIndex new_len = 0; /* length of the UTF-8 buffer */ + PYFRIBIDI_UNICODE_OBJECT *result = NULL; /* Allocate fribidi unicode buffers TODO - Don't copy strings if sizeof(FriBidiChar) == sizeof(Py_UNICODE) @@ -49,9 +58,14 @@ unicode_log2vis (PyUnicodeObject* string, goto cleanup; } +#if PY_MAJOR_VERSION >= 3 + int number_of_copied_wchar = PyUnicode_AsWideChar(string, logical, length); + //logical[number_of_copied_wchar] = (wchar_t) NULL; +#else for (i=0; istr[i]; } +#endif /* Convert to unicode and order visually */ fribidi_set_reorder_nsm(reordernsm); @@ -69,14 +83,21 @@ unicode_log2vis (PyUnicodeObject* string, length = fribidi_remove_bidi_marks (visual, length, NULL, NULL, NULL); } - result = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, length); +#if PY_MAJOR_VERSION >= 3 + result = PyUnicode_FromWideChar(visual, length); + +#else + result = (PYFRIBIDI_UNICODE_OBJECT *) PyUnicode_FromUnicode(NULL, length); +#endif if (result == NULL) { goto cleanup; } - +#if PY_MAJOR_VERSION >= 3 +#else for (i=0; istr[i] = visual[i]; } +#endif cleanup: /* Delete unicode buffers */ @@ -89,7 +110,7 @@ unicode_log2vis (PyUnicodeObject* string, static PyObject * _pyfribidi_log2vis (PyObject * self, PyObject * args, PyObject * kw) { - PyUnicodeObject *logical = NULL; /* input unicode or string object */ + PYFRIBIDI_UNICODE_OBJECT *logical = NULL; /* input unicode or string object */ FriBidiParType base = FRIBIDI_TYPE_RTL; /* optional direction */ int clean = 0; /* optional flag to clean the string */ int reordernsm = 1; /* optional flag to allow reordering of non spacing marks*/ @@ -101,7 +122,6 @@ _pyfribidi_log2vis (PyObject * self, PyObject * args, PyObject * kw) &logical, &base, &clean, &reordernsm)) { return NULL; } - /* Validate base */ if (!(base == FRIBIDI_TYPE_RTL @@ -112,7 +132,9 @@ _pyfribidi_log2vis (PyObject * self, PyObject * args, PyObject * kw) base); } - return unicode_log2vis (logical, base, clean, reordernsm); + PyObject * return_pyobject = unicode_log2vis (logical, base, clean, reordernsm); + + return return_pyobject; } @@ -121,12 +143,40 @@ static PyMethodDef PyfribidiMethods[] = { {NULL, NULL, 0, NULL} }; +#if PY_MAJOR_VERSION >= 3 +#define INITERROR return NULL + +static struct PyModuleDef pyfribidi_moduledef = { + PyModuleDef_HEAD_INIT, /* m_base */ + "pyfribidi", /* m_name */ + NULL, /* m_doc */ + -1, /* m_size == -1 means cannot be reinitialized */ + PyfribidiMethods, // + NULL, // myextension_traverse, + NULL, // myextension_clear, + NULL +}; PyMODINIT_FUNC +PyInit__pyfribidi (void) +#else +#define INITERROR return + +void init_pyfribidi (void) +#endif { +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create (&pyfribidi_moduledef); // "_pyfribidi", PyfribidiMethods); +#else PyObject *module = Py_InitModule ("_pyfribidi", PyfribidiMethods); +#endif + if (module == NULL) INITERROR; + PyModule_AddIntConstant (module, "RTL", (long) FRIBIDI_TYPE_RTL); PyModule_AddIntConstant (module, "LTR", (long) FRIBIDI_TYPE_LTR); PyModule_AddIntConstant (module, "ON", (long) FRIBIDI_TYPE_ON); +#if PY_MAJOR_VERSION >= 3 + return module; +#endif } diff --git a/pyfribidi.py b/pyfribidi.py index 9a5de26..311add2 100644 --- a/pyfribidi.py +++ b/pyfribidi.py @@ -5,6 +5,7 @@ logical order, but the conversion may be wrong in certain cases. """ +from builtins import str __version__ = "0.11.0" from _pyfribidi import LTR, ON, RTL, log2vis as _log2vis @@ -25,8 +26,8 @@ def log2vis(logical, base_direction=RTL, encoding="utf-8", clean=False, reordern - encoding: optional string encoding (ignored for unicode input) """ - if not isinstance(logical, unicode): - logical = unicode(logical, encoding) + if not isinstance(logical, str): + logical = str(logical, encoding) else: encoding = None res = _log2vis(logical, base_direction=base_direction, clean=clean, reordernsm=reordernsm) diff --git a/requirements-27.txt b/requirements-27.txt new file mode 100644 index 0000000..1232594 --- /dev/null +++ b/requirements-27.txt @@ -0,0 +1,2 @@ +future==0.16.0 +six==1.10.0 diff --git a/requirements-36.txt b/requirements-36.txt new file mode 100644 index 0000000..1232594 --- /dev/null +++ b/requirements-36.txt @@ -0,0 +1,2 @@ +future==0.16.0 +six==1.10.0 diff --git a/setup.py b/setup.py index 25881e8..42d9fa0 100755 --- a/setup.py +++ b/setup.py @@ -1,13 +1,14 @@ #!/usr/bin/env python +from past.builtins import execfile import os - from distutils.core import setup from distutils.extension import Extension from distutils.command import build_ext -from my_build_ext import build_ext as _link_build_ext from distutils.util import strtobool +from my_build_ext import build_ext as _link_build_ext + EMBED = strtobool(os.environ.get("EMBED", "1")) if not os.path.exists("fribidi-src"): diff --git a/test_pyfribidi.py b/test_pyfribidi.py index b74fe5d..7ff9101 100755 --- a/test_pyfribidi.py +++ b/test_pyfribidi.py @@ -8,6 +8,7 @@ import sys import unittest + import pyfribidi from pyfribidi import RTL, LTR, ON @@ -21,7 +22,7 @@ class InputTests(unittest.TestCase): def testRequireInputString(self): """ input: require input string or unicode """ self.assertRaises(TypeError, pyfribidi.log2vis) - + def testInvalidInputString(self): """ input: raise TypeError for non string or unicode input """ self.assertRaises(TypeError, pyfribidi.log2vis, 1) @@ -40,8 +41,8 @@ def testInvalidEncodedString(self): """ input: raise UnicodeError for invalid encoded string """ self.assertRaises(UnicodeError, pyfribidi.log2vis, "שלום", encoding='iso8859-8') - - + + class UnicodeTests(unittest.TestCase): def testEmpty(self): @@ -176,10 +177,15 @@ def testUTF16NaturalRTL(self): encoding=charset), u"hello - םולש".encode(charset)) + class Crasher(unittest.TestCase): def test_glibc_free_invalid_next_size(self): - # *** glibc detected *** /home/ralf/py27/bin/python2: free(): invalid next size (fast): 0x00000000011cff00 *** - pyfribidi.log2vis('\xf0\x90\x8e\xa2\xf0\x90\x8e\xaf\xf0\x90\x8e\xb4\xf0\x90\x8e\xa1\xf0\x90\x8f\x83') + # *** glibc detected *** /home/ralf/py27/bin/python2: free(): invalid + # next size (fast): 0x00000000011cff00 *** + pyfribidi.log2vis( + '\xf0\x90\x8e\xa2\xf0\x90\x8e\xaf\xf0\x90\x8e\xb4\xf0\x90\x8e\xa1' + '\xf0\x90\x8f\x83') + if __name__ == '__main__': suite = unittest.defaultTestLoader.loadTestsFromName(__name__) diff --git a/time_pyfribidi.py b/time_pyfribidi.py index e5a0b58..805d2f6 100644 --- a/time_pyfribidi.py +++ b/time_pyfribidi.py @@ -3,7 +3,9 @@ """ time pyfribidi with all hebrew encodings. """ +from __future__ import print_function +from builtins import range import syspath import timeit import pyfribidi @@ -30,24 +32,24 @@ def timeEncoding(encoding, tests): timer = timeit.Timer(code, setup) seconds = timer.timeit(number=tests) microseconds = 1000000 * seconds / tests - print "%12s: %.8f seconds (%.2f usec/pass)" % (encoding, seconds, - microseconds) + print("%12s: %.8f seconds (%.2f usec/pass)" % (encoding, seconds, + microseconds)) # warm up caches -for i in xrange(100000): +for i in range(100000): pyfribidi.log2vis(u'Some text to warm up the caches') lines = 50 # typical screen of text -print -print 'time to rerorder %s lines:' % lines -print +print() +print('time to rerorder %s lines:' % lines) +print() for encoding in hebrew_encodings: timeEncoding(encoding, lines) lines = 100000 -print -print 'time to rerorder %s lines:' % lines -print +print() +print('time to rerorder %s lines:' % lines) +print() for encoding in hebrew_encodings: timeEncoding(encoding, lines)