Add travis / codecov

.gitignore

@@ -5,6 +5,7 @@ cover/
 *.egg
 /env/
 /htmlcov/
+.idea/
 .project
 *.pyc
 .pydevproject

.travis.yml

@@ -0,0 +1,24 @@
+sudo: required
+language: python
+env:
+  # newest everything
+  - CDH=cdh5 PRESTO=0.118 SQLALCHEMY=1.0.8
+  # stale stuff we're still actively using
+  - CDH=cdh4 PRESTO=0.97 SQLALCHEMY=0.5.8
+  # every version of sqlalchemy with special code
+  - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.5.8
+  - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.6.9
+  - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.7.10
+  - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.8.7
+  - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=1.0.8
+python:
+  - "2.7"
+before_install:
+  - ./scripts/travis-install.sh
+  - pip install codecov
+install:
+  - pip install -e .
+  - pip install sqlalchemy==$SQLALCHEMY
+  - pip install -r dev_requirements.txt
+script: py.test
+after_success: codecov

README.rst

@@ -48,6 +48,10 @@ There's also a `third party Conda package <https://binstar.org/blaze/pyhive>`_.
 
 Testing
 =======
+.. image:: https://travis-ci.org/dropbox/PyHive.svg
+    :target: https://travis-ci.org/dropbox/PyHive
+.. image:: http://codecov.io/github/dropbox/PyHive/coverage.svg?branch=master
+    :target: http://codecov.io/github/dropbox/PyHive?branch=master
 
 Run the following in an environment with Hive/Presto::
 

dev_requirements.txt

@@ -4,5 +4,6 @@ pytest-cov
 pytest-random
 requests>=1.0.0
 sasl>=0.1.3
-sqlalchemy>=0.9.4
+sqlalchemy>=0.5.8
 thrift>=0.8.0
+thrift_sasl>=0.1.0

pyhive/sqlalchemy_hive.py

@@ -250,7 +250,8 @@ def get_columns(self, connection, table_name, schema=None, **kw):
                 break
             # Take out the more detailed type information
             # e.g. 'map<int,int>' -> 'map'
-            col_type = col_type.partition('<')[0]
+            #      'decimal(10,1)' -> decimal
+            col_type = re.search(r'^\w+', col_type).group(0)
             try:
                 coltype = _type_map[col_type]
             except KeyError:
@@ -309,7 +310,7 @@ def _check_unicode_description(self, connection):
         # We decode everything as UTF-8
         return True
 
-if StrictVersion(sqlalchemy.__version__) < StrictVersion('0.6.0'):
+if StrictVersion(sqlalchemy.__version__) < StrictVersion('0.7.0'):
     from pyhive import sqlalchemy_backports
 
     def reflecttable(self, connection, table, include_columns=None, exclude_columns=None):

pyhive/sqlalchemy_presto.py

@@ -165,7 +165,7 @@ def _check_unicode_description(self, connection):
         # requests gives back Unicode strings
         return True
 
-if StrictVersion(sqlalchemy.__version__) < StrictVersion('0.6.0'):
+if StrictVersion(sqlalchemy.__version__) < StrictVersion('0.7.0'):
     from pyhive import sqlalchemy_backports
 
     def reflecttable(self, connection, table, include_columns=None, exclude_columns=None):

pyhive/tests/sqlachemy_test_case.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import absolute_import
 from __future__ import unicode_literals
+from distutils.version import StrictVersion
 import sqlalchemy
 from sqlalchemy.exc import NoSuchTableError
 from sqlalchemy.schema import Index
@@ -10,6 +11,7 @@
 import abc
 import contextlib
 import functools
+import unittest
 
 
 def with_engine_connection(fn):
@@ -95,13 +97,17 @@ def test_unicode(self, engine, connection):
         ).scalar()
         self.assertEqual(returned_str, unicode_str)
 
+    @unittest.skipIf(StrictVersion(sqlalchemy.__version__) < StrictVersion('0.8.0'),
+                     "inspect not available yet")
     @with_engine_connection
     def test_reflect_schemas(self, engine, connection):
         insp = sqlalchemy.inspect(engine)
         schemas = insp.get_schema_names()
         self.assertIn('pyhive_test_database', schemas)
         self.assertIn('default', schemas)
 
+    @unittest.skipIf(StrictVersion(sqlalchemy.__version__) < StrictVersion('0.8.0'),
+                     "inspect not available yet")
     @with_engine_connection
     def test_get_table_names(self, engine, connection):
         meta = MetaData()
@@ -120,6 +126,8 @@ def test_has_table(self, engine, connection):
         self.assertTrue(Table('one_row', MetaData(bind=engine)).exists())
         self.assertFalse(Table('this_table_does_not_exist', MetaData(bind=engine)).exists())
 
+    @unittest.skipIf(StrictVersion(sqlalchemy.__version__) < StrictVersion('0.6.0'),
+                     "visitor stuff for changing char_length -> length not available yet")
     @with_engine_connection
     def test_char_length(self, engine, connection):
         one_row_complex = Table('one_row_complex', MetaData(bind=engine), autoload=True)

pyhive/tests/test_hive.py

@@ -12,6 +12,7 @@
 from pyhive.tests.dbapi_test_case import with_cursor
 import contextlib
 import mock
+import os
 import unittest
 
 _HOST = 'localhost'
@@ -26,14 +27,14 @@ def connect(self):
     @with_cursor
     def test_description(self, cursor):
         cursor.execute('SELECT * FROM one_row')
+
         desc = [('number_of_rows', 'INT_TYPE', None, None, None, None, True)]
         self.assertEqual(cursor.description, desc)
-        self.assertEqual(cursor.description, desc)
 
     @with_cursor
     def test_complex(self, cursor):
         cursor.execute('SELECT * FROM one_row_complex')
-        # TODO Presto drops the union and decimal fields
+        stringly_typed = 'STRING_TYPE' if os.environ.get('CDH') == 'cdh4' else None
         self.assertEqual(cursor.description, [
             ('boolean', 'BOOLEAN_TYPE', None, None, None, None, True),
             ('tinyint', 'TINYINT_TYPE', None, None, None, None, True),
@@ -45,10 +46,10 @@ def test_complex(self, cursor):
             ('string', 'STRING_TYPE', None, None, None, None, True),
             ('timestamp', 'TIMESTAMP_TYPE', None, None, None, None, True),
             ('binary', 'BINARY_TYPE', None, None, None, None, True),
-            ('array', 'STRING_TYPE', None, None, None, None, True),
-            ('map', 'STRING_TYPE', None, None, None, None, True),
-            ('struct', 'STRING_TYPE', None, None, None, None, True),
-            ('union', 'STRING_TYPE', None, None, None, None, True),
+            ('array', stringly_typed or 'ARRAY_TYPE', None, None, None, None, True),
+            ('map', stringly_typed or 'MAP_TYPE', None, None, None, None, True),
+            ('struct', stringly_typed or 'STRUCT_TYPE', None, None, None, None, True),
+            ('union', stringly_typed or 'UNION_TYPE', None, None, None, None, True),
             ('decimal', 'DECIMAL_TYPE', None, None, None, None, True),
         ])
         self.assertEqual(cursor.fetchall(), [[
@@ -93,13 +94,18 @@ def test_escape(self):
 
     def test_newlines(self):
         """Verify that newlines are passed through in a way that doesn't fail parsing"""
-        # Hive thrift translates newlines into multiple rows. WTF.
         cursor = self.connect().cursor()
+        orig = ' \r\n \r \n '
         cursor.execute(
             'SELECT %s FROM one_row',
-            (' \r\n \r \n ',)
+            (orig,)
         )
-        self.assertEqual(cursor.fetchall(), [[' '], [' '], [' '], [' ']])
+        result = cursor.fetchall()
+        if os.environ.get('CDH') == 'cdh4':
+            # Hive thrift translates newlines into multiple rows. WTF.
+            self.assertEqual(result, [[' '], [' '], [' '], [' ']])
+        else:
+            self.assertEqual(result, [[orig]])
 
     @with_cursor
     def test_no_result_set(self, cursor):

pyhive/tests/test_sqlalchemy_hive.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
+from distutils.version import StrictVersion
 from pyhive.sqlalchemy_hive import HiveDate
 from pyhive.sqlalchemy_hive import HiveDecimal
 from pyhive.sqlalchemy_hive import HiveTimestamp
@@ -12,6 +13,7 @@
 import contextlib
 import datetime
 import decimal
+import os
 import sqlalchemy.types
 import unittest
 
@@ -82,6 +84,8 @@ def test_switch_database(self):
         finally:
             engine.dispose()
 
+    @unittest.skipIf(StrictVersion(sqlalchemy.__version__) < StrictVersion('0.7.0'),
+                     "features not available yet")
     @with_engine_connection
     def test_lots_of_types(self, engine, connection):
         # Presto doesn't have raw CREATE TABLE support, so we ony test hive
@@ -105,6 +109,7 @@ def test_lots_of_types(self, engine, connection):
         table.create()
         connection.execute('SET mapred.job.tracker=local')
         connection.execute('USE pyhive_test_database')
+        big_number = 10 ** 10 - 1
         connection.execute("""
         INSERT OVERWRITE TABLE test_table
         SELECT
@@ -114,15 +119,17 @@ def test_lots_of_types(self, engine, connection):
             "a", 1, 1,
             0.1, 0.1, 0, 0, 0, "a",
             false, "a", "a",
-            0, 0.1, 123 + 2000
+            0, %d, 123 + 2000
         FROM default.one_row
-        """)
+        """, big_number)
         row = connection.execute(table.select()).fetchone()
         self.assertEqual(row.hive_date, datetime.date(1970, 1, 1))
-        self.assertEqual(row.hive_decimal, decimal.Decimal('0.1'),)
+        self.assertEqual(row.hive_decimal, decimal.Decimal(big_number))
         self.assertEqual(row.hive_timestamp, datetime.datetime(1970, 1, 1, 0, 0, 2, 123))
         table.drop()
 
+    @unittest.skipIf(StrictVersion(sqlalchemy.__version__) < StrictVersion('0.8.0'),
+                     "from_select not available yet")
     @with_engine_connection
     def test_insert_select(self, engine, connection):
         one_row = Table('one_row', MetaData(bind=engine), autoload=True)
@@ -139,6 +146,7 @@ def test_insert_select(self, engine, connection):
         expected = [(1,)]
         self.assertEqual(result, expected)
 
+    @unittest.skipIf(os.environ.get('CDH') == 'cdh4', "not supported on hive 0.10")
     @with_engine_connection
     def test_insert_values(self, engine, connection):
         table = Table('insert_test', MetaData(bind=engine),
@@ -151,3 +159,8 @@ def test_insert_values(self, engine, connection):
         result = table.select().execute().fetchall()
         expected = [(1,), (2,)]
         self.assertEqual(result, expected)
+
+    @unittest.skipIf(os.environ.get('CDH') == 'cdh4',
+                     "Hive 0.10 doesn't distinguish partition columns in DESCRIBE")
+    def test_reflect_partitions(self):
+        super(TestSqlAlchemyHive, self).test_reflect_partitions()

scripts/make_one_row_complex.sh

@@ -1,4 +1,12 @@
 #!/bin/bash -eux
+
+if [ "${CDH:-}" == 'cdh4' ]
+then
+    DECIMAL_TYPE='DECIMAL'
+else
+    DECIMAL_TYPE='DECIMAL(10,1)'
+fi
+
 hive -e '
 set mapred.job.tracker=local;
 DROP TABLE IF EXISTS one_row_complex;
@@ -17,7 +25,7 @@ CREATE TABLE one_row_complex (
     `map` MAP<int, int>,
     `struct` STRUCT<a: int, b: int>,
     `union` UNIONTYPE<int, string>,
-    `decimal` DECIMAL
+    `decimal` '"$DECIMAL_TYPE"'
 );
 INSERT OVERWRITE TABLE one_row_complex SELECT
     true,

scripts/travis-conf/hive/hive-site.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>hive.metastore.uris</name>
+    <value>thrift://localhost:9083</value>
+  </property>
+  <property>
+    <name>javax.jdo.option.ConnectionURL</name>
+    <value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value>
+  </property>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>file:///</value>
+  </property>
+  <property>
+    <name>hive.metastore.warehouse.dir</name>
+    <value>/tmp/hive/warehouse</value>
+  </property>
+  <!--
+  TODO tests rely having result set column names unprefixed
+  This could be improved by having an option to strip out prefixes when it would not result in
+  ambiguity.
+  -->
+  <property>
+    <name>hive.resultset.use.unique.column.names</name>
+    <value>false</value>
+  </property>
+</configuration>

scripts/travis-conf/presto/catalog/hive.properties

@@ -0,0 +1,2 @@
+connector.name=hive-%CDH%
+hive.metastore.uri=thrift://localhost:9083

scripts/travis-conf/presto/config.properties

@@ -0,0 +1,7 @@
+coordinator=true
+node-scheduler.include-coordinator=true
+http-server.http.port=8080
+query.max-memory=100MB
+query.max-memory-per-node=100MB
+discovery-server.enabled=true
+discovery.uri=http://localhost:8080

scripts/travis-conf/presto/node.properties

@@ -0,0 +1,3 @@
+node.environment=production
+node.id=ffffffff-ffff-ffff-ffff-ffffffffffff
+node.data-dir=/tmp/presto/data

scripts/travis-install.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -eux
+
+sudo wget "http://archive.cloudera.com/$CDH/ubuntu/precise/amd64/cdh/cloudera.list" \
+    -O /etc/apt/sources.list.d/cloudera.list
+# work around broken list
+sudo sed -i 's mirror.infra.cloudera.com/archive archive.cloudera.com g' \
+    /etc/apt/sources.list.d/cloudera.list
+sudo apt-get update
+
+#
+# Hive
+#
+
+sudo apt-get install -y --force-yes hive
+sudo cp $(dirname $0)/travis-conf/hive/* /etc/hive/conf
+sudo -u hive mkdir /tmp/hive && sudo chmod 777 /tmp/hive
+sudo apt-get install -y --force-yes hive-metastore hive-server2
+
+sudo -Eu hive $(dirname $0)/make_test_tables.sh
+
+#
+# Presto
+#
+
+sudo apt-get install -y oracle-java8-installer
+sudo update-java-alternatives -s java-8-oracle
+
+curl https://repo1.maven.org/maven2/com/facebook/presto/presto-server/$PRESTO/presto-server-$PRESTO.tar.gz \
+    | tar zxf -
+
+cp -r $(dirname $0)/travis-conf/presto presto-server-$PRESTO/etc
+sed -i s/%CDH%/$CDH/g presto-server-$PRESTO/etc/catalog/hive.properties
+
+if [ "$PRESTO" == "0.97" ]
+then
+    sed -i '/query.max-memory/d' presto-server-$PRESTO/etc/config.properties
+fi
+
+./presto-server-$PRESTO/bin/launcher start