Test fixing Long type in theschema and int32 parquet type

lyft · Oct 12, 2021 · dc38be5 · dc38be5
1 parent c94d53a
commit dc38be5
Showing 1 changed file with 2 additions and 5 deletions.
diff --git a/.../java/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorUpdaterFactory.java b/.../java/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorUpdaterFactory.java
@@ -72,11 +72,8 @@ public ParquetVectorUpdater getUpdater(ColumnDescriptor descriptor, DataType spa
       case INT32:
         if (sparkType == DataTypes.IntegerType || canReadAsIntDecimal(descriptor, sparkType)) {
           return new IntegerUpdater();
-        } else if (sparkType == DataTypes.LongType && isUnsignedIntTypeMatched(32)) {
-          // In `ParquetToSparkSchemaConverter`, we map parquet UINT32 to our LongType.
-          // For unsigned int32, it stores as plain signed int32 in Parquet when dictionary
-          // fallbacks. We read them as long values.
-          return new UnsignedIntegerUpdater();
+        } else if (sparkType == DataTypes.LongType) {
+          return new IntegerUpdater();
         } else if (sparkType == DataTypes.ByteType) {
           return new ByteUpdater();
         } else if (sparkType == DataTypes.ShortType) {