diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java index 69fbe5bf99c2..8e8ce639bc68 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java @@ -89,7 +89,6 @@ public class BaseHiveIcebergMetaHook implements HiveMetaHook { ); private static final Set PARAMETERS_TO_REMOVE = ImmutableSet .of(InputFormatConfig.TABLE_SCHEMA, Catalogs.LOCATION, Catalogs.NAME, InputFormatConfig.PARTITION_SPEC); - static final String ORC_FILES_ONLY = "iceberg.orc.files.only"; private static final String ZORDER_FIELDS_JSON_KEY = "zorderFields"; protected final Configuration conf; @@ -197,8 +196,6 @@ public void preCreateTable(CreateTableRequest request) { assertFileFormat(tableProperties.getProperty(TableProperties.DEFAULT_FILE_FORMAT)); - // Set whether the format is ORC, to be used during vectorization. - setOrcOnlyFilesParam(hmsTable); // Remove hive primary key columns from table request, as iceberg doesn't support hive primary key. request.setPrimaryKeys(null); setSortOrder(hmsTable, schema, tableProperties); @@ -456,14 +453,6 @@ protected static PartitionSpec spec(Configuration configuration, Schema schema, return HMSTablePropertyHelper.getPartitionSpec(hmsTable.getParameters(), schema); } - protected void setOrcOnlyFilesParam(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - hmsTable.getParameters().put(ORC_FILES_ONLY, String.valueOf(isOrcOnlyFiles(hmsTable))); - } - - protected boolean isOrcOnlyFiles(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - return !"FALSE".equalsIgnoreCase(hmsTable.getParameters().get(ORC_FILES_ONLY)) && isOrcFileFormat(hmsTable); - } - static boolean isOrcFileFormat(org.apache.hadoop.hive.metastore.api.Table hmsTable) { return hmsTable.getSd().getInputFormat() != null && hmsTable.getSd().getInputFormat().toUpperCase() .contains(org.apache.iceberg.FileFormat.ORC.name()) || org.apache.iceberg.FileFormat.ORC.name() diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java index 5c9781132a35..9b284a71593b 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java @@ -46,7 +46,6 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.iceberg.FileScanTask; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -249,13 +248,13 @@ public VectorizedSupport.Support[] getSupportedFeatures() { @Override public VectorizedSupport.Support[] getSupportedFeatures(HiveConf hiveConf, TableDesc tableDesc) { - // disabling VectorizedSupport.Support.DECIMAL_64 for Parquet as it doesn't support it - boolean isORCOnly = - Boolean.parseBoolean(tableDesc.getProperties().getProperty(HiveIcebergMetaHook.DECIMAL64_VECTORIZATION)) && - Boolean.parseBoolean(tableDesc.getProperties().getProperty(HiveIcebergMetaHook.ORC_FILES_ONLY)) && - org.apache.iceberg.FileFormat.ORC.name() - .equalsIgnoreCase(tableDesc.getProperties().getProperty(TableProperties.DEFAULT_FILE_FORMAT)); - if (!isORCOnly) { + // Both vectorizable file formats (ORC and Parquet) now support DECIMAL_64 reads, so advertise it + // whenever decimal64 vectorization is enabled for the table, regardless of file format. + boolean decimal64Enabled = + Boolean.parseBoolean(tableDesc.getProperties().getProperty(HiveIcebergMetaHook.DECIMAL64_VECTORIZATION)); + if (!decimal64Enabled) { + // Keep the LLAP ORC reader from emitting decimal64 so it stays consistent with the full-decimal + // operator pipeline; consumed in HiveVectorizedReader#orcRecordReader. final String vectorizationConfName = getVectorizationConfName(tableDesc.getTableName()); LOG.debug("Setting {} for table: {} to true", vectorizationConfName, tableDesc.getTableName()); hiveConf.set(vectorizationConfName, "true"); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 58c2d19373dd..92195078d47b 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -311,8 +311,6 @@ private void doPreAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable // If so, we will create the iceberg table in commitAlterTable and go ahead with the migration assertTableCanBeMigrated(hmsTable); isTableMigration = true; - // Set whether the format is ORC, to be used during vectorization. - setOrcOnlyFilesParam(hmsTable); StorageDescriptor sd = hmsTable.getSd(); preAlterTableProperties = new PreAlterTableProperties(); @@ -375,13 +373,6 @@ private void doPreAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable assertNotCrossTableMetadataLocationChange(hmsTable.getParameters(), context); } - // Migration case is already handled above, in case of migration we don't have all the properties set till this - // point. - if (!isTableMigration) { - // Set whether the format is ORC, to be used during vectorization. - setOrcOnlyFilesParam(hmsTable); - } - } /** diff --git a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_multitable.q b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_multitable.q index 349b02f706c4..73c6c05fe8f1 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_multitable.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_multitable.q @@ -11,8 +11,6 @@ insert into customer_ice values (10); create external table orders(o_orderkey int, o_custkey int) stored as orc; insert into orders values (10, 10); -alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'false'); - select sum(1 - l_discount) as revenue FROM customer_ice, orders, lineitem WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20; @@ -21,16 +19,6 @@ create external table lineitem_ice(l_discount decimal(15,2), l_orderkey int) STO TBLPROPERTIES ('iceberg.decimal64.vectorization'='true'); insert into lineitem_ice values (100.2, 10); -select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem_ice -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20; - -alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'true'); - -select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20; - select sum(1 - l_discount) as revenue FROM customer_ice, orders, lineitem_ice WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out index 910e48e4214e..346f8b8cd108 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out @@ -206,7 +206,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000},{\"name\":\"c\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 7 @@ -482,7 +481,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000},{\"name\":\"c\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 7 @@ -758,7 +756,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000},{\"name\":\"c\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 7 @@ -1098,7 +1095,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000},{\"name\":\"c\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001},{\"name\":\"d\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 10 @@ -1545,7 +1541,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000},{\"name\":\"c\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001},{\"name\":\"d\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 10 @@ -1992,7 +1987,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000},{\"name\":\"c\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001},{\"name\":\"d\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 10 diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out index 55bfee6eb031..813ec51708e8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out @@ -163,7 +163,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 4 @@ -441,7 +440,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 7 @@ -796,7 +794,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 7 @@ -1151,7 +1148,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 7 @@ -1452,7 +1448,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out index 88ad3396dd70..6198ce9405d4 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out @@ -113,7 +113,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"5\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"5\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 1 @@ -284,7 +283,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"5\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"5\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 1 @@ -455,7 +453,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"5\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"5\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out index 287c95f18fa8..e64b74f0cffb 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out @@ -30,7 +30,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out index a26573047807..69e18f01cb86 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out @@ -36,7 +36,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -109,7 +108,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -182,7 +180,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -255,7 +252,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -323,7 +319,6 @@ Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} dummy dummy_value format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out index 287c95f18fa8..e64b74f0cffb 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out @@ -30,7 +30,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out index 3e6850812ff3..6ce589351ccc 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out @@ -30,7 +30,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out index 0d1700ff07a9..0f60cd08c28b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out @@ -303,7 +303,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"a_bucket\",\"transform\":\"bucket[16]\",\"source-id\":1,\"field-id\":1000},{\"name\":\"b_trunc\",\"transform\":\"truncate[3]\",\"source-id\":2,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/ctlt_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/ctlt_iceberg.q.out index 93b114614928..f4ab854b0ed5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/ctlt_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/ctlt_iceberg.q.out @@ -43,7 +43,6 @@ TBLPROPERTIES ( 'created_with_ctlt'='true', 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"int"}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'snapshot-count'='0', @@ -129,7 +128,6 @@ TBLPROPERTIES ( 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"company","required":false,"type":"string"}]}', 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"company","transform":"identity","source-id":2,"field-id":1000}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', @@ -174,7 +172,6 @@ TBLPROPERTIES ( 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"company","required":false,"type":"string"}]}', 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"company","transform":"identity","source-id":2,"field-id":1000}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'snapshot-count'='0', @@ -247,7 +244,6 @@ TBLPROPERTIES ( 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"company","required":false,"type":"string"}]}', 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"company","transform":"identity","source-id":2,"field-id":1000}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'snapshot-count'='0', diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_all_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_all_iceberg.q.out index 58dde9ecad2a..cacb4e27adc7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_all_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_all_iceberg.q.out @@ -117,7 +117,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# format-version 2 iceberg.delete.skiprowdata false - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -269,7 +268,6 @@ Table Parameters: current-snapshot-summary {\"deleted-data-files\":\"5\",\"deleted-records\":\"20\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"0\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"0\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -421,7 +419,6 @@ Table Parameters: current-snapshot-summary {\"deleted-data-files\":\"5\",\"deleted-records\":\"20\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"0\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"0\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out index fb1cdbcaf12a..185fd2a27322 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out @@ -78,7 +78,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"i\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"s\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":4,\"name\":\"d\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -150,7 +149,6 @@ Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"year_field\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"month_field\",\"required\":false,\"type\":\"date\"},{\"id\":3,\"name\":\"day_field\",\"required\":false,\"type\":\"date\"},{\"id\":4,\"name\":\"hour_field\",\"required\":false,\"type\":\"timestamp\"},{\"id\":5,\"name\":\"truncate_field\",\"required\":false,\"type\":\"string\"},{\"id\":6,\"name\":\"bucket_field\",\"required\":false,\"type\":\"int\"},{\"id\":7,\"name\":\"identity_field\",\"required\":false,\"type\":\"int\"}]} default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"year_field_year\",\"transform\":\"year\",\"source-id\":1,\"field-id\":1000},{\"name\":\"month_field_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001},{\"name\":\"day_field_day\",\"transform\":\"day\",\"source-id\":3,\"field-id\":1002},{\"name\":\"hour_field_hour\",\"transform\":\"hour\",\"source-id\":4,\"field-id\":1003},{\"name\":\"truncate_field_trunc\",\"transform\":\"truncate[2]\",\"source-id\":5,\"field-id\":1004},{\"name\":\"bucket_field_bucket\",\"transform\":\"bucket[2]\",\"source-id\":6,\"field-id\":1005},{\"name\":\"identity_field\",\"transform\":\"identity\",\"source-id\":7,\"field-id\":1006}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -223,7 +221,6 @@ Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"year_field\",\"required\":false,\"type\":\"date\"},{\"id\":3,\"name\":\"month_field\",\"required\":false,\"type\":\"date\"},{\"id\":4,\"name\":\"day_field\",\"required\":false,\"type\":\"date\"},{\"id\":5,\"name\":\"hour_field\",\"required\":false,\"type\":\"timestamp\"},{\"id\":6,\"name\":\"truncate_field\",\"required\":false,\"type\":\"string\"},{\"id\":7,\"name\":\"bucket_field\",\"required\":false,\"type\":\"int\"},{\"id\":8,\"name\":\"identity_field\",\"required\":false,\"type\":\"int\"}]} default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"year_field_year\",\"transform\":\"year\",\"source-id\":2,\"field-id\":1000},{\"name\":\"month_field_month\",\"transform\":\"month\",\"source-id\":3,\"field-id\":1001},{\"name\":\"day_field_day\",\"transform\":\"day\",\"source-id\":4,\"field-id\":1002},{\"name\":\"hour_field_hour\",\"transform\":\"hour\",\"source-id\":5,\"field-id\":1003},{\"name\":\"truncate_field_trunc\",\"transform\":\"truncate[2]\",\"source-id\":6,\"field-id\":1004},{\"name\":\"bucket_field_bucket\",\"transform\":\"bucket[2]\",\"source-id\":7,\"field-id\":1005},{\"name\":\"identity_field\",\"transform\":\"identity\",\"source-id\":8,\"field-id\":1006}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -278,7 +275,6 @@ Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"}]} default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index c68cb256cd60..1f417296d01d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -505,7 +505,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 4 @@ -1705,7 +1704,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"country\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"state\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 4 @@ -2762,7 +2760,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"country\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"state\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 4 @@ -3456,7 +3453,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 3 @@ -4150,7 +4146,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 3 @@ -4844,7 +4839,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 3 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out index fcd1c17b24ed..758c774405d9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out @@ -636,7 +636,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_year\",\"transform\":\"year\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 1 @@ -1332,7 +1331,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 1 @@ -2028,7 +2026,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_day\",\"transform\":\"day\",\"source-id\":1,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 5 @@ -2483,7 +2480,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_trunc\",\"transform\":\"truncate[2]\",\"source-id\":1,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 1 @@ -2922,7 +2918,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_bucket\",\"transform\":\"bucket[16]\",\"source-id\":1,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 1 @@ -3149,7 +3144,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_bucket\",\"transform\":\"bucket[16]\",\"source-id\":1,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out index de49a0d3b366..956d89494fc4 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out @@ -188,7 +188,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"b_trunc_2\",\"transform\":\"truncate[2]\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 063b6389863e..c19c6331c074 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -287,7 +287,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 2 @@ -1261,7 +1260,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"country\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"state\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 4 @@ -1741,7 +1739,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 2 @@ -2209,7 +2206,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 2 @@ -2677,7 +2673,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 2 @@ -3145,7 +3140,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out index 12745b17f098..09bff4eaf55c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out @@ -632,7 +632,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_year\",\"transform\":\"year\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 1 @@ -1302,7 +1301,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 1 @@ -1976,7 +1974,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_day\",\"transform\":\"day\",\"source-id\":1,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 5 @@ -2431,7 +2428,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"pcol_trunc\",\"transform\":\"truncate[2]\",\"source-id\":1,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles #Masked# numPartitions 1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_v2_deletes.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_v2_deletes.q.out index a00317017d43..60e4125a31c0 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_v2_deletes.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_v2_deletes.q.out @@ -28,7 +28,6 @@ TBLPROPERTIES ( 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]}', 'format-version'='2', 'iceberg.delete.skiprowdata'='false', - 'iceberg.orc.files.only'='true', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', @@ -140,7 +139,6 @@ TBLPROPERTIES ( 'current-snapshot-timestamp-ms'='#Masked#', 'format-version'='2', 'iceberg.delete.skiprowdata'='true', - 'iceberg.orc.files.only'='true', #### A masked pattern was here #### 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', @@ -288,7 +286,6 @@ TBLPROPERTIES ( 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"part","transform":"identity","source-id":2,"field-id":1000}]}', 'format-version'='2', 'iceberg.delete.skiprowdata'='true', - 'iceberg.orc.files.only'='true', #### A masked pattern was here #### 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_v3_deletion_vectors.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_v3_deletion_vectors.q.out index cd64a41b9422..81ddec580730 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_v3_deletion_vectors.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_v3_deletion_vectors.q.out @@ -27,7 +27,6 @@ TBLPROPERTIES ( 'bucketing_version'='2', 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]}', 'format-version'='3', - 'iceberg.orc.files.only'='true', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out index 5036ca420f88..146cda56a500 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out @@ -114,7 +114,6 @@ Table Parameters: bucketing_version 2 format-version 2 iceberg.catalog ice01 - iceberg.orc.files.only true numFiles 10 numPartitions 10 numRows 21 @@ -197,7 +196,6 @@ Table Parameters: bucketing_version 2 format-version 2 iceberg.catalog ice01 - iceberg.orc.files.only true numFiles 10 numPartitions 10 numRows 21 @@ -377,7 +375,6 @@ Table Parameters: bucketing_version 2 format-version 2 iceberg.catalog location_based_table - iceberg.orc.files.only true numFiles 10 numPartitions 10 numRows 21 @@ -452,7 +449,6 @@ Table Parameters: bucketing_version 2 format-version 2 iceberg.catalog location_based_table - iceberg.orc.files.only true numFiles 10 numRows 21 rawDataSize 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_alter_locally_ordered_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_alter_locally_ordered_table.q.out index 74a2945b82d6..0d40faea7aa3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_alter_locally_ordered_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_alter_locally_ordered_table.q.out @@ -30,7 +30,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":4,\"name\":\"city\",\"required\":false,\"type\":\"string\"}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 0 numRows 0 @@ -88,7 +87,6 @@ Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":4,\"name\":\"city\",\"required\":false,\"type\":\"string\"}]} default-sort-order {\"order-id\":1,\"fields\":[{\"transform\":\"identity\",\"source-id\":1,\"direction\":\"desc\",\"null-order\":\"nulls-first\"},{\"transform\":\"identity\",\"source-id\":2,\"direction\":\"asc\",\"null-order\":\"nulls-last\"}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_alter_locally_zordered_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_alter_locally_zordered_table.q.out index 53293d3798c4..cd7174eaed3f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_alter_locally_zordered_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_alter_locally_zordered_table.q.out @@ -30,7 +30,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":4,\"name\":\"city\",\"required\":false,\"type\":\"string\"}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 0 numRows 0 @@ -192,7 +191,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":4,\"name\":\"city\",\"required\":false,\"type\":\"string\"}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_clustered_by.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_clustered_by.q.out index 53b11da616ce..cce9b0da04de 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_clustered_by.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_clustered_by.q.out @@ -41,7 +41,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"age\",\"required\":false,\"type\":\"int\"}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### numFiles #Masked# numRows 0 @@ -285,7 +284,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"customer_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"order_id\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"product\",\"required\":false,\"type\":\"string\"}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### numFiles #Masked# numRows 0 @@ -496,7 +494,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"2\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"#Masked#\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles #Masked# numRows 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_ordered_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_ordered_table.q.out index ed49513445f1..305451686c30 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_ordered_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_ordered_table.q.out @@ -39,7 +39,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"9\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"9\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 9 @@ -134,7 +133,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-sort-order {\"order-id\":1,\"fields\":[{\"transform\":\"identity\",\"source-id\":1,\"direction\":\"desc\",\"null-order\":\"nulls-first\"},{\"transform\":\"identity\",\"source-id\":2,\"direction\":\"asc\",\"null-order\":\"nulls-last\"}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 9 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out index 42f0631140fc..3636b845c563 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out @@ -38,7 +38,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"text\",\"required\":false,\"type\":\"string\"}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 0 numRows 0 @@ -283,7 +282,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"text\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"bool_val\",\"required\":false,\"type\":\"boolean\"},{\"id\":4,\"name\":\"date_val\",\"required\":false,\"type\":\"date\"}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 0 numRows 0 @@ -409,7 +407,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":2,\"name\":\"dd\",\"required\":false,\"type\":\"double\"},{\"id\":3,\"name\":\"ll\",\"required\":false,\"type\":\"long\"}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 0 numRows 0 @@ -543,7 +540,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"20\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"20\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 20 @@ -663,7 +659,6 @@ Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"ts\",\"required\":false,\"type\":\"timestamp\"},{\"id\":2,\"name\":\"dd\",\"required\":false,\"type\":\"double\"},{\"id\":3,\"name\":\"ll\",\"required\":false,\"type\":\"int\"}]} default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"ll_bucket\",\"transform\":\"bucket[4]\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index 981b8269ad51..6a28128817c9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -301,7 +301,6 @@ Table Parameters: default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 hive.compactor.worker.pool iceberg - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 17 numRows 17 @@ -898,7 +897,6 @@ Table Parameters: default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 hive.compactor.worker.pool iceberg - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 4 numPartitions 3 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out index 3af63c58b6c2..a541f9350bf1 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out @@ -165,7 +165,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 4 numRows 4 @@ -252,7 +251,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 2 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out index 924c6d9953fa..835a2af5e577 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out @@ -159,7 +159,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 4 numPartitions 3 @@ -309,7 +308,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 3 numPartitions 3 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out index 61a931d74a33..cf41867c1d2b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out @@ -255,7 +255,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 13 numPartitions 8 @@ -366,7 +365,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 8 numPartitions 8 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out index bc786c1e8dc8..d391b9ebd3d6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out @@ -207,7 +207,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 8 numPartitions 5 @@ -320,7 +319,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 8 numPartitions 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out index 5508bdca124c..435a56d23294 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out @@ -200,7 +200,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 11 numRows 11 @@ -303,7 +302,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 2 numPartitions 2 @@ -541,7 +539,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 14 numRows 16 @@ -648,7 +645,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 2 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out index 26d7eca677b5..970c81e6e1c0 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out @@ -99,7 +99,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# external.table.purge true format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 7 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out index 440f6334f114..365e446af9ba 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out @@ -236,7 +236,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 10 numRows 10 @@ -340,7 +339,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 2 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out index 4e120fb8c50d..5efe2d0f536f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out @@ -210,7 +210,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 11 numRows 11 @@ -322,7 +321,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 6 numPartitions 2 @@ -441,7 +439,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 2 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out index 1d1143f4b635..8b9310b71a33 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out @@ -191,7 +191,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 8 numPartitions 4 @@ -308,7 +307,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 7 numPartitions 4 @@ -425,7 +423,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 6 numPartitions 4 @@ -541,7 +538,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 5 numPartitions 3 @@ -657,7 +653,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1000},{\"name\":\"city\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1001},{\"name\":\"registration_date\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1002}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 4 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out index b01185bb6911..0eb2ba637a43 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out @@ -134,7 +134,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"a\",\"transform\":\"identity\",\"source-id\":1,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 4 numPartitions 2 @@ -228,7 +227,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"a\",\"transform\":\"identity\",\"source-id\":1,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 4 numPartitions 2 @@ -336,7 +334,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"a\",\"transform\":\"identity\",\"source-id\":1,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 3 numPartitions 2 @@ -444,7 +441,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"a\",\"transform\":\"identity\",\"source-id\":1,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 5 numPartitions 5 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out index 7bf7f084572f..2868b83339da 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned.q.out @@ -188,7 +188,6 @@ Table Parameters: current-snapshot-summary {\"deleted-data-files\":\"3\",\"deleted-records\":\"3\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"11\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"11\",\"total-delete-files\":\"7\",\"total-position-deletes\":\"7\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 11 numRows 11 @@ -301,7 +300,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"deleted-data-files\":\"11\",\"removed-position-delete-files\":\"7\",\"removed-delete-files\":\"7\",\"added-records\":\"4\",\"deleted-records\":\"11\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"7\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_ordered.q.out index 978928209269..9f6cb1107730 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_ordered.q.out @@ -85,7 +85,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# format-version 2 hive.compactor.worker.pool iceberg - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 4 @@ -199,7 +198,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# format-version 2 hive.compactor.worker.pool iceberg - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_w_filter.q.out index ce004c9547ef..5fe81cf1f72a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_unpartitioned_w_filter.q.out @@ -188,7 +188,6 @@ Table Parameters: current-snapshot-summary {\"deleted-data-files\":\"3\",\"deleted-records\":\"3\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"11\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"11\",\"total-delete-files\":\"7\",\"total-position-deletes\":\"7\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 11 numRows 11 @@ -280,7 +279,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"deleted-data-files\":\"11\",\"removed-position-delete-files\":\"7\",\"removed-delete-files\":\"7\",\"added-records\":\"4\",\"deleted-records\":\"11\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"7\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out index fa5dcd05e101..a177af309c6a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out @@ -93,7 +93,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"key_bucket_8\",\"transform\":\"bucket[8]\",\"source-id\":2,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### numFiles 7 numPartitions 6 @@ -211,7 +210,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"key_bucket_8\",\"transform\":\"bucket[8]\",\"source-id\":2,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### numFiles 8 numPartitions 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out index 62070278505d..c372ea96c222 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out @@ -120,7 +120,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 4 numPartitions 1 @@ -205,7 +204,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 4 numPartitions 1 @@ -314,7 +312,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 2 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out index fabbe1a82c76..2b782f5a0f15 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out @@ -96,7 +96,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 3 numRows 7 @@ -180,7 +179,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"deleted-data-files\":\"3\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 7 @@ -278,7 +276,6 @@ Table Parameters: current-snapshot-summary {\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"1\",\"removed-delete-files\":\"1\",\"added-records\":\"7\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### numFiles 1 numRows 7 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 8bba659e8fd1..3dbb6fbb605b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -86,7 +86,6 @@ TBLPROPERTIES ( 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"company_id","transform":"identity","source-id":5,"field-id":1000}]}', 'format-version'='2', 'iceberg.catalog'='ice01', - 'iceberg.orc.files.only'='true', #### A masked pattern was here #### 'name'='ice_rest.ice_orc2', 'parquet.compression'='zstd', @@ -142,7 +141,6 @@ Table Parameters: default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000}]} format-version 2 iceberg.catalog ice01 - iceberg.orc.files.only true #### A masked pattern was here #### name ice_rest.ice_orc2 numFiles 1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 409eb484480b..c87c07aa45ed 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -86,7 +86,6 @@ TBLPROPERTIES ( 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"company_id","transform":"identity","source-id":5,"field-id":1000}]}', 'format-version'='2', 'iceberg.catalog'='ice01', - 'iceberg.orc.files.only'='true', #### A masked pattern was here #### 'name'='ice_rest.ice_orc2', 'parquet.compression'='zstd', @@ -142,7 +141,6 @@ Table Parameters: default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000}]} format-version 2 iceberg.catalog ice01 - iceberg.orc.files.only true #### A masked pattern was here #### name ice_rest.ice_orc2 numFiles 1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index 5f31e752db3f..5d197263b37b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -259,8 +259,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false @@ -507,7 +507,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2), 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] Select Vectorization: className: VectorSelectOperator native: true @@ -516,7 +516,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, col 9:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -531,8 +531,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false @@ -540,9 +540,9 @@ STAGE PLANS: rowBatchContext: dataColumnCount: 10 includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2) + dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [] + scratchColumnTypeNames: [decimal(4,2)] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: @@ -940,8 +940,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_multitable.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_multitable.q.out index 7ec8f5c23b97..7728fb532cc9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_multitable.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_multitable.q.out @@ -52,14 +52,6 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@orders POSTHOOK: Lineage: orders.o_custkey SCRIPT [] POSTHOOK: Lineage: orders.o_orderkey SCRIPT [] -PREHOOK: query: alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'false') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@customer_ice -PREHOOK: Output: default@customer_ice -POSTHOOK: query: alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'false') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@customer_ice -POSTHOOK: Output: default@customer_ice PREHOOK: query: select sum(1 - l_discount) as revenue FROM customer_ice, orders, lineitem WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 @@ -112,45 +104,3 @@ POSTHOOK: Input: default@lineitem_ice POSTHOOK: Input: default@orders #### A masked pattern was here #### -99.20 -PREHOOK: query: alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'true') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@customer_ice -PREHOOK: Output: default@customer_ice -POSTHOOK: query: alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'true') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@customer_ice -POSTHOOK: Output: default@customer_ice -PREHOOK: query: select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_ice -PREHOOK: Input: default@lineitem -PREHOOK: Input: default@orders -#### A masked pattern was here #### -POSTHOOK: query: select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_ice -POSTHOOK: Input: default@lineitem -POSTHOOK: Input: default@orders -#### A masked pattern was here #### --99.20 -PREHOOK: query: select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem_ice -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_ice -PREHOOK: Input: default@lineitem_ice -PREHOOK: Input: default@orders -#### A masked pattern was here #### -POSTHOOK: query: select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem_ice -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_ice -POSTHOOK: Input: default@lineitem_ice -POSTHOOK: Input: default@orders -#### A masked pattern was here #### --99.20 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index 2feda580b67a..3b0d1940c1be 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -150,8 +150,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false @@ -348,7 +348,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2), 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] Select Vectorization: className: VectorSelectOperator native: true @@ -357,7 +357,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, col 9:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -372,8 +372,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false @@ -381,9 +381,9 @@ STAGE PLANS: rowBatchContext: dataColumnCount: 10 includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2) + dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [] + scratchColumnTypeNames: [decimal(4,2)] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: @@ -582,8 +582,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out index 7ea7605467ca..e58b7ffd846a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out @@ -73,7 +73,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} format-version 1 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numPartitions 2 @@ -165,7 +164,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out index 770cc967d0bc..c9b3f1af8d82 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out @@ -75,7 +75,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b_bucket\",\"transform\":\"bucket[16]\",\"source-id\":1,\"field-id\":1000},{\"name\":\"c_trunc\",\"transform\":\"truncate[3]\",\"source-id\":2,\"field-id\":1001}]} format-version 1 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numPartitions 2 @@ -169,7 +168,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b_bucket\",\"transform\":\"bucket[16]\",\"source-id\":1,\"field-id\":1000},{\"name\":\"c_trunc\",\"transform\":\"truncate[3]\",\"source-id\":2,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numPartitions 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index a44d9394025b..e46dfab99143 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -111,7 +111,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"p1\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"p2\",\"transform\":\"identity\",\"source-id\":6,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 10 numPartitions 10 @@ -203,7 +202,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"p1\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"p2\",\"transform\":\"identity\",\"source-id\":6,\"field-id\":1001}]} format-version 2 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 10 numPartitions 10 diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out index 3ecbc531d271..5c67ce2129f6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out @@ -34,7 +34,6 @@ TBLPROPERTIES ( 'bucketing_version'='2', 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"s","required":false,"type":"string"},{"id":3,"name":"ts","required":false,"type":"timestamp"},{"id":4,"name":"d","required":false,"type":"date"}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', @@ -82,7 +81,6 @@ TBLPROPERTIES ( 'bucketing_version'='2', 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"s","required":false,"type":"string"},{"id":3,"name":"ts","required":false,"type":"timestamp"},{"id":4,"name":"d","required":false,"type":"date"}]}', 'format-version'='1', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', @@ -127,7 +125,6 @@ TBLPROPERTIES ( 'bucketing_version'='2', 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"s","required":false,"type":"string"},{"id":3,"name":"ts","required":false,"type":"timestamp"},{"id":4,"name":"d","required":false,"type":"date"}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', @@ -187,7 +184,6 @@ TBLPROPERTIES ( 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"year_field","required":false,"type":"date"},{"id":2,"name":"month_field","required":false,"type":"date"},{"id":3,"name":"day_field","required":false,"type":"date"},{"id":4,"name":"hour_field","required":false,"type":"timestamp"},{"id":5,"name":"truncate_field","required":false,"type":"string"},{"id":6,"name":"bucket_field","required":false,"type":"int"},{"id":7,"name":"identity_field","required":false,"type":"int"}]}', 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"year_field_year","transform":"year","source-id":1,"field-id":1000},{"name":"month_field_month","transform":"month","source-id":2,"field-id":1001},{"name":"day_field_day","transform":"day","source-id":3,"field-id":1002},{"name":"hour_field_hour","transform":"hour","source-id":4,"field-id":1003},{"name":"truncate_field_trunc","transform":"truncate[2]","source-id":5,"field-id":1004},{"name":"bucket_field_bucket","transform":"bucket[2]","source-id":6,"field-id":1005},{"name":"identity_field","transform":"identity","source-id":7,"field-id":1006}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', @@ -248,7 +244,6 @@ TBLPROPERTIES ( 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"year_field","required":false,"type":"date"},{"id":3,"name":"month_field","required":false,"type":"date"},{"id":4,"name":"day_field","required":false,"type":"date"},{"id":5,"name":"hour_field","required":false,"type":"timestamp"},{"id":6,"name":"truncate_field","required":false,"type":"string"},{"id":7,"name":"bucket_field","required":false,"type":"int"},{"id":8,"name":"identity_field","required":false,"type":"int"}]}', 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"year_field_year","transform":"year","source-id":2,"field-id":1000},{"name":"month_field_month","transform":"month","source-id":3,"field-id":1001},{"name":"day_field_day","transform":"day","source-id":4,"field-id":1002},{"name":"hour_field_hour","transform":"hour","source-id":5,"field-id":1003},{"name":"truncate_field_trunc","transform":"truncate[2]","source-id":6,"field-id":1004},{"name":"bucket_field_bucket","transform":"bucket[2]","source-id":7,"field-id":1005},{"name":"identity_field","transform":"identity","source-id":8,"field-id":1006}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', @@ -297,7 +292,6 @@ TBLPROPERTIES ( 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"int"},{"id":2,"name":"b","required":false,"type":"string"}]}', 'default-partition-spec'='{"spec-id":0,"fields":[{"name":"b","transform":"identity","source-id":2,"field-id":1000}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', @@ -354,7 +348,6 @@ TBLPROPERTIES ( 'current-snapshot-summary'='{"added-data-files":"1","added-records":"3","added-files-size":"#Masked#","changed-partition-count":"1","total-records":"3","total-files-size":"#Masked#","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"#Masked#"}', 'current-snapshot-timestamp-ms'='#Masked#', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'previous_metadata_location'='hdfs://### HDFS PATH ###', @@ -401,7 +394,6 @@ TBLPROPERTIES ( 'bucketing_version'='2', 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"s","required":false,"type":"string"}]}', 'format-version'='2', - 'iceberg.orc.files.only'='false', 'metadata_location'='hdfs://### HDFS PATH ###', 'parquet.compression'='zstd', 'serialization.format'='1', diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_iceberg_materialized_views.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_iceberg_materialized_views.q.out index 1dc569be87c9..756a634026dc 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/show_iceberg_materialized_views.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/show_iceberg_materialized_views.q.out @@ -366,7 +366,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"key\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"value\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ds\",\"required\":false,\"type\":\"string\"}]} format-version 1 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -419,7 +418,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"key\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"value\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ds\",\"required\":false,\"type\":\"string\"}]} format-version 1 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -471,7 +469,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"key\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"value\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"ds\",\"required\":false,\"type\":\"string\"}]} format-version 1 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -577,7 +574,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]} format-version 1 - iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out index 44d2c059a19e..ba1b3d724782 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out @@ -96,7 +96,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# external.table.purge false format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 3 @@ -172,7 +171,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# external.table.purge false format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out index 6c5831383bab..56897e0ea67f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out @@ -96,7 +96,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# external.table.purge true format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 3 @@ -172,7 +171,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# external.table.purge true format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 0 @@ -246,7 +244,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# external.table.purge true format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 1 @@ -322,7 +319,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# external.table.purge true format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 0 @@ -414,7 +410,6 @@ Table Parameters: current-snapshot-timestamp-ms #Masked# external.table.purge false format-version 2 - iceberg.orc.files.only true #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out index be765c27120a..f28166ec3b6b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out @@ -107,7 +107,6 @@ Table Parameters: default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} external.table.purge true format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 4 @@ -215,7 +214,6 @@ Table Parameters: default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"b\",\"transform\":\"identity\",\"source-id\":2,\"field-id\":1000}]} external.table.purge true format-version 2 - iceberg.orc.files.only false #### A masked pattern was here #### metadata_location hdfs://### HDFS PATH ### numFiles 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/use_basic_stats_from_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/use_basic_stats_from_iceberg.q.out index a0ba02dbf037..a9ad0fc8da75 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/use_basic_stats_from_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/use_basic_stats_from_iceberg.q.out @@ -161,7 +161,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 @@ -206,7 +205,6 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"b\",\"required\":false,\"type\":\"int\"}]} format-version 2 - iceberg.orc.files.only false metadata_location hdfs://### HDFS PATH ### numFiles 0 numRows 0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index cd0ce562a725..e91e55d0d034 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -220,8 +220,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false @@ -467,7 +467,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2), 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] Select Vectorization: className: VectorSelectOperator native: true @@ -476,7 +476,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, col 9:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -490,8 +490,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false @@ -499,9 +499,9 @@ STAGE PLANS: rowBatchContext: dataColumnCount: 10 includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2) + dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [] + scratchColumnTypeNames: [decimal(4,2)] Reducer 2 Execution mode: vectorized Reduce Vectorization: @@ -820,8 +820,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_multitable.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_multitable.q.out index 2f8ce7ad42b1..fa939bb08ba4 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_multitable.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_multitable.q.out @@ -52,14 +52,6 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@orders POSTHOOK: Lineage: orders.o_custkey SCRIPT [] POSTHOOK: Lineage: orders.o_orderkey SCRIPT [] -PREHOOK: query: alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'false') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@customer_ice -PREHOOK: Output: default@customer_ice -POSTHOOK: query: alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'false') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@customer_ice -POSTHOOK: Output: default@customer_ice PREHOOK: query: select sum(1 - l_discount) as revenue FROM customer_ice, orders, lineitem WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 @@ -112,45 +104,3 @@ POSTHOOK: Input: default@lineitem_ice POSTHOOK: Input: default@orders POSTHOOK: Output: hdfs://### HDFS PATH ### -99.20 -PREHOOK: query: alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'true') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@customer_ice -PREHOOK: Output: default@customer_ice -POSTHOOK: query: alter table customer_ice set tblproperties ( 'iceberg.orc.files.only' = 'true') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@customer_ice -POSTHOOK: Output: default@customer_ice -PREHOOK: query: select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_ice -PREHOOK: Input: default@lineitem -PREHOOK: Input: default@orders -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_ice -POSTHOOK: Input: default@lineitem -POSTHOOK: Input: default@orders -POSTHOOK: Output: hdfs://### HDFS PATH ### --99.20 -PREHOOK: query: select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem_ice -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_ice -PREHOOK: Input: default@lineitem_ice -PREHOOK: Input: default@orders -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select sum(1 - l_discount) as revenue -FROM customer_ice, orders, lineitem_ice -WHERE c_custkey = o_custkey and l_orderkey = o_orderkey limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_ice -POSTHOOK: Input: default@lineitem_ice -POSTHOOK: Input: default@orders -POSTHOOK: Output: hdfs://### HDFS PATH ### --99.20 diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index acc7794e12ce..ba3f66f093c0 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -112,8 +112,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false @@ -271,7 +271,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2), 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] Select Vectorization: className: VectorSelectOperator native: true @@ -280,7 +280,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, col 9:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -294,8 +294,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false @@ -303,9 +303,9 @@ STAGE PLANS: rowBatchContext: dataColumnCount: 10 includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2) + dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [] + scratchColumnTypeNames: [decimal(4,2)] Reducer 2 Execution mode: vectorized Reduce Vectorization: @@ -503,8 +503,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java index 2a3bccb6d9a5..ce0065a3ad6b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java @@ -115,6 +115,6 @@ public boolean validateInput(FileSystem fs, HiveConf conf, List file @Override public VectorizedSupport.Support[] getSupportedFeatures() { - return null; + return new VectorizedSupport.Support[] { VectorizedSupport.Support.DECIMAL_64 }; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java index 348406d9cfac..c94dd493ae87 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java @@ -104,6 +104,32 @@ public interface ParquetDataColumnReader { */ byte[] readDecimal(); + /** + * True when this reader can supply DECIMAL_64 values as raw unscaled longs at the column's scale, + * with no per-row HiveDecimal/byte[] conversion -- i.e. an INT32/INT64- or byte-array-backed decimal + * whose file scale equals the requested Hive scale and whose value fits a long. When true, the + * long-backed reader may call {@link #readDecimal64()} / {@link #readDecimal64(int)} instead of + * {@link #readDecimal()} / {@link #readDecimal(int)}. + */ + default boolean isFastDecimal64() { + return false; + } + + /** + * @return the next value as a raw unscaled decimal64 long. Only valid when {@link #isFastDecimal64()}. + * {@link #isValid()} is set false when the value does not fit the Hive precision (caller -> NULL). + */ + default long readDecimal64() { + throw new UnsupportedOperationException(); + } + + /** + * @return the dictionary value at {@code id} as a raw unscaled decimal64 long. See {@link #readDecimal64()}. + */ + default long readDecimal64(int id) { + throw new UnsupportedOperationException(); + } + /** * @return the next Double from the page */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java index 6b44459b8064..472deeca41f8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java @@ -382,6 +382,19 @@ byte[] validatedScaledDecimal(int inpScale) { } } + // Validate a raw unscaled decimal64 long (already at the Hive scale) against the Hive precision. + // Sets isValid; returns the value unchanged when in range, else 0 (caller marks the entry NULL). + // Used by the Decimal64 identity fast path; bounds via HiveDecimalWritable to avoid hand-rolling. + long validatedDecimal64(long unscaledValue) { + long absMax = HiveDecimalWritable.getDecimal64AbsMax(hivePrecision); + if (unscaledValue >= -absMax && unscaledValue <= absMax) { + this.isValid = true; + return unscaledValue; + } + this.isValid = false; + return 0; + } + /** * Helper function to validate double data. Sets the isValid to true if the data is valid * for the type it will be read in, otherwise false. @@ -1304,17 +1317,21 @@ private static byte[] convertToBytes(Timestamp value) { * and returned as per the type defined in HMS. */ public static class TypesFromDecimalPageReader extends DefaultParquetDataColumnReader { - private short scale; + // Parquet file decimal precision and scale. + private final int precision; + private final short scale; - public TypesFromDecimalPageReader(ValuesReader realReader, int length, short scale, + public TypesFromDecimalPageReader(ValuesReader realReader, int length, int precision, short scale, int hivePrecision, int hiveScale) { super(realReader, length, hivePrecision, hiveScale); + this.precision = precision; this.scale = scale; } - public TypesFromDecimalPageReader(Dictionary dict, int length, short scale, int hivePrecision, - int hiveScale) { + public TypesFromDecimalPageReader(Dictionary dict, int length, int precision, short scale, + int hivePrecision, int hiveScale) { super(dict, length, hivePrecision, hiveScale); + this.precision = precision; this.scale = scale; } @@ -1460,6 +1477,47 @@ public byte[] readDecimal(int id) { hiveDecimalWritable.set(dict.decodeToBinary(id).getBytesUnsafe(), scale); return super.validatedScaledDecimal(scale); } + + /** + * Decimal64 identity fast path for FIXED_LEN_BYTE_ARRAY/BINARY decimals: the stored big-endian + * two's-complement unscaled value is read straight into a long with only a bounds check, instead + * of the per-row HiveDecimal materialization {@link #readDecimal()} performs via + * {@code fastSetFromBigIntegerBytesAndScale}. Rescaled or wider decimals fall back to + * {@link #readDecimal()}. + */ + @Override + public boolean isFastDecimal64() { + // Valid only with no rescale (file scale == Hive scale) and precision <= 18, so the unscaled + // value fits a long. + return scale == hiveScale && HiveDecimalWritable.isPrecisionDecimal64(precision); + } + + @Override + public long readDecimal64() { + return validatedDecimal64(binaryToUnscaledLong(valuesReader.readBytes())); + } + + @Override + public long readDecimal64(int id) { + return validatedDecimal64(binaryToUnscaledLong(dict.decodeToBinary(id))); + } + + /** + * Big-endian two's-complement bytes -> unscaled long (the low 64 bits). The fast path runs only + * for file precision <= 18 (see isFastDecimal64), so a value conforming to that precision fits a + * long; the leading bytes of a wider-than-8-byte array are sign extension and shift out losslessly. + */ + private static long binaryToUnscaledLong(Binary value) { + ByteBuffer buf = value.toByteBuffer(); + int pos = buf.position(); + int len = buf.remaining(); + // Sign-extend from the most significant byte so negative values are reconstructed correctly. + long v = (len > 0 && buf.get(pos) < 0) ? -1L : 0L; + for (int i = 0; i < len; i++) { + v = (v << 8) | (buf.get(pos + i) & 0xFF); + } + return v; + } } /** @@ -1469,7 +1527,7 @@ public byte[] readDecimal(int id) { * and returned as per the type defined in HMS. */ public static class TypesFromInt32DecimalPageReader extends DefaultParquetDataColumnReader { - private short scale; + private final short scale; public TypesFromInt32DecimalPageReader(ValuesReader realReader, int length, short scale, int hivePrecision, int hiveScale) { @@ -1622,6 +1680,23 @@ public byte[] readDecimal(int id) { hiveDecimalWritable.set(hiveDecimal); return super.validatedScaledDecimal(scale); } + + @Override + public boolean isFastDecimal64() { + // Identity fast path: the file scale equals the Hive scale, so the stored unscaled value IS the + // Decimal64 value -- no rescale/rounding, only a precision bounds check. + return scale == hiveScale; + } + + @Override + public long readDecimal64() { + return validatedDecimal64(valuesReader.readInteger()); + } + + @Override + public long readDecimal64(int id) { + return validatedDecimal64(dict.decodeToInt(id)); + } } /** @@ -1631,7 +1706,7 @@ public byte[] readDecimal(int id) { * and returned as per the type defined in HMS. */ public static class TypesFromInt64DecimalPageReader extends DefaultParquetDataColumnReader { - private short scale; + private final short scale; public TypesFromInt64DecimalPageReader(ValuesReader realReader, int length, short scale, int hivePrecision, int hiveScale) { @@ -1784,6 +1859,23 @@ public byte[] readDecimal(int id) { hiveDecimalWritable.set(hiveDecimal); return super.validatedScaledDecimal(scale); } + + @Override + public boolean isFastDecimal64() { + // Identity fast path: the file scale equals the Hive scale, so the stored unscaled long IS the + // Decimal64 value -- no rescale/rounding, only a precision bounds check. + return scale == hiveScale; + } + + @Override + public long readDecimal64() { + return validatedDecimal64(valuesReader.readLong()); + } + + @Override + public long readDecimal64(int id) { + return validatedDecimal64(dict.decodeToLong(id)); + } } /** @@ -1956,15 +2048,16 @@ private static ParquetDataColumnReader getConvertorFromBinary(boolean isDict, } Optional reader = parquetType.getLogicalTypeAnnotation() - .accept(new LogicalTypeAnnotationVisitor() { + .accept(new LogicalTypeAnnotationVisitor<>() { @Override public Optional visit( DecimalLogicalTypeAnnotation logicalTypeAnnotation) { + final int precision = logicalTypeAnnotation.getPrecision(); final short scale = (short) logicalTypeAnnotation.getScale(); return isDict ? Optional - .of(new TypesFromDecimalPageReader(dictionary, length, scale, hivePrecision, - hiveScale)) : Optional - .of(new TypesFromDecimalPageReader(valuesReader, length, scale, hivePrecision, - hiveScale)); + .of(new TypesFromDecimalPageReader(dictionary, length, precision, scale, + hivePrecision, hiveScale)) : Optional + .of(new TypesFromDecimalPageReader(valuesReader, length, precision, scale, + hivePrecision, hiveScale)); } @Override public Optional visit( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java index c25475733ee4..05f4268a4852 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java @@ -16,6 +16,7 @@ import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DateColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -136,7 +137,11 @@ private void readBatchHelper( readFloats(num, (DoubleColumnVector) column, rowId); break; case DECIMAL: - readDecimal(num, (DecimalColumnVector) column, rowId); + if (column instanceof Decimal64ColumnVector) { + readDecimal64(num, (Decimal64ColumnVector) column, rowId); + } else { + readDecimal(num, (DecimalColumnVector) column, rowId); + } break; case TIMESTAMP: readTimestamp(num, (TimestampColumnVector) column, rowId); @@ -153,10 +158,7 @@ private static void setNullValue(ColumnVector c, int rowId) { c.noNulls = false; } - private void readDictionaryIDs( - int total, - LongColumnVector c, - int rowId) throws IOException { + private void readDictionaryIDs(int total, LongColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -172,10 +174,7 @@ private void readDictionaryIDs( } } - private void readIntegers( - int total, - LongColumnVector c, - int rowId) throws IOException { + private void readIntegers(int total, LongColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -196,10 +195,7 @@ private void readIntegers( } } - private void readSmallInts( - int total, - LongColumnVector c, - int rowId) throws IOException { + private void readSmallInts(int total, LongColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -220,10 +216,7 @@ private void readSmallInts( } } - private void readTinyInts( - int total, - LongColumnVector c, - int rowId) throws IOException { + private void readTinyInts(int total, LongColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -244,10 +237,7 @@ private void readTinyInts( } } - private void readDoubles( - int total, - DoubleColumnVector c, - int rowId) throws IOException { + private void readDoubles(int total, DoubleColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -268,10 +258,7 @@ private void readDoubles( } } - private void readBooleans( - int total, - LongColumnVector c, - int rowId) throws IOException { + private void readBooleans(int total, LongColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -287,10 +274,7 @@ private void readBooleans( } } - private void readLongs( - int total, - LongColumnVector c, - int rowId) throws IOException { + private void readLongs(int total, LongColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -311,10 +295,7 @@ private void readLongs( } } - private void readFloats( - int total, - DoubleColumnVector c, - int rowId) throws IOException { + private void readFloats(int total, DoubleColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -335,17 +316,9 @@ private void readFloats( } } - private void readDecimal( - int total, - DecimalColumnVector c, - int rowId) throws IOException { - - DecimalLogicalTypeAnnotation decimalLogicalType = null; - if (type.getLogicalTypeAnnotation() instanceof DecimalLogicalTypeAnnotation) { - decimalLogicalType = (DecimalLogicalTypeAnnotation) type.getLogicalTypeAnnotation(); - } - byte[] decimalData = null; - fillDecimalPrecisionScale(decimalLogicalType, c); + private void readDecimal(int total, DecimalColumnVector c, int rowId) { + byte[] decimalData; + fillDecimalPrecisionScale(c); int left = total; while (left > 0) { @@ -367,10 +340,7 @@ private void readDecimal( } } - private void readString( - int total, - BytesColumnVector c, - int rowId) throws IOException { + private void readString(int total, BytesColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -387,10 +357,7 @@ private void readString( } } - private void readChar( - int total, - BytesColumnVector c, - int rowId) throws IOException { + private void readChar(int total, BytesColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -407,10 +374,7 @@ private void readChar( } } - private void readVarchar( - int total, - BytesColumnVector c, - int rowId) throws IOException { + private void readVarchar(int total, BytesColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -427,10 +391,7 @@ private void readVarchar( } } - private void readBinaries( - int total, - BytesColumnVector c, - int rowId) throws IOException { + private void readBinaries(int total, BytesColumnVector c, int rowId) { int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); @@ -447,10 +408,7 @@ private void readBinaries( } } - private void readDate( - int total, - DateColumnVector c, - int rowId) throws IOException { + private void readDate(int total, DateColumnVector c, int rowId) { c.setUsingProlepticCalendar(true); int left = total; while (left > 0) { @@ -651,14 +609,42 @@ private void decodeDictionaryIds( } break; case DECIMAL: - DecimalLogicalTypeAnnotation decimalLogicalType = null; - if (type.getLogicalTypeAnnotation() instanceof DecimalLogicalTypeAnnotation) { - decimalLogicalType = (DecimalLogicalTypeAnnotation) type.getLogicalTypeAnnotation(); + if (column instanceof Decimal64ColumnVector dec64) { + fillDecimal64PrecisionScale(dec64); + boolean fast = dictionary.isFastDecimal64(); + short valueScale = getEncodedDecimalScale(); + for (int i = rowId; i < rowId + num; ++i) { + if (!column.isNull[i]) { + int id = (int) dictionaryIds.vector[i]; + boolean stored; + if (fast) { + // Identity fast path: store the raw unscaled long directly. + long v = dictionary.readDecimal64(id); + stored = dictionary.isValid(); + if (stored) { + dec64.vector[i] = v; + } + } else { + // set() enforces the column precision/scale and marks the entry NULL on overflow. + byte[] bytes = dictionary.readDecimal(id); + stored = dictionary.isValid(); + if (stored) { + dec64.set(i, bytes, valueScale); + stored = !dec64.isNull[i]; + } + } + if (!stored) { + setNullValue(column, i); + dec64.vector[i] = 0; + } + } + } + break; } DecimalColumnVector decimalColumnVector = ((DecimalColumnVector) column); - byte[] decimalData = null; + byte[] decimalData; - fillDecimalPrecisionScale(decimalLogicalType, decimalColumnVector); + fillDecimalPrecisionScale(decimalColumnVector); for (int i = rowId; i < rowId + num; ++i) { if (!column.isNull[i]) { @@ -686,27 +672,100 @@ private void decodeDictionaryIds( } } + private void fillDecimalPrecisionScale(DecimalColumnVector c) { + short[] ps = getDecimalPrecisionScale(); + c.precision = ps[0]; + c.scale = ps[1]; + } + /** - * The decimal precision and scale is filled into decimalColumnVector. If the data in - * Parquet is in decimal, the precision and scale will come in from decimalLogicalType. If parquet - * is not in decimal, then this call is made because HMS shows the type as decimal. So, the - * precision and scale are picked from hiveType. - * - * @param decimalLogicalType - * @param decimalColumnVector + * Fill a long-backed {@link Decimal64ColumnVector} at the Hive (table) scale -- the scale every + * consumer reads {@code c.vector} at, and the only scale at which the unscaled value fits the long. + * NOT the Parquet file scale from {@link #getDecimalPrecisionScale()}: under schema evolution that + * scale can be larger (e.g. a DECIMAL(38,37) file read as DECIMAL(16,8)) and would overflow. */ - private void fillDecimalPrecisionScale(DecimalLogicalTypeAnnotation decimalLogicalType, - DecimalColumnVector decimalColumnVector) { - if (decimalLogicalType != null) { - decimalColumnVector.precision = (short) decimalLogicalType.getPrecision(); - decimalColumnVector.scale = (short) decimalLogicalType.getScale(); + private void fillDecimal64PrecisionScale(Decimal64ColumnVector c) { + if (hiveType instanceof DecimalTypeInfo dti) { + c.precision = (short) dti.getPrecision(); + c.scale = (short) dti.getScale(); + } else { + short[] ps = getDecimalPrecisionScale(); + c.precision = ps[0]; + c.scale = ps[1]; + } + } + + /** + * Scale at which {@code dataColumn}/{@code dictionary}.readDecimal() encodes the unscaled bytes: + * the Parquet physical scale when stored as decimal, else the Hive scale. Pass this to + * {@link Decimal64ColumnVector#set(int, byte[], int)} so the bytes are read at the right scale + * before re-scaling to the column scale. + */ + private short getEncodedDecimalScale() { + return getDecimalPrecisionScale()[1]; + } + + /** + * Precision/scale for this decimal column: from the Parquet decimal logical type when present, + * otherwise from the Hive type (Parquet stores it as a non-decimal physical type but HMS reports + * decimal). + */ + private short[] getDecimalPrecisionScale() { + if (type.getLogicalTypeAnnotation() instanceof DecimalLogicalTypeAnnotation d) { + return new short[] { (short) d.getPrecision(), (short) d.getScale() }; } else if (TypeInfoUtils.getBaseName(hiveType.getTypeName()) .equalsIgnoreCase(serdeConstants.DECIMAL_TYPE_NAME)) { - decimalColumnVector.precision = (short) ((DecimalTypeInfo) hiveType).getPrecision(); - decimalColumnVector.scale = (short) ((DecimalTypeInfo) hiveType).getScale(); - } else { - throw new UnsupportedOperationException( - "The underlying Parquet type cannot be converted to Hive Decimal type: " + type); + DecimalTypeInfo dti = (DecimalTypeInfo) hiveType; + return new short[] { (short) dti.getPrecision(), (short) dti.getScale() }; + } + throw new UnsupportedOperationException( + "The underlying Parquet type cannot be converted to Hive Decimal type: " + type); + } + + /** + * Decimal64 fast path: read the unscaled value straight into the long-backed vector instead of + * materializing a HiveDecimal per row. Only for columns the vectorizer tagged DECIMAL_64 + * (precision <= 18); higher precision uses {@link #readDecimal}. + */ + private void readDecimal64(int total, Decimal64ColumnVector c, int rowId) { + fillDecimal64PrecisionScale(c); + boolean fast = dataColumn.isFastDecimal64(); + short valueScale = getEncodedDecimalScale(); + int left = total; + while (left > 0) { + readRepetitionAndDefinitionLevels(); + if (definitionLevel >= maxDefLevel) { + boolean stored; + if (fast) { + // Identity fast path: store the raw unscaled long directly (no HiveDecimal/byte[] per row). + long v = dataColumn.readDecimal64(); + stored = dataColumn.isValid(); + if (stored) { + c.vector[rowId] = v; + } + } else { + // set() enforces the column precision/scale and marks the entry NULL if the value does not + // fit (e.g. schema-evolved data whose larger file scale can't be held at the column scale). + byte[] bytes = dataColumn.readDecimal(); + stored = dataColumn.isValid(); + if (stored) { + c.isNull[rowId] = false; + c.set(rowId, bytes, valueScale); + stored = !c.isNull[rowId]; + } + } + if (stored) { + c.isNull[rowId] = false; + c.isRepeating = c.isRepeating && (c.vector[0] == c.vector[rowId]); + } else { + c.vector[rowId] = 0; + setNullValue(c, rowId); + } + } else { + setNullValue(c, rowId); + } + rowId++; + left--; } } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java index 0a0867fff9f1..0e8cb81f25c6 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java @@ -119,6 +119,31 @@ public void decimalRead() throws Exception { stringReadDecimal(isDictionaryEncoding); } + @Test + public void testDecimal64Read() throws Exception { + decimal64Read(isDictionaryEncoding); + } + + @Test + public void testDecimal64ReadInt32() throws Exception { + decimal64ReadInt32(); + } + + @Test + public void testDecimal64ReadInt64() throws Exception { + decimal64ReadInt64(); + } + + @Test + public void testDecimal64ReadScaleEvolution() throws Exception { + decimal64ReadScaleEvolution(); + } + + @Test + public void testDecimal64ReadPrecisionNarrowing() throws Exception { + decimal64ReadPrecisionNarrowing(); + } + @Test public void verifyBatchOffsets() throws Exception { super.verifyBatchOffsets(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedDictionaryEncodingColumnReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedDictionaryEncodingColumnReader.java index 32d27d95477f..d80abc34d094 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedDictionaryEncodingColumnReader.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedDictionaryEncodingColumnReader.java @@ -96,4 +96,19 @@ public void decimalRead() throws Exception { decimalRead(isDictionaryEncoding); stringReadDecimal(isDictionaryEncoding); } + + @Test + public void testDecimal64Read() throws Exception { + decimal64Read(isDictionaryEncoding); + } + + @Test + public void testDecimal64ReadInt32() throws Exception { + decimal64ReadInt32(); + } + + @Test + public void testDecimal64ReadInt64() throws Exception { + decimal64ReadInt64(); + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java index c9c858932550..dcc3f2ffb8b4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java @@ -25,8 +25,10 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -60,9 +62,13 @@ import org.apache.parquet.hadoop.example.GroupReadSupport; import org.apache.parquet.hadoop.example.GroupWriteSupport; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; import java.io.IOException; +import java.math.BigInteger; import java.util.Arrays; import java.util.List; import java.util.TimeZone; @@ -258,19 +264,36 @@ protected static boolean isNull(int index) { public static VectorizedParquetRecordReader createTestParquetReader(String schemaString, Configuration conf) throws IOException, InterruptedException, HiveException { + return createTestParquetReader(schemaString, conf, null); + } + + public static VectorizedParquetRecordReader createTestParquetReader(String schemaString, Configuration conf, + DataTypePhysicalVariation[] rowDataTypePhysicalVariations) + throws IOException, InterruptedException, HiveException { + return createTestParquetReader(schemaString, conf, rowDataTypePhysicalVariations, file); + } + + public static VectorizedParquetRecordReader createTestParquetReader(String schemaString, Configuration conf, + DataTypePhysicalVariation[] rowDataTypePhysicalVariations, Path inputFile) + throws IOException, InterruptedException, HiveException { conf.set(PARQUET_READ_SCHEMA, schemaString); HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); HiveConf.setVar(conf, HiveConf.ConfVars.PLAN, "//tmp"); Job vectorJob = new Job(conf, "read vector"); - ParquetInputFormat.setInputPaths(vectorJob, file); - initialVectorizedRowBatchCtx(conf); - return new VectorizedParquetRecordReader(getFileSplit(vectorJob), new JobConf(conf)); + ParquetInputFormat.setInputPaths(vectorJob, inputFile); + initialVectorizedRowBatchCtx(conf, rowDataTypePhysicalVariations); + return new VectorizedParquetRecordReader(getFileSplit(vectorJob, inputFile), new JobConf(conf)); } protected static FileSplit getFileSplit(Job vectorJob) throws IOException, InterruptedException { + return getFileSplit(vectorJob, file); + } + + protected static FileSplit getFileSplit(Job vectorJob, Path inputFile) + throws IOException, InterruptedException { ParquetInputFormat parquetInputFormat = new ParquetInputFormat(GroupReadSupport.class); InputSplit split = (InputSplit) parquetInputFormat.getSplits(vectorJob).get(0); - FileSplit fsplit = new FileSplit(file, 0L, split.getLength(), split.getLocations()); + FileSplit fsplit = new FileSplit(inputFile, 0L, split.getLength(), split.getLocations()); return fsplit; } @@ -344,14 +367,368 @@ protected static void writeData(ParquetWriter writer, boolean isDictionar } protected static void initialVectorizedRowBatchCtx(Configuration conf) throws HiveException { + initialVectorizedRowBatchCtx(conf, null); + } + + protected static void initialVectorizedRowBatchCtx(Configuration conf, + DataTypePhysicalVariation[] rowDataTypePhysicalVariations) throws HiveException { MapWork mapWork = new MapWork(); VectorizedRowBatchCtx rbCtx = new VectorizedRowBatchCtx(); rbCtx.init(createStructObjectInspector(conf), new String[0]); + if (rowDataTypePhysicalVariations != null) { + rbCtx.setRowDataTypePhysicalVariations(rowDataTypePhysicalVariations); + } mapWork.setVectorMode(true); mapWork.setVectorizedRowBatchCtx(rbCtx); Utilities.setMapWork(conf, mapWork); } + /** + * Verifies the Decimal64 read path: when the decimal column is tagged DECIMAL_64 (as the + * vectorizer does once {@code MapredParquetInputFormat} advertises it), the reader must fill a + * {@link Decimal64ColumnVector} (long-backed) with the correct unscaled values. + */ + protected void decimal64Read(boolean isDictionaryEncoding) throws Exception { + Configuration readerConf = new Configuration(); + readerConf.set(IOConstants.COLUMNS, "value"); + readerConf.set(IOConstants.COLUMNS_TYPES, "decimal(5,2)"); + readerConf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false); + readerConf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0"); + VectorizedParquetRecordReader reader = createTestParquetReader( + "message hive_schema { required value (DECIMAL(5,2));}", readerConf, + new DataTypePhysicalVariation[] { DataTypePhysicalVariation.DECIMAL_64 }); + VectorizedRowBatch previous = reader.createValue(); + try { + int c = 0; + while (reader.next(NullWritable.get(), previous)) { + assertTrue("expected Decimal64ColumnVector but got " + previous.cols[0].getClass().getSimpleName(), + previous.cols[0] instanceof Decimal64ColumnVector); + Decimal64ColumnVector vector = (Decimal64ColumnVector) previous.cols[0]; + assertTrue(vector.noNulls); + assertEquals((short) 5, vector.precision); + assertEquals((short) 2, vector.scale); + for (int i = 0; i < vector.vector.length; i++) { + if (c == nElements) { + break; + } + long expected = + new HiveDecimalWritable(getDecimal(isDictionaryEncoding, c).setScale(2)).serialize64(2); + assertEquals("Check failed at pos " + c, expected, vector.vector[i]); + assertFalse(vector.isNull[i]); + c++; + } + } + assertEquals(nElements, c); + } finally { + reader.close(); + } + } + + // Unscaled values (scale=2) used by the INT32/INT64-backed Decimal64 tests. The element at + // DECIMAL64_NULL_INDEX is ignored because those rows are written as null. 1001 -> 10.01, + // 1234 -> 12.34, -550 -> -5.50, 9999 -> 99.99. These would all decode WRONG (truncated integer + // part, e.g. 1001 -> 10) on the buggy readInteger()/readLong() path, and to the correct unscaled + // long on the fixed reader path. At scale 2 that equals the literal value, which is what the + // tests assert. + private static final long[] DECIMAL64_UNSCALED = { 1001L, 1234L, -550L, 0L, 9999L }; + private static final int DECIMAL64_NULL_INDEX = 3; + private static final int DECIMAL64_ROWS = 200; + // Dedicated file so the per-test write never clobbers the class-level {@link #file} fixture that + // other @Test methods read via the @BeforeClass writeData(); JUnit @Test ordering is undefined. + private static final Path DECIMAL64_FILE = + new Path(System.getProperty("java.io.tmpdir"), "testDecimal64ParquetFile"); + + /** + * Writes a parquet file with a single DECIMAL(5,2) column physically stored as the given + * primitive ({@link PrimitiveTypeName#INT32} or {@link PrimitiveTypeName#INT64}), cycling + * through the unscaled values in {@link #DECIMAL64_UNSCALED} (index {@link #DECIMAL64_NULL_INDEX} + * is written as null), then reads it back tagged DECIMAL_64 and asserts the long-backed + * {@link Decimal64ColumnVector} holds the correct unscaled longs (NOT the truncated/zero values + * the buggy reader produced). + *

+ * Runs with {@code dictionaryEncoding} both true and false so that BOTH reader paths are + * exercised: with dictionary encoding on (few distinct values, huge dictionary page) Parquet + * dictionary-encodes the column and the read goes through {@code decodeDictionaryIds}; with it off + * every value is plain-encoded and the read goes through {@code readDecimal64}. Both populate the + * {@link Decimal64ColumnVector} via {@link Decimal64ColumnVector#set(int, byte[], int)}. + */ + protected void decimal64ReadFromPrimitive(PrimitiveTypeName physical, boolean dictionaryEncoding) + throws Exception { + final int scale = 2; + final int precision = 5; + MessageType writeSchema = Types.buildMessage() + .optional(physical).as(LogicalTypeAnnotation.decimalType(scale, precision)).named("value") + .named("hive_schema"); + + FileSystem fs = DECIMAL64_FILE.getFileSystem(conf); + if (fs.exists(DECIMAL64_FILE)) { + fs.delete(DECIMAL64_FILE, true); + } + GroupWriteSupport.setSchema(writeSchema, conf); + try (ParquetWriter writer = new ParquetWriter<>( + DECIMAL64_FILE, new GroupWriteSupport(), GZIP, 1024 * 1024, 1024, 1024 * 1024, + dictionaryEncoding, false, PARQUET_1_0, conf)) { + SimpleGroupFactory f = new SimpleGroupFactory(writeSchema); + for (int i = 0; i < DECIMAL64_ROWS; i++) { + int idx = i % DECIMAL64_UNSCALED.length; + Group group = f.newGroup(); + if (idx != DECIMAL64_NULL_INDEX) { + if (physical == PrimitiveTypeName.INT32) { + group.append("value", (int) DECIMAL64_UNSCALED[idx]); + } else { + group.append("value", DECIMAL64_UNSCALED[idx]); + } + } + writer.write(group); + } + } + + Configuration readerConf = new Configuration(); + readerConf.set(IOConstants.COLUMNS, "value"); + readerConf.set(IOConstants.COLUMNS_TYPES, "decimal(" + precision + "," + scale + ")"); + readerConf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false); + readerConf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0"); + VectorizedParquetRecordReader reader = createTestParquetReader( + writeSchema.toString(), readerConf, + new DataTypePhysicalVariation[] { DataTypePhysicalVariation.DECIMAL_64 }, DECIMAL64_FILE); + VectorizedRowBatch previous = reader.createValue(); + String label = physical + (dictionaryEncoding ? " (dict)" : " (plain)"); + try { + int c = 0; + boolean sawNull = false; + while (reader.next(NullWritable.get(), previous)) { + assertTrue("expected Decimal64ColumnVector but got " + previous.cols[0].getClass().getSimpleName(), + previous.cols[0] instanceof Decimal64ColumnVector); + Decimal64ColumnVector vector = (Decimal64ColumnVector) previous.cols[0]; + assertEquals((short) precision, vector.precision); + assertEquals((short) scale, vector.scale); + for (int i = 0; i < previous.size; i++) { + if (c == DECIMAL64_ROWS) { + break; + } + int idx = c % DECIMAL64_UNSCALED.length; + if (idx == DECIMAL64_NULL_INDEX) { + assertTrue("Expected null at pos " + c + " for " + label, vector.isNull[i]); + sawNull = true; + } else { + assertFalse("Unexpected null at pos " + c + " for " + label, vector.isNull[i]); + // A Decimal64 vector must hold the FULL unscaled long: 10.01 -> 1001, not 10 or 0. + // For scale 2 the expected unscaled long is exactly DECIMAL64_UNSCALED[idx], so the + // buggy truncating reader (returning the integer part 10, or 0) cannot pass this. + assertEquals("Decimal64 must keep the full unscaled value at pos " + c + " for " + label, + DECIMAL64_UNSCALED[idx], vector.vector[i]); + } + c++; + } + } + assertEquals("Did not read all rows for " + label, DECIMAL64_ROWS, c); + assertTrue("Null row was never exercised for " + label, sawNull); + } finally { + reader.close(); + if (fs.exists(DECIMAL64_FILE)) { + fs.delete(DECIMAL64_FILE, true); + } + } + } + + protected void decimal64ReadInt32() throws Exception { + decimal64ReadFromPrimitive(PrimitiveTypeName.INT32, true); + decimal64ReadFromPrimitive(PrimitiveTypeName.INT32, false); + } + + protected void decimal64ReadInt64() throws Exception { + decimal64ReadFromPrimitive(PrimitiveTypeName.INT64, true); + decimal64ReadFromPrimitive(PrimitiveTypeName.INT64, false); + } + + /** + * Gate test for the Decimal64 identity fast path. When the Parquet file scale differs from the Hive + * table scale (schema evolution), the fast path must NOT engage: each value must be rescaled + * (rounded, or NULLed when it no longer fits) exactly as HiveDecimal would -- never copied verbatim, + * which would be off by a power of ten. Writes DECIMAL(9,4), reads as DECIMAL(6,2), and asserts every + * result against a HiveDecimal oracle. (The {@code decimal64Read*} tests cover the file==table-scale + * identity fast path; this covers the mismatched-scale fallback and proves the gate.) + */ + protected void decimal64ReadScaleEvolution(PrimitiveTypeName physical, boolean dictionaryEncoding) + throws Exception { + final int fileScale = 4; + final int filePrecision = 9; + final int readScale = 2; + final int readPrecision = 6; + // Unscaled values at fileScale=4 (index DECIMAL64_NULL_INDEX is written as null): + // 1234567 -> 123.4567 (rounds to 123.46), 100 -> 0.0100, -98765 -> -9.8765 (-> -9.88), + // 99999999 -> 9999.9999 (rounds to 10000.00 -> exceeds DECIMAL(6,2) -> NULL). + long[] unscaled = { 1234567L, 100L, -98765L, 0L, 99999999L }; + + FileSystem fs = DECIMAL64_FILE.getFileSystem(conf); + if (fs.exists(DECIMAL64_FILE)) { + fs.delete(DECIMAL64_FILE, true); + } + MessageType writeSchema = Types.buildMessage() + .optional(physical).as(LogicalTypeAnnotation.decimalType(fileScale, filePrecision)).named("value") + .named("hive_schema"); + GroupWriteSupport.setSchema(writeSchema, conf); + try (ParquetWriter writer = new ParquetWriter<>( + DECIMAL64_FILE, new GroupWriteSupport(), GZIP, 1024 * 1024, 1024, 1024 * 1024, + dictionaryEncoding, false, PARQUET_1_0, conf)) { + SimpleGroupFactory f = new SimpleGroupFactory(writeSchema); + for (int i = 0; i < DECIMAL64_ROWS; i++) { + int idx = i % unscaled.length; + Group group = f.newGroup(); + if (idx != DECIMAL64_NULL_INDEX) { + if (physical == PrimitiveTypeName.INT32) { + group.append("value", (int) unscaled[idx]); + } else { + group.append("value", unscaled[idx]); + } + } + writer.write(group); + } + } + + Configuration readerConf = new Configuration(); + readerConf.set(IOConstants.COLUMNS, "value"); + readerConf.set(IOConstants.COLUMNS_TYPES, "decimal(" + readPrecision + "," + readScale + ")"); + readerConf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false); + readerConf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0"); + VectorizedParquetRecordReader reader = createTestParquetReader( + writeSchema.toString(), readerConf, + new DataTypePhysicalVariation[] { DataTypePhysicalVariation.DECIMAL_64 }, DECIMAL64_FILE); + VectorizedRowBatch previous = reader.createValue(); + String label = physical + (dictionaryEncoding ? " (dict)" : " (plain)") + " scale-evolution"; + HiveDecimalWritable oracle = new HiveDecimalWritable(); + try { + int c = 0; + while (reader.next(NullWritable.get(), previous)) { + Decimal64ColumnVector vector = (Decimal64ColumnVector) previous.cols[0]; + // The long is stored at the Hive (table) scale, NOT the file scale -- proves the gate. + assertEquals((short) readScale, vector.scale); + for (int i = 0; i < previous.size; i++) { + if (c == DECIMAL64_ROWS) { + break; + } + int idx = c % unscaled.length; + if (idx == DECIMAL64_NULL_INDEX) { + assertTrue("Expected null at pos " + c + " for " + label, vector.isNull[i]); + } else { + // Oracle: interpret the unscaled value at the file scale, enforce to the read type. + oracle.set(HiveDecimal.create(BigInteger.valueOf(unscaled[idx]), fileScale)); + oracle.mutateEnforcePrecisionScale(readPrecision, readScale); + if (!oracle.isSet()) { + assertTrue("Expected NULL (out of range) at pos " + c + " for " + label, vector.isNull[i]); + } else { + assertFalse("Unexpected null at pos " + c + " for " + label, vector.isNull[i]); + assertEquals("Scale-evolved Decimal64 must match HiveDecimal at pos " + c + " for " + label, + oracle.serialize64(readScale), vector.vector[i]); + } + } + c++; + } + } + assertEquals("Did not read all rows for " + label, DECIMAL64_ROWS, c); + } finally { + reader.close(); + if (fs.exists(DECIMAL64_FILE)) { + fs.delete(DECIMAL64_FILE, true); + } + } + } + + protected void decimal64ReadScaleEvolution() throws Exception { + decimal64ReadScaleEvolution(PrimitiveTypeName.INT32, true); + decimal64ReadScaleEvolution(PrimitiveTypeName.INT32, false); + decimal64ReadScaleEvolution(PrimitiveTypeName.INT64, true); + decimal64ReadScaleEvolution(PrimitiveTypeName.INT64, false); + } + + /** + * Bounds test for the Decimal64 identity fast path: file scale == table scale (so the fast path + * engages) but the file precision is wider than the table precision. Values outside the table + * precision -- and the pathological {@link Long#MIN_VALUE} -- must be NULLed by the fast path's + * bounds check, exactly as the enforced path would. Writes DECIMAL(filePrecision,2), reads DECIMAL(5,2). + */ + protected void decimal64ReadPrecisionNarrowing(PrimitiveTypeName physical, boolean dictionaryEncoding) + throws Exception { + final int scale = 2; + final int filePrecision = (physical == PrimitiveTypeName.INT32) ? 9 : 18; + final int readPrecision = 5; // max abs unscaled value = 99999 + final long absMax = 99999L; + // Unscaled @ scale 2: 1001 -> 10.01 (fits), 99999 -> 999.99 (boundary, fits), + // 100000 -> 1000.00 (precision 6 > 5 -> NULL), then an out-of-range negative -> NULL. + long[] unscaled = (physical == PrimitiveTypeName.INT32) + ? new long[] { 1001L, 99999L, 100000L, -100000L } + : new long[] { 1001L, 99999L, 100000L, Long.MIN_VALUE }; + + FileSystem fs = DECIMAL64_FILE.getFileSystem(conf); + if (fs.exists(DECIMAL64_FILE)) { + fs.delete(DECIMAL64_FILE, true); + } + MessageType writeSchema = Types.buildMessage() + .optional(physical).as(LogicalTypeAnnotation.decimalType(scale, filePrecision)).named("value") + .named("hive_schema"); + GroupWriteSupport.setSchema(writeSchema, conf); + try (ParquetWriter writer = new ParquetWriter<>( + DECIMAL64_FILE, new GroupWriteSupport(), GZIP, 1024 * 1024, 1024, 1024 * 1024, + dictionaryEncoding, false, PARQUET_1_0, conf)) { + SimpleGroupFactory f = new SimpleGroupFactory(writeSchema); + for (int i = 0; i < DECIMAL64_ROWS; i++) { + long v = unscaled[i % unscaled.length]; + Group group = f.newGroup(); + if (physical == PrimitiveTypeName.INT32) { + group.append("value", (int) v); + } else { + group.append("value", v); + } + writer.write(group); + } + } + + Configuration readerConf = new Configuration(); + readerConf.set(IOConstants.COLUMNS, "value"); + readerConf.set(IOConstants.COLUMNS_TYPES, "decimal(" + readPrecision + "," + scale + ")"); + readerConf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false); + readerConf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0"); + VectorizedParquetRecordReader reader = createTestParquetReader( + writeSchema.toString(), readerConf, + new DataTypePhysicalVariation[] { DataTypePhysicalVariation.DECIMAL_64 }, DECIMAL64_FILE); + VectorizedRowBatch previous = reader.createValue(); + String label = physical + (dictionaryEncoding ? " (dict)" : " (plain)") + " precision-narrowing"; + try { + int c = 0; + while (reader.next(NullWritable.get(), previous)) { + Decimal64ColumnVector vector = (Decimal64ColumnVector) previous.cols[0]; + assertEquals((short) scale, vector.scale); + for (int i = 0; i < previous.size; i++) { + if (c == DECIMAL64_ROWS) { + break; + } + long v = unscaled[c % unscaled.length]; + if (v < -absMax || v > absMax) { + assertTrue("Expected NULL (out of table precision) at pos " + c + " for " + label, vector.isNull[i]); + } else { + assertFalse("Unexpected null at pos " + c + " for " + label, vector.isNull[i]); + assertEquals("In-range Decimal64 must be stored verbatim at pos " + c + " for " + label, + v, vector.vector[i]); + } + c++; + } + } + assertEquals("Did not read all rows for " + label, DECIMAL64_ROWS, c); + } finally { + reader.close(); + if (fs.exists(DECIMAL64_FILE)) { + fs.delete(DECIMAL64_FILE, true); + } + } + } + + protected void decimal64ReadPrecisionNarrowing() throws Exception { + decimal64ReadPrecisionNarrowing(PrimitiveTypeName.INT32, true); + decimal64ReadPrecisionNarrowing(PrimitiveTypeName.INT32, false); + decimal64ReadPrecisionNarrowing(PrimitiveTypeName.INT64, true); + decimal64ReadPrecisionNarrowing(PrimitiveTypeName.INT64, false); + } + private static StructObjectInspector createStructObjectInspector(Configuration conf) { // Create row related objects String columnNames = conf.get(IOConstants.COLUMNS); diff --git a/ql/src/test/queries/clientpositive/parquet_decimal64.q b/ql/src/test/queries/clientpositive/parquet_decimal64.q new file mode 100644 index 000000000000..ce06dee45c54 --- /dev/null +++ b/ql/src/test/queries/clientpositive/parquet_decimal64.q @@ -0,0 +1,15 @@ +--! qt:replace:/(\s+Statistics: Num rows: \d+)/#Masked#/ +set hive.vectorized.execution.enabled=true; +set hive.explain.user=false; + +drop table if exists dec64_parquet; + +create table dec64_parquet (k int, d decimal(7,2)) stored as parquet; +insert into dec64_parquet values + (1, 1.10), (1, 2.20), (2, 3.30), (2, 4.40), (3, 5.50), (3, cast(null as decimal(7,2))); + +-- Verifies that the Parquet vectorized reader engages the DECIMAL_64 path: +explain vectorization detail +select k, sum(d) from dec64_parquet group by k; + +select k, sum(d) from dec64_parquet group by k order by k; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/llap/parquet_complex_types_vectorization.q.out b/ql/src/test/results/clientpositive/llap/parquet_complex_types_vectorization.q.out index 907c374f1841..289cc55a5834 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_complex_types_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_complex_types_vectorization.q.out @@ -146,8 +146,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -267,8 +267,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -454,8 +454,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -575,8 +575,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -762,8 +762,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -883,8 +883,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_decimal64.q.out b/ql/src/test/results/clientpositive/llap/parquet_decimal64.q.out new file mode 100644 index 000000000000..cb52b3c0393d --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/parquet_decimal64.q.out @@ -0,0 +1,171 @@ +PREHOOK: query: drop table if exists dec64_parquet +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists dec64_parquet +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create table dec64_parquet (k int, d decimal(7,2)) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dec64_parquet +POSTHOOK: query: create table dec64_parquet (k int, d decimal(7,2)) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dec64_parquet +PREHOOK: query: insert into dec64_parquet values + (1, 1.10), (1, 2.20), (2, 3.30), (2, 4.40), (3, 5.50), (3, cast(null as decimal(7,2))) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dec64_parquet +POSTHOOK: query: insert into dec64_parquet values + (1, 1.10), (1, 2.20), (2, 3.30), (2, 4.40), (3, 5.50), (3, cast(null as decimal(7,2))) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dec64_parquet +POSTHOOK: Lineage: dec64_parquet.d SCRIPT [] +POSTHOOK: Lineage: dec64_parquet.k SCRIPT [] +PREHOOK: query: explain vectorization detail +select k, sum(d) from dec64_parquet group by k +PREHOOK: type: QUERY +PREHOOK: Input: default@dec64_parquet +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail +select k, sum(d) from dec64_parquet group by k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dec64_parquet +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: dec64_parquet +#Masked# Data size: 696 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:k:int, 1:d:decimal(7,2)/DECIMAL_64, 2:ROW__ID:struct, 3:ROW__IS__DELETED:boolean] + Select Operator + expressions: k (type: int), d (type: decimal(7,2)) + outputColumnNames: k, d + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] +#Masked# Data size: 696 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(d) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64(col 1:decimal(7,2)/DECIMAL_64) -> decimal(17,2)/DECIMAL_64 + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 0:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + keys: k (type: int) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 +#Masked# Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 0:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 1:decimal(17,2) +#Masked# Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(17,2)) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + includeColumns: [0, 1] + dataColumns: k:int, d:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true + reduceColumnNullOrder: z + reduceColumnSortOrder: + + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + dataColumns: KEY._col0:int, VALUE._col0:decimal(17,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64(col 1:decimal(17,2)/DECIMAL_64) -> decimal(17,2)/DECIMAL_64 + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + keyExpressions: col 0:int + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumnNums: [0] + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 +#Masked# Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false +#Masked# Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select k, sum(d) from dec64_parquet group by k order by k +PREHOOK: type: QUERY +PREHOOK: Input: default@dec64_parquet +#### A masked pattern was here #### +POSTHOOK: query: select k, sum(d) from dec64_parquet group by k order by k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dec64_parquet +#### A masked pattern was here #### +1 3.30 +2 7.70 +3 5.50 diff --git a/ql/src/test/results/clientpositive/llap/parquet_map_type_vectorization.q.out b/ql/src/test/results/clientpositive/llap/parquet_map_type_vectorization.q.out index 8de33e7c480b..458ddf3c5297 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_map_type_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_map_type_vectorization.q.out @@ -158,8 +158,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -283,8 +283,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_struct_type_vectorization.q.out b/ql/src/test/results/clientpositive/llap/parquet_struct_type_vectorization.q.out index 20c6f428a8c2..faca95103c2f 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_struct_type_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_struct_type_vectorization.q.out @@ -134,8 +134,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -291,8 +291,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_0.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_0.q.out index 3d01120514a3..41f017ab2f0b 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_0.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_0.q.out @@ -123,8 +123,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -251,8 +251,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -443,8 +443,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -571,8 +571,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -763,8 +763,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -891,8 +891,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -1084,8 +1084,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_1.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_1.q.out index a00c56715b32..85b2feec2ee4 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_1.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_1.q.out @@ -111,8 +111,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_10.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_10.q.out index 87b54a394c53..28a89994b74f 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_10.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_10.q.out @@ -102,8 +102,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_11.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_11.q.out index 96e5976e66de..f7fb95307d02 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_11.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_11.q.out @@ -84,8 +84,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_12.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_12.q.out index 02fde23144f2..1a797059afd5 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_12.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_12.q.out @@ -138,8 +138,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_13.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_13.q.out index 7125704c33d2..cdeb5e543056 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_13.q.out @@ -150,8 +150,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -507,8 +507,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_14.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_14.q.out index 5acc12c3b71d..f5780b5bbe0f 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_14.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_14.q.out @@ -140,8 +140,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_15.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_15.q.out index 5979bc2dbb9a..41cfc82d2aab 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_15.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_15.q.out @@ -136,8 +136,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_16.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_16.q.out index b3c24ec4c133..969bf4fb798d 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_16.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_16.q.out @@ -113,8 +113,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_17.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_17.q.out index 1300d7beb851..93d249b7ef87 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_17.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_17.q.out @@ -105,8 +105,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_2.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_2.q.out index 4e8b06ab91e6..3f32b3cbd15b 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_2.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_2.q.out @@ -115,8 +115,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_3.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_3.q.out index 25e7197cedb8..529d61eb1db3 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_3.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_3.q.out @@ -120,8 +120,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_4.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_4.q.out index aecd787c793c..7bcf796c86ea 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_4.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_4.q.out @@ -115,8 +115,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_5.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_5.q.out index 342674ecd771..f905323dba6f 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_5.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_5.q.out @@ -108,8 +108,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_6.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_6.q.out index 48d7026eb630..37a7e111dc73 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_6.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_6.q.out @@ -96,8 +96,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_7.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_7.q.out index 02d8275b94e8..cfde16992112 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_7.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_7.q.out @@ -120,8 +120,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -369,8 +369,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_8.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_8.q.out index 22a2e1922e37..8f6bb22b2814 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_8.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_8.q.out @@ -116,8 +116,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -352,8 +352,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_9.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_9.q.out index b3c24ec4c133..969bf4fb798d 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_9.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_9.q.out @@ -113,8 +113,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_decimal_date.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_decimal_date.q.out index 05e4d277465d..0bab42017f29 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_decimal_date.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_decimal_date.q.out @@ -77,8 +77,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_div0.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_div0.q.out index 9f9a40bac222..5cc571489008 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_div0.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_div0.q.out @@ -58,8 +58,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -256,8 +256,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -489,8 +489,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_limit.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_limit.q.out index 1167c5d43e92..09d31bdd4c41 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_limit.q.out @@ -52,8 +52,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -177,8 +177,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -334,8 +334,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -502,8 +502,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -666,8 +666,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -870,8 +870,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_offset_limit.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_offset_limit.q.out index a65dddfbdcd6..e14c57ffbed4 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_offset_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_offset_limit.q.out @@ -50,8 +50,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -172,8 +172,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_part_project.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_part_project.q.out index 4f13a49cc63a..bb56a81f6e61 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_part_project.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_part_project.q.out @@ -99,8 +99,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_pushdown.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_pushdown.q.out index d90110e7b662..170da8b19b52 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_pushdown.q.out @@ -52,8 +52,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out index 80c87d730585..630a39fab04d 100644 --- a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out @@ -2980,8 +2980,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -3174,8 +3174,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -3697,8 +3697,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -3970,8 +3970,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -4212,8 +4212,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -4759,8 +4759,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -5032,8 +5032,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -5274,8 +5274,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out b/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out index ae3717090477..9948978d5a05 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out @@ -101,8 +101,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -204,8 +204,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -785,8 +785,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -888,8 +888,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/vectorization_numeric_overflows.q.out b/ql/src/test/results/clientpositive/llap/vectorization_numeric_overflows.q.out index acc0073a7a0c..374b0b572553 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_numeric_overflows.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_numeric_overflows.q.out @@ -168,8 +168,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -295,8 +295,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -420,8 +420,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -545,8 +545,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -672,8 +672,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -797,8 +797,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -1090,8 +1090,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false @@ -1313,8 +1313,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: true usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/vectorization_parquet_projection.q.out b/ql/src/test/results/clientpositive/llap/vectorization_parquet_projection.q.out index e3bc4547b9dd..e0d69c927f32 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_parquet_projection.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_parquet_projection.q.out @@ -247,8 +247,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -344,8 +344,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -563,8 +563,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out b/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out index 78eb986611ab..d0b011f3ef24 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out @@ -174,8 +174,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out b/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out index 51a5fc25c098..f14be4078046 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out @@ -311,7 +311,7 @@ STAGE PLANS: Group By Operator aggregations: max(_col1), min(_col2), count(_col3), sum(_col4), count(_col4), sum(_col7), sum(_col5), count(_col5), max(_col6) Group By Vectorization: - aggregators: VectorUDAFMaxLong(col 0:int) -> int, VectorUDAFMinLong(col 2:smallint) -> smallint, VectorUDAFCount(col 5:string) -> bigint, VectorUDAFSumDouble(col 3:float) -> double, VectorUDAFCount(col 3:float) -> bigint, VectorUDAFSumDouble(col 13:double) -> double, VectorUDAFSumDouble(col 4:double) -> double, VectorUDAFCount(col 4:double) -> bigint, VectorUDAFMaxDecimal(col 10:decimal(4,2)) -> decimal(4,2) + aggregators: VectorUDAFMaxLong(col 0:int) -> int, VectorUDAFMinLong(col 2:smallint) -> smallint, VectorUDAFCount(col 5:string) -> bigint, VectorUDAFSumDouble(col 3:float) -> double, VectorUDAFCount(col 3:float) -> bigint, VectorUDAFSumDouble(col 13:double) -> double, VectorUDAFSumDouble(col 4:double) -> double, VectorUDAFCount(col 4:double) -> bigint, VectorUDAFMaxDecimal64(col 10:decimal(4,2)/DECIMAL_64) -> decimal(4,2)/DECIMAL_64 className: VectorGroupByOperator groupByMode: HASH keyExpressions: col 1:tinyint @@ -339,8 +339,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false @@ -652,8 +652,8 @@ STAGE PLANS: Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat allNative: false usesVectorUDFAdaptor: false diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java index 3753d337d8b6..8c3152a02868 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java @@ -63,13 +63,27 @@ public void fill(HiveDecimal value) { */ public void set(int elementNum, HiveDecimalWritable writable) { scratchHiveDecWritable.set(writable); - scratchHiveDecWritable.mutateEnforcePrecisionScale(precision, scale); - if (!scratchHiveDecWritable.isSet()) { - noNulls = false; - isNull[elementNum] = true; - } else { - vector[elementNum] = scratchHiveDecWritable.serialize64(scale); - } + enforceScaleAndSet(elementNum); + } + + /** + * Set a Decimal64 field straight from the unscaled big-integer {@code bigIntegerBytes} (encoded at + * {@code valueScale}), without materializing a HiveDecimal per row -- unlike the sibling + * {@link #set(int, HiveDecimalWritable)}. + * + * {@code valueScale} may differ from this vector's {@link #scale} under schema evolution (e.g. + * reading a DECIMAL(38,37) file column as DECIMAL(16,8)); the stored long is always at this + * vector's scale, where the value fits the 64-bit backing store and downstream consumers read it. + * + * FAST version: assumes elementNum is already adjusted for isRepeating and the isNull entry is set. + * + * @param elementNum + * @param bigIntegerBytes + * @param valueScale + */ + public void set(int elementNum, byte[] bigIntegerBytes, int valueScale) { + scratchHiveDecWritable.set(bigIntegerBytes, valueScale); + enforceScaleAndSet(elementNum); } /** @@ -87,6 +101,14 @@ public void set(int elementNum, HiveDecimalWritable writable) { */ public void set(int elementNum, HiveDecimal hiveDec) { scratchHiveDecWritable.set(hiveDec); + enforceScaleAndSet(elementNum); + } + + /** + * Apply this vector's precision/scale to the value in scratchHiveDecWritable and store it as a + * Decimal64 long, marking the entry NULL if out of range. + */ + private void enforceScaleAndSet(int elementNum) { scratchHiveDecWritable.mutateEnforcePrecisionScale(precision, scale); if (!scratchHiveDecWritable.isSet()) { noNulls = false;