[WIP] HIVE-29641: Upgrade Calcite to 1.42.0

druid-handler/pom.xml

@@ -409,6 +409,17 @@
                 </configuration>
               </execution>
             </executions>
+            <dependencies>
+              <dependency>
+                <groupId>org.ow2.asm</groupId>
+                <artifactId>asm</artifactId>
+                <version>9.6</version> </dependency>
+              <dependency>
+                <groupId>org.ow2.asm</groupId>
+                <artifactId>asm-commons</artifactId>
+                <version>9.6</version>
+              </dependency>
+            </dependencies>
           </plugin>
         </plugins>
       </build>

hbase-handler/src/test/results/positive/hbase_queries.q.out

@@ -148,7 +148,7 @@ STAGE PLANS:
             alias: hbase_table_1
             Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
+              predicate: key is not null (type: boolean)
               Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: UDFToDouble(key) (type: double)
@@ -604,7 +604,7 @@ STAGE PLANS:
             alias: hbase_table_1
             Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
+              predicate: key is not null (type: boolean)
               Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double)

iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out

@@ -621,7 +621,7 @@ STAGE PLANS:
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
                 Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE
                 Filter Operator
-                  predicate: ((((_col5 <> _col8) or (_col5 <= 100)) and _col5 is not null) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean)
+                  predicate: ((_col5 <> _col8) or (_col5 <= 100) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean)
                   Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE
                   Select Operator
                     expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)

itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out

@@ -87,7 +87,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')]))
+FROM TABLE(INLINE(ARRAY ((1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505'))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1

itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out

@@ -51,7 +51,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1)]))
+FROM TABLE(INLINE(ARRAY ((1))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5

itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out

@@ -105,7 +105,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')]))
+FROM TABLE(INLINE(ARRAY ((1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505'))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1

itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out

@@ -59,7 +59,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1)]))
+FROM TABLE(INLINE(ARRAY ((1))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5

kudu-handler/src/test/results/positive/kudu_complex_queries.q.out

@@ -85,10 +85,10 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: kv_table
-                  filterExpr: UDFToDouble(key) is not null (type: boolean)
+                  filterExpr: key is not null (type: boolean)
                   Statistics: Num rows: 309 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: UDFToDouble(key) is not null (type: boolean)
+                    predicate: key is not null (type: boolean)
                     Statistics: Num rows: 309 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: UDFToDouble(key) (type: double)
@@ -246,7 +246,7 @@ STAGE PLANS:
                   filterExpr: (key > 100) (type: boolean)
                   Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: (((key % 2) = 0) and UDFToDouble(key) is not null) (type: boolean)
+                    predicate: ((key % 2) = 0) (type: boolean)
                     Statistics: Num rows: 154 Data size: 14630 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double)

pom.xml

@@ -113,7 +113,7 @@
     <apache-directory-server.version>2.0.0-M24</apache-directory-server.version>
     <avro.version>1.12.0</avro.version>
     <bcprov-jdk18on.version>1.82</bcprov-jdk18on.version>
-    <calcite.version>1.33.0</calcite.version>
+    <calcite.version>1.42.0</calcite.version>
     <datanucleus-api-jdo.version>6.0.5</datanucleus-api-jdo.version>
     <datanucleus-core.version>6.0.11</datanucleus-core.version>
     <datanucleus-jdo.version>3.2.0-release</datanucleus-jdo.version>

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -28,6 +28,7 @@
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.calcite.util.ConversionUtil;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.ObjectUtils;
@@ -569,6 +570,10 @@ private Hive(HiveConf c, boolean doRegisterAllFns) throws HiveException {
     conf = c;
     // turn off calcite rexnode normalization
     System.setProperty("calcite.enable.rexnode.digest.normalize", "false");
+    // update calcite default charset
+    // TODO is this the right place? It's not considered in cases like TestHiveRelJsonReader
+    System.setProperty("calcite.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
+    System.setProperty("calcite.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
     if (doRegisterAllFns) {
       registerAllFunctionsOnce();
     }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java

@@ -48,9 +48,12 @@
 import org.apache.calcite.rex.RexDynamicParam;
 import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLambda;
+import org.apache.calcite.rex.RexLambdaRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexLocalRef;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
 import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexPatternFieldRef;
 import org.apache.calcite.rex.RexShuttle;
@@ -1139,6 +1142,21 @@ public Boolean visitSubQuery(RexSubQuery subQuery) {
     public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
       return false;
     }
+
+    @Override
+    public Boolean visitLambda(RexLambda lambda) {
+      return false;
+    }
+
+    @Override
+    public Boolean visitLambdaRef(RexLambdaRef lambdaRef) {
+      return false;
+    }
+
+    @Override
+    public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+      return false;
+    }
   }
 
   public static Set<Integer> getInputRefs(RexNode expr) {

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java

@@ -89,7 +89,8 @@ public static RelBuilderFactory proto(final Context context) {
     return new RelBuilderFactory() {
       @Override
       public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) {
-        Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED));
+        Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED)
+            .withSimplifyValues(false)); // disabled to avoid simplifications that can create non-empty HiveValues
         return new HiveRelBuilder(Contexts.chain(context, confContext), cluster, schema);
       }
     };

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java

@@ -24,6 +24,7 @@
 import org.apache.calcite.rel.core.TableFunctionScan;
 import org.apache.calcite.rel.core.TableScan;
 import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalAsofJoin;
 import org.apache.calcite.rel.logical.LogicalCalc;
 import org.apache.calcite.rel.logical.LogicalCorrelate;
 import org.apache.calcite.rel.logical.LogicalExchange;
@@ -33,6 +34,7 @@
 import org.apache.calcite.rel.logical.LogicalMatch;
 import org.apache.calcite.rel.logical.LogicalMinus;
 import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalRepeatUnion;
 import org.apache.calcite.rel.logical.LogicalSort;
 import org.apache.calcite.rel.logical.LogicalTableModify;
 import org.apache.calcite.rel.logical.LogicalUnion;
@@ -177,6 +179,16 @@ public RelNode visit(HiveTableScan scan) {
     public RelNode visit(HiveJdbcConverter conv) {
         return visitChild(conv, 0, conv.getInput());
     }
+
+    @Override
+    public RelNode visit(LogicalAsofJoin logicalAsofJoin) {
+        return visitChildren(logicalAsofJoin);
+    }
+
+    @Override
+    public RelNode visit(LogicalRepeatUnion logicalRepeatUnion) {
+        return visitChildren(logicalRepeatUnion);
+    }
 }
 
 // End RelShuttleImpl.java

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java

@@ -53,7 +53,7 @@ public class HiveTypeSystemImpl extends RelDataTypeSystemImpl {
   public int getMaxScale(SqlTypeName typeName) {
     switch (typeName) {
     case DECIMAL:
-      return getMaxNumericScale();
+      return MAX_DECIMAL_SCALE;
     case INTERVAL_YEAR:
     case INTERVAL_MONTH:
     case INTERVAL_YEAR_MONTH:
@@ -126,7 +126,7 @@ public int getDefaultPrecision(SqlTypeName typeName) {
   public int getMaxPrecision(SqlTypeName typeName) {
     switch (typeName) {
     case DECIMAL:
-      return getMaxNumericPrecision();
+      return MAX_DECIMAL_PRECISION;
     case VARCHAR:
     case CHAR:
       return MAX_CHAR_PRECISION;
@@ -157,16 +157,6 @@ public int getMaxPrecision(SqlTypeName typeName) {
     }
   }
 
-  @Override
-  public int getMaxNumericScale() {
-    return MAX_DECIMAL_SCALE;
-  }
-
-  @Override
-  public int getMaxNumericPrecision() {
-    return MAX_DECIMAL_PRECISION;
-  }
-
   @Override
   public boolean isSchemaCaseSensitive() {
     return false;

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumAggFunction.java

@@ -108,7 +108,7 @@ public AggregateCall topSplit(RexBuilder rexBuilder,
         break;
       case 2:
         node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, merges);
-        node = rexBuilder.makeAbstractCast(aggregateCall.type, node);
+        node = rexBuilder.makeAbstractCast(aggregateCall.type, node, false);
         break;
       default:
         throw new AssertionError("unexpected count " + merges);
@@ -120,6 +120,7 @@ public AggregateCall topSplit(RexBuilder rexBuilder,
           false,
           false,
           false,
+          List.of(),
           ImmutableList.of(ordinal),
           -1,
           aggregateCall.distinctKeys,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java

@@ -75,8 +75,8 @@ private static AggregateCall replaceAggFunction(RelDataType rowType, AggregateCa
         SqlTypeUtil.projectTypes(rowType, aggCall.getArgList()),
         aggCall.getType());
     return AggregateCall.create(aggFunction, aggCall.isDistinct(), aggCall.isApproximate(), aggCall.ignoreNulls(),
-        aggCall.getArgList(), aggCall.filterArg, aggCall.distinctKeys, aggCall.getCollation(), aggCall.getType(),
-        aggCall.getName());
+        aggCall.rexList, aggCall.getArgList(), aggCall.filterArg, aggCall.distinctKeys, aggCall.getCollation(),
+        aggCall.getType(), aggCall.getName());
   }
 
   @Override

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFieldTrimmerRule.java

@@ -21,6 +21,7 @@
 import org.apache.calcite.plan.RelOptRuleCall;
 import org.apache.calcite.plan.hep.HepRelVertex;
 import org.apache.calcite.rel.RelNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
 
 
@@ -57,7 +58,7 @@ public void onMatch(RelOptRuleCall call) {
       return;
     }
     // The node is the root, release the kraken!
-    node = HiveHepExtractRelNodeRule.execute(node);
+    node = HiveCalciteUtil.stripHepVertices(node);
     call.transformTo(trim(call, node));
     triggered = true;
   }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveHepExtractRelNodeRule.java

@@ -25,19 +25,32 @@
 import org.apache.calcite.plan.hep.HepRelVertex;
 import org.apache.calcite.rel.RelNode;
 
-
 /**
  * The goal of this rule is to extract the RelNode from the
  * HepRelVertex node so rules do tree traversal can be applied correctly.
  * @see HiveFieldTrimmerRule
  * @see org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveAggregateInsertDeleteIncrementalRewritingRule
  */
+// TODO remove
 public class HiveHepExtractRelNodeRule extends RelOptRule {
 
   public static RelNode execute(RelNode node) {
     final HepPlanner tmpPlanner = new HepPlanner(PROGRAM);
     tmpPlanner.setRoot(node);
     return tmpPlanner.findBestExp();
+
+    // TODO cleanup
+    /* RelNode res = node.accept(new RelShuttleImpl() {
+      @Override
+      public RelNode visit(RelNode other) {
+        RelNode innerNode = other.stripped();
+        if (other != innerNode) {
+          return innerNode.accept(this);
+        }
+        return super.visit(other);
+      }
+    });
+    return res; */
   }
 
   private static final HepProgram PROGRAM = new HepProgramBuilder()

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java

@@ -163,15 +163,15 @@ public void onMatch(RelOptRuleCall call) {
         }
         // Then, we start by trying to reference only left side in top projections
         List<RexNode> swappedTopProjExprs = topProjExprs.stream()
-            .map(projExpr -> projExpr.accept(new RexPermuteInputsShuttle(mappingRL, call.rel(1))))
+            .map(projExpr -> projExpr.accept(new RexPermuteInputsShuttle(mappingRL, false, call.rel(1))))
             .collect(Collectors.toList());
         rightInputPotentialFK = RelOptUtil.InputFinder.bits(swappedTopProjExprs, null).intersects(rightBits);
         if (!rightInputPotentialFK) {
           topProjExprs = swappedTopProjExprs;
         } else {
           // If it did not work, we try to reference only right side in top projections
           swappedTopProjExprs = topProjExprs.stream()
-              .map(projExpr -> projExpr.accept(new RexPermuteInputsShuttle(mappingLR, call.rel(1))))
+              .map(projExpr -> projExpr.accept(new RexPermuteInputsShuttle(mappingLR, false, call.rel(1))))
               .collect(Collectors.toList());
           leftInputPotentialFK = RelOptUtil.InputFinder.bits(swappedTopProjExprs, null).intersects(leftBits);
           if (!leftInputPotentialFK) {

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java

@@ -549,9 +549,8 @@ private ImmutableBitSet generateNewGroupset(Aggregate aggregate, ImmutableBitSet
    */
   private Aggregate rewriteGBConstantKeys(Aggregate aggregate, ImmutableBitSet fieldsUsed,
       ImmutableBitSet aggCallFields) {
-    if ((aggregate.getIndicatorCount() > 0)
-        || (aggregate.getGroupSet().isEmpty())
-        || fieldsUsed.contains(aggregate.getGroupSet())) {
+    if (aggregate.getGroupSet().isEmpty()
+        || fieldsUsed.intersects(aggregate.getGroupSet())) {
       return aggregate;
     }
 

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java

@@ -31,10 +31,9 @@
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.tools.RelBuilder;
 import org.apache.hadoop.hive.ql.ddl.view.materialized.alter.rebuild.AlterMaterializedViewRebuildAnalyzer;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
 
-import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveHepExtractRelNodeRule;
-
 /**
  * This rule will perform a rewriting to prepare the plan for incremental
  * view maintenance in case there exist aggregation operator, so we can
@@ -118,7 +117,7 @@ protected IncrementalComputePlanWithDeletedRows createJoinRightInput(RelOptRuleC
     RelNode aggInput = aggregate.getInput();
 
     // Propagate rowIsDeleted column
-    aggInput = HiveHepExtractRelNodeRule.execute(aggInput);
+    aggInput = HiveCalciteUtil.stripHepVertices(aggInput);
     aggInput = new HiveRowIsDeletedPropagator(relBuilder).propagate(aggInput);
 
     // The row schema has two additional columns after propagation:

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAugmentSnapshotMaterializationRule.java

@@ -146,6 +146,12 @@ public void onMatch(RelOptRuleCall call) {
 
     final RelBuilder relBuilder = call.builder();
     relBuilder.push(tableScan);
+    if (snapshotId == null) {
+      // Avoid creating an incorrect expression $snapshotIdInputRef <= NULL
+      // which may be problematic for Calcite later on; instead use a special value -1,
+      // which will be later interpreted by HivePushdownSnapshotFilterRule (and removed)
+      snapshotId = -1L;
+    }
     final RexNode snapshotIdLiteral = rexBuilder.makeLiteral(
         snapshotId, snapshotIdType(relBuilder.getTypeFactory()), false);
     final RexNode predicateWithSnapShotId = rexBuilder.makeCall(