From ff59663e60cb1d587415a422b4cfa1a6729e7869 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Tue, 10 Nov 2026 16:56:34 +0800 Subject: [PATCH] [test](variant) Add compaction coverage for sparse path promotion CIR-22588: compaction promotes sparse paths to materialized subcolumns by a raw-byte lower_bound. Sparse keys written in component order were missed, filled with NULL, and removed from sparse, so the non-null values disappeared from the output file. The writer now publishes byte-ordered sparse keys; this suite verifies that after a full compaction the promoted tag_id/tag_name values are still present, using the customer's six-row shape and control rows. --- .../sparse_promotion_compaction.out | 17 +++++ .../sparse_promotion_compaction.groovy | 74 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 regression-test/data/variant_p0/sparse_promotion_compaction.out create mode 100644 regression-test/suites/variant_p0/sparse_promotion_compaction.groovy diff --git a/regression-test/data/variant_p0/sparse_promotion_compaction.out b/regression-test/data/variant_p0/sparse_promotion_compaction.out new file mode 100644 index 00000000000000..22a784d9bce306 --- /dev/null +++ b/regression-test/data/variant_p0/sparse_promotion_compaction.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_compaction -- +1 id open {"text_info":{"open":{"tag_id":"id","tag_name":"open"}}} +2 \N \N \N + +-- !after_compaction -- +1 id open {"text_info":{"open":{"tag_id":"id","tag_name":"open"}}} +2 \N \N \N +3 id3 open3 {"text_info":{"open":{"tag_id":"id3","tag_name":"open3"}}} +4 id4 open4 {"text_info":{"open":{"tag_id":"id4","tag_name":"open4"}}} +5 id5 open5 {"text_info":{"open":{"tag_id":"id5","tag_name":"open5"}}} +6 id6 open6 {"text_info":{"open":{"tag_id":"id6","tag_name":"open6"}}} + +-- !newer_rows -- +3 id3 open3 +4 id4 open4 + diff --git a/regression-test/suites/variant_p0/sparse_promotion_compaction.groovy b/regression-test/suites/variant_p0/sparse_promotion_compaction.groovy new file mode 100644 index 00000000000000..48559d48a20172 --- /dev/null +++ b/regression-test/suites/variant_p0/sparse_promotion_compaction.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("sparse_promotion_compaction", "p0") { + // CIR-22588: compaction promotes sparse paths to materialized subcolumns by a raw-byte + // lower_bound. Sparse keys written in component order are missed, filled with NULL, and + // then removed from sparse, so the values disappear from the output file. The writer must + // publish sparse keys in raw-byte order; after full compaction the promoted values must + // still be present. + setFeConfigTemporary([enable_variant_v2: true]) { + sql "SET default_variant_enable_doc_mode = false" + sql "SET enable_sql_cache = false" + + def tableName = "sparse_promotion_compaction" + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + id INT NOT NULL, + v VARIANT + ) ENGINE = OLAP + UNIQUE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true" + ) + """ + + sql """INSERT INTO ${tableName} VALUES + (1, parse_to_variant('{"hot_a":1,"hot_b":1,"tag-labeling":{"text_info":{"open":{"tag_id":"id","tag_name":"open"}}},"tag-labeling-0-1-2":{"x":1,"y":2,"z":3}}')), + (2, parse_to_variant('{"hot_a":2,"hot_b":2}')) + """ + + def projection = """SELECT id, + CAST(v['tag-labeling']['text_info']['open']['tag_id'] AS STRING) AS tag_id, + CAST(v['tag-labeling']['text_info']['open']['tag_name'] AS STRING) AS tag_name, + CAST(v['tag-labeling'] AS STRING) AS parent + FROM ${tableName} ORDER BY id""" + qt_before_compaction projection + + // More rows make the promoted paths frequent enough for full compaction to rewrite + // them; row 1 is not updated or overwritten. + sql """INSERT INTO ${tableName} VALUES + (3, parse_to_variant('{"tag-labeling":{"text_info":{"open":{"tag_id":"id3","tag_name":"open3"}}}}')), + (4, parse_to_variant('{"tag-labeling":{"text_info":{"open":{"tag_id":"id4","tag_name":"open4"}}}}')), + (5, parse_to_variant('{"tag-labeling":{"text_info":{"open":{"tag_id":"id5","tag_name":"open5"}}}}')), + (6, parse_to_variant('{"tag-labeling":{"text_info":{"open":{"tag_id":"id6","tag_name":"open6"}}}}')) + """ + trigger_and_wait_compaction(tableName, "full", 1800) + + qt_after_compaction projection + + def newerRows = """SELECT id, + CAST(v['tag-labeling']['text_info']['open']['tag_id'] AS STRING) AS tag_id, + CAST(v['tag-labeling']['text_info']['open']['tag_name'] AS STRING) AS tag_name + FROM ${tableName} WHERE id IN (3, 4) ORDER BY id""" + qt_newer_rows newerRows + } +}