Clickhouse migration fails with unknown field #7814
Description
Environment
Kubernetes, Self hosted
Version 26.2.1
Steps to Reproduce
On a clean CH cluster run the migration snuba migrations migrate --force
Expected Result
Successfully run all migrations
Actual Result
DDL Statement:
CREATE MATERIALIZED VIEW IF NOT EXISTS eap_trace_item_attrs_mv ON CLUSTER 'sentry' TO eap_trace_item_attrs_local (project_id UInt64, item_type String, date Date CODEC (DoubleDelta, ZSTD(1)), retention_days UInt16, attrs_string Map(String, String), attrs_bool Array(String), attrs_int64 Array(String), attrs_float64 Array(String), key_val_hash UInt64) AS
SELECT
project_id,
'span',
toDate(_sort_timestamp) AS date,
retention_days as retention_days,
mapConcat(attr_str_0, attr_str_1, attr_str_2, attr_str_3, attr_str_4, attr_str_5, attr_str_6, attr_str_7, attr_str_8, attr_str_9, attr_str_10, attr_str_11, attr_str_12, attr_str_13, attr_str_14, attr_str_15, attr_str_16, attr_str_17, attr_str_18, attr_str_19) AS attrs_string, -- `attrs_string` Map(String, String),
array() AS attrs_bool, -- bool
array() AS attrs_int64, -- int64
arrayConcat(mapKeys(attr_num_0), mapKeys(attr_num_1), mapKeys(attr_num_2), mapKeys(attr_num_3), mapKeys(attr_num_4), mapKeys(attr_num_5), mapKeys(attr_num_6), mapKeys(attr_num_7), mapKeys(attr_num_8), mapKeys(attr_num_9), mapKeys(attr_num_10), mapKeys(attr_num_11), mapKeys(attr_num_12), mapKeys(attr_num_13), mapKeys(attr_num_14), mapKeys(attr_num_15), mapKeys(attr_num_16), mapKeys(attr_num_17), mapKeys(attr_num_18), mapKeys(attr_num_19)) AS attrs_float64, -- float
-- a hash of all the attribute key,val pairs of the item in sorted order
-- this lets us deduplicate rows with merges
cityHash64(mapSort(
mapConcat(
mapApply((k, v) -> (k, ''), attr_num_0),
mapApply((k, v) -> (k, ''), attr_num_1),
mapApply((k, v) -> (k, ''), attr_num_2),
mapApply((k, v) -> (k, ''), attr_num_3),
mapApply((k, v) -> (k, ''), attr_num_4),
mapApply((k, v) -> (k, ''), attr_num_5),
mapApply((k, v) -> (k, ''), attr_num_6),
mapApply((k, v) -> (k, ''), attr_num_7),
mapApply((k, v) -> (k, ''), attr_num_8),
mapApply((k, v) -> (k, ''), attr_num_9),
mapApply((k, v) -> (k, ''), attr_num_10),
mapApply((k, v) -> (k, ''), attr_num_11),
mapApply((k, v) -> (k, ''), attr_num_12),
mapApply((k, v) -> (k, ''), attr_num_13),
mapApply((k, v) -> (k, ''), attr_num_14),
mapApply((k, v) -> (k, ''), attr_num_15),
mapApply((k, v) -> (k, ''), attr_num_16),
mapApply((k, v) -> (k, ''), attr_num_17),
mapApply((k, v) -> (k, ''), attr_num_18),
mapApply((k, v) -> (k, ''), attr_num_19),
attr_str_0,
attr_str_1,
attr_str_2,
attr_str_3,
attr_str_4,
attr_str_5,
attr_str_6,
attr_str_7,
attr_str_8,
attr_str_9,
attr_str_10,
attr_str_11,
attr_str_12,
attr_str_13,
attr_str_14,
attr_str_15,
attr_str_16,
attr_str_17,
attr_str_18,
attr_str_19
)
)) AS key_val_hash
FROM eap_spans_2_local
CH Error:
DB::Exception: SELECT query outputs column with name ''span'', which is not found in the target table. Use 'AS' to assign alias that matches a column name
CH Stacktrace:
{"module": "snuba.migrations.operations", "event": "Failed to execute operation on StorageSetKey.EVENTS_ANALYTICS_PLATFORM, target: OperationTarget.LOCAL\nCREATE MATERIALIZED VIEW IF NOT EXISTS eap_trace_item_attrs_mv ON CLUSTER 'sentry' TO eap_trace_item_attrs_local (project_id UInt64, item_type String, date Date CODEC (DoubleDelta, ZSTD(1)), retention_days UInt16, attrs_string Map(String, String), attrs_bool Array(String), attrs_int64 Array(String), attrs_float64 Array(String), key_val_hash UInt64) AS \nSELECT\n project_id,\n 'span',\n toDate(_sort_timestamp) AS date,\n retention_days as retention_days,\n mapConcat(attr_str_0, attr_str_1, attr_str_2, attr_str_3, attr_str_4, attr_str_5, attr_str_6, attr_str_7, attr_str_8, attr_str_9, attr_str_10, attr_str_11, attr_str_12, attr_str_13, attr_str_14, attr_str_15, attr_str_16, attr_str_17, attr_str_18, attr_str_19) AS attrs_string, -- `attrs_string` Map(String, String),\n array() AS attrs_bool, -- bool\n array() AS attrs_int64, -- int64\n arrayConcat(mapKeys(attr_num_0), mapKeys(attr_num_1), mapKeys(attr_num_2), mapKeys(attr_num_3), mapKeys(attr_num_4), mapKeys(attr_num_5), mapKeys(attr_num_6), mapKeys(attr_num_7), mapKeys(attr_num_8), mapKeys(attr_num_9), mapKeys(attr_num_10), mapKeys(attr_num_11), mapKeys(attr_num_12), mapKeys(attr_num_13), mapKeys(attr_num_14), mapKeys(attr_num_15), mapKeys(attr_num_16), mapKeys(attr_num_17), mapKeys(attr_num_18), mapKeys(attr_num_19)) AS attrs_float64, -- float\n -- a hash of all the attribute key,val pairs of the item in sorted order\n -- this lets us deduplicate rows with merges\n cityHash64(mapSort(\n mapConcat(\n mapApply((k, v) -> (k, ''), attr_num_0),\n mapApply((k, v) -> (k, ''), attr_num_1),\n mapApply((k, v) -> (k, ''), attr_num_2),\n mapApply((k, v) -> (k, ''), attr_num_3),\n mapApply((k, v) -> (k, ''), attr_num_4),\n mapApply((k, v) -> (k, ''), attr_num_5),\n mapApply((k, v) -> (k, ''), attr_num_6),\n mapApply((k, v) -> (k, ''), attr_num_7),\n mapApply((k, v) -> (k, ''), attr_num_8),\n mapApply((k, v) -> (k, ''), attr_num_9),\n mapApply((k, v) -> (k, ''), attr_num_10),\n mapApply((k, v) -> (k, ''), attr_num_11),\n mapApply((k, v) -> (k, ''), attr_num_12),\n mapApply((k, v) -> (k, ''), attr_num_13),\n mapApply((k, v) -> (k, ''), attr_num_14),\n mapApply((k, v) -> (k, ''), attr_num_15),\n mapApply((k, v) -> (k, ''), attr_num_16),\n mapApply((k, v) -> (k, ''), attr_num_17),\n mapApply((k, v) -> (k, ''), attr_num_18),\n mapApply((k, v) -> (k, ''), attr_num_19),\n attr_str_0,\n attr_str_1,\n attr_str_2,\n attr_str_3,\n attr_str_4,\n attr_str_5,\n attr_str_6,\n attr_str_7,\n attr_str_8,\n attr_str_9,\n attr_str_10,\n attr_str_11,\n attr_str_12,\n attr_str_13,\n attr_str_14,\n attr_str_15,\n attr_str_16,\n attr_str_17,\n attr_str_18,\n attr_str_19\n )\n )) AS key_val_hash\nFROM eap_spans_2_local\n\n\n;\nNone", "severity": "error", "exception": "Traceback (most recent call last):\n File \"/usr/src/snuba/snuba/clickhouse/native.py\", line 177, in execute\n result_data = query_execute()\n ^^^^^^^^^^^^^^^\n File \"/usr/src/snuba/snuba/clickhouse/native.py\", line 160, in query_execute\n return conn.execute( # type: ignore\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/.venv/lib/python3.11/site-packages/clickhouse_driver/client.py\", line 382, in execute\n rv = self.process_ordinary_query(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/.venv/lib/python3.11/site-packages/clickhouse_driver/client.py\", line 580, in process_ordinary_query\n return self.receive_result(with_column_types=with_column_types,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/.venv/lib/python3.11/site-packages/sentry_sdk/integrations/clickhouse_driver.py\", line 110, in _inner_end\n res = f(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^\n File \"/.venv/lib/python3.11/site-packages/clickhouse_driver/client.py\", line 212, in receive_result\n return result.get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"/.venv/lib/python3.11/site-packages/clickhouse_driver/result.py\", line 50, in get_result\n for packet in self.packet_generator:\n File \"/.venv/lib/python3.11/site-packages/clickhouse_driver/client.py\", line 228, in packet_generator\n packet = self.receive_packet()\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/.venv/lib/python3.11/site-packages/clickhouse_driver/client.py\", line 245, in receive_packet\n raise packet.exception\nclickhouse_driver.errors.ServerException: Code: 8.\nDB::Exception: SELECT query outputs column with name ''span'', which is not found in the target table. Use 'AS' to assign alias that matches a column name. Stack trace:\n\n0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x00000000166da3ea\n1. DB::Exception::Exception(String&&, int, String, bool) @ 0x000000000e55a34e\n2. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x000000000e559c89\n3. DB::Exception::Exception<String const&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type>, String const&) @ 0x000000000e56ddf6\n4. DB::InterpreterCreateQuery::createTable(DB::ASTCreateQuery&) @ 0x000000001b5e5a42\n5. DB::InterpreterCreateQuery::execute() @ 0x000000001b5f9818\n6. DB::executeQueryImpl(char const*, char const*, std::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum, std::unique_ptr<DB::ReadBuffer, std::default_delete<DB::ReadBuffer>>&, boost::intrusive_ptr<DB::IAST>&, std::shared_ptr<DB::ImplicitTransactionControlExecutor>, std::function<void ()>, DB::QueryResultDetails&) @ 0x000000001ba815fe\n7. DB::executeQuery(String const&, std::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum) @ 0x000000001ba7b782\n8. DB::TCPHandler::runImpl() @ 0x000000001d20b3cd\n9. DB::TCPHandler::run() @ 0x000000001d22dbe4\n10. Poco::Net::TCPServerConnection::start() @ 0x000000002277af87\n11. Poco::Net::TCPServerDispatcher::run() @ 0x000000002277b502\n12. Poco::PooledThread::run() @ 0x0000000022739c85\n13. Poco::ThreadImpl::runnableEntry(void*) @ 0x000000002273820f\n14. ? @ 0x0000000000094ac3\n15. ? @ 0x00000000001268d0\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/usr/src/snuba/snuba/migrations/operations.py\", line 136, in execute\n connection.execute(sql, settings=self._settings)\n File \"/usr/src/snuba/snuba/clickhouse/native.py\", line 246, in execute\n raise ClickhouseError(e.message, code=e.code) from e\nsnuba.clickhouse.errors.ClickhouseError: DB::Exception: SELECT query outputs column with name ''span'', which is not found in the target table. Use 'AS' to assign alias that matches a column name. Stack trace:\n\n0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x00000000166da3ea\n1. DB::Exception::Exception(String&&, int, String, bool) @ 0x000000000e55a34e\n2. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x000000000e559c89\n3. DB::Exception::Exception<String const&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type>, String const&) @ 0x000000000e56ddf6\n4. DB::InterpreterCreateQuery::createTable(DB::ASTCreateQuery&) @ 0x000000001b5e5a42\n5. DB::InterpreterCreateQuery::execute() @ 0x000000001b5f9818\n6. DB::executeQueryImpl(char const*, char const*, std::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum, std::unique_ptr<DB::ReadBuffer, std::default_delete<DB::ReadBuffer>>&, boost::intrusive_ptr<DB::IAST>&, std::shared_ptr<DB::ImplicitTransactionControlExecutor>, std::function<void ()>, DB::QueryResultDetails&) @ 0x000000001ba815fe\n7. DB::executeQuery(String const&, std::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum) @ 0x000000001ba7b782\n8. DB::TCPHandler::runImpl() @ 0x000000001d20b3cd\n9. DB::TCPHandler::run() @ 0x000000001d22dbe4\n10. Poco::Net::TCPServerConnection::start() @ 0x000000002277af87\n11. Poco::Net::TCPServerDispatcher::run() @ 0x000000002277b502\n12. Poco::PooledThread::run() @ 0x0000000022739c85\n13. Poco::ThreadImpl::runnableEntry(void*) @ 0x000000002273820f\n14. ? @ 0x0000000000094ac3\n15. ? @ 0x00000000001268d0\n", "timestamp": "2026-03-13T16:35:07.335126Z"}