diff --git a/dbms/src/DataStreams/AddExtraTableIDColumnTransformAction.cpp b/dbms/src/DataStreams/AddExtraTableIDColumnTransformAction.cpp index 47b52d9eb98..a819743e9c7 100644 --- a/dbms/src/DataStreams/AddExtraTableIDColumnTransformAction.cpp +++ b/dbms/src/DataStreams/AddExtraTableIDColumnTransformAction.cpp @@ -21,7 +21,7 @@ namespace DB Block AddExtraTableIDColumnTransformAction::buildHeader(const Block & inner_header_, int extra_table_id_index) { auto header = inner_header_.cloneEmpty(); - if (extra_table_id_index != InvalidColumnID) + if (extra_table_id_index != MutSup::invalid_col_id) { const auto & extra_table_id_col_define = DM::getExtraTableIDColumnDefine(); ColumnWithTypeAndName col{ @@ -67,7 +67,7 @@ bool AddExtraTableIDColumnTransformAction::transform(Block & block, TableID phys if (unlikely(!block)) return true; - if (extra_table_id_index != InvalidColumnID) + if (extra_table_id_index != MutSup::invalid_col_id) { const auto & extra_table_id_col_define = DM::getExtraTableIDColumnDefine(); ColumnWithTypeAndName col{ diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index 939e1827a8a..864f388e870 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -24,6 +24,8 @@ #include #include +#include + #if __SSE2__ #include #endif @@ -517,4 +519,14 @@ String DataTypeString::getNullableDefaultName() return fmt::format("Nullable({})", getDefaultName()); } +std::span> DataTypeString::getTiDBPkColumnStringNameAndTypes() +{ + static const auto name_and_types = std::array{ + std::make_pair(NameV2, DataTypeFactory::instance().getOrSet(NameV2)), + std::make_pair(LegacyName, DataTypeFactory::instance().getOrSet(LegacyName)), + }; + // Minus one for ignoring SerdesFormat::None. + static_assert(magic_enum::enum_count() - 1 == name_and_types.size()); + return name_and_types; +} } // namespace DB diff --git a/dbms/src/DataTypes/DataTypeString.h b/dbms/src/DataTypes/DataTypeString.h index 8687e945864..c0611a58895 100644 --- a/dbms/src/DataTypes/DataTypeString.h +++ b/dbms/src/DataTypes/DataTypeString.h @@ -13,9 +13,10 @@ // limitations under the License. #pragma once - #include +#include + namespace DB { class DataTypeString final : public IDataType @@ -98,6 +99,8 @@ class DataTypeString final : public IDataType static String getDefaultName(); static String getNullableDefaultName(); + static std::span> getTiDBPkColumnStringNameAndTypes(); + explicit DataTypeString(SerdesFormat serdes_fmt_ = SerdesFormat::None); private: diff --git a/dbms/src/Databases/test/gtest_database.cpp b/dbms/src/Databases/test/gtest_database.cpp index 8e13e69b558..f5021874de2 100644 --- a/dbms/src/Databases/test/gtest_database.cpp +++ b/dbms/src/Databases/test/gtest_database.cpp @@ -173,7 +173,7 @@ try auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -257,7 +257,7 @@ try auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -272,7 +272,7 @@ try auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -373,7 +373,7 @@ try auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -391,7 +391,7 @@ try auto storage = db2->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -581,7 +581,7 @@ try auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -597,7 +597,7 @@ try auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -632,7 +632,7 @@ try // Get storage from database auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -744,7 +744,7 @@ try auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); @@ -793,7 +793,7 @@ try const auto * tbl_name = "t_45"; auto storage = db->tryGetTable(*ctx, tbl_name); ASSERT_NE(storage, nullptr); - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), tbl_name); auto managed_storage = std::dynamic_pointer_cast(storage); diff --git a/dbms/src/Debug/MockExecutor/TableScanBinder.cpp b/dbms/src/Debug/MockExecutor/TableScanBinder.cpp index 833c3b064fe..fc33bdc6500 100644 --- a/dbms/src/Debug/MockExecutor/TableScanBinder.cpp +++ b/dbms/src/Debug/MockExecutor/TableScanBinder.cpp @@ -61,7 +61,7 @@ TableID TableScanBinder::getTableId() const void TableScanBinder::setTipbColumnInfo(tipb::ColumnInfo * ci, const DAGColumnInfo & dag_column_info) const { auto names = splitQualifiedName(dag_column_info.first); - if (names.column_name == MutableSupport::tidb_pk_column_name) + if (names.column_name == MutSup::extra_handle_column_name) ci->set_column_id(-1); else ci->set_column_id(table_info.getColumnID(names.column_name)); @@ -142,10 +142,10 @@ ExecutorBinderPtr compileTableScan( { TiDB::ColumnInfo ci; ci.tp = TiDB::TypeLongLong; - ci.id = TiDBPkColumnID; + ci.id = MutSup::extra_handle_id; ci.setPriKeyFlag(); ci.setNotNullFlag(); - ts_output.emplace_back(std::make_pair(MutableSupport::tidb_pk_column_name, std::move(ci))); + ts_output.emplace_back(std::make_pair(MutSup::extra_handle_column_name, std::move(ci))); } return std::make_shared(executor_index, ts_output, table_info, keep_order); diff --git a/dbms/src/Debug/dbgQueryCompiler.cpp b/dbms/src/Debug/dbgQueryCompiler.cpp index b8bd24253b0..1acc4ef7634 100644 --- a/dbms/src/Debug/dbgQueryCompiler.cpp +++ b/dbms/src/Debug/dbgQueryCompiler.cpp @@ -125,7 +125,7 @@ std::pair compileQueryBlock( { if (auto * identifier = typeid_cast(expr.get())) { - if (identifier->getColumnName() == MutableSupport::tidb_pk_column_name) + if (identifier->getColumnName() == MutSup::extra_handle_column_name) { append_pk_column = true; } @@ -179,7 +179,7 @@ std::pair compileQueryBlock( if (auto * identifier = typeid_cast(expr.get())) { auto [db_name, table_name, column_name] = splitQualifiedName(identifier->getColumnName()); - if (column_name == MutableSupport::tidb_pk_column_name) + if (column_name == MutSup::extra_handle_column_name) { if (table_name.empty()) { @@ -423,4 +423,4 @@ QueryTasks queryPlanToQueryTasks( } return tasks; } -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index 0dd45d77f5b..8fa8ac976be 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -557,7 +557,7 @@ std::tuple compareColumns( for (const auto & column : columns) { - // Exclude virtual columns, including EXTRA_HANDLE_COLUMN_ID, VERSION_COLUMN_ID,TAG_COLUMN_ID,EXTRA_TABLE_ID_COLUMN_ID + // Exclude virtual columns, including MutSup::extra_handle_id, MutSup::version_col_id,MutSup::delmark_col_id,MutSup::extra_table_id_col_id if (column.id < 0) { continue; @@ -1482,7 +1482,7 @@ std::unordered_map DAG std::pair> DAGStorageInterpreter::getColumnsForTableScan() { // Get handle column name. - String handle_column_name = MutableSupport::tidb_pk_column_name; + String handle_column_name = MutSup::extra_handle_column_name; if (auto pk_handle_col = storage_for_logical_table->getTableInfo().getPKHandleColumn()) handle_column_name = pk_handle_col->get().name; @@ -1504,10 +1504,10 @@ std::pair> DAGStorageInterpreter::getColumnsForTableSc } // Column ID -1 return the handle column String name; - if (cid == TiDBPkColumnID) + if (cid == MutSup::extra_handle_id) name = handle_column_name; - else if (cid == ExtraTableIDColumnID) - name = MutableSupport::extra_table_id_column_name; + else if (cid == MutSup::extra_table_id_col_id) + name = MutSup::extra_table_id_column_name; else name = storage_for_logical_table->getTableInfo().getColumnName(cid); required_columns_tmp.emplace_back(std::move(name)); diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index d7df2262cfb..b37a2984602 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -26,13 +26,13 @@ namespace DataTypePtr getPkType(const TiDB::ColumnInfo & column_info) { const auto & pk_data_type = getDataTypeByColumnInfoForComputingLayer(column_info); - /// primary key type must be tidb_pk_column_int_type or tidb_pk_column_string_type. + /// primary key type must be getTiDBPkColumnIntType or getTiDBPkColumnStringType. RUNTIME_CHECK( - pk_data_type->equals(*MutableSupport::tidb_pk_column_int_type) - || pk_data_type->equals(*MutableSupport::tidb_pk_column_string_type), + pk_data_type->equals(*MutSup::getExtraHandleColumnIntType()) + || pk_data_type->equals(*MutSup::getExtraHandleColumnStringType()), pk_data_type->getName(), - MutableSupport::tidb_pk_column_int_type->getName(), - MutableSupport::tidb_pk_column_string_type->getName()); + MutSup::getExtraHandleColumnIntType()->getName(), + MutSup::getExtraHandleColumnStringType()->getName()); return pk_data_type; } } // namespace @@ -70,13 +70,11 @@ NamesAndTypes genNamesAndTypes(const TiDB::ColumnInfos & column_infos, const Str const auto & column_info = column_infos[i]; switch (column_info.id) { - case TiDBPkColumnID: - names_and_types.emplace_back(MutableSupport::tidb_pk_column_name, getPkType(column_info)); + case MutSup::extra_handle_id: + names_and_types.emplace_back(MutSup::extra_handle_column_name, getPkType(column_info)); break; - case ExtraTableIDColumnID: - names_and_types.emplace_back( - MutableSupport::extra_table_id_column_name, - MutableSupport::extra_table_id_column_type); + case MutSup::extra_table_id_col_id: + names_and_types.emplace_back(MutSup::extra_table_id_column_name, MutSup::getExtraTableIdColumnType()); break; default: names_and_types.emplace_back( @@ -94,7 +92,7 @@ NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan, const StringRef std::tuple genColumnDefinesForDisaggregatedRead(const TiDBTableScan & table_scan) { auto column_defines = std::make_shared(); - int extra_table_id_index = InvalidColumnID; + int extra_table_id_index = MutSup::invalid_col_id; column_defines->reserve(table_scan.getColumnSize()); for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) { @@ -106,18 +104,18 @@ std::tuple genColumnDefinesForDisaggregatedRead(const const auto output_name = genNameForExchangeReceiver(i); switch (column_info.id) { - case TiDBPkColumnID: + case MutSup::extra_handle_id: column_defines->emplace_back(DM::ColumnDefine{ - TiDBPkColumnID, - output_name, // MutableSupport::tidb_pk_column_name + MutSup::extra_handle_id, + output_name, // MutSup::extra_handle_column_name getPkType(column_info)}); break; - case ExtraTableIDColumnID: + case MutSup::extra_table_id_col_id: { column_defines->emplace_back(DM::ColumnDefine{ - ExtraTableIDColumnID, - output_name, // MutableSupport::extra_table_id_column_name - MutableSupport::extra_table_id_column_type}); + MutSup::extra_table_id_col_id, + output_name, // MutSup::extra_table_id_column_name + MutSup::getExtraTableIdColumnType()}); extra_table_id_index = i; break; } diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp index a6a05feb09f..13bca3793db 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp @@ -44,7 +44,7 @@ RemoteRequest RemoteRequest::build( auto * mutable_table_scan = ts_exec->mutable_tbl_scan(); table_scan.constructTableScanForRemoteRead(mutable_table_scan, table_info.id); - String handle_column_name = MutableSupport::tidb_pk_column_name; + String handle_column_name = MutSup::extra_handle_column_name; if (auto pk_handle_col = table_info.getPKHandleColumn()) handle_column_name = pk_handle_col->get().name; @@ -53,7 +53,7 @@ RemoteRequest RemoteRequest::build( const auto & col = table_scan.getColumns()[i]; auto col_id = col.id; - if (col_id == DB::TiDBPkColumnID) + if (col_id == MutSup::extra_handle_id) { TiDB::ColumnInfo ci; ci.tp = TiDB::TypeLongLong; @@ -61,11 +61,11 @@ RemoteRequest RemoteRequest::build( ci.setNotNullFlag(); schema.emplace_back(std::make_pair(handle_column_name, std::move(ci))); } - else if (col_id == ExtraTableIDColumnID) + else if (col_id == MutSup::extra_table_id_col_id) { TiDB::ColumnInfo ci; ci.tp = TiDB::TypeLongLong; - schema.emplace_back(std::make_pair(MutableSupport::extra_table_id_column_name, std::move(ci))); + schema.emplace_back(std::make_pair(MutSup::extra_table_id_column_name, std::move(ci))); } else { diff --git a/dbms/src/Flash/Mpp/tests/gtest_trigger_pipeline_writer.cpp b/dbms/src/Flash/Mpp/tests/gtest_trigger_pipeline_writer.cpp index f4c1db357ba..b453aa69f11 100644 --- a/dbms/src/Flash/Mpp/tests/gtest_trigger_pipeline_writer.cpp +++ b/dbms/src/Flash/Mpp/tests/gtest_trigger_pipeline_writer.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include diff --git a/dbms/src/Flash/tests/bench_aggregation_hash_map.cpp b/dbms/src/Flash/tests/bench_aggregation_hash_map.cpp index 39c711a03c5..21a339c2a1f 100644 --- a/dbms/src/Flash/tests/bench_aggregation_hash_map.cpp +++ b/dbms/src/Flash/tests/bench_aggregation_hash_map.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -310,7 +311,7 @@ try LOG_DEBUG(log, "build_side_watch: {}, hashmap size: {}", build_side_watch.elapsed(), data_variants->size()); std::vector variants{data_variants}; - auto merging_buckets = aggregator->mergeAndConvertToBlocks(variants, /*final=*/true, /*max_thread=*/1); + auto merging_buckets = aggregator->mergeAndConvertToBlocks(variants, /*final=*/true, /*max_threads=*/1); std::vector res_block; Stopwatch probe_side_watch; diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 765e8e224cf..8601df0110b 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -524,7 +524,7 @@ void ExpressionAnalyzer::analyzeAggregation() { NameSet unique_keys; ASTs & group_asts = select_query->group_expression_list->children; - for (ssize_t i = 0; i < ssize_t(group_asts.size()); ++i) + for (ssize_t i = 0; i < static_cast(group_asts.size()); ++i) { ssize_t size = group_asts.size(); getRootActions(group_asts[i], true, false, temp_actions); @@ -1037,7 +1037,7 @@ void ExpressionAnalyzer::normalizeTreeImpl( if (storage && select_query && !select_query->raw_for_mutable) { // LOG_DEBUG(&Poco::Logger::get("ExpressionAnalyzer"), "Filter hidden columns for mutable table."); - filtered_names = MutableSupport::instance().hiddenColumns(storage->getName()); + filtered_names = MutSup::instance().hiddenColumns(storage->getName()); } } @@ -1392,7 +1392,7 @@ void ExpressionAnalyzer::optimizeGroupBy() } select_query->group_expression_list = std::make_shared(); - select_query->group_expression_list->children.emplace_back(std::make_shared(UInt64(unused_column))); + select_query->group_expression_list->children.emplace_back(std::make_shared(unused_column)); } } diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 240b43042c0..01c00793a03 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -106,7 +106,7 @@ void InterpreterAlterQuery::parseAlter( { const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); - OrderedNameSet filtered_names = MutableSupport::instance().hiddenColumns(table->getName()); + OrderedNameSet filtered_names = MutSup::instance().hiddenColumns(table->getName()); for (const auto & params : params_container) { @@ -117,8 +117,7 @@ void InterpreterAlterQuery::parseAlter( const auto & ast_col_decl = typeid_cast(*params.col_decl); - if (ast_col_decl.name == MutableSupport::version_column_name - || ast_col_decl.name == MutableSupport::delmark_column_name) + if (ast_col_decl.name == MutSup::version_column_name || ast_col_decl.name == MutSup::delmark_column_name) throw Exception( "Internal column name can not be used: " + ast_col_decl.name, ErrorCodes::ARGUMENT_OUT_OF_BOUND); @@ -150,8 +149,7 @@ void InterpreterAlterQuery::parseAlter( const Field & column_name = typeid_cast(*(params.column)).name; - if (column_name == MutableSupport::version_column_name - || column_name == MutableSupport::delmark_column_name) + if (column_name == MutSup::version_column_name || column_name == MutSup::delmark_column_name) { FieldVisitorToString to_string; auto err_msg = "Internal column name can not be dropped: " + applyVisitor(to_string, column_name); @@ -170,8 +168,8 @@ void InterpreterAlterQuery::parseAlter( command.type = AlterCommand::DROP_COLUMN; command.column_name = typeid_cast(*(params.column)).name; - if (command.column_name == MutableSupport::version_column_name - || command.column_name == MutableSupport::delmark_column_name) + if (command.column_name == MutSup::version_column_name + || command.column_name == MutSup::delmark_column_name) throw Exception( "Internal column name can not be dropped: " + command.column_name, ErrorCodes::ARGUMENT_OUT_OF_BOUND); @@ -186,8 +184,7 @@ void InterpreterAlterQuery::parseAlter( const auto & ast_col_decl = typeid_cast(*params.col_decl); - if (ast_col_decl.name == MutableSupport::version_column_name - || ast_col_decl.name == MutableSupport::delmark_column_name) + if (ast_col_decl.name == MutSup::version_column_name || ast_col_decl.name == MutSup::delmark_column_name) throw Exception( "Internal column name can not be modified: " + ast_col_decl.name, ErrorCodes::ARGUMENT_OUT_OF_BOUND); diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index 36e4b9363aa..b7a87734ef8 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -71,7 +71,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() ColumnDefaults column_defaults; StoragePtr table; - auto table_expression = typeid_cast(ast.table_expression.get()); + const auto * table_expression = typeid_cast(ast.table_expression.get()); if (table_expression->subquery) { @@ -82,7 +82,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() { if (table_expression->table_function) { - auto table_function = typeid_cast(table_expression->table_function.get()); + const auto * table_function = typeid_cast(table_expression->table_function.get()); /// Get the table function TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_function->name, context); /// Run it and remember the result @@ -123,7 +123,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() Block sample_block = getSampleBlock(); MutableColumns res_columns = sample_block.cloneEmptyColumns(); - const OrderedNameSet filtered_names = MutableSupport::instance().hiddenColumns(table->getName()); + const OrderedNameSet filtered_names = MutSup::instance().hiddenColumns(table->getName()); for (const auto & column : columns) { diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 71ea49e3e02..8a3b4253715 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -111,9 +111,6 @@ BlockIO InterpreterInsertQuery::execute() checkAccess(query); StoragePtr table = getTable(query); - // if (table->getName() == MutableSupport::txn_storage_name) - // throw Exception(MutableSupport::txn_storage_name + " doesn't support Insert", ErrorCodes::LOGICAL_ERROR); - auto table_lock = table->lockStructureForShare(context.getCurrentQueryId()); NamesAndTypesList required_columns = table->getColumns().getAllPhysical(); diff --git a/dbms/src/Interpreters/InterpreterManageQuery.cpp b/dbms/src/Interpreters/InterpreterManageQuery.cpp index 765e87480e7..b572853720e 100644 --- a/dbms/src/Interpreters/InterpreterManageQuery.cpp +++ b/dbms/src/Interpreters/InterpreterManageQuery.cpp @@ -28,7 +28,7 @@ BlockIO InterpreterManageQuery::execute() StoragePtr table = context.getTable(ast.database, ast.table); StorageDeltaMerge * manageable_storage; - if (table->getName() == MutableSupport::delta_tree_storage_name) + if (table->getName() == MutSup::delta_tree_storage_name) { manageable_storage = &dynamic_cast(*table); } diff --git a/dbms/src/Server/DTTool/DTToolBench.cpp b/dbms/src/Server/DTTool/DTToolBench.cpp index c5085d7b3e6..040a668708c 100644 --- a/dbms/src/Server/DTTool/DTToolBench.cpp +++ b/dbms/src/Server/DTTool/DTToolBench.cpp @@ -108,7 +108,7 @@ DB::Block createBlock( Block block; //PK { - ColumnWithTypeAndName pk_col(nullptr, EXTRA_HANDLE_COLUMN_INT_TYPE, "id", EXTRA_HANDLE_COLUMN_ID); + ColumnWithTypeAndName pk_col(nullptr, MutSup::getExtraHandleColumnIntType(), "id", MutSup::extra_handle_id); IColumn::MutablePtr m_col = pk_col.type->createColumn(); for (size_t i = 0; i < row_number; i++) { @@ -121,7 +121,11 @@ DB::Block createBlock( } // Version { - ColumnWithTypeAndName version_col({}, VERSION_COLUMN_TYPE, VERSION_COLUMN_NAME, VERSION_COLUMN_ID); + ColumnWithTypeAndName version_col( + {}, + MutSup::getVersionColumnType(), + MutSup::version_column_name, + MutSup::version_col_id); IColumn::MutablePtr m_col = version_col.type->createColumn(); for (size_t i = 0; i < row_number; ++i) { @@ -135,7 +139,11 @@ DB::Block createBlock( //Tag { - ColumnWithTypeAndName tag_col(nullptr, TAG_COLUMN_TYPE, TAG_COLUMN_NAME, TAG_COLUMN_ID); + ColumnWithTypeAndName tag_col( + nullptr, + MutSup::getDelmarkColumnType(), + MutSup::delmark_column_name, + MutSup::delmark_col_id); IColumn::MutablePtr m_col = tag_col.type->createColumn(); auto & column_data = typeid_cast &>(*m_col).getData(); column_data.resize(row_number); diff --git a/dbms/src/Server/DTTool/DTToolInspect.cpp b/dbms/src/Server/DTTool/DTToolInspect.cpp index 4ab155b5c95..c906d15c36b 100644 --- a/dbms/src/Server/DTTool/DTToolInspect.cpp +++ b/dbms/src/Server/DTTool/DTToolInspect.cpp @@ -150,7 +150,8 @@ int inspectServiceMain(DB::Context & context, const InspectArgs & args) { for (const auto & c : all_cols) { - if (c.id == DB::TiDBPkColumnID || c.id == DB::VersionColumnID || c.id == DB::DelMarkColumnID) + if (c.id == DB::MutSup::extra_handle_id || c.id == DB::MutSup::version_col_id + || c.id == DB::MutSup::delmark_col_id) cols_to_dump.emplace_back(c); } } diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.h b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.h index eb9f6f90f01..0e2b515bb07 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.h +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.h @@ -175,14 +175,14 @@ class ColumnFileBigReader : public ColumnFileReader { if (col_defs_->size() == 1) { - if ((*col_defs)[0].id == EXTRA_HANDLE_COLUMN_ID) + if ((*col_defs)[0].id == MutSup::extra_handle_id) { pk_ver_only = true; } } else if (col_defs_->size() == 2) { - if ((*col_defs)[0].id == EXTRA_HANDLE_COLUMN_ID && (*col_defs)[1].id == VERSION_COLUMN_ID) + if ((*col_defs)[0].id == MutSup::extra_handle_id && (*col_defs)[1].id == MutSup::version_col_id) { pk_ver_only = true; } diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp index fe07ad9da12..59d96035d5c 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -20,7 +21,6 @@ #include #include - namespace DB::DM { diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexWriter.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexWriter.cpp index f7a8e014847..5903092652f 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexWriter.cpp +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexWriter.cpp @@ -144,7 +144,7 @@ ColumnFileTinyPtr ColumnFileTinyVectorIndexWriter::buildIndexForFile( break; RUNTIME_CHECK(block.columns() == read_columns->size()); - RUNTIME_CHECK(block.getByPosition(0).column_id == TAG_COLUMN_ID); + RUNTIME_CHECK(block.getByPosition(0).column_id == MutSup::delmark_col_id); auto del_mark_col = block.safeGetByPosition(0).column; RUNTIME_CHECK(del_mark_col != nullptr); @@ -218,7 +218,7 @@ ColumnFileTinys ColumnFileTinyVectorIndexWriter::build(ProceedCheckFn should_pro column_defines = getColumnDefinesFromBlock(schema->getSchema()); const auto del_cd_iter = std::find_if(column_defines.cbegin(), column_defines.cend(), [](const ColumnDefine & cd) { - return cd.id == TAG_COLUMN_ID; + return cd.id == MutSup::delmark_col_id; }); RUNTIME_CHECK_MSG( del_cd_iter != column_defines.cend(), diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index d24404ecc74..897b2e6c119 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -40,7 +40,7 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream , log(Logger::get(tracing_id)) { children.emplace_back(input); - delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); + delete_col_pos = input->getHeader().getPositionByName(MutSup::delmark_column_name); } ~DMDeleteFilterBlockInputStream() override { diff --git a/dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h b/dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h index 40d00927d01..d4c63be93ce 100644 --- a/dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h +++ b/dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h @@ -62,9 +62,9 @@ class DMVersionFilterBlockInputStream : public IBlockInputStream auto input_header = input->getHeader(); - handle_col_pos = input_header.getPositionByName(EXTRA_HANDLE_COLUMN_NAME); - version_col_pos = input_header.getPositionByName(VERSION_COLUMN_NAME); - delete_col_pos = input_header.getPositionByName(TAG_COLUMN_NAME); + handle_col_pos = input_header.getPositionByName(MutSup::extra_handle_column_name); + version_col_pos = input_header.getPositionByName(MutSup::version_column_name); + delete_col_pos = input_header.getPositionByName(MutSup::delmark_column_name); } ~DMVersionFilterBlockInputStream() override diff --git a/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToDTFilesOutputStream.cpp b/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToDTFilesOutputStream.cpp index 3a4a47975a7..53136529b60 100644 --- a/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToDTFilesOutputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToDTFilesOutputStream.cpp @@ -275,8 +275,8 @@ void SSTFilesToDTFilesOutputStream::write() { // Check whether rows are sorted by handle & version in ascending order. SortDescription sort; - sort.emplace_back(MutableSupport::tidb_pk_column_name, 1, 0); - sort.emplace_back(MutableSupport::version_column_name, 1, 0); + sort.emplace_back(MutSup::extra_handle_column_name, 1, 0); + sort.emplace_back(MutSup::version_column_name, 1, 0); if (unlikely(block.rows() > 1 && !isAlreadySorted(block, sort))) { @@ -288,8 +288,8 @@ void SSTFilesToDTFilesOutputStream::write() const size_t nrows = block.rows(); for (size_t i = 0; i < nrows; ++i) { - const auto & pk_col = block.getByName(MutableSupport::tidb_pk_column_name); - const auto & ver_col = block.getByName(MutableSupport::version_column_name); + const auto & pk_col = block.getByName(MutSup::extra_handle_column_name); + const auto & ver_col = block.getByName(MutSup::version_column_name); LOG_ERROR( log, "[Row={}/{}] [pk={}] [ver={}]", @@ -385,7 +385,7 @@ void SSTFilesToDTFilesOutputStream::updateRangeFromNonEmptyBlock(Bl RUNTIME_CHECK(block.rows() > 0); - const auto & pk_col = block.getByName(MutableSupport::tidb_pk_column_name); + const auto & pk_col = block.getByName(MutSup::extra_handle_column_name); const auto rowkey_column = RowKeyColumnContainer(pk_col.column, schema_snap->is_common_handle); auto & current_file_range = ingest_files_range.back(); diff --git a/dbms/src/Storages/DeltaMerge/DeltaMerge.h b/dbms/src/Storages/DeltaMerge/DeltaMerge.h index c3178e71825..167c7a794b9 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMerge.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMerge.h @@ -217,8 +217,8 @@ class DeltaMergeBlockInputStream final { // In some cases like Segment::getSplitPointSlow, only handle column in block. if (block.columns() < 2 // - || block.getByPosition(0).column_id != EXTRA_HANDLE_COLUMN_ID // - || block.getByPosition(1).column_id != VERSION_COLUMN_ID) + || block.getByPosition(0).column_id != MutSup::extra_handle_id // + || block.getByPosition(1).column_id != MutSup::version_col_id) return; ++num_read; diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.cpp new file mode 100644 index 00000000000..66f38de11ab --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.cpp @@ -0,0 +1,95 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace DB::DM +{ + +const ColumnDefine & getExtraIntHandleColumnDefine() +{ + static const ColumnDefine int_handle_cd{ + MutSup::extra_handle_id, + MutSup::extra_handle_column_name, + MutSup::getExtraHandleColumnIntType()}; + return int_handle_cd; +} + +namespace +{ +auto getExtraStringHandleColumnDefines() +{ + const auto names_and_types = ::DB::DataTypeString::getTiDBPkColumnStringNameAndTypes(); + std::vector> name_and_column_defines; + name_and_column_defines.reserve(names_and_types.size()); + for (const auto & name_and_type : names_and_types) + { + name_and_column_defines.emplace_back( + name_and_type.first, + ColumnDefine{MutSup::extra_handle_id, MutSup::extra_handle_column_name, name_and_type.second}); + } + return name_and_column_defines; +} +} // namespace +const ColumnDefine & getExtraStringHandleColumnDefine() +{ + static const auto name_and_column_defines = getExtraStringHandleColumnDefines(); + + const auto default_name = ::DB::DataTypeString::getDefaultName(); + auto itr = std::find_if( + name_and_column_defines.begin(), + name_and_column_defines.end(), + [&default_name](const auto & name_and_column_define) { return name_and_column_define.first == default_name; }); + RUNTIME_CHECK_MSG( + itr != name_and_column_defines.end(), + "Cannot find '{}' for TiDB primary key column string type", + default_name); + return itr->second; +} + +const ColumnDefine & getExtraHandleColumnDefine(bool is_common_handle) +{ + if (is_common_handle) + return getExtraStringHandleColumnDefine(); + return getExtraIntHandleColumnDefine(); +} + +const ColumnDefine & getVersionColumnDefine() +{ + static const ColumnDefine version_cd{ + MutSup::version_col_id, + MutSup::version_column_name, + MutSup::getVersionColumnType()}; + return version_cd; +} + +const ColumnDefine & getTagColumnDefine() +{ + static const ColumnDefine delmark_cd{ + MutSup::delmark_col_id, + MutSup::delmark_column_name, + MutSup::getDelmarkColumnType()}; + return delmark_cd; +} + +const ColumnDefine & getExtraTableIDColumnDefine() +{ + static const ColumnDefine table_id_cd{ + MutSup::extra_table_id_col_id, + MutSup::extra_table_id_column_name, + MutSup::getExtraTableIdColumnType()}; + return table_id_cd; +} +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h index 3c8c7256a00..3a95b7e4aee 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h @@ -16,20 +16,15 @@ #include #include -#include -#include +#include #include -#include +#include #include -#include -#include #include #include -#include #include #include -#include #include namespace TiDB @@ -37,9 +32,7 @@ namespace TiDB struct TableInfo; } // namespace TiDB -namespace DB -{ -namespace DM +namespace DB::DM { /// S: Scale factor of delta tree node. /// M: Capacity factor of leaf node. @@ -49,8 +42,6 @@ static constexpr size_t DT_S = 3; static constexpr size_t DT_M = 55; static constexpr size_t DT_F = 20; -using Ids = std::vector; - template > class DeltaTree; @@ -60,7 +51,6 @@ class DTEntryIterator; template > class DTEntriesCopy; - struct EmptyValueSpace { void removeFromInsert(UInt64) {} @@ -70,16 +60,10 @@ using EntryIterator = DTEntryIterator; using DefaultDeltaTree = DeltaTree; using DeltaTreePtr = std::shared_ptr; using BlockPtr = std::shared_ptr; - -using RowId = UInt64; using ColId = DB::ColumnID; -using Handle = DB::HandleID; - using ColIds = std::vector; -using HandlePair = std::pair; - +using Handle = DB::HandleID; using RowsAndBytes = std::pair; - using OptionTableInfoConstRef = std::optional>; struct ColumnDefine @@ -97,70 +81,21 @@ struct ColumnDefine {} }; -using ColumnMap = std::unordered_map; -using MutableColumnMap = std::unordered_map; -using LockGuard = std::lock_guard; - -inline static const UInt64 INITIAL_EPOCH = 0; - -// TODO maybe we should use those variables instead of macros? -#define EXTRA_HANDLE_COLUMN_NAME ::DB::MutableSupport::tidb_pk_column_name -#define VERSION_COLUMN_NAME ::DB::MutableSupport::version_column_name -#define TAG_COLUMN_NAME ::DB::MutableSupport::delmark_column_name -#define EXTRA_TABLE_ID_COLUMN_NAME ::DB::MutableSupport::extra_table_id_column_name - -#define EXTRA_HANDLE_COLUMN_ID ::DB::TiDBPkColumnID -#define VERSION_COLUMN_ID ::DB::VersionColumnID -#define TAG_COLUMN_ID ::DB::DelMarkColumnID -#define EXTRA_TABLE_ID_COLUMN_ID ::DB::ExtraTableIDColumnID - -#define EXTRA_HANDLE_COLUMN_INT_TYPE ::DB::MutableSupport::tidb_pk_column_int_type -#define EXTRA_HANDLE_COLUMN_STRING_TYPE ::DB::MutableSupport::tidb_pk_column_string_type -#define VERSION_COLUMN_TYPE ::DB::MutableSupport::version_column_type -#define TAG_COLUMN_TYPE ::DB::MutableSupport::delmark_column_type -#define EXTRA_TABLE_ID_COLUMN_TYPE ::DB::MutableSupport::extra_table_id_column_type - -inline const ColumnDefine & getExtraIntHandleColumnDefine() -{ - static ColumnDefine EXTRA_HANDLE_COLUMN_DEFINE_{ - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_INT_TYPE}; - return EXTRA_HANDLE_COLUMN_DEFINE_; -} -inline const ColumnDefine & getExtraStringHandleColumnDefine() +inline auto format_as(const ColumnDefine & cd) { - static ColumnDefine EXTRA_HANDLE_COLUMN_DEFINE_{ - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_STRING_TYPE}; - return EXTRA_HANDLE_COLUMN_DEFINE_; -} -inline const ColumnDefine & getExtraHandleColumnDefine(bool is_common_handle) -{ - if (is_common_handle) - return getExtraStringHandleColumnDefine(); - return getExtraIntHandleColumnDefine(); -} -inline const ColumnDefine & getVersionColumnDefine() -{ - static ColumnDefine VERSION_COLUMN_DEFINE_{VERSION_COLUMN_ID, VERSION_COLUMN_NAME, VERSION_COLUMN_TYPE}; - return VERSION_COLUMN_DEFINE_; -} -inline const ColumnDefine & getTagColumnDefine() -{ - static ColumnDefine TAG_COLUMN_DEFINE_{TAG_COLUMN_ID, TAG_COLUMN_NAME, TAG_COLUMN_TYPE}; - return TAG_COLUMN_DEFINE_; -} -inline const ColumnDefine & getExtraTableIDColumnDefine() -{ - static ColumnDefine EXTRA_TABLE_ID_COLUMN_DEFINE_{ - EXTRA_TABLE_ID_COLUMN_ID, - EXTRA_TABLE_ID_COLUMN_NAME, - EXTRA_TABLE_ID_COLUMN_TYPE}; - return EXTRA_TABLE_ID_COLUMN_DEFINE_; + return fmt::format("{}/{}", cd.id, cd.type->getName()); } +inline static constexpr UInt64 INITIAL_EPOCH = 0; +inline static constexpr bool DM_RUN_CHECK = true; + +ALWAYS_INLINE const ColumnDefine & getExtraIntHandleColumnDefine(); +ALWAYS_INLINE const ColumnDefine & getExtraStringHandleColumnDefine(); +ALWAYS_INLINE const ColumnDefine & getExtraHandleColumnDefine(bool is_common_handle); +ALWAYS_INLINE const ColumnDefine & getVersionColumnDefine(); +ALWAYS_INLINE const ColumnDefine & getTagColumnDefine(); +ALWAYS_INLINE const ColumnDefine & getExtraTableIDColumnDefine(); + static_assert( static_cast(static_cast(std::numeric_limits::min())) == std::numeric_limits::min(), "Unsupported compiler!"); @@ -168,8 +103,6 @@ static_assert( static_cast(static_cast(std::numeric_limits::max())) == std::numeric_limits::max(), "Unsupported compiler!"); -static constexpr bool DM_RUN_CHECK = true; - struct Attr { String col_name; @@ -177,16 +110,4 @@ struct Attr DataTypePtr type; }; using Attrs = std::vector; - -} // namespace DM -} // namespace DB - -template <> -struct fmt::formatter -{ - template - auto format(const DB::DM::ColumnDefine & cd, FormatContext & ctx) const -> decltype(ctx.out()) - { - return fmt::format_to(ctx.out(), "{}/{}", cd.id, cd.type->getName()); - } -}; +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h index 0c0cc49c54e..ca50f38bde5 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h @@ -79,7 +79,7 @@ inline SortDescription getPkSort(const ColumnDefine & handle) { SortDescription sort; sort.emplace_back(handle.name, 1, 1); - sort.emplace_back(VERSION_COLUMN_NAME, 1, 1); + sort.emplace_back(MutSup::version_column_name, 1, 1); return sort; } diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 17a3cc29df6..3b61f57b502 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -199,7 +200,8 @@ ColumnDefinesPtr generateStoreColumns(const ColumnDefines & table_columns, bool // Add other columns for (const auto & col : table_columns) { - if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != VERSION_COLUMN_NAME && col.name != TAG_COLUMN_NAME) + if (col.name != MutSup::extra_handle_column_name && col.name != MutSup::version_column_name + && col.name != MutSup::delmark_column_name) columns->emplace_back(col); } return columns; @@ -265,7 +267,8 @@ DeltaMergeStore::DeltaMergeStore( original_table_columns.emplace_back(getTagColumnDefine()); for (const auto & col : columns) { - if (col.id != original_table_handle_define.id && col.id != VERSION_COLUMN_ID && col.id != TAG_COLUMN_ID) + if (col.id != original_table_handle_define.id && col.id != MutSup::version_col_id + && col.id != MutSup::delmark_col_id) original_table_columns.emplace_back(col); } @@ -559,15 +562,15 @@ Block DeltaMergeStore::addExtraColumnIfNeed( { if (pkIsHandle(handle_define)) { - if (!EXTRA_HANDLE_COLUMN_INT_TYPE->equals(*handle_define.type)) + if (!MutSup::getExtraHandleColumnIntType()->equals(*handle_define.type)) { auto handle_pos = getPosByColumnId(block, handle_define.id); addColumnToBlock( block, // - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_INT_TYPE, - EXTRA_HANDLE_COLUMN_INT_TYPE->createColumn()); + MutSup::extra_handle_id, + MutSup::extra_handle_column_name, + MutSup::getExtraHandleColumnIntType(), + MutSup::getExtraHandleColumnIntType()->createColumn()); // Fill the new handle column with data in column[handle_pos] by applying cast. DefaultExecutable(FunctionToInt64::create(db_context)).execute(block, {handle_pos}, block.columns() - 1); } @@ -580,9 +583,9 @@ Block DeltaMergeStore::addExtraColumnIfNeed( ColumnPtr handle_column = pk_column->cloneResized(pk_column->size()); addColumnToBlock( block, // - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_id, + MutSup::extra_handle_column_name, + MutSup::getExtraHandleColumnIntType(), handle_column); } } @@ -607,7 +610,7 @@ DM::WriteResult DeltaMergeStore::write( if (db_context.getSettingsRef().dt_log_record_version) { - const auto & ver_col = block.getByName(VERSION_COLUMN_NAME).column; + const auto & ver_col = block.getByName(MutSup::version_column_name).column; const auto * ver = toColumnVectorDataPtr(ver_col); std::unordered_set dedup_ver; for (auto v : *ver) @@ -616,7 +619,7 @@ DM::WriteResult DeltaMergeStore::write( } std::unordered_set dedup_handles; - auto extra_handle_col = tryGetByColumnId(block, EXTRA_HANDLE_COLUMN_ID); + auto extra_handle_col = tryGetByColumnId(block, MutSup::extra_handle_id); if (extra_handle_col.column) { if (extra_handle_col.type->getTypeId() == TypeIndex::Int64) @@ -643,8 +646,8 @@ DM::WriteResult DeltaMergeStore::write( { // Sort the block by handle & version in ascending order. SortDescription sort; - sort.emplace_back(EXTRA_HANDLE_COLUMN_NAME, 1, 0); - sort.emplace_back(VERSION_COLUMN_NAME, 1, 0); + sort.emplace_back(MutSup::extra_handle_column_name, 1, 0); + sort.emplace_back(MutSup::version_column_name, 1, 0); if (rows > 1 && !isAlreadySorted(block, sort)) stableSortBlock(block, sort); @@ -654,7 +657,7 @@ DM::WriteResult DeltaMergeStore::write( size_t offset = 0; size_t limit; - const auto handle_column = block.getByName(EXTRA_HANDLE_COLUMN_NAME).column; + const auto handle_column = block.getByName(MutSup::extra_handle_column_name).column; auto rowkey_column = RowKeyColumnContainer(handle_column, is_common_handle); // Write block by segments @@ -1986,7 +1989,8 @@ void DeltaMergeStore::applySchemaChanges(TiDB::TableInfo & table_info) while (iter != new_original_table_columns.end()) { // remove the extra columns - if (iter->id == EXTRA_HANDLE_COLUMN_ID || iter->id == VERSION_COLUMN_ID || iter->id == TAG_COLUMN_ID) + if (iter->id == MutSup::extra_handle_id || iter->id == MutSup::version_col_id + || iter->id == MutSup::delmark_col_id) { iter++; continue; diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h index d72041946df..df8bbb90924 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h @@ -435,7 +435,7 @@ class DeltaMergeStore size_t num_streams, bool keep_order, const SegmentIdSet & read_segments = {}, - size_t extra_table_id_index = InvalidColumnID); + size_t extra_table_id_index = MutSup::invalid_col_id); /// Read all rows without MVCC filtering void readRaw( @@ -447,7 +447,7 @@ class DeltaMergeStore size_t num_streams, bool keep_order, const SegmentIdSet & read_segments = {}, - size_t extra_table_id_index = InvalidColumnID); + size_t extra_table_id_index = MutSup::invalid_col_id); /// Read rows in two modes: /// when is_fast_scan == false, we will read rows with MVCC filtering, del mark !=0 filter and sorted merge. @@ -468,7 +468,7 @@ class DeltaMergeStore bool is_fast_scan = false, size_t expected_block_size = DEFAULT_BLOCK_SIZE, const SegmentIdSet & read_segments = {}, - size_t extra_table_id_index = InvalidColumnID, + size_t extra_table_id_index = MutSup::invalid_col_id, ScanContextPtr scan_context = nullptr); @@ -493,7 +493,7 @@ class DeltaMergeStore bool is_fast_scan = false, size_t expected_block_size = DEFAULT_BLOCK_SIZE, const SegmentIdSet & read_segments = {}, - size_t extra_table_id_index = InvalidColumnID, + size_t extra_table_id_index = MutSup::invalid_col_id, ScanContextPtr scan_context = nullptr); Remote::DisaggPhysicalTableReadSnapshotPtr writeNodeBuildRemoteReadSnapshot( @@ -618,7 +618,7 @@ class DeltaMergeStore const String & tracing_id = "", ScanContextPtr scan_context = nullptr); - static bool pkIsHandle(const ColumnDefine & handle_define) { return handle_define.id != EXTRA_HANDLE_COLUMN_ID; } + static bool pkIsHandle(const ColumnDefine & handle_define) { return handle_define.id != MutSup::extra_handle_id; } /// Try to stall the writing. It will suspend the current thread if flow control is necessary. /// There are roughly two flow control mechanisms: diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp index 55354d28078..02aa0d8d3bb 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/File/ColumnStat.h b/dbms/src/Storages/DeltaMerge/File/ColumnStat.h index 13eb4078e01..a6a6cd8fadf 100644 --- a/dbms/src/Storages/DeltaMerge/File/ColumnStat.h +++ b/dbms/src/Storages/DeltaMerge/File/ColumnStat.h @@ -14,10 +14,12 @@ #pragma once +#include #include #include #include #include +#include #include namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp index 21048578a4b..b9750c77983 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp index a5f3f9810e6..231809c6662 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include - namespace DB::ErrorCodes { extern const int CORRUPTED_DATA; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp index d18c79108d8..a532c9d1ad9 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -21,7 +22,6 @@ #include #include - namespace DB::DM { @@ -56,7 +56,7 @@ DMFilePackFilterResultPtr DMFilePackFilter::load() auto read_all_packs = (rowkey_ranges.size() == 1 && rowkey_ranges[0].all()) || rowkey_ranges.empty(); if (!read_all_packs) { - tryLoadIndex(result.param, EXTRA_HANDLE_COLUMN_ID); + tryLoadIndex(result.param, MutSup::extra_handle_id); std::vector handle_filters; for (auto & rowkey_range : rowkey_ranges) handle_filters.emplace_back(toFilter(rowkey_range)); @@ -65,7 +65,7 @@ DMFilePackFilterResultPtr DMFilePackFilter::load() if (!rowkey_ranges.empty()) { bool is_common_handle = rowkey_ranges.begin()->is_common_handle; - auto handle_col_type = dmfile->getColumnStat(EXTRA_HANDLE_COLUMN_ID).type; + auto handle_col_type = dmfile->getColumnStat(MutSup::extra_handle_id).type; if (is_common_handle) RUNTIME_CHECK_MSG( handle_col_type->getTypeId() == TypeIndex::String, diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.cpp b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.cpp index b5778efeb86..9d809930761 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.h b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.h index dadf054fd88..49314f13205 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.h @@ -46,9 +46,9 @@ class DMFilePackFilterResult const FileProviderPtr & file_provider, const ScanContextPtr & scan_context) const { - if (!param.indexes.count(EXTRA_HANDLE_COLUMN_ID)) - tryLoadIndex(dmfile, EXTRA_HANDLE_COLUMN_ID, file_provider, scan_context); - auto & minmax_index = param.indexes.find(EXTRA_HANDLE_COLUMN_ID)->second.minmax; + if (!param.indexes.count(MutSup::extra_handle_id)) + tryLoadIndex(dmfile, MutSup::extra_handle_id, file_provider, scan_context); + auto & minmax_index = param.indexes.find(MutSup::extra_handle_id)->second.minmax; return minmax_index->getIntMinMax(pack_id).first; } @@ -58,9 +58,9 @@ class DMFilePackFilterResult const FileProviderPtr & file_provider, const ScanContextPtr & scan_context) const { - if (!param.indexes.count(EXTRA_HANDLE_COLUMN_ID)) - tryLoadIndex(dmfile, EXTRA_HANDLE_COLUMN_ID, file_provider, scan_context); - auto & minmax_index = param.indexes.find(EXTRA_HANDLE_COLUMN_ID)->second.minmax; + if (!param.indexes.count(MutSup::extra_handle_id)) + tryLoadIndex(dmfile, MutSup::extra_handle_id, file_provider, scan_context); + auto & minmax_index = param.indexes.find(MutSup::extra_handle_id)->second.minmax; return minmax_index->getStringMinMax(pack_id).first; } @@ -70,9 +70,9 @@ class DMFilePackFilterResult const FileProviderPtr & file_provider, const ScanContextPtr & scan_context) const { - if (!param.indexes.count(VERSION_COLUMN_ID)) - tryLoadIndex(dmfile, VERSION_COLUMN_ID, file_provider, scan_context); - auto & minmax_index = param.indexes.find(VERSION_COLUMN_ID)->second.minmax; + if (!param.indexes.count(MutSup::version_col_id)) + tryLoadIndex(dmfile, MutSup::version_col_id, file_provider, scan_context); + auto & minmax_index = param.indexes.find(MutSup::version_col_id)->second.minmax; return minmax_index->getUInt64MinMax(pack_id).second; } diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index b0afccc6d91..e8b64bb10f4 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -227,7 +227,7 @@ Block DMFileReader::readWithFilter(const IColumn::Filter & filter) bool DMFileReader::isCacheableColumn(const ColumnDefine & cd) { - return cd.id == EXTRA_HANDLE_COLUMN_ID || cd.id == VERSION_COLUMN_ID; + return cd.id == MutSup::extra_handle_id || cd.id == MutSup::version_col_id; } Block DMFileReader::read() @@ -257,7 +257,7 @@ Block DMFileReader::readImpl(const ReadBlockInfo & read_info) /// 2. Find packs can do clean read. const bool need_read_extra_columns = std::any_of(read_columns.cbegin(), read_columns.cend(), [](const auto & cd) { - return cd.id == EXTRA_HANDLE_COLUMN_ID || cd.id == TAG_COLUMN_ID || cd.id == VERSION_COLUMN_ID; + return cd.id == MutSup::extra_handle_id || cd.id == MutSup::delmark_col_id || cd.id == MutSup::version_col_id; }); const auto & pack_stats = dmfile->getPackStats(); const auto & pack_properties = dmfile->getPackProperties(); @@ -334,13 +334,13 @@ Block DMFileReader::readImpl(const ReadBlockInfo & read_info) // For handle, tag and version column, we can try to do clean read. switch (cd.id) { - case EXTRA_HANDLE_COLUMN_ID: + case MutSup::extra_handle_id: col = readExtraColumn(cd, start_pack_id, pack_count, read_rows, handle_column_clean_read_packs); break; - case TAG_COLUMN_ID: + case MutSup::delmark_col_id: col = readExtraColumn(cd, start_pack_id, pack_count, read_rows, del_column_clean_read_packs); break; - case VERSION_COLUMN_ID: + case MutSup::version_col_id: col = readExtraColumn(cd, start_pack_id, pack_count, read_rows, version_column_clean_read_packs); break; default: @@ -371,7 +371,7 @@ ColumnPtr DMFileReader::cleanRead( { switch (cd.id) { - case EXTRA_HANDLE_COLUMN_ID: + case MutSup::extra_handle_id: { if (is_common_handle) { @@ -384,11 +384,11 @@ ColumnPtr DMFileReader::cleanRead( return cd.type->createColumnConst(rows_count, Field(min_handle)); } } - case TAG_COLUMN_ID: + case MutSup::delmark_col_id: { return cd.type->createColumnConst(rows_count, Field(static_cast(pack_stats[range.first].first_tag))); } - case VERSION_COLUMN_ID: + case MutSup::version_col_id: { return cd.type->createColumnConst( rows_count, @@ -412,11 +412,11 @@ ColumnPtr DMFileReader::readExtraColumn( size_t read_rows, const std::vector & clean_read_packs) { - assert(cd.id == EXTRA_HANDLE_COLUMN_ID || cd.id == TAG_COLUMN_ID || cd.id == VERSION_COLUMN_ID); + assert(cd.id == MutSup::extra_handle_id || cd.id == MutSup::delmark_col_id || cd.id == MutSup::version_col_id); const auto & pack_stats = dmfile->getPackStats(); auto read_strategy = ColumnCache::getCleanReadStrategy(start_pack_id, pack_count, clean_read_packs); - if (read_strategy.size() != 1 && cd.id == EXTRA_HANDLE_COLUMN_ID) + if (read_strategy.size() != 1 && cd.id == MutSup::extra_handle_id) { // If size of read_strategy is not 1, handle can not do clean read. read_strategy.clear(); diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp index f25090b390a..6e1f987ddbc 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -21,7 +22,6 @@ #include #include - namespace DB::ErrorCodes { extern const int S3_ERROR; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexWriter.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexWriter.cpp index 78f415206a4..3b69242cd8e 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexWriter.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexWriter.cpp @@ -90,7 +90,7 @@ size_t DMFileVectorIndexWriter::buildIndexForFile(const DMFilePtr & dm_file_muta { const auto column_defines = dm_file_mutable->getColumnDefines(); const auto del_cd_iter = std::find_if(column_defines.cbegin(), column_defines.cend(), [](const ColumnDefine & cd) { - return cd.id == TAG_COLUMN_ID; + return cd.id == MutSup::delmark_col_id; }); RUNTIME_CHECK_MSG( del_cd_iter != column_defines.cend(), @@ -175,7 +175,7 @@ size_t DMFileVectorIndexWriter::buildIndexForFile(const DMFilePtr & dm_file_muta break; RUNTIME_CHECK(block.columns() == num_cols); - RUNTIME_CHECK(block.getByPosition(0).column_id == TAG_COLUMN_ID); + RUNTIME_CHECK(block.getByPosition(0).column_id == MutSup::delmark_col_id); auto del_mark_col = block.safeGetByPosition(0).column; RUNTIME_CHECK(del_mark_col != nullptr); diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp index a7e65ab9461..b1c07a04275 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include - namespace DB::DM { diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp index fe8a6b3d505..43300ce3cda 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp @@ -62,7 +62,7 @@ DMFileWriter::DMFileWriter( // TODO: currently we only generate index for Integers, Date, DateTime types, and this should be configurable by user. /// for handle column always generate index auto type = removeNullable(cd.type); - bool do_index = cd.id == EXTRA_HANDLE_COLUMN_ID || type->isInteger() || type->isDateOrDateTime(); + bool do_index = cd.id == MutSup::extra_handle_id || type->isInteger() || type->isDateOrDateTime(); addStreams(cd.id, cd.type, do_index); dmfile->meta->getColumnStats().emplace( @@ -142,7 +142,7 @@ void DMFileWriter::write(const Block & block, const BlockProperty & block_proper stat.not_clean = block_property.not_clean_rows; stat.bytes = block.bytes(); // This is bytes of pack data in memory. - auto del_mark_column = tryGetByColumnId(block, TAG_COLUMN_ID).column; + auto del_mark_column = tryGetByColumnId(block, MutSup::delmark_col_id).column; const ColumnVector * del_mark = !del_mark_column ? nullptr : static_cast *>(del_mark_column.get()); @@ -152,9 +152,9 @@ void DMFileWriter::write(const Block & block, const BlockProperty & block_proper const auto & col = getByColumnId(block, cd.id).column; writeColumn(cd.id, *cd.type, *col, del_mark); - if (cd.id == VERSION_COLUMN_ID) + if (cd.id == MutSup::version_col_id) stat.first_version = col->get64(0); - else if (cd.id == TAG_COLUMN_ID) + else if (cd.id == MutSup::delmark_col_id) stat.first_tag = static_cast(col->get64(0)); } @@ -204,12 +204,12 @@ void DMFileWriter::writeColumn( auto & stream = column_streams.at(name); if (stream->minmaxes) { - // For EXTRA_HANDLE_COLUMN_ID, we ignore del_mark when add minmax index. + // For MutSup::extra_handle_id, we ignore del_mark when add minmax index. // Because we need all rows which satisfy a certain range when place delta index no matter whether the row is a delete row. // For TAG Column, we also ignore del_mark when add minmax index. stream->minmaxes->addPack( column, - (col_id == EXTRA_HANDLE_COLUMN_ID || col_id == TAG_COLUMN_ID) ? nullptr : del_mark); + (col_id == MutSup::extra_handle_id || col_id == MutSup::delmark_col_id) ? nullptr : del_mark); } /// There could already be enough data to compress into the new block. diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h index 1871c9bef29..ce7693e6dea 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -23,7 +24,6 @@ #include #include - namespace DB::DM { diff --git a/dbms/src/Storages/DeltaMerge/Filter/FilterHelper.h b/dbms/src/Storages/DeltaMerge/Filter/FilterHelper.h index 396fb5e3941..1e322e62457 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/FilterHelper.h +++ b/dbms/src/Storages/DeltaMerge/Filter/FilterHelper.h @@ -24,9 +24,10 @@ namespace DB::DM inline RSOperatorPtr toFilter(RowKeyRange & rowkey_range) { Attr handle_attr = { - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - rowkey_range.is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + rowkey_range.is_common_handle ? MutSup::getExtraHandleColumnStringType() + : MutSup::getExtraHandleColumnIntType(), }; if (rowkey_range.is_common_handle) { diff --git a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp index de666f3e622..006dcb8ddf8 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp +++ b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp @@ -87,9 +87,11 @@ PushDownExecutorPtr PushDownExecutor::build( source_columns_of_analyzer.emplace_back(col_name, data_type); continue; } - if (cid == EXTRA_TABLE_ID_COLUMN_ID) + if (cid == MutSup::extra_table_id_col_id) { - source_columns_of_analyzer.emplace_back(EXTRA_TABLE_ID_COLUMN_NAME, EXTRA_TABLE_ID_COLUMN_TYPE); + source_columns_of_analyzer.emplace_back( + MutSup::extra_table_id_column_name, + MutSup::getExtraTableIdColumnType()); continue; } RUNTIME_CHECK_MSG(columns_to_read_map.contains(cid), "ColumnID({}) not found in columns_to_read_map", cid); @@ -146,7 +148,7 @@ PushDownExecutorPtr PushDownExecutor::build( for (size_t i = 0; i < table_scan_column_info.size(); ++i) { if (table_scan_column_info[i].hasGeneratedColumnFlag() - || table_scan_column_info[i].id == EXTRA_TABLE_ID_COLUMN_ID) + || table_scan_column_info[i].id == MutSup::extra_table_id_col_id) continue; auto col = columns_to_read_map.at(table_scan_column_info[i].id); RUNTIME_CHECK_MSG( diff --git a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp index 16d3d493a5b..7a8b35a7c27 100644 --- a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,6 @@ #include #include #include - namespace DB::DM { diff --git a/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h b/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h index fa24b1cca16..1debb7b5a8b 100644 --- a/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h +++ b/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h @@ -176,8 +176,8 @@ class PKSquashingBlockInputStream final : public IBlockInputStream { // Sort by handle & version in ascending order. static SortDescription sort{ - SortColumnDescription{EXTRA_HANDLE_COLUMN_NAME, 1, 0}, - SortColumnDescription{VERSION_COLUMN_NAME, 1, 0}}; + SortColumnDescription{MutSup::extra_handle_column_name, 1, 0}, + SortColumnDescription{MutSup::version_column_name, 1, 0}}; if (block.rows() > 1 && !isAlreadySorted(block, sort)) stableSortBlock(block, sort); } diff --git a/dbms/src/Storages/DeltaMerge/RowKeyRange.h b/dbms/src/Storages/DeltaMerge/RowKeyRange.h index 5387439ca86..0fe50ebcfe1 100644 --- a/dbms/src/Storages/DeltaMerge/RowKeyRange.h +++ b/dbms/src/Storages/DeltaMerge/RowKeyRange.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index e139e1e8bf6..3a23b522188 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -1009,9 +1010,9 @@ bool Segment::useCleanRead(const SegmentSnapshotPtr & segment_snap, const Column { return segment_snap->delta->getRows() == 0 // && segment_snap->delta->getDeletes() == 0 // - && !hasColumn(columns_to_read, EXTRA_HANDLE_COLUMN_ID) // - && !hasColumn(columns_to_read, VERSION_COLUMN_ID) // - && !hasColumn(columns_to_read, TAG_COLUMN_ID); + && !hasColumn(columns_to_read, MutSup::extra_handle_id) // + && !hasColumn(columns_to_read, MutSup::version_col_id) // + && !hasColumn(columns_to_read, MutSup::delmark_col_id); } BlockInputStreamPtr Segment::getInputStreamModeNormal( @@ -1148,7 +1149,7 @@ BlockInputStreamPtr Segment::getInputStreamForDataExport( { data_stream = std::make_shared>( data_stream, - EXTRA_HANDLE_COLUMN_ID, + MutSup::extra_handle_id, is_common_handle); } data_stream = std::make_shared>( @@ -1204,11 +1205,11 @@ BlockInputStreamPtr Segment::getInputStreamModeFast( for (const auto & c : columns_to_read) { - if (c.id == EXTRA_HANDLE_COLUMN_ID) + if (c.id == MutSup::extra_handle_id) { enable_handle_clean_read = false; } - else if (c.id == TAG_COLUMN_ID) + else if (c.id == MutSup::delmark_col_id) { enable_del_clean_read = false; } @@ -1280,7 +1281,7 @@ BlockInputStreamPtr Segment::getInputStreamModeRaw( for (const auto & c : columns_to_read) { - if (c.id != EXTRA_HANDLE_COLUMN_ID) + if (c.id != MutSup::extra_handle_id) new_columns_to_read->push_back(c); } @@ -2130,7 +2131,7 @@ std::optional Segment::prepareSplitPhysical( // my_data = std::make_shared>(my_data, my_ranges, 0); my_data - = std::make_shared>(my_data, EXTRA_HANDLE_COLUMN_ID, is_common_handle); + = std::make_shared>(my_data, MutSup::extra_handle_id, is_common_handle); my_data = std::make_shared>( my_data, *read_info.read_columns, @@ -2161,7 +2162,7 @@ std::optional Segment::prepareSplitPhysical( // other_data = std::make_shared>(other_data, other_ranges, 0); other_data = std::make_shared>( other_data, - EXTRA_HANDLE_COLUMN_ID, + MutSup::extra_handle_id, is_common_handle); other_data = std::make_shared>( other_data, @@ -2362,7 +2363,7 @@ StableValueSpacePtr Segment::prepareMerge( stream = std::make_shared>(stream, rowkey_ranges, 0); stream = std::make_shared>( stream, - EXTRA_HANDLE_COLUMN_ID, + MutSup::extra_handle_id, dm_context.is_common_handle); stream = std::make_shared>( stream, @@ -2703,7 +2704,7 @@ ColumnDefinesPtr Segment::arrangeReadColumns(const ColumnDefine & handle, const for (const auto & c : columns_to_read) { - if (c.id != handle.id && c.id != VERSION_COLUMN_ID && c.id != TAG_COLUMN_ID) + if (c.id != handle.id && c.id != MutSup::version_col_id && c.id != MutSup::delmark_col_id) new_columns_to_read.push_back(c); } @@ -3310,9 +3311,9 @@ SkippableBlockInputStreamPtr Segment::getConcatSkippableBlockInputStream( { static constexpr bool NeedRowID = false; // set `is_fast_scan` to true to try to enable clean read - auto enable_handle_clean_read = !hasColumn(columns_to_read, EXTRA_HANDLE_COLUMN_ID); + auto enable_handle_clean_read = !hasColumn(columns_to_read, MutSup::extra_handle_id); constexpr auto is_fast_scan = true; - auto enable_del_clean_read = !hasColumn(columns_to_read, TAG_COLUMN_ID); + auto enable_del_clean_read = !hasColumn(columns_to_read, MutSup::version_col_id); SkippableBlockInputStreamPtr stable_stream = segment_snap->stable->getInputStream( dm_context, @@ -3366,9 +3367,9 @@ std::tuple Segment::getConcatVectorIndexBloc { static constexpr bool NeedRowID = false; // set `is_fast_scan` to true to try to enable clean read - auto enable_handle_clean_read = !hasColumn(columns_to_read, EXTRA_HANDLE_COLUMN_ID); + auto enable_handle_clean_read = !hasColumn(columns_to_read, MutSup::extra_handle_id); constexpr auto is_fast_scan = true; - auto enable_del_clean_read = !hasColumn(columns_to_read, TAG_COLUMN_ID); + auto enable_del_clean_read = !hasColumn(columns_to_read, MutSup::delmark_col_id); SkippableBlockInputStreamPtr stable_stream = segment_snap->stable->tryGetInputStreamWithVectorIndex( dm_context, diff --git a/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp index a2fada6afe3..39e8fa54b29 100644 --- a/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp +++ b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,6 @@ #include #include #include - namespace DB { namespace ErrorCodes diff --git a/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h b/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h index 8ba2a9e29ef..fadeacd785e 100644 --- a/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h +++ b/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h @@ -170,7 +170,8 @@ class DMTestEnv columns->emplace_back(getExtraHandleColumnDefine(/*is_common_handle=*/true)); break; case PkType::PkIsHandleInt64: - columns->emplace_back(ColumnDefine{PK_ID_PK_IS_HANDLE, PK_NAME_PK_IS_HANDLE, EXTRA_HANDLE_COLUMN_INT_TYPE}); + columns->emplace_back( + ColumnDefine{PK_ID_PK_IS_HANDLE, PK_NAME_PK_IS_HANDLE, MutSup::getExtraHandleColumnIntType()}); break; case PkType::PkIsHandleInt32: columns->emplace_back( @@ -196,18 +197,19 @@ class DMTestEnv switch (pk_type) { case PkType::HiddenTiDBRowID: - columns.push_back({EXTRA_HANDLE_COLUMN_NAME, EXTRA_HANDLE_COLUMN_INT_TYPE}); + columns.push_back({MutSup::extra_handle_column_name, MutSup::getExtraHandleColumnIntType()}); break; case PkType::CommonHandle: columns.push_back( {PK_NAME_PK_IS_HANDLE, - EXTRA_HANDLE_COLUMN_STRING_TYPE}); // For common handle, there must be a user-given primary key. + MutSup:: + getExtraHandleColumnStringType()}); // For common handle, there must be a user-given primary key. columns.push_back( - {EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_STRING_TYPE}); // For common handle, a _tidb_rowid is also constructed. + {MutSup::extra_handle_column_name, + MutSup::getExtraHandleColumnStringType()}); // For common handle, a _tidb_rowid is also constructed. break; case PkType::PkIsHandleInt64: - columns.emplace_back(PK_NAME_PK_IS_HANDLE, EXTRA_HANDLE_COLUMN_INT_TYPE); + columns.emplace_back(PK_NAME_PK_IS_HANDLE, MutSup::getExtraHandleColumnIntType()); break; case PkType::PkIsHandleInt32: throw Exception("PkIsHandleInt32 is unsupported"); @@ -266,10 +268,10 @@ class DMTestEnv switch (pk_type) { case PkType::HiddenTiDBRowID: - name = EXTRA_HANDLE_COLUMN_NAME; + name = MutSup::extra_handle_column_name; break; case PkType::CommonHandle: - name = EXTRA_HANDLE_COLUMN_NAME; + name = MutSup::extra_handle_column_name; break; case PkType::PkIsHandleInt64: name = PK_NAME_PK_IS_HANDLE; @@ -297,8 +299,8 @@ class DMTestEnv bool reversed, UInt64 tso = 2, const String & pk_name_ = pk_name, - ColumnID pk_col_id = EXTRA_HANDLE_COLUMN_ID, - DataTypePtr pk_type = EXTRA_HANDLE_COLUMN_INT_TYPE, + ColumnID pk_col_id = MutSup::extra_handle_id, + DataTypePtr pk_type = MutSup::getExtraHandleColumnIntType(), bool is_common_handle = false, size_t rowkey_column_size = 1, bool with_internal_columns = true, @@ -327,15 +329,15 @@ class DMTestEnv pk_name_, pk_col_id}); // add extra column if need - if (pk_col_id != EXTRA_HANDLE_COLUMN_ID) + if (pk_col_id != MutSup::extra_handle_id) { block.insert(ColumnWithTypeAndName{ DB::tests::makeColumn( - EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::getExtraHandleColumnIntType(), createNumbers(beg, end, reversed)), - EXTRA_HANDLE_COLUMN_INT_TYPE, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID}); + MutSup::getExtraHandleColumnIntType(), + MutSup::extra_handle_column_name, + MutSup::extra_handle_id}); } } if (with_internal_columns) @@ -343,13 +345,13 @@ class DMTestEnv // version_col block.insert(DB::tests::createColumn( std::vector(num_rows, tso), - VERSION_COLUMN_NAME, - VERSION_COLUMN_ID)); + MutSup::version_column_name, + MutSup::version_col_id)); // tag_col block.insert(DB::tests::createColumn( std::vector(num_rows, is_deleted), - TAG_COLUMN_NAME, - TAG_COLUMN_ID)); + MutSup::delmark_column_name, + MutSup::delmark_col_id)); } if (with_nullable_uint64) { @@ -369,8 +371,8 @@ class DMTestEnv /*reversed*/ false, /*tso*/ 2, pk_name, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnIntType(), /* is_common_handle */ false, /* rowkey_column_size */ 1, /*with_internal_columns*/ true, @@ -401,9 +403,9 @@ class DMTestEnv end, reversed, tso, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnIntType(), false, 1, with_internal_columns); @@ -413,9 +415,9 @@ class DMTestEnv end, reversed, tso, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, 1, with_internal_columns); @@ -427,7 +429,7 @@ class DMTestEnv tso, PK_NAME_PK_IS_HANDLE, PK_ID_PK_IS_HANDLE, - EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::getExtraHandleColumnIntType(), false, 1, with_internal_columns); @@ -459,17 +461,18 @@ class DMTestEnv Block block; const size_t num_rows = (ts_end - ts_beg); // int64 pk_col - block.insert(DB::tests::createColumn(std::vector(num_rows, pk), pk_name, EXTRA_HANDLE_COLUMN_ID)); + block.insert( + DB::tests::createColumn(std::vector(num_rows, pk), pk_name, MutSup::extra_handle_id)); // version_col block.insert(DB::tests::createColumn( createNumbers(ts_beg, ts_end, reversed), - VERSION_COLUMN_NAME, - VERSION_COLUMN_ID)); + MutSup::version_column_name, + MutSup::version_col_id)); // tag_col block.insert(DB::tests::createColumn( std::vector(num_rows, deleted ? 1 : 0), - TAG_COLUMN_NAME, - TAG_COLUMN_ID)); + MutSup::delmark_column_name, + MutSup::delmark_col_id)); return block; } @@ -490,22 +493,24 @@ class DMTestEnv if (is_common_handle) { Strings values{genMockCommonHandle(pk, rowkey_column_size)}; - block.insert(DB::tests::createColumn(std::move(values), pk_name, EXTRA_HANDLE_COLUMN_ID)); + block.insert(DB::tests::createColumn(std::move(values), pk_name, MutSup::extra_handle_id)); } else { // int64 pk_col block.insert( - DB::tests::createColumn(std::vector(num_rows, pk), pk_name, EXTRA_HANDLE_COLUMN_ID)); + DB::tests::createColumn(std::vector(num_rows, pk), pk_name, MutSup::extra_handle_id)); } // version_col block.insert(DB::tests::createColumn( std::vector(num_rows, tso), - VERSION_COLUMN_NAME, - VERSION_COLUMN_ID)); + MutSup::version_column_name, + MutSup::version_col_id)); // tag_col - block.insert( - DB::tests::createColumn(std::vector(num_rows, mark), TAG_COLUMN_NAME, TAG_COLUMN_ID)); + block.insert(DB::tests::createColumn( + std::vector(num_rows, mark), + MutSup::delmark_column_name, + MutSup::delmark_col_id)); // string column block.insert(DB::tests::createColumn( Strings{value}, @@ -542,15 +547,18 @@ class DMTestEnv // int64 pk_col block.insert(DB::tests::createColumn( createNumbers(start_pk, start_pk + rows), - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID)); + MutSup::extra_handle_column_name, + MutSup::extra_handle_id)); // version_col block.insert(DB::tests::createColumn( createNumbers(start_ts, start_ts + rows), - VERSION_COLUMN_NAME, - VERSION_COLUMN_ID)); + MutSup::version_column_name, + MutSup::version_col_id)); // tag_col - block.insert(DB::tests::createColumn(std::vector(rows, 0), TAG_COLUMN_NAME, TAG_COLUMN_ID)); + block.insert(DB::tests::createColumn( + std::vector(rows, 0), + MutSup::delmark_column_name, + MutSup::delmark_col_id)); return block; } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp index 2701ed4f11f..795315d797a 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -48,9 +48,9 @@ class DebugBlockInputStream : public BlocksListBlockInputStream BlockInputStreamPtr genColumnProjInputStream(BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) { ColumnDefine handle_define( - TiDBPkColumnID, + MutSup::extra_handle_id, DMTestEnv::pk_name, - is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE); + is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType()); return std::make_shared( std::make_shared(blocks, is_common_handle), @@ -63,9 +63,9 @@ BlockInputStreamPtr genDeleteFilterInputStream( bool is_common_handle) { ColumnDefine handle_define( - TiDBPkColumnID, + MutSup::extra_handle_id, DMTestEnv::pk_name, - is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE); + is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType()); return std::make_shared( std::make_shared(blocks, is_common_handle), diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp index 2c563b5b0f8..04fd8e69645 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp @@ -61,13 +61,13 @@ TEST(PKSquashTest, WithExtraSort) // Sorted by pk, tso asc SortDescription sort // = SortDescription{// - SortColumnDescription{EXTRA_HANDLE_COLUMN_NAME, 1, 0}, - SortColumnDescription{VERSION_COLUMN_NAME, 1, 0}}; + SortColumnDescription{MutSup::extra_handle_column_name, 1, 0}, + SortColumnDescription{MutSup::version_column_name, 1, 0}}; { auto in = std::make_shared>( std::make_shared(blocks.begin(), blocks.end()), - TiDBPkColumnID, + MutSup::extra_handle_id, false); size_t num_blocks_read = 0; size_t num_rows_read = 0; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp index 6873685a4a6..9d63a7aef04 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp @@ -129,14 +129,14 @@ try while (Block in = column_file_big_reader->readNextBlock()) { ASSERT_EQ(in.columns(), column_defines->size()); - ASSERT_TRUE(in.has(EXTRA_HANDLE_COLUMN_NAME)); - ASSERT_TRUE(in.has(VERSION_COLUMN_NAME)); - auto & pk_column = in.getByName(EXTRA_HANDLE_COLUMN_NAME).column; + ASSERT_TRUE(in.has(MutSup::extra_handle_column_name)); + ASSERT_TRUE(in.has(MutSup::version_column_name)); + auto & pk_column = in.getByName(MutSup::extra_handle_column_name).column; for (size_t i = 0; i < pk_column->size(); i++) { ASSERT_EQ(pk_column->getInt(i), num_rows_read + i); } - auto & version_column = in.getByName(VERSION_COLUMN_NAME).column; + auto & version_column = in.getByName(MutSup::version_column_name).column; for (size_t i = 0; i < version_column->size(); i++) { ASSERT_EQ(version_column->getInt(i), tso_value); @@ -156,14 +156,14 @@ try while (Block in = column_file_big_reader2->readNextBlock()) { ASSERT_EQ(in.columns(), column_defines_pk_and_del->size()); - ASSERT_TRUE(in.has(EXTRA_HANDLE_COLUMN_NAME)); - ASSERT_TRUE(in.has(TAG_COLUMN_NAME)); - auto & pk_column = in.getByName(EXTRA_HANDLE_COLUMN_NAME).column; + ASSERT_TRUE(in.has(MutSup::extra_handle_column_name)); + ASSERT_TRUE(in.has(MutSup::delmark_column_name)); + auto & pk_column = in.getByName(MutSup::extra_handle_column_name).column; for (size_t i = 0; i < pk_column->size(); i++) { ASSERT_EQ(pk_column->getInt(i), num_rows_read + i); } - auto & del_column = in.getByName(TAG_COLUMN_NAME).column; + auto & del_column = in.getByName(MutSup::delmark_column_name).column; for (size_t i = 0; i < del_column->size(); i++) { ASSERT_EQ(del_column->getInt(i), 0); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index 6cc96bdc63d..56c7cccb7fe 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -90,9 +90,9 @@ try { // check handle column of store const auto & h = store->getHandle(); - ASSERT_EQ(h.name, EXTRA_HANDLE_COLUMN_NAME); - ASSERT_EQ(h.id, EXTRA_HANDLE_COLUMN_ID); - ASSERT_TRUE(h.type->equals(*EXTRA_HANDLE_COLUMN_INT_TYPE)); + ASSERT_EQ(h.name, MutSup::extra_handle_column_name); + ASSERT_EQ(h.id, MutSup::extra_handle_id); + ASSERT_TRUE(h.type->equals(*MutSup::getExtraHandleColumnIntType())); } { // check column structure of store @@ -358,7 +358,7 @@ try store->getRowKeyColumnSize()); block1 = DeltaMergeStore::addExtraColumnIfNeed(*db_context, store->getHandle(), std::move(block1)); ASSERT_EQ(block1.rows(), nrows); - ASSERT_TRUE(block1.has(EXTRA_HANDLE_COLUMN_NAME)); + ASSERT_TRUE(block1.has(MutSup::extra_handle_column_name)); ASSERT_NO_THROW({ block1.checkNumberOfRows(); }); // Make a block that is overlapped with `block1` and it should be squashed by `PKSquashingBlockInputStream` @@ -375,14 +375,14 @@ try store->getRowKeyColumnSize()); block2 = DeltaMergeStore::addExtraColumnIfNeed(*db_context, store->getHandle(), std::move(block2)); ASSERT_EQ(block2.rows(), nrows_2); - ASSERT_TRUE(block2.has(EXTRA_HANDLE_COLUMN_NAME)); + ASSERT_TRUE(block2.has(MutSup::extra_handle_column_name)); ASSERT_NO_THROW({ block2.checkNumberOfRows(); }); BlockInputStreamPtr stream = std::make_shared(BlocksList{block1, block2}); stream = std::make_shared>( stream, - EXTRA_HANDLE_COLUMN_ID, + MutSup::extra_handle_id, store->isCommonHandle()); ASSERT_INPUTSTREAM_NROWS(stream, nrows + nrows_2); @@ -562,7 +562,7 @@ try /* is_fast_scan= */ false, /* expected_block_size= */ 1024, /* read_segments */ {}, - /* extra_table_id_index */ InvalidColumnID, + /* extra_table_id_index */ MutSup::invalid_col_id, /* scan_context */ scan_context)[0]; in->readPrefix(); while (in->read()) {}; @@ -590,7 +590,7 @@ try /* is_fast_scan= */ false, /* expected_block_size= */ 1024, /* read_segments */ {}, - /* extra_table_id_index */ InvalidColumnID, + /* extra_table_id_index */ MutSup::invalid_col_id, /* scan_context */ scan_context)[0]; in->readPrefix(); @@ -1307,7 +1307,7 @@ try BlockInputStreamPtr in = ins[0]; ASSERT_UNORDERED_INPUTSTREAM_COLS_UR( in, - Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + Strings({DMTestEnv::pk_name, MutSup::version_column_name}), createColumns({ createColumn(createNumbers(0, 32)), createColumn(std::vector(32, tso1)), @@ -1336,7 +1336,7 @@ try BlockInputStreamPtr in = ins[0]; ASSERT_UNORDERED_INPUTSTREAM_COLS_UR( in, - Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + Strings({DMTestEnv::pk_name, MutSup::version_column_name}), createColumns({ createColumn(createNumbers(0, 32)), createColumn(std::vector(32, tso1)), @@ -1472,7 +1472,7 @@ try BlockInputStreamPtr in = ins[0]; ASSERT_UNORDERED_INPUTSTREAM_COLS_UR( in, - Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + Strings({DMTestEnv::pk_name, MutSup::version_column_name}), createColumns({ createColumn(createNumbers(0, 32)), createColumn(std::vector(32, tso1)), @@ -1501,7 +1501,7 @@ try BlockInputStreamPtr in = ins[0]; ASSERT_UNORDERED_INPUTSTREAM_COLS_UR( in, - Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + Strings({DMTestEnv::pk_name, MutSup::version_column_name}), createColumns({ createColumn(createNumbers(0, 32)), createColumn(std::vector(32, tso1)), @@ -1578,7 +1578,8 @@ try { break; } - const auto & handle = *toColumnVectorDataPtr(block.getByName(EXTRA_HANDLE_COLUMN_NAME).column); + const auto & handle + = *toColumnVectorDataPtr(block.getByName(MutSup::extra_handle_column_name).column); const auto & value = *toColumnVectorDataPtr(block.getByName(value_col_name).column); for (size_t i = 0; i < block.rows(); i++) { @@ -2748,9 +2749,9 @@ try { // check handle column of store const auto & h = store->getHandle(); - ASSERT_EQ(h.name, EXTRA_HANDLE_COLUMN_NAME); - ASSERT_EQ(h.id, EXTRA_HANDLE_COLUMN_ID); - ASSERT_TRUE(h.type->equals(*EXTRA_HANDLE_COLUMN_STRING_TYPE)); + ASSERT_EQ(h.name, MutSup::extra_handle_column_name); + ASSERT_EQ(h.id, MutSup::extra_handle_id); + ASSERT_TRUE(h.type->equals(*MutSup::getExtraHandleColumnStringType())); } { // check column structure of store @@ -2946,9 +2947,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, rowkey_column_size); // Add a column of col2:String for test @@ -3052,9 +3053,9 @@ try 1 * num_write_rows, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, rowkey_column_size); Block block2 = DMTestEnv::prepareSimpleWriteBlock( @@ -3062,9 +3063,9 @@ try 2 * num_write_rows, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, rowkey_column_size); Block block3 = DMTestEnv::prepareSimpleWriteBlock( @@ -3072,9 +3073,9 @@ try 3 * num_write_rows, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, rowkey_column_size); store->write(*db_context, db_context->getSettingsRef(), block1); @@ -3129,9 +3130,9 @@ try 1 * num_write_rows, false, tso1, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, rowkey_column_size); Block block2 = DMTestEnv::prepareSimpleWriteBlock( @@ -3139,9 +3140,9 @@ try 2 * num_write_rows, false, tso1, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, rowkey_column_size); Block block3 = DMTestEnv::prepareSimpleWriteBlock( @@ -3149,9 +3150,9 @@ try num_write_rows / 2 + num_write_rows, false, tso2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, rowkey_column_size); store->write(*db_context, db_context->getSettingsRef(), block1); @@ -3251,9 +3252,9 @@ try 128, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, rowkey_column_size); store->write(*db_context, db_context->getSettingsRef(), block); @@ -3857,8 +3858,8 @@ try false, std::numeric_limits::max(), // max version to make sure it's the latest pk_name, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnIntType(), false, 1, true, @@ -3879,7 +3880,8 @@ try ColumnDefines real_columns; for (const auto & col : columns) { - if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != TAG_COLUMN_NAME && col.name != VERSION_COLUMN_NAME) + if (col.name != MutSup::extra_handle_column_name && col.name != MutSup::delmark_column_name + && col.name != MutSup::version_column_name) { real_columns.emplace_back(col); } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_scan.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_scan.cpp index 11adc800029..3d613cfe8f4 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_scan.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_scan.cpp @@ -819,8 +819,8 @@ try false, 3, DMTestEnv::pk_name, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnIntType(), false, 1, true, @@ -1303,7 +1303,7 @@ try ColumnDefines real_columns; for (const auto & col : columns) { - if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != TAG_COLUMN_NAME) + if (col.name != MutSup::extra_handle_column_name && col.name != MutSup::delmark_column_name) { real_columns.emplace_back(col); } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_vector_index.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_vector_index.cpp index 83fe42b5038..0072eb0bd89 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_vector_index.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_vector_index.cpp @@ -85,9 +85,9 @@ class DeltaMergeStoreVectorTest end, false, /*tso= */ 3, - /*pk_name_=*/EXTRA_HANDLE_COLUMN_NAME, - /*pk_col_id=*/EXTRA_HANDLE_COLUMN_ID, - /*pk_type=*/EXTRA_HANDLE_COLUMN_INT_TYPE, + /*pk_name_=*/MutSup::extra_handle_column_name, + /*pk_col_id=*/MutSup::extra_handle_id, + /*pk_type=*/MutSup::getExtraHandleColumnIntType(), /*is_common_handle=*/false, /*rowkey_column_size=*/1, /*with_internal_columns=*/true, diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index 3f2919e5ef3..e6ed053c1f7 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -2025,9 +2025,9 @@ try num_rows_write / 2, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); Block block2 = DMTestEnv::prepareSimpleWriteBlock( @@ -2035,9 +2035,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); auto stream = std::make_shared(dbContext(), dm_file, *cols); @@ -2104,9 +2104,9 @@ try pk_end, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); stream->write(block, block_property); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_meta_version.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_meta_version.cpp index 92db6f9daeb..250a558684d 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_meta_version.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_meta_version.cpp @@ -148,7 +148,7 @@ try auto dm_file = prepareDMFile(/* file_id= */ 1); ASSERT_EQ(0, dm_file->metaVersion()); ASSERT_EQ(4, dm_file->meta->getColumnStats().size()); - ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str()); + ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str()); // Write new metadata auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{ @@ -158,7 +158,7 @@ try .path_pool = path_pool, .disagg_ctx = db_context->getSharedContextDisagg(), }); - dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test"; + dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test"; ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({})); iw->finalize(); @@ -173,7 +173,7 @@ try ASSERT_EQ(0, dm_file->metaVersion()); ASSERT_EQ(4, dm_file->meta->getColumnStats().size()); - ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str()); + ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str()); // Read out meta version = 1 dm_file = DMFile::restore( @@ -186,7 +186,7 @@ try ASSERT_EQ(1, dm_file->metaVersion()); ASSERT_EQ(4, dm_file->meta->getColumnStats().size()); - ASSERT_STREQ("test", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str()); + ASSERT_STREQ("test", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str()); } CATCH @@ -202,7 +202,7 @@ try .path_pool = path_pool, .disagg_ctx = db_context->getSharedContextDisagg(), }); - dm_file_for_write->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test"; + dm_file_for_write->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test"; ASSERT_EQ(1, dm_file_for_write->meta->bumpMetaVersion({})); iw->finalize(); @@ -215,7 +215,7 @@ try /* meta_version= */ 1); ASSERT_STREQ( "test", - dm_file_for_read_v1->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + dm_file_for_read_v1->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); // Write a new meta with a new version = 2 iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{ @@ -225,14 +225,14 @@ try .path_pool = path_pool, .disagg_ctx = db_context->getSharedContextDisagg(), }); - dm_file_for_write->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test2"; + dm_file_for_write->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test2"; ASSERT_EQ(2, dm_file_for_write->meta->bumpMetaVersion({})); iw->finalize(); // Current DMFile instance does not affect ASSERT_STREQ( "test", - dm_file_for_read_v1->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + dm_file_for_read_v1->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); // Read out meta version = 2 auto dm_file_for_read_v2 = DMFile::restore( @@ -244,7 +244,7 @@ try /* meta_version= */ 2); ASSERT_STREQ( "test2", - dm_file_for_read_v2->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + dm_file_for_read_v2->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); } CATCH @@ -261,7 +261,7 @@ try .path_pool = path_pool, .disagg_ctx = db_context->getSharedContextDisagg(), }); - dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test"; + dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test"; ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({})); iw->finalize(); @@ -281,7 +281,7 @@ try .path_pool = path_pool, .disagg_ctx = db_context->getSharedContextDisagg(), }); - dm_file_2->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test_overwrite"; + dm_file_2->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test_overwrite"; ASSERT_EQ(1, dm_file_2->meta->bumpMetaVersion({})); ASSERT_NO_THROW({ iw->finalize(); @@ -297,7 +297,7 @@ try /* meta_version= */ 1); ASSERT_STREQ( "test_overwrite", - dm_file_for_read->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + dm_file_for_read->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); } CATCH @@ -307,7 +307,7 @@ try auto dm_file = prepareDMFile(/* file_id= */ 1); ASSERT_EQ(0, dm_file->metaVersion()); ASSERT_EQ(4, dm_file->meta->getColumnStats().size()); - ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str()); + ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str()); // Write new metadata auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{ @@ -317,7 +317,7 @@ try .path_pool = path_pool, .disagg_ctx = db_context->getSharedContextDisagg(), }); - dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test"; + dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test"; dm_file->meta->bumpMetaVersion({}); iw->finalize(); @@ -432,7 +432,7 @@ try ASSERT_EQ(0, dm_file->metaVersion()); ASSERT_EQ(4, dm_file->meta->getColumnStats().size()); - ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str()); + ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str()); // Write new metadata auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{ @@ -442,7 +442,7 @@ try .path_pool = path_pool, .disagg_ctx = db_context->getSharedContextDisagg(), }); - dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test"; + dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test"; ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({})); iw->finalize(); @@ -457,12 +457,12 @@ try /* page_id= */ 0); auto cn_dmf = token->restore(DMFileMeta::ReadMode::all(), 0); ASSERT_EQ(0, cn_dmf->metaVersion()); - ASSERT_STREQ("", cn_dmf->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + ASSERT_STREQ("", cn_dmf->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); // Read out meta version = 1 cn_dmf = token->restore(DMFileMeta::ReadMode::all(), 1); ASSERT_EQ(1, cn_dmf->metaVersion()); - ASSERT_STREQ("test", cn_dmf->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + ASSERT_STREQ("test", cn_dmf->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); } CATCH @@ -480,7 +480,7 @@ try ASSERT_EQ(0, dm_file->metaVersion()); ASSERT_EQ(4, dm_file->meta->getColumnStats().size()); - ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str()); + ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str()); // Write new metadata auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{ @@ -490,7 +490,7 @@ try .path_pool = path_pool, .disagg_ctx = db_context->getSharedContextDisagg(), }); - dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test"; + dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test"; ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({})); iw->finalize(); @@ -510,7 +510,7 @@ try /* page_id= */ 0); auto cn_dmf = token->restore(DMFileMeta::ReadMode::all(), 0); ASSERT_EQ(0, cn_dmf->metaVersion()); - ASSERT_STREQ("", cn_dmf->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + ASSERT_STREQ("", cn_dmf->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); { auto * file_cache = FileCache::instance(); @@ -520,7 +520,7 @@ try // Read out meta version = 1 cn_dmf = token->restore(DMFileMeta::ReadMode::all(), 1); ASSERT_EQ(1, cn_dmf->metaVersion()); - ASSERT_STREQ("test", cn_dmf->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + ASSERT_STREQ("test", cn_dmf->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); SCOPE_EXIT({ FileCache::shutdown(); }); } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index 3e0ede20007..dcafc0f22a7 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -1129,8 +1129,8 @@ try false, 3, "_tidb_rowid", - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnIntType(), false, 1, true, diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp index ea3659931a7..a7c22265717 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp @@ -136,9 +136,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); // write to segment @@ -190,9 +190,9 @@ try num_rows_write + num_rows_write_2, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -235,9 +235,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); // write to segment @@ -292,9 +292,9 @@ try num_rows_write + num_rows_write_2, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -340,9 +340,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -410,9 +410,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -488,9 +488,9 @@ try num_rows_write / 2, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -507,9 +507,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -580,9 +580,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -790,9 +790,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -923,9 +923,9 @@ try num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); @@ -981,9 +981,9 @@ try num_batches_written * num_rows_per_write + num_rows_per_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), is_common_handle, rowkey_column_size); segment->write(dmContext(), std::move(block)); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_simple_pk_test_basic.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_simple_pk_test_basic.cpp index 919ad3c830e..d25d6f80957 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_simple_pk_test_basic.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_simple_pk_test_basic.cpp @@ -192,8 +192,8 @@ Block SimplePKTestBasic::fillBlock(const FillBlockOptions & options) false, version, DMTestEnv::pk_name, - EXTRA_HANDLE_COLUMN_ID, - is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_id, + is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType(), is_common_handle, 1, true, diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp index fcfa1bd3b96..d0ef2c15bf0 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp @@ -698,7 +698,7 @@ try runtime_filter_ids, 0, ctx->getTimezoneInfo()); - Names read_columns = {"col1", EXTRA_TABLE_ID_COLUMN_NAME, "col2"}; + Names read_columns = {"col1", MutSup::extra_table_id_column_name, "col2"}; BlockInputStreams ins = storage->read(read_columns, query_info, *ctx, stage2, 8192, 1); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; @@ -706,7 +706,7 @@ try in, Block({ createColumn(createNumbers(0, num_rows_write), "col1"), - createConstColumn>(num_rows_write, table_id, EXTRA_TABLE_ID_COLUMN_NAME), + createConstColumn>(num_rows_write, table_id, MutSup::extra_table_id_column_name), createColumn(Strings(num_rows_write, "a"), "col2"), })); @@ -818,7 +818,7 @@ try runtime_filter_ids, 0, ctx->getTimezoneInfo()); - Names read_columns = {"col1", EXTRA_TABLE_ID_COLUMN_NAME, "col2"}; + Names read_columns = {"col1", MutSup::extra_table_id_column_name, "col2"}; BlockInputStreams ins = storage->read(read_columns, query_info, *ctx, stage2, 8192, 1); return getInputStreamNRows(ins[0]); }; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp index d1a9a17748c..549b3a18b6d 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp @@ -62,9 +62,9 @@ TEST(RowKeyFilterTest, FilterSortedBlockCommonHandle) num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, 1); auto filtered_block = RowKeyFilter::filterSorted(ranges, std::move(block), 0); @@ -84,9 +84,9 @@ TEST(RowKeyFilterTest, FilterUnsortedBlockCommonHandle) num_rows_write, false, 2, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_STRING_TYPE, + MutSup::extra_handle_column_name, + MutSup::extra_handle_id, + MutSup::getExtraHandleColumnStringType(), true, 1); auto filtered_block = RowKeyFilter::filterUnsorted(ranges, std::move(block), 0); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp index 0391ff1100b..2d783ce3b1d 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp @@ -485,7 +485,7 @@ try // does not have index at all. { auto ann_query_info = std::make_shared(); - ann_query_info->set_column_id(EXTRA_HANDLE_COLUMN_ID); + ann_query_info->set_column_id(MutSup::extra_handle_id); ann_query_info->set_distance_metric(tipb::VectorDistanceMetric::L2); ann_query_info->set_top_k(1); ann_query_info->set_ref_vec_f32(encodeVectorFloat32({1.0, 2.0, 3.8})); @@ -1159,7 +1159,7 @@ class VectorIndexSegmentTestBase { auto options = SegmentTestBasic::SegmentTestOptions{}; if (enable_column_cache_long_term) - options.pk_col_id = EXTRA_HANDLE_COLUMN_ID; + options.pk_col_id = MutSup::extra_handle_id; SegmentTestBasic::SetUp(options); } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task.cpp index 92fd9b04dc5..964482c1702 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task.cpp @@ -129,7 +129,7 @@ class SegmentReadTaskTest : public SegmentTestBasic { blks.push_back(blk); } - auto handle_col1 = vstackBlocks(std::move(blks)).getByName(EXTRA_HANDLE_COLUMN_NAME).column; + auto handle_col1 = vstackBlocks(std::move(blks)).getByName(MutSup::extra_handle_column_name).column; auto handle_col2 = getSegmentHandle(task->segment->segmentId(), {task->segment->getRowKeyRange()}); ASSERT_TRUE(sequenceEqual( toColumnVectorDataPtr(handle_col2)->data(), diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task_pool.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task_pool.cpp index 9c686ef750b..8a931ec7b69 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task_pool.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task_pool.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_replace_stable_data.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_replace_stable_data.cpp index 07854c0f0a0..1fd2affc570 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_replace_stable_data.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_replace_stable_data.cpp @@ -100,8 +100,8 @@ class SegmentReplaceStableData .disagg_ctx = dm_context->global_context.getSharedContextDisagg(), }); auto & column_stats = new_dm_file->meta->getColumnStats(); - RUNTIME_CHECK(column_stats.find(::DB::TiDBPkColumnID) != column_stats.end()); - column_stats[::DB::TiDBPkColumnID].additional_data_for_test = pk_additiona_data; + RUNTIME_CHECK(column_stats.find(MutSup::extra_handle_id) != column_stats.end()); + column_stats[MutSup::extra_handle_id].additional_data_for_test = pk_additiona_data; new_dm_file->meta->bumpMetaVersion({}); iw->finalize(); @@ -156,9 +156,9 @@ class SegmentReplaceStableData auto file = files[0]; auto column_stats = file->meta->getColumnStats(); - RUNTIME_CHECK(column_stats.find(::DB::TiDBPkColumnID) != column_stats.end()); + RUNTIME_CHECK(column_stats.find(MutSup::extra_handle_id) != column_stats.end()); - auto meta_value = column_stats[::DB::TiDBPkColumnID].additional_data_for_test; + auto meta_value = column_stats[MutSup::extra_handle_id].additional_data_for_test; // Read again using a fresh DMFile restore, to ensure that this value is // indeed persisted. @@ -172,8 +172,8 @@ class SegmentReplaceStableData RUNTIME_CHECK(file2 != nullptr); column_stats = file2->meta->getColumnStats(); - RUNTIME_CHECK(column_stats.find(::DB::TiDBPkColumnID) != column_stats.end()); - RUNTIME_CHECK(column_stats[::DB::TiDBPkColumnID].additional_data_for_test == meta_value); + RUNTIME_CHECK(column_stats.find(MutSup::extra_handle_id) != column_stats.end()); + RUNTIME_CHECK(column_stats[MutSup::extra_handle_id].additional_data_for_test == meta_value); return meta_value; } @@ -570,8 +570,8 @@ try .disagg_ctx = wn_dm_context->global_context.getSharedContextDisagg(), }); auto & column_stats = new_dm_file->meta->getColumnStats(); - RUNTIME_CHECK(column_stats.find(::DB::TiDBPkColumnID) != column_stats.end()); - column_stats[::DB::TiDBPkColumnID].additional_data_for_test = "tiflash_foo"; + RUNTIME_CHECK(column_stats.find(MutSup::extra_handle_id) != column_stats.end()); + column_stats[MutSup::extra_handle_id].additional_data_for_test = "tiflash_foo"; new_dm_file->meta->bumpMetaVersion({}); iw->finalize(); @@ -588,7 +588,7 @@ try auto cn_files = snapshot->stable->getDMFiles(); ASSERT_EQ(1, cn_files.size()); ASSERT_EQ(0, cn_files[0]->metaVersion()); - ASSERT_STREQ("", cn_files[0]->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + ASSERT_STREQ("", cn_files[0]->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); } // Read meta v1 in CN @@ -600,7 +600,7 @@ try ASSERT_EQ(1, cn_files[0]->metaVersion()); ASSERT_STREQ( "tiflash_foo", - cn_files[0]->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + cn_files[0]->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); } // Read meta v0 again in CN @@ -610,7 +610,7 @@ try auto cn_files = snapshot->stable->getDMFiles(); ASSERT_EQ(1, cn_files.size()); ASSERT_EQ(0, cn_files[0]->metaVersion()); - ASSERT_STREQ("", cn_files[0]->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str()); + ASSERT_STREQ("", cn_files[0]->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str()); } } CATCH diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp index f2c10e2a677..d8fc3ae12b1 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp @@ -363,8 +363,8 @@ Block SegmentTestBasic::prepareWriteBlockImpl(Int64 start_key, Int64 end_key, bo false, version, DMTestEnv::pk_name, - EXTRA_HANDLE_COLUMN_ID, - options.is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_id, + options.is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType(), options.is_common_handle, 1, true, @@ -381,8 +381,8 @@ Block sortvstackBlocks(std::vector && blocks) auto accumulated_block = vstackBlocks(std::move(blocks)); SortDescription sort; - sort.emplace_back(EXTRA_HANDLE_COLUMN_NAME, 1, 0); - sort.emplace_back(VERSION_COLUMN_NAME, 1, 0); + sort.emplace_back(MutSup::extra_handle_column_name, 1, 0); + sort.emplace_back(MutSup::version_column_name, 1, 0); stableSortBlock(accumulated_block, sort); return accumulated_block; @@ -1048,7 +1048,7 @@ ColumnPtr SegmentTestBasic::getSegmentRowId(PageIdU64 segment_id, const RowKeyRa else { auto block = mergeSegmentRowIds(std::move(blks)); - RUNTIME_CHECK(!block.has(EXTRA_HANDLE_COLUMN_NAME)); + RUNTIME_CHECK(!block.has(MutSup::extra_handle_column_name)); RUNTIME_CHECK(block.segmentRowIdCol() != nullptr); return block.segmentRowIdCol(); } @@ -1065,9 +1065,9 @@ ColumnPtr SegmentTestBasic::getSegmentHandle(PageIdU64 segment_id, const RowKeyR else { auto block = vstackBlocks(std::move(blks)); - RUNTIME_CHECK(block.has(EXTRA_HANDLE_COLUMN_NAME)); + RUNTIME_CHECK(block.has(MutSup::extra_handle_column_name)); RUNTIME_CHECK(block.segmentRowIdCol() == nullptr); - return block.getByName(EXTRA_HANDLE_COLUMN_NAME).column; + return block.getByName(MutSup::extra_handle_column_name).column; } } @@ -1105,47 +1105,51 @@ try version = 0; auto block = prepareWriteBlockInSegmentRange(*s1_id, 10); ASSERT_COLUMN_EQ( - block.getByName(EXTRA_HANDLE_COLUMN_NAME), + block.getByName(MutSup::extra_handle_column_name), createColumn({10, 11, 12, 13, 14, 15, 16, 17, 18, 19})); - ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn({1, 1, 1, 1, 1, 1, 1, 1, 1, 1})); + ASSERT_COLUMN_EQ( + block.getByName(MutSup::version_column_name), + createColumn({1, 1, 1, 1, 1, 1, 1, 1, 1, 1})); } { // write_rows > segment_rows, start_key not specified version = 0; auto block = prepareWriteBlockInSegmentRange(*s1_id, 13); ASSERT_COLUMN_EQ( - block.getByName(EXTRA_HANDLE_COLUMN_NAME), + block.getByName(MutSup::extra_handle_column_name), createColumn({10, 10, 11, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19})); ASSERT_COLUMN_EQ( - block.getByName(VERSION_COLUMN_NAME), + block.getByName(MutSup::version_column_name), createColumn({1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1})); } { // start_key specified, end_key - start_key < write_rows version = 0; auto block = prepareWriteBlockInSegmentRange(*s1_id, 2, /* at */ 16); - ASSERT_COLUMN_EQ(block.getByName(EXTRA_HANDLE_COLUMN_NAME), createColumn({16, 17})); - ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn({1, 1})); + ASSERT_COLUMN_EQ(block.getByName(MutSup::extra_handle_column_name), createColumn({16, 17})); + ASSERT_COLUMN_EQ(block.getByName(MutSup::version_column_name), createColumn({1, 1})); } { version = 0; auto block = prepareWriteBlockInSegmentRange(*s1_id, 4, /* at */ 16); - ASSERT_COLUMN_EQ(block.getByName(EXTRA_HANDLE_COLUMN_NAME), createColumn({16, 17, 18, 19})); - ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn({1, 1, 1, 1})); + ASSERT_COLUMN_EQ(block.getByName(MutSup::extra_handle_column_name), createColumn({16, 17, 18, 19})); + ASSERT_COLUMN_EQ(block.getByName(MutSup::version_column_name), createColumn({1, 1, 1, 1})); } { version = 0; auto block = prepareWriteBlockInSegmentRange(*s1_id, 5, /* at */ 16); - ASSERT_COLUMN_EQ(block.getByName(EXTRA_HANDLE_COLUMN_NAME), createColumn({16, 16, 17, 18, 19})); - ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn({1, 2, 1, 1, 1})); + ASSERT_COLUMN_EQ(block.getByName(MutSup::extra_handle_column_name), createColumn({16, 16, 17, 18, 19})); + ASSERT_COLUMN_EQ(block.getByName(MutSup::version_column_name), createColumn({1, 2, 1, 1, 1})); } { version = 0; auto block = prepareWriteBlockInSegmentRange(*s1_id, 10, /* at */ 16); ASSERT_COLUMN_EQ( - block.getByName(EXTRA_HANDLE_COLUMN_NAME), + block.getByName(MutSup::extra_handle_column_name), createColumn({16, 16, 16, 17, 17, 17, 18, 18, 19, 19})); - ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn({1, 2, 3, 1, 2, 3, 1, 2, 1, 2})); + ASSERT_COLUMN_EQ( + block.getByName(MutSup::version_column_name), + createColumn({1, 2, 3, 1, 2, 3, 1, 2, 1, 2})); } { // write rows < segment rows, start key not specified, should choose a random start. @@ -1158,7 +1162,7 @@ try for (size_t i = 0; i < 100; i++) { auto block = prepareWriteBlockInSegmentRange(*s1_id, 3); - if (block.getByName(EXTRA_HANDLE_COLUMN_NAME).column->getInt(0) == 12) + if (block.getByName(MutSup::extra_handle_column_name).column->getInt(0) == 12) start_from_12++; } ASSERT_TRUE(start_from_12 > 0); // We should hit at least 1 times in 100 iters. diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_sst_files_stream.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_sst_files_stream.cpp index 016683bc2df..af787a1c5f8 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_sst_files_stream.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_sst_files_stream.cpp @@ -120,7 +120,7 @@ try auto blocks = prepareBlocks(-10, 5, 1000); ASSERT_EQ(blocks.size(), 1); auto block = blocks[0]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({-10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4})); } { @@ -128,27 +128,27 @@ try ASSERT_EQ(blocks.size(), 5); { auto block = blocks[0]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({1, 2, 3})); } { auto block = blocks[1]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({4, 5, 6})); } { auto block = blocks[2]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({7, 8, 9})); } { auto block = blocks[3]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({10, 11, 12})); } { auto block = blocks[4]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({13})); } } @@ -157,17 +157,17 @@ try ASSERT_EQ(blocks.size(), 3); { auto block = blocks[0]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({1})); } { auto block = blocks[1]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({2})); } { auto block = blocks[2]; - auto col = block.getByName(MutableSupport::tidb_pk_column_name); + auto col = block.getByName(MutSup::extra_handle_column_name); ASSERT_COLUMN_EQ(col, createColumn({3})); } } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp index aee463ff075..6c41bd1e2dd 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp @@ -73,9 +73,9 @@ BlockInputStreamPtr getVersionFilterInputStream( bool is_common_handle) { ColumnDefine handle_define( - TiDBPkColumnID, + MutSup::extra_handle_id, DMTestEnv::pk_name, - is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE); + is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType()); return std::make_shared>( std::make_shared(blocks, is_common_handle), columns, @@ -92,9 +92,9 @@ BlockInputStreamPtr getVersionFilterInputStreamWithRowKeyFilterStream( const RowKeyRanges & read_ranges) { ColumnDefine handle_define( - TiDBPkColumnID, + MutSup::extra_handle_id, DMTestEnv::pk_name, - is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE); + is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType()); BlockInputStreamPtr rowkey_filter_stream = std::make_shared>( std::make_shared(blocks, is_common_handle), diff --git a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp index 0a14c9df287..6b1aeebde5f 100644 --- a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp @@ -118,9 +118,9 @@ uint64_t DTWorkload::updateBlock(Block & block, uint64_t key) auto ts = ts_gen->get(); { auto & cd = (*table_info->columns)[1]; - if (cd.id != VERSION_COLUMN_ID) + if (cd.id != MutSup::version_col_id) { - LOG_ERROR(log, "Column id not match: {} but {} is required", cd.id, VERSION_COLUMN_ID); + LOG_ERROR(log, "Column id not match: {} but {} is required", cd.id, MutSup::version_col_id); throw std::invalid_argument("Column id not match"); } diff --git a/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp index 6a6ee198044..5752f0b33d7 100644 --- a/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -73,12 +74,12 @@ class RandomDataGenerator : public DataGenerator uint64_t ts = ts_gen.get(); { auto & col_def = (*table_info.columns)[1]; - if (col_def.id != VERSION_COLUMN_ID) + if (col_def.id != MutSup::version_col_id) { throw std::invalid_argument(fmt::format( - "(*table_info.columns)[1].id is {} not VERSION_COLUMN_ID {}.", + "(*table_info.columns)[1].id is {} not MutSup::version_col_id {}.", col_def.id, - VERSION_COLUMN_ID)); + MutSup::version_col_id)); } ColumnWithTypeAndName col({}, col_def.type, col_def.name, col_def.id); IColumn::MutablePtr mut_col = col.type->createColumn(); @@ -91,10 +92,12 @@ class RandomDataGenerator : public DataGenerator // Generate 'delete mark' { auto & col_def = (*table_info.columns)[2]; - if (col_def.id != TAG_COLUMN_ID) + if (col_def.id != MutSup::delmark_col_id) { - throw std::invalid_argument( - fmt::format("(*table_info.columns)[2].id is {} not TAG_COLUMN_ID {}.", col_def.id, TAG_COLUMN_ID)); + throw std::invalid_argument(fmt::format( + "(*table_info.columns)[2].id is {} not MutSup::delmark_col_id {}.", + col_def.id, + MutSup::delmark_col_id)); } ColumnWithTypeAndName col({}, col_def.type, col_def.name, col_def.id); IColumn::MutablePtr mut_col = col.type->createColumn(); @@ -113,7 +116,8 @@ class RandomDataGenerator : public DataGenerator continue; } auto & col_def = (*table_info.columns)[i]; - if (col_def.id == table_info.handle.id || col_def.id == VERSION_COLUMN_ID || col_def.id == TAG_COLUMN_ID) + if (col_def.id == table_info.handle.id || col_def.id == MutSup::version_col_id + || col_def.id == MutSup::delmark_col_id) { continue; } diff --git a/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp index 611b69ed71b..e0c6f9b9ea6 100644 --- a/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp @@ -13,7 +13,9 @@ // limitations under the License. #include +#include #include +#include #include #include #include @@ -104,7 +106,7 @@ class TablePkType columns->emplace_back(getExtraHandleColumnDefine(/*is_common_handle=*/true)); break; case PkType::PkIsHandleInt64: - columns->emplace_back(ColumnDefine{2, PK_NAME_PK_IS_HANDLE, EXTRA_HANDLE_COLUMN_INT_TYPE}); + columns->emplace_back(ColumnDefine{2, PK_NAME_PK_IS_HANDLE, MutSup::getExtraHandleColumnIntType()}); break; case PkType::PkIsHandleInt32: columns->emplace_back(ColumnDefine{2, PK_NAME_PK_IS_HANDLE, DataTypeFactory::instance().get("Int32")}); diff --git a/dbms/src/Storages/DeltaMerge/workload/Utils.cpp b/dbms/src/Storages/DeltaMerge/workload/Utils.cpp index 9917001f991..7a45a605acc 100644 --- a/dbms/src/Storages/DeltaMerge/workload/Utils.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Utils.cpp @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include #include #include @@ -119,4 +121,4 @@ std::string blockToString(const Block & block) return s; } -} // namespace DB::DM::tests \ No newline at end of file +} // namespace DB::DM::tests diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index b7e089dca83..0a087c3b0e9 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -84,8 +84,8 @@ Block ITableDeclaration::getSampleBlockForColumns(const Names & column_names) co for (const auto & name : column_names) { - auto col = name == MutableSupport::extra_table_id_column_name - ? NameAndTypePair(name, MutableSupport::extra_table_id_column_type) + auto col = name == MutSup::extra_table_id_column_name + ? NameAndTypePair(name, MutSup::getExtraTableIdColumnType()) : getColumn(name); res.insert({col.type->createColumn(), col.type, name}); } diff --git a/dbms/src/Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.cpp b/dbms/src/Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.cpp index b615e288fd8..fdfdfe84200 100644 --- a/dbms/src/Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.cpp +++ b/dbms/src/Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -41,13 +42,13 @@ DecodingStorageSchemaSnapshot::DecodingStorageSchemaSnapshot( for (size_t i = 0; i < column_defines->size(); i++) { auto & cd = (*column_defines)[i]; - if (cd.id != VersionColumnID || with_version_column) + if (cd.id != MutSup::version_col_id || with_version_column) { col_id_to_block_pos.insert({cd.id, index_in_block++}); } col_id_to_def_pos.insert({cd.id, i}); - if (cd.id != TiDBPkColumnID && cd.id != VersionColumnID && cd.id != DelMarkColumnID) + if (cd.id != MutSup::extra_handle_id && cd.id != MutSup::version_col_id && cd.id != MutSup::delmark_col_id) { const auto & columns = table_info_.columns; column_infos.push_back(columns[column_lut.at(cd.id)]); @@ -131,11 +132,11 @@ Block createBlockSortByColumnID(DecodingStorageSchemaSnapshotConstPtr schema_sna { // col_id == cd.id // Including some internal columns: - // - (VersionColumnID, _INTERNAL_VERSION, u64) - // - (DelMarkColumnID, _INTERNAL_DELMARK, u8) - // - (TiDBPkColumnID, _tidb_rowid, i64) + // - (MutSup::version_col_id, _INTERNAL_VERSION, u64) + // - (MutSup::delmark_col_id, _INTERNAL_DELMARK, u8) + // - (MutSup::extra_handle_id, _tidb_rowid, i64) auto & cd = (*(schema_snapshot->column_defines))[def_pos]; - if (!with_version_column && cd.id == VersionColumnID) + if (!with_version_column && cd.id == MutSup::version_col_id) continue; block.insert({cd.type->createColumn(), cd.type, cd.name, col_id}); } diff --git a/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp b/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp index d4d5c2515d0..e31892dc9b9 100644 --- a/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp +++ b/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp @@ -201,17 +201,17 @@ bool RegionBlockReader::readImpl(Block & block, const ReadList & data_list, bool while (raw_delmark_col == nullptr || version_col_resolver.needBuild() || extra_handle_column_pos == invalid_column_pos) { - if (column_ids_iter->first == DelMarkColumnID) + if (column_ids_iter->first == MutSup::delmark_col_id) { raw_delmark_col = static_cast(const_cast(block.getByPosition(next_column_pos).column.get())); } - else if (column_ids_iter->first == VersionColumnID) + else if (column_ids_iter->first == MutSup::version_col_id) { version_col_resolver.build( static_cast(const_cast(block.getByPosition(next_column_pos).column.get()))); } - else if (column_ids_iter->first == TiDBPkColumnID) + else if (column_ids_iter->first == MutSup::extra_handle_id) { extra_handle_column_pos = next_column_pos; } @@ -299,12 +299,12 @@ bool RegionBlockReader::readImpl(Block & block, const ReadList & data_list, bool if constexpr (pk_type == TMTPKType::INT64) static_cast(raw_pk_column)->getData().push_back(handle_value); else if constexpr (pk_type == TMTPKType::UINT64) - static_cast(raw_pk_column)->getData().push_back(UInt64(handle_value)); + static_cast(raw_pk_column)->getData().push_back(static_cast(handle_value)); else { // The pk_type must be Int32/UInt32 or more narrow type // so cannot tell its' exact type here, just use `insert(Field)` - raw_pk_column->insert(Field(handle_value)); + raw_pk_column->insert(Field{handle_value}); if (unlikely(raw_pk_column->getInt(index) != handle_value)) { if (!force_decode) diff --git a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp index 94aa55ace5d..e0b8ce26862 100644 --- a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp +++ b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp @@ -159,7 +159,7 @@ static inline std::tuple executeTransform // Read from SSTs and refine the boundary of blocks output to DTFiles auto bounded_stream = std::make_shared( sst_stream, - ::DB::TiDBPkColumnID, + MutSup::extra_handle_id, opts.schema_snap, split_id); diff --git a/dbms/src/Storages/KVStore/Types.h b/dbms/src/Storages/KVStore/Types.h index be11a1b28be..ace9c7365bb 100644 --- a/dbms/src/Storages/KVStore/Types.h +++ b/dbms/src/Storages/KVStore/Types.h @@ -57,14 +57,6 @@ enum : IndexID EmptyIndexID = 0, }; -// Constants for column id, prevent conflict with TiDB. -static constexpr ColumnID TiDBPkColumnID = -1; -static constexpr ColumnID ExtraTableIDColumnID = -3; -static constexpr ColumnID VersionColumnID = -1024; -static constexpr ColumnID DelMarkColumnID = -1025; -static constexpr ColumnID InvalidColumnID = -10000; - - using HandleID = Int64; using Timestamp = UInt64; diff --git a/dbms/src/Storages/KVStore/tests/bench_region_block_reader.cpp b/dbms/src/Storages/KVStore/tests/bench_region_block_reader.cpp index 28d2798bb49..8aa9b1d047a 100644 --- a/dbms/src/Storages/KVStore/tests/bench_region_block_reader.cpp +++ b/dbms/src/Storages/KVStore/tests/bench_region_block_reader.cpp @@ -74,7 +74,7 @@ class RegionBlockReaderBenchTest : public benchmark::Fixture WriteBufferFromOwnString pk_buf; if (table_info.is_common_handle) { - auto & primary_index_info = table_info.getPrimaryIndexInfo(); + const auto & primary_index_info = table_info.getPrimaryIndexInfo(); for (size_t i = 0; i < primary_index_info.idx_cols.size(); i++) { auto idx = column_name_columns_index_map[primary_index_info.idx_cols[i].name]; @@ -135,7 +135,7 @@ BENCHMARK_DEFINE_F(RegionBlockReaderBenchTest, CommonHandle) auto [table_info, fields] = getNormalTableInfoFields({2, 3, 4}, true); encodeColumns(table_info, fields, RowEncodeVersion::RowV2, num_rows); auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); - for (auto _ : state) + for (auto _ : state) // NOLINT { decodeColumns(decoding_schema, true); } @@ -146,10 +146,10 @@ BENCHMARK_DEFINE_F(RegionBlockReaderBenchTest, PKIsNotHandle) (benchmark::State & state) { size_t num_rows = state.range(0); - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV2, num_rows); auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); - for (auto _ : state) + for (auto _ : state) // NOLINT { decodeColumns(decoding_schema, true); } @@ -162,7 +162,7 @@ BENCHMARK_DEFINE_F(RegionBlockReaderBenchTest, PKIsHandle) auto [table_info, fields] = getNormalTableInfoFields({2}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV2, num_rows); auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); - for (auto _ : state) + for (auto _ : state) // NOLINT { decodeColumns(decoding_schema, true); } diff --git a/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp b/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp index d76fbe6c078..8902e6305e0 100644 --- a/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp +++ b/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp @@ -153,7 +153,7 @@ class RegionBlockReaderTest : public ::testing::Test { ASSERT_EQ(column_element.column->size(), rows); } - if (column_element.name == EXTRA_HANDLE_COLUMN_NAME) + if (column_element.name == MutSup::extra_handle_column_name) { if (decoding_schema->is_common_handle) { @@ -165,11 +165,11 @@ class RegionBlockReaderTest : public ::testing::Test ASSERT_FIELD_EQ((*column_element.column)[row], Field(handle_value)) << gen_error_log(); } } - else if (column_element.name == VERSION_COLUMN_NAME) + else if (column_element.name == MutSup::version_column_name) { ASSERT_FIELD_EQ((*column_element.column)[row], Field(version_value)) << gen_error_log(); } - else if (column_element.name == TAG_COLUMN_NAME) + else if (column_element.name == MutSup::delmark_column_name) { ASSERT_FIELD_EQ((*column_element.column)[row], Field(NearestFieldType::Type(del_mark_value))) << gen_error_log(); @@ -312,7 +312,7 @@ String bytesFromHexString(std::string_view hex_str) TEST_F(RegionBlockReaderTest, PKIsNotHandle) { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); ASSERT_EQ(table_info.is_common_handle, false); ASSERT_EQ(table_info.pk_is_handle, false); ASSERT_FALSE(table_info.getColumnInfo(2).hasPriKeyFlag()); @@ -350,27 +350,27 @@ TEST_F(RegionBlockReaderTest, CommonHandle) TEST_F(RegionBlockReaderTest, MissingColumnRowV2) { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV2); - auto new_table_info = getTableInfoWithMoreColumns({EXTRA_HANDLE_COLUMN_ID}, false); + auto new_table_info = getTableInfoWithMoreColumns({MutSup::extra_handle_id}, false); auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info); ASSERT_TRUE(decodeAndCheckColumns(new_decoding_schema, false)); } TEST_F(RegionBlockReaderTest, MissingColumnRowV1) { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV1); - auto new_table_info = getTableInfoWithMoreColumns({EXTRA_HANDLE_COLUMN_ID}, false); + auto new_table_info = getTableInfoWithMoreColumns({MutSup::extra_handle_id}, false); auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info); ASSERT_TRUE(decodeAndCheckColumns(new_decoding_schema, false)); } TEST_F(RegionBlockReaderTest, ExtraColumnRowV2) { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV2); - auto new_table_info = getTableInfoWithLessColumns({EXTRA_HANDLE_COLUMN_ID}, false); + auto new_table_info = getTableInfoWithLessColumns({MutSup::extra_handle_id}, false); auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info); ASSERT_FALSE(decodeAndCheckColumns(new_decoding_schema, false)); ASSERT_TRUE(decodeAndCheckColumns(new_decoding_schema, true)); @@ -378,9 +378,9 @@ TEST_F(RegionBlockReaderTest, ExtraColumnRowV2) TEST_F(RegionBlockReaderTest, ExtraColumnRowV1) { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV1); - auto new_table_info = getTableInfoWithLessColumns({EXTRA_HANDLE_COLUMN_ID}, false); + auto new_table_info = getTableInfoWithLessColumns({MutSup::extra_handle_id}, false); auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info); ASSERT_FALSE(decodeAndCheckColumns(new_decoding_schema, false)); ASSERT_TRUE(decodeAndCheckColumns(new_decoding_schema, true)); @@ -388,9 +388,9 @@ TEST_F(RegionBlockReaderTest, ExtraColumnRowV1) TEST_F(RegionBlockReaderTest, OverflowColumnRowV2) { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV2); - auto new_table_info = getTableInfoWithMoreNarrowIntType({EXTRA_HANDLE_COLUMN_ID}, false); + auto new_table_info = getTableInfoWithMoreNarrowIntType({MutSup::extra_handle_id}, false); auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info); ASSERT_FALSE(decodeAndCheckColumns(new_decoding_schema, false)); ASSERT_ANY_THROW(decodeAndCheckColumns(new_decoding_schema, true)); @@ -401,9 +401,9 @@ TEST_F(RegionBlockReaderTest, OverflowColumnRowV2) TEST_F(RegionBlockReaderTest, OverflowColumnRowV1) { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV1); - auto new_table_info = getTableInfoWithMoreNarrowIntType({EXTRA_HANDLE_COLUMN_ID}, false); + auto new_table_info = getTableInfoWithMoreNarrowIntType({MutSup::extra_handle_id}, false); auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info); ASSERT_FALSE(decodeAndCheckColumns(new_decoding_schema, false)); ASSERT_ANY_THROW(decodeAndCheckColumns(new_decoding_schema, true)); @@ -415,12 +415,12 @@ TEST_F(RegionBlockReaderTest, OverflowColumnRowV1) TEST_F(RegionBlockReaderTest, InvalidNULLRowV2) try { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); ASSERT_FALSE(table_info.getColumnInfo(11).hasNotNullFlag()); // col 11 is nullable encodeColumns(table_info, fields, RowEncodeVersion::RowV2); - auto new_table_info = getTableInfoFieldsForInvalidNULLTest({EXTRA_HANDLE_COLUMN_ID}, false); + auto new_table_info = getTableInfoFieldsForInvalidNULLTest({MutSup::extra_handle_id}, false); invalid_null_column_ids.emplace(11); ASSERT_TRUE(new_table_info.getColumnInfo(11).hasNotNullFlag()); // col 11 is not null @@ -432,10 +432,10 @@ CATCH TEST_F(RegionBlockReaderTest, InvalidNULLRowV1) { - auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false); + auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false); encodeColumns(table_info, fields, RowEncodeVersion::RowV1); - auto new_table_info = getTableInfoFieldsForInvalidNULLTest({EXTRA_HANDLE_COLUMN_ID}, false); + auto new_table_info = getTableInfoFieldsForInvalidNULLTest({MutSup::extra_handle_id}, false); invalid_null_column_ids.emplace(11); ASSERT_TRUE(new_table_info.getColumnInfo(11).hasNotNullFlag()); // col 11 is not null diff --git a/dbms/src/Storages/KVStore/tests/gtest_spill.cpp b/dbms/src/Storages/KVStore/tests/gtest_spill.cpp index e4533c43583..3a50e77ae03 100644 --- a/dbms/src/Storages/KVStore/tests/gtest_spill.cpp +++ b/dbms/src/Storages/KVStore/tests/gtest_spill.cpp @@ -52,7 +52,7 @@ class KVStoreSpillTest : public KVStoreTestBase protected: StorageDeltaMergePtr storage; - TableID table_id; + TableID table_id{0}; }; TEST_F(KVStoreSpillTest, CreateBlock) @@ -67,8 +67,8 @@ try { auto [schema_snapshot, block] = storage->getSchemaSnapshotAndBlockForDecoding(table_lock, true, false); UNUSED(schema_snapshot); - EXPECT_NO_THROW(block->getPositionByName(MutableSupport::delmark_column_name)); - EXPECT_THROW(block->getPositionByName(MutableSupport::version_column_name), Exception); + EXPECT_NO_THROW(block->getPositionByName(MutSup::delmark_column_name)); + EXPECT_THROW(block->getPositionByName(MutSup::version_column_name), Exception); ASSERT_EQ(block->columns(), 3); } { diff --git a/dbms/src/Storages/KVStore/tests/gtest_sync_status.cpp b/dbms/src/Storages/KVStore/tests/gtest_sync_status.cpp index 24b84305e81..36a9df765bf 100644 --- a/dbms/src/Storages/KVStore/tests/gtest_sync_status.cpp +++ b/dbms/src/Storages/KVStore/tests/gtest_sync_status.cpp @@ -154,7 +154,7 @@ TableID createDBAndTable(String db_name, String table_name) StorageDeltaMergePtr storage_ptr = std::static_pointer_cast(storage); table_id = storage_ptr->getTableInfo().id; - EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name); + EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name); EXPECT_EQ(storage->getTableName(), table_name); auto managed_storage = std::dynamic_pointer_cast(storage); diff --git a/dbms/src/Storages/MutableSupport.cpp b/dbms/src/Storages/MutableSupport.cpp index d8b91220375..c4376dca872 100644 --- a/dbms/src/Storages/MutableSupport.cpp +++ b/dbms/src/Storages/MutableSupport.cpp @@ -12,27 +12,50 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include namespace DB { -const String MutableSupport::mmt_storage_name = "MutableMergeTree"; -const String MutableSupport::txn_storage_name = "TxnMergeTree"; -const String MutableSupport::delta_tree_storage_name = "DeltaMerge"; - -const String MutableSupport::tidb_pk_column_name = "_tidb_rowid"; -const String MutableSupport::version_column_name = "_INTERNAL_VERSION"; -const String MutableSupport::delmark_column_name = "_INTERNAL_DELMARK"; -const String MutableSupport::extra_table_id_column_name = "_tidb_tid"; - -const DataTypePtr MutableSupport::tidb_pk_column_int_type = DataTypeFactory::instance().get("Int64"); -const DataTypePtr MutableSupport::tidb_pk_column_string_type - = DataTypeFactory::instance().get(DataTypeString::getDefaultName()); -const DataTypePtr MutableSupport::version_column_type = DataTypeFactory::instance().get("UInt64"); -const DataTypePtr MutableSupport::delmark_column_type = DataTypeFactory::instance().get("UInt8"); -/// it should not be nullable, but TiDB does not set not null flag for extra_table_id_column_type, so has to align with TiDB -const DataTypePtr MutableSupport::extra_table_id_column_type = DataTypeFactory::instance().get("Nullable(Int64)"); -; +const DataTypePtr & MutSup::getExtraHandleColumnIntType() +{ + static const auto type = DataTypeFactory::instance().getOrSet("Int64"); + return type; +} + +const DataTypePtr & MutSup::getExtraHandleColumnStringType() +{ + static const auto name_and_types = DataTypeString::getTiDBPkColumnStringNameAndTypes(); + + const auto default_name = DataTypeString::getDefaultName(); + auto itr = std::find_if(name_and_types.begin(), name_and_types.end(), [&default_name](const auto & name_and_type) { + return name_and_type.first == default_name; + }); + RUNTIME_CHECK_MSG( + itr != name_and_types.end(), + "Cannot find '{}' for TiDB primary key column string type", + default_name); + return itr->second; +} + +const DataTypePtr & MutSup::getVersionColumnType() +{ + static const auto type = DataTypeFactory::instance().getOrSet("UInt64"); + return type; +} + +const DataTypePtr & MutSup::getDelmarkColumnType() +{ + static const auto type = DataTypeFactory::instance().getOrSet("UInt8"); + return type; +} + +const DataTypePtr & MutSup::getExtraTableIdColumnType() +{ + /// it should not be nullable, but TiDB does not set not null flag for extra_table_id_column_type, so has to align with TiDB + static const auto type = DataTypeFactory::instance().getOrSet("Nullable(Int64)"); + return type; +} } // namespace DB diff --git a/dbms/src/Storages/MutableSupport.h b/dbms/src/Storages/MutableSupport.h index 6ac3b262e3b..5a098ccc313 100644 --- a/dbms/src/Storages/MutableSupport.h +++ b/dbms/src/Storages/MutableSupport.h @@ -14,84 +14,53 @@ #pragma once -#include -#include #include -#include -#include -#include +#include +#include #include namespace DB { -class MutableSupport : public ext::Singleton +class MutSup : public ext::Singleton { public: - MutableSupport() + MutSup() { mutable_hidden.push_back(version_column_name); mutable_hidden.push_back(delmark_column_name); - //mutable_hidden.push_back(tidb_pk_column_name); - - all_hidden.insert(all_hidden.end(), mutable_hidden.begin(), mutable_hidden.end()); + //mutable_hidden.push_back(extra_handle_column_name); } const OrderedNameSet & hiddenColumns(const String & table_type_name) { - if (mmt_storage_name == table_type_name || txn_storage_name == table_type_name - || delta_tree_storage_name == table_type_name) + if (delta_tree_storage_name == table_type_name) return mutable_hidden; return empty; } - static const String mmt_storage_name; - static const String txn_storage_name; - static const String delta_tree_storage_name; - - static const String tidb_pk_column_name; - static const String version_column_name; - static const String delmark_column_name; - static const String extra_table_id_column_name; - - static const DataTypePtr tidb_pk_column_int_type; - static const DataTypePtr tidb_pk_column_string_type; - static const DataTypePtr version_column_type; - static const DataTypePtr delmark_column_type; - static const DataTypePtr extra_table_id_column_type; - - /// mark that ColumnId of those columns are defined in dbms/src/Storages/KVStore/Types.h - - // TODO: detail doc about these delete marks - struct DelMark - { - static const UInt8 NONE = 0x00; - static const UInt8 INTERNAL_DEL = 0x01; - static const UInt8 DEFINITE_DEL = (INTERNAL_DEL << 1); - static const UInt8 DEL_MASK = (INTERNAL_DEL | DEFINITE_DEL); - static const UInt8 DATA_MASK = ~DEL_MASK; - - static UInt8 getData(UInt8 raw_data) { return raw_data & DATA_MASK; } - - static bool isDel(UInt8 raw_data) { return raw_data & DEL_MASK; } + inline static const String delta_tree_storage_name = "DeltaMerge"; - static bool isDefiniteDel(UInt8 raw_data) { return raw_data & DEFINITE_DEL; } + inline static constexpr ColumnID extra_handle_id = -1; + inline static constexpr ColumnID extra_table_id_col_id = -3; + inline static constexpr ColumnID version_col_id = -1024; + inline static constexpr ColumnID delmark_col_id = -1025; + inline static constexpr ColumnID invalid_col_id = -10000; - static UInt8 genDelMark(bool internal_del, bool definite_del, UInt8 src_data) - { - return (internal_del ? INTERNAL_DEL : NONE) | (definite_del ? DEFINITE_DEL : NONE) | getData(src_data); - } + inline static const String extra_handle_column_name = "_tidb_rowid"; + inline static const String version_column_name = "_INTERNAL_VERSION"; + inline static const String delmark_column_name = "_INTERNAL_DELMARK"; + inline static const String extra_table_id_column_name = "_tidb_tid"; - static UInt8 genDelMark(bool internal_del, bool definite_del = false) - { - return genDelMark(internal_del, definite_del, 0); - } - }; + ALWAYS_INLINE static const DataTypePtr & getExtraHandleColumnIntType(); + ALWAYS_INLINE static const DataTypePtr & getExtraHandleColumnStringType(); + ALWAYS_INLINE static const DataTypePtr & getVersionColumnType(); + ALWAYS_INLINE static const DataTypePtr & getDelmarkColumnType(); + ALWAYS_INLINE static const DataTypePtr & getExtraTableIdColumnType(); private: OrderedNameSet empty; OrderedNameSet mutable_hidden; - OrderedNameSet all_hidden; }; } // namespace DB diff --git a/dbms/src/Storages/S3/FileCache.cpp b/dbms/src/Storages/S3/FileCache.cpp index 9ea59004bc5..c47797d86a5 100644 --- a/dbms/src/Storages/S3/FileCache.cpp +++ b/dbms/src/Storages/S3/FileCache.cpp @@ -601,11 +601,11 @@ FileType FileCache::getFileTypeOfColData(const std::filesystem::path & p) auto col_id = std::stol(str_col_id); switch (col_id) { - case EXTRA_HANDLE_COLUMN_ID: + case MutSup::extra_handle_id: return FileType::HandleColData; - case VERSION_COLUMN_ID: + case MutSup::version_col_id: return FileType::VersionColData; - case TAG_COLUMN_ID: + case MutSup::delmark_col_id: return FileType::DeleteMarkColData; default: return FileType::ColData; diff --git a/dbms/src/Storages/S3/tests/gtest_filecache.cpp b/dbms/src/Storages/S3/tests/gtest_filecache.cpp index e88571e79ab..5dca3b79dce 100644 --- a/dbms/src/Storages/S3/tests/gtest_filecache.cpp +++ b/dbms/src/Storages/S3/tests/gtest_filecache.cpp @@ -192,8 +192,7 @@ class FileCacheTest : public ::testing::Test void waitForBgDownload(const FileCache & file_cache) { Stopwatch sw; - UInt64 downloading = 0; - while ((downloading = file_cache.bg_downloading_count.load(std::memory_order_relaxed)) > 0) + while (file_cache.bg_downloading_count.load(std::memory_order_relaxed) > 0) { std::this_thread::sleep_for(1000ms); } @@ -430,15 +429,19 @@ try auto handle_fname = fmt::format( "{}/{}.dat", s3_fname, - IDataType::getFileNameForStream(std::to_string(EXTRA_HANDLE_COLUMN_ID), {})); + IDataType::getFileNameForStream(std::to_string(MutSup::extra_handle_id), {})); ASSERT_EQ(FileCache::getFileType(handle_fname), FileType::HandleColData); auto vec_index_fname = fmt::format("{}/idx_{}.vector", s3_fname, /*index_id*/ 50); // DMFile::vectorIndexFileName ASSERT_EQ(FileCache::getFileType(vec_index_fname), FileType::VectorIndex); - auto version_fname - = fmt::format("{}/{}.dat", s3_fname, IDataType::getFileNameForStream(std::to_string(VERSION_COLUMN_ID), {})); + auto version_fname = fmt::format( + "{}/{}.dat", + s3_fname, + IDataType::getFileNameForStream(std::to_string(MutSup::version_col_id), {})); ASSERT_EQ(FileCache::getFileType(version_fname), FileType::VersionColData); - auto delmark_fname - = fmt::format("{}/{}.dat", s3_fname, IDataType::getFileNameForStream(std::to_string(TAG_COLUMN_ID), {})); + auto delmark_fname = fmt::format( + "{}/{}.dat", + s3_fname, + IDataType::getFileNameForStream(std::to_string(MutSup::delmark_col_id), {})); ASSERT_EQ(FileCache::getFileType(delmark_fname), FileType::DeleteMarkColData); auto unknow_fname0 = fmt::format("{}/123456", s3_fname); ASSERT_EQ(FileCache::getFileType(unknow_fname0), FileType::Unknow); diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index 685d57071b6..5e9bbd07643 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -149,8 +149,8 @@ void StorageDeltaMerge::updateTableColumnInfo() } else { - pks.emplace(EXTRA_HANDLE_COLUMN_NAME); - pk_column_names.emplace_back(EXTRA_HANDLE_COLUMN_NAME); + pks.emplace(MutSup::extra_handle_column_name); + pk_column_names.emplace_back(MutSup::extra_handle_column_name); } } else @@ -188,8 +188,8 @@ void StorageDeltaMerge::updateTableColumnInfo() col_def.default_value = itr->defaultValueToField(); } - if (col_def.id != TiDBPkColumnID && col_def.id != VersionColumnID && col_def.id != DelMarkColumnID - && tidb_table_info.getColumnInfo(col_def.id).hasPriKeyFlag()) + if (col_def.id != MutSup::extra_handle_id && col_def.id != MutSup::version_col_id + && col_def.id != MutSup::delmark_col_id && tidb_table_info.getColumnInfo(col_def.id).hasPriKeyFlag()) { rowkey_column_defines.push_back(col_def); } @@ -225,15 +225,15 @@ void StorageDeltaMerge::updateTableColumnInfo() table_column_defines.push_back(col_def); } - if (!new_columns.materialized.contains(VERSION_COLUMN_NAME)) + if (!new_columns.materialized.contains(MutSup::version_column_name)) { - hidden_columns.emplace_back(VERSION_COLUMN_NAME); - new_columns.materialized.emplace_back(VERSION_COLUMN_NAME, VERSION_COLUMN_TYPE); + hidden_columns.emplace_back(MutSup::version_column_name); + new_columns.materialized.emplace_back(MutSup::version_column_name, MutSup::getVersionColumnType()); } - if (!new_columns.materialized.contains(TAG_COLUMN_NAME)) + if (!new_columns.materialized.contains(MutSup::delmark_column_name)) { - hidden_columns.emplace_back(TAG_COLUMN_NAME); - new_columns.materialized.emplace_back(TAG_COLUMN_NAME, TAG_COLUMN_TYPE); + hidden_columns.emplace_back(MutSup::delmark_column_name); + new_columns.materialized.emplace_back(MutSup::delmark_column_name, MutSup::getDelmarkColumnType()); } if (pks.size() > 1) @@ -242,13 +242,15 @@ void StorageDeltaMerge::updateTableColumnInfo() { throw Exception("Should not reach here: common handle with pk size > 1", ErrorCodes::LOGICAL_ERROR); } - handle_column_define.id = EXTRA_HANDLE_COLUMN_ID; - handle_column_define.name = EXTRA_HANDLE_COLUMN_NAME; - handle_column_define.type = EXTRA_HANDLE_COLUMN_INT_TYPE; - if (!new_columns.materialized.contains(EXTRA_HANDLE_COLUMN_NAME)) + handle_column_define.id = MutSup::extra_handle_id; + handle_column_define.name = MutSup::extra_handle_column_name; + handle_column_define.type = MutSup::getExtraHandleColumnIntType(); + if (!new_columns.materialized.contains(MutSup::extra_handle_column_name)) { - hidden_columns.emplace_back(EXTRA_HANDLE_COLUMN_NAME); - new_columns.materialized.emplace_back(EXTRA_HANDLE_COLUMN_NAME, EXTRA_HANDLE_COLUMN_INT_TYPE); + hidden_columns.emplace_back(MutSup::extra_handle_column_name); + new_columns.materialized.emplace_back( + MutSup::extra_handle_column_name, + MutSup::getExtraHandleColumnIntType()); } } @@ -357,12 +359,12 @@ Block StorageDeltaMerge::buildInsertBlock(bool is_import, bool is_delete, const if (!is_import) { // Remove the default columns generated by InterpreterInsertQuery - if (to_write.has(EXTRA_HANDLE_COLUMN_NAME)) - to_write.erase(EXTRA_HANDLE_COLUMN_NAME); - if (to_write.has(VERSION_COLUMN_NAME)) - to_write.erase(VERSION_COLUMN_NAME); - if (to_write.has(TAG_COLUMN_NAME)) - to_write.erase(TAG_COLUMN_NAME); + if (to_write.has(MutSup::extra_handle_column_name)) + to_write.erase(MutSup::extra_handle_column_name); + if (to_write.has(MutSup::version_column_name)) + to_write.erase(MutSup::version_column_name); + if (to_write.has(MutSup::delmark_column_name)) + to_write.erase(MutSup::delmark_column_name); } auto & store = getAndMaybeInitStore(); @@ -391,18 +393,18 @@ Block StorageDeltaMerge::buildInsertBlock(bool is_import, bool is_delete, const addColumnToBlock( to_write, - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_NAME, - is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE, + MutSup::extra_handle_id, + MutSup::extra_handle_column_name, + is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType(), std::move(handle_column)); } auto block = DeltaMergeStore::addExtraColumnIfNeed(global_context, store->getHandle(), std::move(to_write)); // add version column - if (!block.has(VERSION_COLUMN_NAME)) + if (!block.has(MutSup::version_column_name)) { - auto column = VERSION_COLUMN_TYPE->createColumn(); + auto column = MutSup::getVersionColumnType()->createColumn(); auto & column_data = typeid_cast &>(*column).getData(); column_data.resize(rows); for (size_t i = 0; i < rows; ++i) @@ -410,13 +412,18 @@ Block StorageDeltaMerge::buildInsertBlock(bool is_import, bool is_delete, const column_data[i] = next_version++; } - addColumnToBlock(block, VERSION_COLUMN_ID, VERSION_COLUMN_NAME, VERSION_COLUMN_TYPE, std::move(column)); + addColumnToBlock( + block, + MutSup::version_col_id, + MutSup::version_column_name, + MutSup::getVersionColumnType(), + std::move(column)); } // add tag column (upsert / delete) - if (!block.has(TAG_COLUMN_NAME)) + if (!block.has(MutSup::delmark_column_name)) { - auto column = TAG_COLUMN_TYPE->createColumn(); + auto column = MutSup::getDelmarkColumnType()->createColumn(); auto & column_data = typeid_cast &>(*column).getData(); column_data.resize(rows); UInt8 tag = is_delete ? 1 : 0; @@ -425,19 +432,24 @@ Block StorageDeltaMerge::buildInsertBlock(bool is_import, bool is_delete, const column_data[i] = tag; } - addColumnToBlock(block, TAG_COLUMN_ID, TAG_COLUMN_NAME, TAG_COLUMN_TYPE, std::move(column)); + addColumnToBlock( + block, + MutSup::delmark_col_id, + MutSup::delmark_column_name, + MutSup::getDelmarkColumnType(), + std::move(column)); } // Set the real column id. auto header = store->getHeader(); for (auto & col : block) { - if (col.name == EXTRA_HANDLE_COLUMN_NAME) - col.column_id = EXTRA_HANDLE_COLUMN_ID; - else if (col.name == VERSION_COLUMN_NAME) - col.column_id = VERSION_COLUMN_ID; - else if (col.name == TAG_COLUMN_NAME) - col.column_id = TAG_COLUMN_ID; + if (col.name == MutSup::extra_handle_column_name) + col.column_id = MutSup::extra_handle_id; + else if (col.name == MutSup::version_column_name) + col.column_id = MutSup::version_col_id; + else if (col.name == MutSup::delmark_column_name) + col.column_id = MutSup::delmark_col_id; else col.column_id = header->getByName(col.name).column_id; } @@ -536,25 +548,25 @@ WriteResult StorageDeltaMerge::write( { name = col.name; cid = col.column_id; - if (col.name == EXTRA_HANDLE_COLUMN_NAME) + if (col.name == MutSup::extra_handle_column_name) { - if (col.column_id != EXTRA_HANDLE_COLUMN_ID) + if (col.column_id != MutSup::extra_handle_id) { ok = false; break; } } - else if (col.name == VERSION_COLUMN_NAME) + else if (col.name == MutSup::version_column_name) { - if (col.column_id != VERSION_COLUMN_ID) + if (col.column_id != MutSup::version_col_id) { ok = false; break; } } - else if (col.name == TAG_COLUMN_NAME) + else if (col.name == MutSup::delmark_column_name) { - if (col.column_id != TAG_COLUMN_ID) + if (col.column_id != MutSup::delmark_col_id) { ok = false; break; @@ -655,13 +667,13 @@ void setColumnsToRead( for (size_t i = 0; i < column_names.size(); i++) { ColumnDefine col_define; - if (column_names[i] == EXTRA_HANDLE_COLUMN_NAME) + if (column_names[i] == MutSup::extra_handle_column_name) col_define = store->getHandle(); - else if (column_names[i] == VERSION_COLUMN_NAME) + else if (column_names[i] == MutSup::version_column_name) col_define = getVersionColumnDefine(); - else if (column_names[i] == TAG_COLUMN_NAME) + else if (column_names[i] == MutSup::delmark_column_name) col_define = getTagColumnDefine(); - else if (column_names[i] == EXTRA_TABLE_ID_COLUMN_NAME) + else if (column_names[i] == MutSup::extra_table_id_column_name) { extra_table_id_index = i; continue; @@ -800,7 +812,7 @@ BlockInputStreams StorageDeltaMerge::read( // in `Coprocessor.TableScan.columns`, or rough set filter could be // failed to parsed. ColumnDefines columns_to_read; - size_t extra_table_id_index = InvalidColumnID; + size_t extra_table_id_index = MutSup::invalid_col_id; setColumnsToRead(store, columns_to_read, extra_table_id_index, column_names); const ASTSelectQuery & select_query = typeid_cast(*query_info.query); @@ -881,7 +893,7 @@ void StorageDeltaMerge::read( // in `Coprocessor.TableScan.columns`, or rough set filter could be // failed to parsed. ColumnDefines columns_to_read; - size_t extra_table_id_index = InvalidColumnID; + size_t extra_table_id_index = MutSup::invalid_col_id; setColumnsToRead(store, columns_to_read, extra_table_id_index, column_names); const ASTSelectQuery & select_query = typeid_cast(*query_info.query); @@ -959,7 +971,7 @@ DM::Remote::DisaggPhysicalTableReadSnapshotPtr StorageDeltaMerge::writeNodeBuild { auto & store = getAndMaybeInitStore(); ColumnDefines columns_to_read; - size_t extra_table_id_index = InvalidColumnID; + size_t extra_table_id_index = MutSup::invalid_col_id; setColumnsToRead(store, columns_to_read, extra_table_id_index, column_names); auto tracing_logger = log->getChild(query_info.req_id); @@ -1353,9 +1365,9 @@ NamesAndTypes getColumnsFromTableInfo(const TiDB::TableInfo & table_info) { // Make primary key as a column, and make the handle column as the primary key. if (table_info.is_common_handle) - columns.emplace_back(MutableSupport::tidb_pk_column_name, std::make_shared()); + columns.emplace_back(MutSup::extra_handle_column_name, std::make_shared()); else - columns.emplace_back(MutableSupport::tidb_pk_column_name, std::make_shared()); + columns.emplace_back(MutSup::extra_handle_column_name, std::make_shared()); } return columns; @@ -1449,8 +1461,8 @@ ColumnDefines StorageDeltaMerge::getStoreColumnDefines() const cols.emplace_back(getTagColumnDefine()); for (const auto & col : table_column_info->table_column_defines) { - if (col.id != table_column_info->handle_column_define.id && col.id != VERSION_COLUMN_ID - && col.id != TAG_COLUMN_ID) + if (col.id != table_column_info->handle_column_define.id && col.id != MutSup::version_col_id + && col.id != MutSup::delmark_col_id) { cols.emplace_back(col); } @@ -1460,7 +1472,7 @@ ColumnDefines StorageDeltaMerge::getStoreColumnDefines() const String StorageDeltaMerge::getName() const { - return MutableSupport::delta_tree_storage_name; + return MutSup::delta_tree_storage_name; } void StorageDeltaMerge::rename( @@ -1605,7 +1617,7 @@ void updateDeltaMergeTableCreateStatement( "Wrong arguments num: {} in table: {} with engine={}", args.children.size(), table_name, - MutableSupport::delta_tree_storage_name), + MutSup::delta_tree_storage_name), ErrorCodes::BAD_ARGUMENTS); } }; diff --git a/dbms/src/Storages/System/StorageSystemDTLocalIndexes.cpp b/dbms/src/Storages/System/StorageSystemDTLocalIndexes.cpp index afd14ff6381..6934d40272b 100644 --- a/dbms/src/Storages/System/StorageSystemDTLocalIndexes.cpp +++ b/dbms/src/Storages/System/StorageSystemDTLocalIndexes.cpp @@ -101,7 +101,7 @@ BlockInputStreams StorageSystemDTLocalIndexes::read( { const auto & table_name = it->name(); auto & storage = it->table(); - if (storage->getName() != MutableSupport::delta_tree_storage_name) + if (storage->getName() != MutSup::delta_tree_storage_name) continue; auto dm_storage = std::dynamic_pointer_cast(storage); diff --git a/dbms/src/Storages/System/StorageSystemDTSegments.cpp b/dbms/src/Storages/System/StorageSystemDTSegments.cpp index 0502501a891..d87d38d0cec 100644 --- a/dbms/src/Storages/System/StorageSystemDTSegments.cpp +++ b/dbms/src/Storages/System/StorageSystemDTSegments.cpp @@ -101,7 +101,7 @@ BlockInputStreams StorageSystemDTSegments::read( { const auto & table_name = it->name(); auto & storage = it->table(); - if (storage->getName() != MutableSupport::delta_tree_storage_name) + if (storage->getName() != MutSup::delta_tree_storage_name) continue; auto dm_storage = std::dynamic_pointer_cast(storage); diff --git a/dbms/src/Storages/System/StorageSystemDTTables.cpp b/dbms/src/Storages/System/StorageSystemDTTables.cpp index 33c8813128d..5143a0290b0 100644 --- a/dbms/src/Storages/System/StorageSystemDTTables.cpp +++ b/dbms/src/Storages/System/StorageSystemDTTables.cpp @@ -135,7 +135,7 @@ BlockInputStreams StorageSystemDTTables::read( { const auto & table_name = it->name(); auto & storage = it->table(); - if (storage->getName() != MutableSupport::delta_tree_storage_name) + if (storage->getName() != MutSup::delta_tree_storage_name) continue; auto dm_storage = std::dynamic_pointer_cast(storage); diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 18984b0c2c2..8d0e40f80ce 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -225,8 +225,7 @@ BlockInputStreams StorageSystemTables::read( String tidb_table_name; TableID table_id = -1; Timestamp tombstone = 0; - if (engine_name == MutableSupport::txn_storage_name - || engine_name == MutableSupport::delta_tree_storage_name) + if (engine_name == MutSup::delta_tree_storage_name) { auto managed_storage = std::dynamic_pointer_cast(iterator->table()); if (managed_storage) @@ -242,10 +241,10 @@ BlockInputStreams StorageSystemTables::read( res_columns[j++]->insert(tidb_database_name); res_columns[j++]->insert(tidb_table_name); - res_columns[j++]->insert(Int64(table_id)); - res_columns[j++]->insert(UInt64(tombstone)); + res_columns[j++]->insert(static_cast(table_id)); + res_columns[j++]->insert(static_cast(tombstone)); - res_columns[j++]->insert(UInt64(0)); + res_columns[j++]->insert(static_cast(0)); res_columns[j++]->insert(iterator->table()->getDataPath()); res_columns[j++]->insert(database->getTableMetadataPath(table_name)); @@ -293,7 +292,7 @@ BlockInputStreams StorageSystemTables::read( res_columns[j++]->insertDefault(); res_columns[j++]->insert(table.first); res_columns[j++]->insert(table.second->getName()); - res_columns[j++]->insert(UInt64(1)); + res_columns[j++]->insert(static_cast(1)); res_columns[j++]->insertDefault(); res_columns[j++]->insertDefault(); diff --git a/dbms/src/TestUtils/InputStreamTestUtils.h b/dbms/src/TestUtils/InputStreamTestUtils.h index 231a7777517..a36258f2392 100644 --- a/dbms/src/TestUtils/InputStreamTestUtils.h +++ b/dbms/src/TestUtils/InputStreamTestUtils.h @@ -74,7 +74,7 @@ ::testing::AssertionResult InputStreamVSBlockUnrestrictlyCompareColumns( // Similar to `InputStreamVSBlockUnrestrictlyCompareColumns` but assume inputstream's blocks are unordered. // It is only used for normal mode. (The blocks of fast mode have overlap.) -// This function read all blocks first and sort them by handle column or column at position 0 if EXTRA_HANDLE_COLUMN_NAME not exist. +// This function read all blocks first and sort them by handle column or column at position 0 if MutSup::extra_handle_column_name not exist. ::testing::AssertionResult UnorderedInputStreamVSBlockUnrestrictlyCompareColumns( const char * stream_expr, const char * colnames_expr, diff --git a/dbms/src/TiDB/Decode/RowCodec.cpp b/dbms/src/TiDB/Decode/RowCodec.cpp index cda807e87a7..14f0cf7f240 100644 --- a/dbms/src/TiDB/Decode/RowCodec.cpp +++ b/dbms/src/TiDB/Decode/RowCodec.cpp @@ -334,7 +334,7 @@ bool appendRowToBlock( { const ColumnInfos & column_infos = schema_snapshot->column_infos; // when pk is handle, we need skip pk column when decoding value - ColumnID pk_handle_id = InvalidColumnID; + ColumnID pk_handle_id = MutSup::invalid_col_id; if (schema_snapshot->pk_is_handle) { pk_handle_id = schema_snapshot->pk_column_ids[0]; diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index 2db35dd3fe7..110d1647155 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -1061,11 +1061,11 @@ std::tuple parseColumnsFromTableInfo(const TiDB::TableIn { // Make primary key as a column, and make the handle column as the primary key. if (table_info.is_common_handle) - columns.emplace_back(MutableSupport::tidb_pk_column_name, std::make_shared()); + columns.emplace_back(MutSup::extra_handle_column_name, std::make_shared()); else - columns.emplace_back(MutableSupport::tidb_pk_column_name, std::make_shared()); + columns.emplace_back(MutSup::extra_handle_column_name, std::make_shared()); primary_keys.clear(); - primary_keys.emplace_back(MutableSupport::tidb_pk_column_name); + primary_keys.emplace_back(MutSup::extra_handle_column_name); } return std::make_tuple(std::move(columns), std::move(primary_keys)); diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp index 2a5dd792044..5f846e1cd4b 100644 --- a/dbms/src/TiDB/Schema/TiDB.cpp +++ b/dbms/src/TiDB/Schema/TiDB.cpp @@ -1183,12 +1183,12 @@ ColumnID TableInfo::getColumnID(const String & name) const } } - if (name == DB::MutableSupport::tidb_pk_column_name) - return DB::TiDBPkColumnID; - else if (name == DB::MutableSupport::version_column_name) - return DB::VersionColumnID; - else if (name == DB::MutableSupport::delmark_column_name) - return DB::DelMarkColumnID; + if (name == DB::MutSup::extra_handle_column_name) + return DB::MutSup::extra_handle_id; + else if (name == DB::MutSup::version_column_name) + return DB::MutSup::version_col_id; + else if (name == DB::MutSup::delmark_column_name) + return DB::MutSup::delmark_col_id; DB::Strings available_columns; for (const auto & c : columns) diff --git a/dbms/src/TiDB/tests/RowCodecTestUtils.h b/dbms/src/TiDB/tests/RowCodecTestUtils.h index fdf88a4d9ea..6b27e51878d 100644 --- a/dbms/src/TiDB/tests/RowCodecTestUtils.h +++ b/dbms/src/TiDB/tests/RowCodecTestUtils.h @@ -225,7 +225,7 @@ std::pair> getTableInfoAndFields( getTableInfoFieldsInternal(column_info_fields, std::forward(column_value_ids)...); TableInfo table_info; std::vector fields; - bool pk_is_handle = pk_col_ids.size() == 1 && pk_col_ids[0] != ::DB::TiDBPkColumnID; + bool pk_is_handle = pk_col_ids.size() == 1 && pk_col_ids[0] != MutSup::extra_handle_id; for (auto & column_info_field : column_info_fields) { @@ -280,22 +280,22 @@ inline DecodingStorageSchemaSnapshotConstPtr getDecodingStorageSchemaSnapshot(co if (table_info.is_common_handle) { DM::ColumnDefine extra_handle_column{ - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_STRING_TYPE}; + MutSup::extra_handle_id, + MutSup::extra_handle_column_name, + MutSup::getExtraHandleColumnStringType()}; store_columns.emplace_back(extra_handle_column); } else { DM::ColumnDefine extra_handle_column{ - EXTRA_HANDLE_COLUMN_ID, - EXTRA_HANDLE_COLUMN_NAME, - EXTRA_HANDLE_COLUMN_INT_TYPE}; + MutSup::extra_handle_id, + MutSup::extra_handle_column_name, + MutSup::getExtraHandleColumnIntType()}; store_columns.emplace_back(extra_handle_column); } - store_columns.emplace_back(VERSION_COLUMN_ID, VERSION_COLUMN_NAME, VERSION_COLUMN_TYPE); - store_columns.emplace_back(TAG_COLUMN_ID, TAG_COLUMN_NAME, TAG_COLUMN_TYPE); - ColumnID handle_id = EXTRA_HANDLE_COLUMN_ID; + store_columns.emplace_back(MutSup::version_col_id, MutSup::version_column_name, MutSup::getVersionColumnType()); + store_columns.emplace_back(MutSup::delmark_col_id, MutSup::delmark_column_name, MutSup::getDelmarkColumnType()); + ColumnID handle_id = MutSup::extra_handle_id; for (const auto & column_info : table_info.columns) { if (table_info.pk_is_handle) @@ -306,7 +306,7 @@ inline DecodingStorageSchemaSnapshotConstPtr getDecodingStorageSchemaSnapshot(co store_columns.emplace_back(column_info.id, column_info.name, DB::getDataTypeByColumnInfo(column_info)); } - if (handle_id != EXTRA_HANDLE_COLUMN_ID) + if (handle_id != MutSup::extra_handle_id) { auto iter = std::find_if(store_columns.begin(), store_columns.end(), [&](const ColumnDefine & cd) { return cd.id == handle_id; @@ -357,7 +357,7 @@ template std::tuple getValueLengthByRowV2(const T & v) { using NearestType = typename NearestFieldType::Type; - auto [table_info, fields] = getTableInfoAndFields({EXTRA_HANDLE_COLUMN_ID}, false, ColumnIDValue(1, v)); + auto [table_info, fields] = getTableInfoAndFields({MutSup::extra_handle_id}, false, ColumnIDValue(1, v)); auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); WriteBufferFromOwnString ss; encodeRowV2(table_info, fields, ss); @@ -372,7 +372,7 @@ template T getValueByRowV1(const T & v) { using NearestType = typename NearestFieldType::Type; - auto [table_info, fields] = getTableInfoAndFields({EXTRA_HANDLE_COLUMN_ID}, false, ColumnIDValue(1, v)); + auto [table_info, fields] = getTableInfoAndFields({MutSup::extra_handle_id}, false, ColumnIDValue(1, v)); auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); WriteBufferFromOwnString ss; encodeRowV1(table_info, fields, ss); diff --git a/dbms/src/TiDB/tests/gtest_row_v2.cpp b/dbms/src/TiDB/tests/gtest_row_v2.cpp index 5341468e8ce..be01a42b04a 100644 --- a/dbms/src/TiDB/tests/gtest_row_v2.cpp +++ b/dbms/src/TiDB/tests/gtest_row_v2.cpp @@ -82,7 +82,7 @@ TEST(RowV2Suite, FloatValue) ASSERT_FLOAT_VALUE(std::numeric_limits::max()); } -#define ASSERT_STRING_VALUE_LENGTH(b, s) ASSERT_EQ(getValueLengthByRowV2(s), std::make_tuple(s, s.length())) +#define ASSERT_STRING_VALUE_LENGTH(b, s) ASSERT_EQ(getValueLengthByRowV2(s), std::make_tuple(s, (s).length())) TEST(RowV2Suite, StringValueLength) { @@ -101,20 +101,20 @@ TEST(RowV2Suite, DecimalValueLength) ASSERT_DECIMAL_VALUE(DecimalField(ToDecimal(1234.56789, 2), 2)); } -#define ASSERT_ROW_VALUE(is_big, ...) \ - { \ - auto [table_info, fields] = getTableInfoAndFields({EXTRA_HANDLE_COLUMN_ID}, false, __VA_ARGS__); \ - auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); \ - WriteBufferFromOwnString ss; \ - encodeRowV2(table_info, fields, ss); \ - auto encoded = ss.str(); \ - ASSERT_EQ(is_big, isBig(encoded)); \ - auto block = decodeRowToBlock(encoded, decoding_schema); \ - ASSERT_EQ(fields.size(), block.columns()); \ - for (size_t i = 0; i < fields.size(); i++) \ - { \ - ASSERT_EQ(fields[i], ((*block.getByPosition(i).column)[0])); \ - } \ +#define ASSERT_ROW_VALUE(is_big, ...) \ + { \ + auto [table_info, fields] = getTableInfoAndFields({MutSup::extra_handle_id}, false, __VA_ARGS__); \ + auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); \ + WriteBufferFromOwnString ss; \ + encodeRowV2(table_info, fields, ss); \ + auto encoded = ss.str(); \ + ASSERT_EQ(is_big, isBig(encoded)); \ + auto block = decodeRowToBlock(encoded, decoding_schema); \ + ASSERT_EQ(fields.size(), block.columns()); \ + for (size_t i = 0; i < fields.size(); i++) \ + { \ + ASSERT_EQ(fields[i], ((*block.getByPosition(i).column)[0])); \ + } \ } TEST(RowV2Suite, SmallRow)