forked from ydb-platform/ydb
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Scheme correction for indexes usage (ydb-platform#1180)
* correct schemeshard usage for index operations * fix build * fix build * correction * fix build * fix build * fix ut build * fix tests
- Loading branch information
1 parent
19d7298
commit a7dc805
Showing
48 changed files
with
1,760 additions
and
746 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
49 changes: 49 additions & 0 deletions
49
ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.cpp
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| #include "upsert_index.h" | ||
| #include <util/string/type.h> | ||
| #include <library/cpp/json/json_reader.h> | ||
|
|
||
| namespace NKikimr::NKqp { | ||
|
|
||
| TConclusionStatus TUpsertIndexOperation::DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) { | ||
| { | ||
| auto fValue = features.Extract("NAME"); | ||
| if (!fValue) { | ||
| return TConclusionStatus::Fail("can't find alter parameter NAME"); | ||
| } | ||
| IndexName = *fValue; | ||
| } | ||
| TString indexType; | ||
| { | ||
| auto fValue = features.Extract("TYPE"); | ||
| if (!fValue) { | ||
| return TConclusionStatus::Fail("can't find alter parameter TYPE"); | ||
| } | ||
| indexType = *fValue; | ||
| } | ||
| { | ||
| auto fValue = features.Extract("FEATURES"); | ||
| if (!fValue) { | ||
| return TConclusionStatus::Fail("can't find alter parameter FEATURES"); | ||
| } | ||
| if (!IndexMetaConstructor.Initialize(indexType)) { | ||
| return TConclusionStatus::Fail("can't initialize index meta object for type \"" + indexType + "\""); | ||
| } | ||
| NJson::TJsonValue jsonData; | ||
| if (!NJson::ReadJsonFastTree(*fValue, &jsonData)) { | ||
| return TConclusionStatus::Fail("incorrect json in request FEATURES parameter"); | ||
| } | ||
| auto result = IndexMetaConstructor->DeserializeFromJson(jsonData); | ||
| if (result.IsFail()) { | ||
| return result; | ||
| } | ||
| } | ||
| return TConclusionStatus::Success(); | ||
| } | ||
|
|
||
| void TUpsertIndexOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchema& schemaData) const { | ||
| auto* indexProto = schemaData.AddUpsertIndexes(); | ||
| indexProto->SetName(IndexName); | ||
| IndexMetaConstructor.SerializeToProto(*indexProto); | ||
| } | ||
|
|
||
| } |
23 changes: 23 additions & 0 deletions
23
ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| #include "abstract.h" | ||
| #include <ydb/core/tx/columnshard/engines/scheme/indexes/abstract.h> | ||
|
|
||
| namespace NKikimr::NKqp { | ||
|
|
||
| class TUpsertIndexOperation : public ITableStoreOperation { | ||
| private: | ||
| static TString GetTypeName() { | ||
| return "UPSERT_INDEX"; | ||
| } | ||
|
|
||
| static inline auto Registrator = TFactory::TRegistrator<TUpsertIndexOperation>(GetTypeName()); | ||
| private: | ||
| TString IndexName; | ||
| NBackgroundTasks::TInterfaceProtoContainer<NOlap::NIndexes::IIndexMetaConstructor> IndexMetaConstructor; | ||
| public: | ||
| TConclusionStatus DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) override; | ||
|
|
||
| void DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchema& schemaData) const override; | ||
| }; | ||
|
|
||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
10 changes: 10 additions & 0 deletions
10
ydb/core/tx/columnshard/engines/scheme/indexes/abstract.cpp
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #include "abstract.h" | ||
| #include <ydb/core/tx/columnshard/engines/portions/column_record.h> | ||
| #include <ydb/core/tx/columnshard/engines/portions/portion_info.h> | ||
| #include <ydb/core/tx/columnshard/engines/scheme/index_info.h> | ||
| #include <ydb/core/formats/arrow/hash/xx_hash.h> | ||
| #include <ydb/core/formats/arrow/hash/calcer.h> | ||
|
|
||
| namespace NKikimr::NOlap::NIndexes { | ||
|
|
||
| } // namespace NKikimr::NOlap::NIndexes |
147 changes: 147 additions & 0 deletions
147
ydb/core/tx/columnshard/engines/scheme/indexes/abstract.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| #pragma once | ||
|
|
||
| #include <ydb/core/tx/columnshard/splitter/chunks.h> | ||
| #include <ydb/core/tx/program/program.h> | ||
|
|
||
| #include <ydb/core/protos/flat_scheme_op.pb.h> | ||
| #include <library/cpp/object_factory/object_factory.h> | ||
| #include <contrib/libs/apache/arrow/cpp/src/arrow/record_batch.h> | ||
| #include <ydb/services/bg_tasks/abstract/interface.h> | ||
| #include <util/generic/string.h> | ||
|
|
||
| #include <memory> | ||
| #include <vector> | ||
|
|
||
| namespace NKikimr::NOlap { | ||
| struct TIndexInfo; | ||
| } | ||
|
|
||
| namespace NKikimr::NSchemeShard { | ||
| class TOlapSchema; | ||
| class IErrorCollector; | ||
| } | ||
|
|
||
| namespace NKikimr::NOlap::NIndexes { | ||
|
|
||
| class IIndexChecker { | ||
| protected: | ||
| virtual bool DoCheck(std::vector<TString>&& blobs) const = 0; | ||
| public: | ||
| virtual ~IIndexChecker() = default; | ||
| bool Check(std::vector<TString>&& blobs) const { | ||
| return DoCheck(std::move(blobs)); | ||
| } | ||
| }; | ||
|
|
||
| class TIndexCheckerContainer { | ||
| private: | ||
| YDB_READONLY(ui32, IndexId, 0); | ||
| YDB_READONLY_DEF(std::shared_ptr<IIndexChecker>, Object); | ||
| public: | ||
| TIndexCheckerContainer(const ui32 indexId, const std::shared_ptr<IIndexChecker>& object) | ||
| : IndexId(indexId) | ||
| , Object(object) { | ||
| AFL_VERIFY(IndexId); | ||
| AFL_VERIFY(Object); | ||
| } | ||
|
|
||
| const IIndexChecker* operator->() const { | ||
| return Object.get(); | ||
| } | ||
| }; | ||
|
|
||
| class IIndexMeta { | ||
| protected: | ||
| virtual std::shared_ptr<IPortionDataChunk> DoBuildIndex(const ui32 indexId, std::map<ui32, std::vector<std::shared_ptr<IPortionDataChunk>>>& data, const TIndexInfo& indexInfo) const = 0; | ||
| virtual std::shared_ptr<IIndexChecker> DoBuildIndexChecker(const TProgramContainer& program) const = 0; | ||
| virtual bool DoDeserializeFromProto(const NKikimrSchemeOp::TOlapIndexDescription& proto) = 0; | ||
| virtual void DoSerializeToProto(NKikimrSchemeOp::TOlapIndexDescription& proto) const = 0; | ||
|
|
||
| public: | ||
| using TFactory = NObjectFactory::TObjectFactory<IIndexMeta, TString>; | ||
| using TProto = NKikimrSchemeOp::TOlapIndexDescription; | ||
|
|
||
| virtual ~IIndexMeta() = default; | ||
|
|
||
| std::shared_ptr<IPortionDataChunk> BuildIndex(const ui32 indexId, std::map<ui32, std::vector<std::shared_ptr<IPortionDataChunk>>>& data, const TIndexInfo& indexInfo) const { | ||
| return DoBuildIndex(indexId, data, indexInfo); | ||
| } | ||
|
|
||
| std::shared_ptr<IIndexChecker> BuildIndexChecker(const TProgramContainer& program) const { | ||
| return DoBuildIndexChecker(program); | ||
| } | ||
|
|
||
| bool DeserializeFromProto(const NKikimrSchemeOp::TOlapIndexDescription& proto) { | ||
| return DoDeserializeFromProto(proto); | ||
| } | ||
|
|
||
| void SerializeToProto(NKikimrSchemeOp::TOlapIndexDescription& proto) const { | ||
| return DoSerializeToProto(proto); | ||
| } | ||
|
|
||
| virtual TString GetClassName() const = 0; | ||
| }; | ||
|
|
||
| class IIndexMetaConstructor { | ||
| protected: | ||
| virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonInfo) = 0; | ||
| virtual std::shared_ptr<IIndexMeta> DoCreateIndexMeta(const NSchemeShard::TOlapSchema& currentSchema, NSchemeShard::IErrorCollector& errors) const = 0; | ||
| virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TOlapIndexRequested& proto) = 0; | ||
| virtual void DoSerializeToProto(NKikimrSchemeOp::TOlapIndexRequested& proto) const = 0; | ||
| public: | ||
| using TFactory = NObjectFactory::TObjectFactory<IIndexMetaConstructor, TString>; | ||
| using TProto = NKikimrSchemeOp::TOlapIndexRequested; | ||
|
|
||
| virtual ~IIndexMetaConstructor() = default; | ||
|
|
||
| TConclusionStatus DeserializeFromJson(const NJson::TJsonValue& jsonInfo) { | ||
| return DoDeserializeFromJson(jsonInfo); | ||
| } | ||
|
|
||
| std::shared_ptr<IIndexMeta> CreateIndexMeta(const NSchemeShard::TOlapSchema& currentSchema, NSchemeShard::IErrorCollector& errors) const { | ||
| return DoCreateIndexMeta(currentSchema, errors); | ||
| } | ||
|
|
||
| TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TOlapIndexRequested& proto) { | ||
| return DoDeserializeFromProto(proto); | ||
| } | ||
|
|
||
| void SerializeToProto(NKikimrSchemeOp::TOlapIndexRequested& proto) const { | ||
| return DoSerializeToProto(proto); | ||
| } | ||
|
|
||
| virtual TString GetClassName() const = 0; | ||
| }; | ||
|
|
||
| class TIndexMetaContainer: public NBackgroundTasks::TInterfaceProtoContainer<IIndexMeta> { | ||
| private: | ||
| using TBase = NBackgroundTasks::TInterfaceProtoContainer<IIndexMeta>; | ||
| YDB_READONLY(ui32, IndexId, 0); | ||
| public: | ||
| TIndexMetaContainer() = default; | ||
| TIndexMetaContainer(const ui32 indexId, const std::shared_ptr<IIndexMeta>& object) | ||
| : TBase(object) | ||
| , IndexId(indexId) | ||
| { | ||
| AFL_VERIFY(IndexId); | ||
| AFL_VERIFY(Object); | ||
| } | ||
|
|
||
| bool DeserializeFromProto(const NKikimrSchemeOp::TOlapIndexDescription& proto) { | ||
| if (!TBase::DeserializeFromProto(proto)) { | ||
| return false; | ||
| } | ||
| IndexId = proto.GetId(); | ||
| return true; | ||
| } | ||
|
|
||
| std::optional<TIndexCheckerContainer> BuildIndexChecker(const TProgramContainer& program) const { | ||
| auto checker = GetObjectPtr()->BuildIndexChecker(program); | ||
| if (!checker) { | ||
| return {}; | ||
| } | ||
| return TIndexCheckerContainer(IndexId, checker); | ||
| } | ||
| }; | ||
|
|
||
| } // namespace NKikimr::NOlap::NIndexes |
Oops, something went wrong.