diff --git a/hermes_adapters/filesystem/filesystem.h b/hermes_adapters/filesystem/filesystem.h index 1b189cb3c..4278d8738 100644 --- a/hermes_adapters/filesystem/filesystem.h +++ b/hermes_adapters/filesystem/filesystem.h @@ -82,7 +82,7 @@ class Filesystem : public FilesystemIoClient { HILOG(kDebug, "File not opened before by adapter") // Normalize path strings stat.path_ = stdfs::absolute(path).string(); - auto path_shm = hipc::make_uptr(stat.path_); + auto path_shm = hipc::make_uptr(stat.path_); // Verify the bucket exists if not in CREATE mode if (stat.adapter_mode_ == AdapterMode::kScratch && !stat.hflags_.Any(HERMES_FS_EXISTS) && diff --git a/hermes_adapters/filesystem/filesystem_io_client.h b/hermes_adapters/filesystem/filesystem_io_client.h index 8a24dde59..df140f302 100644 --- a/hermes_adapters/filesystem/filesystem_io_client.h +++ b/hermes_adapters/filesystem/filesystem_io_client.h @@ -308,7 +308,7 @@ class FilesystemIoClient { virtual ~FilesystemIoClient() = default; /** Get initial statistics from the backend */ - virtual size_t GetBackendSize(const chi::charbuf &bkt_name) = 0; + virtual size_t GetBackendSize(const chi::string &bkt_name) = 0; /** Write blob to backend */ virtual void WriteBlob(const std::string &bkt_name, diff --git a/hermes_adapters/mapper/abstract_mapper.h b/hermes_adapters/mapper/abstract_mapper.h index 8bb34164c..6f6df3952 100644 --- a/hermes_adapters/mapper/abstract_mapper.h +++ b/hermes_adapters/mapper/abstract_mapper.h @@ -35,16 +35,16 @@ struct BlobPlacement { size_t blob_size_; /**< Size after offset to read */ /** create a BLOB name from index. */ - static chi::charbuf CreateBlobName(size_t page) { - chi::charbuf buf(sizeof(page)); + static chi::string CreateBlobName(size_t page) { + chi::string buf(sizeof(page)); hipc::LocalSerialize srl(buf); srl << page; return buf; } /** create a BLOB name from index. */ - chi::charbuf CreateBlobName() const { - chi::charbuf buf(sizeof(page_)); + chi::string CreateBlobName() const { + chi::string buf(sizeof(page_)); hipc::LocalSerialize srl(buf); srl << page_; return buf; diff --git a/hermes_adapters/mpiio/mpiio_fs_api.h b/hermes_adapters/mpiio/mpiio_fs_api.h index 6d073d95e..dd81c1fcd 100644 --- a/hermes_adapters/mpiio/mpiio_fs_api.h +++ b/hermes_adapters/mpiio/mpiio_fs_api.h @@ -577,7 +577,7 @@ class MpiioFs : public Filesystem { } /** Get initial statistics from the backend */ - size_t GetBackendSize(const chi::charbuf &bkt_name) override { + size_t GetBackendSize(const chi::string &bkt_name) override { size_t true_size = 0; std::string filename = bkt_name.str(); int fd = open(filename.c_str(), O_RDONLY); diff --git a/hermes_adapters/posix/posix_fs_api.h b/hermes_adapters/posix/posix_fs_api.h index d8d7691b0..d72b04b5d 100644 --- a/hermes_adapters/posix/posix_fs_api.h +++ b/hermes_adapters/posix/posix_fs_api.h @@ -185,7 +185,7 @@ class PosixFs : public hermes::adapter::Filesystem { } /** Get initial statistics from the backend */ - size_t GetBackendSize(const chi::charbuf &bkt_name) override { + size_t GetBackendSize(const chi::string &bkt_name) override { size_t true_size = 0; std::string filename = bkt_name.str(); int fd = real_api_->open(filename.c_str(), O_RDONLY); diff --git a/hermes_adapters/stdio/stdio_fs_api.h b/hermes_adapters/stdio/stdio_fs_api.h index d9cc09302..c6115dc11 100644 --- a/hermes_adapters/stdio/stdio_fs_api.h +++ b/hermes_adapters/stdio/stdio_fs_api.h @@ -177,7 +177,7 @@ class StdioFs : public hermes::adapter::Filesystem { } /** Get initial statistics from the backend */ - size_t GetBackendSize(const chi::charbuf &bkt_name) override { + size_t GetBackendSize(const chi::string &bkt_name) override { size_t true_size = 0; std::string filename = bkt_name.str(); int fd = open(filename.c_str(), O_RDONLY); diff --git a/include/hermes/bucket.h b/include/hermes/bucket.h index 2e00a3558..025e65366 100644 --- a/include/hermes/bucket.h +++ b/include/hermes/bucket.h @@ -47,7 +47,7 @@ class Bucket { mdm_ = &HERMES_CONF->mdm_; id_ = mdm_->GetOrCreateTag( mctx_, chi::DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - chi::charbuf(bkt_name), true, + chi::string(bkt_name), true, backend_size, flags); name_ = bkt_name; } @@ -65,11 +65,11 @@ class Bucket { u32 flags = 0) { mctx_ = mctx; mdm_ = &HERMES_CONF->mdm_; - chi::charbuf x; + chi::string x; id_ = mdm_->GetOrCreateTag( mctx_, chi::DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - chi::charbuf(bkt_name), true, + chi::string(bkt_name), true, backend_size, flags, ctx); name_ = bkt_name; } @@ -148,7 +148,7 @@ class Bucket { * Rename this bucket * */ void Rename(const std::string &new_bkt_name) { - // mdm_->RenameTag(id_, chi::charbuf(new_bkt_name)); + // mdm_->RenameTag(id_, chi::string(new_bkt_name)); } /** @@ -191,7 +191,7 @@ class Bucket { BlobId GetBlobId(const std::string &blob_name) { return mdm_->GetBlobId( mctx_, DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(blob_name)); + id_, chi::string(blob_name)); } /** @@ -246,7 +246,7 @@ class Bucket { BlobId blob_id = orig_blob_id; bitfield32_t hermes_flags; // Put to shared memory - chi::charbuf blob_name_buf(blob_name); + chi::string blob_name_buf(blob_name); if constexpr (!ASYNC) { if (blob_id.IsNull()) { hermes_flags.SetBits(HERMES_GET_BLOB_ID); @@ -435,7 +435,7 @@ class Bucket { const Context &ctx = Context()) { mdm_->AsyncReorganizeBlob( mctx_, DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(name), BlobId::GetNull(), score, true, ctx); + id_, chi::string(name), BlobId::GetNull(), score, true, ctx); } /** @@ -446,7 +446,7 @@ class Bucket { const Context &ctx = Context()) { mdm_->AsyncReorganizeBlob( mctx_, DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(""), blob_id, score, true, ctx); + id_, chi::string(""), blob_id, score, true, ctx); } /** @@ -461,7 +461,7 @@ class Bucket { ctx.node_id_ = node_id; mdm_->AsyncReorganizeBlob( mctx_, DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(""), blob_id, score, true, ctx); + id_, chi::string(""), blob_id, score, true, ctx); } /** @@ -470,7 +470,7 @@ class Bucket { size_t GetBlobSize(const BlobId &blob_id) { return mdm_->GetBlobSize( mctx_, DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(""), blob_id); + id_, chi::string(""), blob_id); } /** @@ -479,7 +479,7 @@ class Bucket { size_t GetBlobSize(const std::string &name) { return mdm_->GetBlobSize( mctx_, DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(name), BlobId::GetNull()); + id_, chi::string(name), BlobId::GetNull()); } /** @@ -502,7 +502,7 @@ class Bucket { LPointer task; task = mdm_->AsyncGetBlob( mctx_, chi::DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(blob_name), + id_, chi::string(blob_name), blob_id, blob_off, blob_size, blob.shm_, hermes_flags.bits_, ctx); @@ -564,7 +564,7 @@ class Bucket { if (blob.size() == 0) { data_size = mdm_->GetBlobSize( mctx_, DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(blob_name), + id_, chi::string(blob_name), orig_blob_id); blob.resize(data_size); } @@ -697,7 +697,7 @@ class Bucket { bool ContainsBlob(const std::string &blob_name) { BlobId new_blob_id = mdm_->GetBlobId( mctx_, DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), - id_, chi::charbuf(blob_name)); + id_, chi::string(blob_name)); return !new_blob_id.IsNull(); } @@ -707,7 +707,7 @@ class Bucket { void RenameBlob(const BlobId &blob_id, std::string new_blob_name, Context &ctx) { - // mdm_->RenameBlob(id_, blob_id, chi::charbuf(new_blob_name)); + // mdm_->RenameBlob(id_, blob_id, chi::string(new_blob_name)); } /** diff --git a/include/hermes/data_stager/binary_stager.h b/include/hermes/data_stager/binary_stager.h index 996179719..d8c1594e6 100644 --- a/include/hermes/data_stager/binary_stager.h +++ b/include/hermes/data_stager/binary_stager.h @@ -37,7 +37,7 @@ class BinaryFileStager : public AbstractStager { static std::string BuildFileParams(size_t page_size, u32 flags = 0, size_t elmt_size = 1) { - chi::charbuf params(32); + chi::string params(32); page_size = (page_size / elmt_size) * elmt_size; hipc::LocalSerialize srl(params); srl << std::string("file"); @@ -103,7 +103,7 @@ class BinaryFileStager : public AbstractStager { client.PutBlob( mctx, chi::DomainQuery::GetDirectHash(chi::SubDomainId::kLocalContainers, 0), tag_id, - chi::charbuf(blob_name), + chi::string(blob_name), hermes::BlobId::GetNull(), 0, real_size, blob.shm_, score, TASK_DATA_OWNER, 0, ctx); diff --git a/include/hermes/data_stager/stager_factory.h b/include/hermes/data_stager/stager_factory.h index a02916d0c..111990a58 100644 --- a/include/hermes/data_stager/stager_factory.h +++ b/include/hermes/data_stager/stager_factory.h @@ -19,7 +19,7 @@ class StagerFactory { srl >> protocol; std::unique_ptr stager; - if (protocol == "file") { + if (protocol == "file" || protocol == "") { stager = std::make_unique(); } else if (protocol == "parquet") { } else if (protocol == "hdf5") { diff --git a/include/hermes/hermes_types.h b/include/hermes/hermes_types.h index 1eada247e..e30cf9fd0 100644 --- a/include/hermes/hermes_types.h +++ b/include/hermes/hermes_types.h @@ -65,7 +65,7 @@ enum class TraitType { }; /** Represents a blob */ -typedef chi::charbuf Blob; +typedef chi::charwrap Blob; /** Supported data placement policies */ enum class PlacementPolicy { @@ -239,7 +239,7 @@ struct BufferInfo : public chi::Block { struct BlobInfo { TagId tag_id_; /**< Tag the blob is on */ BlobId blob_id_; /**< Unique ID of the blob */ - chi::charbuf name_; /**< Name of the blob (without tag_id) */ + chi::string name_; /**< Name of the blob (without tag_id) */ std::vector buffers_; /**< Set of buffers */ std::vector tags_; /**< Set of tags */ size_t blob_size_; /**< The overall size of the blob */ @@ -295,10 +295,10 @@ struct BlobInfo { } /** Get the globally unique blob name */ - static const chi::charbuf GetBlobNameWithBucket( + static const chi::string GetBlobNameWithBucket( const TagId &tag_id, - const chi::charbuf &blob_name) { - chi::charbuf new_name(sizeof(TagId) + blob_name.size()); + const chi::string &blob_name) { + chi::string new_name(sizeof(TagId) + blob_name.size()); hipc::LocalSerialize srl(new_name); srl << tag_id; srl << blob_name; @@ -306,7 +306,7 @@ struct BlobInfo { } /** Return the unique blob name for blob_id_map */ - chi::charbuf GetBlobNameWithBucket() { + chi::string GetBlobNameWithBucket() { return GetBlobNameWithBucket(tag_id_, name_); } }; @@ -314,7 +314,7 @@ struct BlobInfo { /** Data structure used to store Bucket information */ struct TagInfo { TagId tag_id_; - chi::charbuf name_; + chi::string name_; std::list blobs_; std::list traits_; size_t internal_size_; diff --git a/tasks/hermes_core/include/hermes_core/hermes_core.h b/tasks/hermes_core/include/hermes_core/hermes_core.h index fdae9d20e..b420e36c1 100644 --- a/tasks/hermes_core/include/hermes_core/hermes_core.h +++ b/tasks/hermes_core/include/hermes_core/hermes_core.h @@ -60,7 +60,7 @@ class Client : public ModuleClient { HSHM_ALWAYS_INLINE TagId GetOrCreateTag(const hipc::MemContext &mctx, const DomainQuery &dom_query, - const chi::charbuf &tag_name, + const chi::string &tag_name, bool blob_owner, size_t backend_size, u32 flags, @@ -78,7 +78,7 @@ class Client : public ModuleClient { /** Get tag ID */ TagId GetTagId(const hipc::MemContext &mctx, const DomainQuery &dom_query, - const chi::charbuf &tag_name) { + const chi::string &tag_name) { LPointer task = AsyncGetTagId(mctx, dom_query, tag_name); task->Wait(); @@ -89,13 +89,13 @@ class Client : public ModuleClient { CHI_TASK_METHODS(GetTagId); /** Get tag name */ - hshm::string GetTagName(const hipc::MemContext &mctx, + chi::string GetTagName(const hipc::MemContext &mctx, const DomainQuery &dom_query, const TagId &tag_id) { LPointer task = AsyncGetTagName(mctx, dom_query, tag_id); task->Wait(); - hshm::string tag_name(task->tag_name_.str()); + chi::string tag_name(task->tag_name_.str()); CHI_CLIENT->DelTask(mctx, task); return tag_name; } @@ -182,7 +182,7 @@ class Client : public ModuleClient { BlobId GetOrCreateBlob(const hipc::MemContext &mctx, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name) { + const chi::string &blob_name) { LPointer task = AsyncGetOrCreateBlobId(mctx, dom_query, tag_id, blob_name); task->Wait(); @@ -208,7 +208,7 @@ class Client : public ModuleClient { size_t PutBlob(const hipc::MemContext &mctx, const DomainQuery &dom_query, TagId tag_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, const BlobId &blob_id, size_t blob_off, size_t blob_size, const hipc::Pointer &blob, @@ -239,7 +239,7 @@ class Client : public ModuleClient { u32 hermes_flags, const Context &ctx = Context()) { LPointer task = - AsyncGetBlob(mctx, dom_query, tag_id, chi::charbuf(""), + AsyncGetBlob(mctx, dom_query, tag_id, chi::string(""), blob_id, off, data_size, data, hermes_flags, ctx); task->Wait(); data = task->data_; @@ -299,7 +299,7 @@ class Client : public ModuleClient { BlobId GetBlobId(const hipc::MemContext &mctx, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name) { + const chi::string &blob_name) { LPointer task = AsyncGetBlobId(mctx, dom_query, tag_id, blob_name); task->Wait(); @@ -331,7 +331,7 @@ class Client : public ModuleClient { size_t GetBlobSize(const hipc::MemContext &mctx, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, const BlobId &blob_id) { LPointer task = AsyncGetBlobSize(mctx, dom_query, tag_id, blob_name, blob_id); @@ -461,8 +461,8 @@ class Client : public ModuleClient { void RegisterStager(const hipc::MemContext &mctx, const DomainQuery &dom_query, const hermes::BucketId &bkt_id, - const chi::charbuf &tag_name, - const chi::charbuf ¶ms) { + const chi::string &tag_name, + const chi::string ¶ms) { LPointer task = AsyncRegisterStager(mctx, dom_query, bkt_id, tag_name, params); task->Wait(); @@ -487,7 +487,7 @@ class Client : public ModuleClient { void StageIn(const hipc::MemContext &mctx, const DomainQuery &dom_query, const BucketId &bkt_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, float score) { LPointer task = AsyncStageIn(mctx, dom_query, bkt_id, blob_name, score); @@ -501,7 +501,7 @@ class Client : public ModuleClient { void StageOut(const hipc::MemContext &mctx, const DomainQuery &dom_query, const BucketId &bkt_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, const hipc::Pointer &data, size_t data_size, u32 task_flags) { diff --git a/tasks/hermes_core/include/hermes_core/hermes_core_tasks.h b/tasks/hermes_core/include/hermes_core/hermes_core_tasks.h index b148fd175..d8a6633f8 100644 --- a/tasks/hermes_core/include/hermes_core/hermes_core_tasks.h +++ b/tasks/hermes_core/include/hermes_core/hermes_core_tasks.h @@ -87,8 +87,8 @@ typedef chi::Admin::DestroyContainerTask DestroyTask; * Create a tag * */ struct GetOrCreateTagTask : public Task, TaskFlags { - IN chi::string tag_name_; - IN chi::string params_; + IN chi::ipc::string tag_name_; + IN chi::ipc::string params_; IN bool blob_owner_; IN size_t backend_size_; IN bitfield32_t flags_; @@ -105,7 +105,7 @@ struct GetOrCreateTagTask : public Task, TaskFlags { const TaskNode &task_node, const PoolId &pool_id, const DomainQuery &dom_query, - const chi::charbuf &tag_name, + const chi::string &tag_name, bool blob_owner, size_t backend_size, u32 flags, @@ -150,7 +150,7 @@ struct GetOrCreateTagTask : public Task, TaskFlags { /** A task to get a tag id */ struct GetTagIdTask : public Task, TaskFlags { - IN chi::string tag_name_; + IN chi::ipc::string tag_name_; OUT TagId tag_id_; /** SHM default constructor */ @@ -164,7 +164,7 @@ struct GetTagIdTask : public Task, TaskFlags { const TaskNode &task_node, const PoolId &pool_id, const DomainQuery &dom_query, - const chi::charbuf &tag_name) + const chi::string &tag_name) : Task(alloc), tag_name_(alloc, tag_name) { // Initialize task task_node_ = task_node; @@ -197,7 +197,7 @@ struct GetTagIdTask : public Task, TaskFlags { /** A task to get a tag name */ struct GetTagNameTask : public Task, TaskFlags { IN TagId tag_id_; - OUT chi::string tag_name_; + OUT chi::ipc::string tag_name_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -593,7 +593,7 @@ struct TagGetContainedBlobIdsTask : public Task, TaskFlags { * */ struct GetOrCreateBlobIdTask : public Task, TaskFlags { IN TagId tag_id_; - IN chi::charbuf blob_name_; + IN chi::string blob_name_; OUT BlobId blob_id_; /** SHM default constructor */ @@ -608,7 +608,7 @@ struct GetOrCreateBlobIdTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name) + const chi::string &blob_name) : Task(alloc), blob_name_(alloc) { // Initialize task task_node_ = task_node; @@ -648,7 +648,7 @@ struct GetOrCreateBlobIdTask : public Task, TaskFlags { * */ struct GetBlobIdTask : public Task, TaskFlags { IN TagId tag_id_; - IN chi::charbuf blob_name_; + IN chi::string blob_name_; OUT BlobId blob_id_; /** SHM default constructor */ @@ -663,7 +663,7 @@ struct GetBlobIdTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name) + const chi::string &blob_name) : Task(alloc), blob_name_(alloc, blob_name) { // Initialize task task_node_ = task_node; @@ -703,7 +703,7 @@ struct GetBlobIdTask : public Task, TaskFlags { struct GetBlobNameTask : public Task, TaskFlags { IN TagId tag_id_; IN BlobId blob_id_; - OUT chi::string blob_name_; + OUT chi::ipc::string blob_name_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -756,7 +756,7 @@ struct GetBlobNameTask : public Task, TaskFlags { /** Get \a score from \a blob_id BLOB id */ struct GetBlobSizeTask : public Task, TaskFlags { IN TagId tag_id_; - IN chi::charbuf blob_name_; + IN chi::string blob_name_; IN BlobId blob_id_; OUT size_t size_; @@ -772,7 +772,7 @@ struct GetBlobSizeTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, const BlobId &blob_id) : Task(alloc), blob_name_(alloc, blob_name) { // Initialize task @@ -1081,7 +1081,7 @@ struct DestroyBlobTask : public Task, TaskFlags { /** A task to reorganize a blob's composition in the hierarchy */ struct ReorganizeBlobTask : public Task, TaskFlags { - IN chi::charbuf blob_name_; + IN chi::string blob_name_; IN TagId tag_id_; IN BlobId blob_id_; IN float score_; @@ -1100,7 +1100,7 @@ struct ReorganizeBlobTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, const BlobId &blob_id, float score, bool is_user_score, @@ -1200,7 +1200,7 @@ struct TagBlobTask : public Task, TaskFlags { /** A task to put data in a blob */ struct PutBlobTask : public Task, TaskFlags { IN TagId tag_id_; - IN chi::charbuf blob_name_; + IN chi::string blob_name_; IN size_t blob_off_; IN size_t data_size_; IN hipc::Pointer data_; @@ -1220,7 +1220,7 @@ struct PutBlobTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, const BlobId &blob_id, size_t blob_off, size_t data_size, @@ -1287,7 +1287,7 @@ struct PutBlobTask : public Task, TaskFlags { /** A task to get data from a blob */ struct GetBlobTask : public Task, TaskFlags { IN TagId tag_id_; - IN chi::charbuf blob_name_; + IN chi::string blob_name_; INOUT BlobId blob_id_; IN size_t blob_off_; IN hipc::Pointer data_; @@ -1306,7 +1306,7 @@ struct GetBlobTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const TagId &tag_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, const BlobId &blob_id, size_t off, size_t data_size, @@ -1625,8 +1625,8 @@ struct PollTagMetadataTask : public Task, TaskFlags { /** The RegisterStagerTask task */ struct RegisterStagerTask : public Task, TaskFlags { IN hermes::BucketId bkt_id_; - IN chi::string tag_name_; - IN chi::string params_; + IN chi::ipc::string tag_name_; + IN chi::ipc::string params_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -1640,8 +1640,8 @@ struct RegisterStagerTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const hermes::BucketId &bkt_id, - const chi::charbuf &tag_name, - const chi::charbuf ¶ms) + const chi::string &tag_name, + const chi::string ¶ms) : Task(alloc), tag_name_(alloc, tag_name), params_(alloc, params) { // Initialize task task_node_ = task_node; @@ -1723,7 +1723,7 @@ struct UnregisterStagerTask : public Task, TaskFlags { /** The StageInTask task */ struct StageInTask : public Task, TaskFlags { IN hermes::BucketId bkt_id_; - IN chi::charbuf blob_name_; + IN chi::string blob_name_; IN float score_; /** SHM default constructor */ @@ -1738,7 +1738,7 @@ struct StageInTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const BucketId &bkt_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, float score) : Task(alloc), blob_name_(alloc, blob_name) { // Initialize task @@ -1777,7 +1777,7 @@ struct StageInTask : public Task, TaskFlags { /** The StageOutTask task */ struct StageOutTask : public Task, TaskFlags { IN hermes::BucketId bkt_id_; - IN chi::charbuf blob_name_; + IN chi::string blob_name_; IN hipc::Pointer data_; IN size_t data_size_; @@ -1793,7 +1793,7 @@ struct StageOutTask : public Task, TaskFlags { const PoolId &pool_id, const DomainQuery &dom_query, const BucketId &bkt_id, - const chi::charbuf &blob_name, + const chi::string &blob_name, const hipc::Pointer &data, size_t data_size, u32 task_flags) diff --git a/tasks/hermes_core/src/hermes_core.cc b/tasks/hermes_core/src/hermes_core.cc index fe0f1c68e..dc016be8c 100644 --- a/tasks/hermes_core/src/hermes_core.cc +++ b/tasks/hermes_core/src/hermes_core.cc @@ -25,9 +25,9 @@ namespace hermes { #define HERMES_LANES 32 /** Type name simplification for the various map types */ -typedef std::unordered_map TAG_ID_MAP_T; +typedef std::unordered_map TAG_ID_MAP_T; typedef std::unordered_map TAG_MAP_T; -typedef std::unordered_map BLOB_ID_MAP_T; +typedef std::unordered_map BLOB_ID_MAP_T; typedef std::unordered_map BLOB_MAP_T; typedef hipc::mpsc_queue IO_PATTERN_LOG_T; typedef std::unordered_map> STAGER_MAP_T; @@ -55,8 +55,8 @@ class Server : public Module { private: /** Get the globally unique blob name */ - const chi::charbuf GetBlobNameWithBucket( - const TagId &tag_id, const chi::charbuf &blob_name) { + const chi::string GetBlobNameWithBucket( + const TagId &tag_id, const chi::string &blob_name) { return BlobInfo::GetBlobNameWithBucket(tag_id, blob_name); } @@ -162,8 +162,8 @@ class Server : public Module { HSHM_DEFAULT_MEM_CTX, chi::DomainQuery::GetGlobalBcast(), tag_id, - chi::charbuf(task->tag_name_.str()), - chi::charbuf(task->params_.str())); + chi::string(task->tag_name_.str()), + chi::string(task->params_.str())); tag.flags_.SetBits(HERMES_SHOULD_STAGE); } } else { @@ -188,7 +188,7 @@ class Server : public Module { HermesLane &tls = tls_[CHI_CUR_LANE->lane_id_.unique_]; chi::ScopedCoRwReadLock tag_map_lock(tls.tag_map_lock_); TAG_ID_MAP_T &tag_id_map = tls.tag_id_map_; - chi::charbuf tag_name(task->tag_name_); + chi::string tag_name(task->tag_name_); auto it = tag_id_map.find(tag_name); if (it == tag_id_map.end()) { task->tag_id_ = TagId::GetNull(); @@ -383,9 +383,9 @@ class Server : public Module { /** Get or create a blob ID */ BlobId GetOrCreateBlobId(HermesLane &tls, TagId &tag_id, u32 name_hash, - const chi::charbuf &blob_name, + const chi::string &blob_name, bitfield32_t &flags) { - chi::charbuf blob_name_unique = GetBlobNameWithBucket(tag_id, blob_name); + chi::string blob_name_unique = GetBlobNameWithBucket(tag_id, blob_name); BLOB_ID_MAP_T &blob_id_map = tls.blob_id_map_; auto it = blob_id_map.find(blob_name_unique); if (it == blob_id_map.end()) { @@ -412,7 +412,7 @@ class Server : public Module { void GetOrCreateBlobId(GetOrCreateBlobIdTask *task, RunContext &rctx) { HermesLane &tls = tls_[CHI_CUR_LANE->lane_id_.unique_]; chi::ScopedCoRwReadLock blob_map_lock(tls.blob_map_lock_); - chi::charbuf blob_name(task->blob_name_); + chi::string blob_name(task->blob_name_); bitfield32_t flags; task->blob_id_ = GetOrCreateBlobId( tls, task->tag_id_, @@ -429,8 +429,8 @@ class Server : public Module { void GetBlobId(GetBlobIdTask *task, RunContext &rctx) { HermesLane &tls = tls_[CHI_CUR_LANE->lane_id_.unique_]; chi::ScopedCoRwReadLock blob_map_lock(tls.blob_map_lock_); - chi::charbuf blob_name(task->blob_name_); - chi::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); + chi::string blob_name(task->blob_name_); + chi::string blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); BLOB_ID_MAP_T &blob_id_map = tls.blob_id_map_; auto it = blob_id_map.find(blob_name_unique); if (it == blob_id_map.end()) { @@ -475,7 +475,7 @@ class Server : public Module { task->blob_id_ = GetOrCreateBlobId( tls, task->tag_id_, HashBlobName(task->tag_id_, task->blob_name_), - chi::charbuf(task->blob_name_), flags); + chi::string(task->blob_name_), flags); } BLOB_MAP_T &blob_map = tls.blob_map_; auto it = blob_map.find(task->blob_id_); @@ -531,7 +531,7 @@ class Server : public Module { chi::ScopedCoRwReadLock blob_map_lock(tls.blob_map_lock_); // Get blob ID - chi::charbuf blob_name(task->blob_name_); + chi::string blob_name(task->blob_name_); if (task->blob_id_.IsNull()) { task->blob_id_ = GetOrCreateBlobId( tls, task->tag_id_, @@ -721,7 +721,7 @@ class Server : public Module { chi::ScopedCoRwReadLock blob_map_lock(tls.blob_map_lock_); // Get blob struct if (task->blob_id_.IsNull()) { - chi::charbuf blob_name(task->blob_name_); + chi::string blob_name(task->blob_name_); task->blob_id_ = GetOrCreateBlobId( tls, task->tag_id_, HashBlobName(task->tag_id_, blob_name), @@ -881,7 +881,7 @@ class Server : public Module { BLOB_ID_MAP_T &blob_id_map = tls.blob_id_map_; BLOB_MAP_T &blob_map = tls.blob_map_; // Get blob ID - chi::charbuf blob_name(task->blob_name_); + chi::string blob_name(task->blob_name_); if (task->blob_id_.IsNull()) { auto blob_id_map_it = blob_id_map.find(blob_name); if (blob_id_map_it == blob_id_map.end()) { @@ -920,7 +920,7 @@ class Server : public Module { HSHM_DEFAULT_MEM_CTX, chi::DomainQuery::GetDirectHash( chi::SubDomainId::kLocalContainers, 0), - task->tag_id_, chi::charbuf(""), task->blob_id_, + task->tag_id_, chi::string(""), task->blob_id_, 0, blob_info.blob_size_, data.shm_, blob_info.score_, TASK_FIRE_AND_FORGET | TASK_DATA_OWNER, 0); task->SetModuleComplete(); @@ -953,7 +953,7 @@ class Server : public Module { // LPointer reorg_task = // blob_mdm_.AsyncReorganizeBlob(task->task_node_ + 1, // blob_info.tag_id_, -// chi::charbuf(""), +// chi::string(""), // blob_info.blob_id_, // new_score, false, ctx, // TASK_LOW_LATENCY); diff --git a/tasks/old/data_stager/CMakeLists.txt b/tasks/old/data_stager/CMakeLists.txt deleted file mode 100644 index d48611782..000000000 --- a/tasks/old/data_stager/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Hrun Admin Task Library -#------------------------------------------------------------------------------ -include_directories(include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install HRUN Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/old/data_stager/include/data_stager/data_stager.h b/tasks/old/data_stager/include/data_stager/data_stager.h deleted file mode 100644 index 3cfb53b43..000000000 --- a/tasks/old/data_stager/include/data_stager/data_stager.h +++ /dev/null @@ -1,164 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#ifndef HRUN_data_stager_H_ -#define HRUN_data_stager_H_ - -#include "data_stager_tasks.h" - -namespace hermes::data_stager { - -/** Create data_stager requests */ -class Client : public ModuleClient { - public: - /** Default constructor */ - Client() = default; - - /** Destructor */ - ~Client() = default; - - /** Async create a task state */ - HSHM_ALWAYS_INLINE - LPointer AsyncCreate(const TaskNode &task_node, - const DomainQuery &dom_query, - const std::string &state_name, - const PoolId &blob_mdm, - const PoolId &bkt_mdm) { - id_ = PoolId::GetNull(); - QueueManagerInfo &qm = CHI_CLIENT->server_config_.queue_manager_; - std::vector queue_info; - return CHI_ADMIN->AsyncCreateTaskState( - task_node, domain_id, state_name, id_, queue_info, blob_mdm, bkt_mdm); - } - HRUN_TASK_NODE_ROOT(AsyncCreate) - template - HSHM_ALWAYS_INLINE - void Create(Args&& ...args) { - LPointer task = - AsyncCreate(std::forward(args)...); - task->Wait(); - id_ = task->id_; - Init(id_, CHI_ADMIN->queue_id_); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, task); - } - - /** Destroy task state + queue */ - HSHM_ALWAYS_INLINE - void Destroy(const DomainQuery &dom_query) { - CHI_ADMIN->DestroyTaskState(domain_id, id_); - } - - /** Register task state */ - HSHM_ALWAYS_INLINE - void AsyncRegisterStagerConstruct(RegisterStagerTask *task, - const TaskNode &task_node, - const BucketId &bkt_id, - const chi::charbuf &path, - const chi::charbuf ¶ms) { - CHI_CLIENT->ConstructTask( - task, task_node, id_, bkt_id, path, params); - } - HSHM_ALWAYS_INLINE - void RegisterStager(const BucketId &bkt_id, - const chi::charbuf &path, - const chi::charbuf params) { - LPointer task = - AsyncRegisterStager(bkt_id, path, params); - task.ptr_->Wait(); - } - CHI_TASK_METHODS(RegisterStager); - - /** Unregister task state */ - HSHM_ALWAYS_INLINE - void AsyncUnregisterStagerConstruct(UnregisterStagerTask *task, - const TaskNode &task_node, - const BucketId &bkt_id) { - CHI_CLIENT->ConstructTask( - task, task_node, id_, bkt_id); - } - HSHM_ALWAYS_INLINE - void UnregisterStager(const BucketId &bkt_id) { - LPointer task = - AsyncUnregisterStager(bkt_id); - task.ptr_->Wait(); - } - CHI_TASK_METHODS(UnregisterStager); - - /** Stage in data from a remote source */ - HSHM_ALWAYS_INLINE - void AsyncStageInConstruct(StageInTask *task, - const TaskNode &task_node, - const BucketId &bkt_id, - const chi::charbuf &blob_name, - float score, - u32 node_id) { - CHI_CLIENT->ConstructTask( - task, task_node, id_, bkt_id, - blob_name, score, node_id); - } - HSHM_ALWAYS_INLINE - void StageIn(const BucketId &bkt_id, - const chi::charbuf &blob_name, - float score, - u32 node_id) { - LPointer task = - AsyncStageIn(bkt_id, blob_name, score, node_id); - task.ptr_->Wait(); - } - CHI_TASK_METHODS(StageIn); - - /** Stage out data to a remote source */ - HSHM_ALWAYS_INLINE - void AsyncStageOutConstruct(StageOutTask *task, - const TaskNode &task_node, - const BucketId &bkt_id, - const chi::charbuf &blob_name, - const hipc::Pointer &data, - size_t data_size, - u32 task_flags) { - CHI_CLIENT->ConstructTask( - task, task_node, id_, bkt_id, - blob_name, data, data_size, task_flags); - } - HSHM_ALWAYS_INLINE - void StageOut(const BucketId &bkt_id, - const chi::charbuf &blob_name, - const hipc::Pointer &data, - size_t data_size, - u32 task_flags) { - LPointer task = - AsyncStageOut(bkt_id, blob_name, data, data_size, task_flags); - task.ptr_->Wait(); - } - CHI_TASK_METHODS(StageOut); - - /** Stage out data to a remote source */ - HSHM_ALWAYS_INLINE - void AsyncUpdateSizeConstruct(UpdateSizeTask *task, - const TaskNode &task_node, - const BucketId &bkt_id, - const chi::charbuf &blob_name, - size_t blob_off, - size_t data_size, - u32 task_flags) { - CHI_CLIENT->ConstructTask( - task, task_node, id_, bkt_id, - blob_name, blob_off, data_size, task_flags); - } - HSHM_ALWAYS_INLINE - void UpdateSize(const BucketId &bkt_id, - const chi::charbuf &blob_name, - size_t blob_off, - size_t data_size, - u32 task_flags) { - AsyncUpdateSize(bkt_id, blob_name, blob_off, data_size, task_flags); - } - CHI_TASK_METHODS(UpdateSize); - - -}; - -} // namespace hrun - -#endif // HRUN_data_stager_H_ diff --git a/tasks/old/data_stager/include/data_stager/data_stager_lib_exec.h b/tasks/old/data_stager/include/data_stager/data_stager_lib_exec.h deleted file mode 100644 index 1a7d66f73..000000000 --- a/tasks/old/data_stager/include/data_stager/data_stager_lib_exec.h +++ /dev/null @@ -1,406 +0,0 @@ -#ifndef HRUN_DATA_STAGER_LIB_EXEC_H_ -#define HRUN_DATA_STAGER_LIB_EXEC_H_ - -/** Execute a task */ -void Run(u32 method, Task *task, RunContext &rctx) override { - switch (method) { - case Method::kConstruct: { - Construct(reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - Destruct(reinterpret_cast(task), rctx); - break; - } - case Method::kRegisterStager: { - RegisterStager(reinterpret_cast(task), rctx); - break; - } - case Method::kUnregisterStager: { - UnregisterStager(reinterpret_cast(task), rctx); - break; - } - case Method::kStageIn: { - StageIn(reinterpret_cast(task), rctx); - break; - } - case Method::kStageOut: { - StageOut(reinterpret_cast(task), rctx); - break; - } - case Method::kUpdateSize: { - UpdateSize(reinterpret_cast(task), rctx); - break; - } - } -} -/** Execute a task */ -void Monitor(u32 mode, Task *task, RunContext &rctx) override { - switch (task->method_) { - case Method::kConstruct: { - MonitorConstruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - MonitorDestruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kRegisterStager: { - MonitorRegisterStager(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kUnregisterStager: { - MonitorUnregisterStager(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kStageIn: { - MonitorStageIn(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kStageOut: { - MonitorStageOut(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kUpdateSize: { - MonitorUpdateSize(mode, reinterpret_cast(task), rctx); - break; - } - } -} -/** Delete a task */ -void Del(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kRegisterStager: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kUnregisterStager: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kStageIn: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kStageOut: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kUpdateSize: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - } -} -/** Duplicate a task */ -void Dup(u32 method, Task *orig_task, std::vector> &dups) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kRegisterStager: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kUnregisterStager: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kStageIn: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kStageOut: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kUpdateSize: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - } -} -/** Register the duplicate output with the origin task */ -void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kRegisterStager: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kUnregisterStager: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kStageIn: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kStageOut: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kUpdateSize: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - } -} -/** Ensure there is space to store replicated outputs */ -void ReplicateStart(u32 method, u32 count, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kRegisterStager: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kUnregisterStager: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kStageIn: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kStageOut: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kUpdateSize: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - } -} -/** Determine success and handle failures */ -void ReplicateEnd(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kRegisterStager: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kUnregisterStager: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kStageIn: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kStageOut: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kUpdateSize: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - } -} -/** Serialize a task when initially pushing into remote */ -std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRegisterStager: { - ar << *reinterpret_cast(task); - break; - } - case Method::kUnregisterStager: { - ar << *reinterpret_cast(task); - break; - } - case Method::kStageIn: { - ar << *reinterpret_cast(task); - break; - } - case Method::kStageOut: { - ar << *reinterpret_cast(task); - break; - } - case Method::kUpdateSize: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when popping from remote queue */ -TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { - TaskPointer task_ptr; - switch (method) { - case Method::kConstruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kRegisterStager: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kUnregisterStager: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kStageIn: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kStageOut: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kUpdateSize: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - } - return task_ptr; -} -/** Serialize a task when returning from remote queue */ -std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRegisterStager: { - ar << *reinterpret_cast(task); - break; - } - case Method::kUnregisterStager: { - ar << *reinterpret_cast(task); - break; - } - case Method::kStageIn: { - ar << *reinterpret_cast(task); - break; - } - case Method::kStageOut: { - ar << *reinterpret_cast(task); - break; - } - case Method::kUpdateSize: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when returning from remote queue */ -void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kRegisterStager: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kUnregisterStager: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kStageIn: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kStageOut: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kUpdateSize: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - } -} -/** Get the grouping of the task */ -u32 GetGroup(u32 method, Task *task, chi::charbuf &group) override { - switch (method) { - case Method::kConstruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kRegisterStager: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kUnregisterStager: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kStageIn: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kStageOut: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kUpdateSize: { - return reinterpret_cast(task)->GetGroup(group); - } - } - return -1; -} - -#endif // HRUN_DATA_STAGER_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/data_stager/include/data_stager/data_stager_methods.h b/tasks/old/data_stager/include/data_stager/data_stager_methods.h deleted file mode 100644 index a1dabbbf9..000000000 --- a/tasks/old/data_stager/include/data_stager/data_stager_methods.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef HRUN_DATA_STAGER_METHODS_H_ -#define HRUN_DATA_STAGER_METHODS_H_ - -/** The set of methods in the admin task */ -struct Method : public TaskMethod { - TASK_METHOD_T kRegisterStager = kLast + 0; - TASK_METHOD_T kUnregisterStager = kLast + 1; - TASK_METHOD_T kStageIn = kLast + 2; - TASK_METHOD_T kStageOut = kLast + 3; - TASK_METHOD_T kUpdateSize = kLast + 4; -}; - -#endif // HRUN_DATA_STAGER_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/data_stager/include/data_stager/data_stager_methods.yaml b/tasks/old/data_stager/include/data_stager/data_stager_methods.yaml deleted file mode 100644 index 0dec22c7b..000000000 --- a/tasks/old/data_stager/include/data_stager/data_stager_methods.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kRegisterStager: 0 -kUnregisterStager: 1 -kStageIn: 2 -kStageOut: 3 -kUpdateSize: 4 \ No newline at end of file diff --git a/tasks/old/data_stager/include/data_stager/data_stager_tasks.h b/tasks/old/data_stager/include/data_stager/data_stager_tasks.h deleted file mode 100644 index 9a99d490b..000000000 --- a/tasks/old/data_stager/include/data_stager/data_stager_tasks.h +++ /dev/null @@ -1,376 +0,0 @@ -// -// Created by lukemartinlogan on 8/11/23. -// - -#ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ -#define HRUN_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ - -#include "chimaera/api/chimaera_client.h" -#include "chimaera/module_registry/task_lib.h" -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/queue_manager/queue_manager_client.h" -#include "proc_queue/proc_queue.h" -#include "hermes/hermes_types.h" - -namespace hermes::data_stager { - -#include "data_stager_methods.h" -#include "chimaera/chimaera_namespace.h" -using chi::proc_queue::TypedPushTask; -using chi::proc_queue::PushTask; - -/** - * A task to create data_stager - * */ -using chi::Admin::CreateTaskStateTask; -struct ConstructTask : public CreateTaskStateTask { - PoolId blob_mdm_; - PoolId bkt_mdm_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc) - : CreateTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const std::string &state_name, - const PoolId &id, - const std::vector &queue_info, - const PoolId &blob_mdm, - const PoolId &bkt_mdm) - : CreateTaskStateTask(alloc, task_node, domain_id, state_name, - "data_stager", id, queue_info) { - // Custom params - blob_mdm_ = blob_mdm; - bkt_mdm_ = bkt_mdm; - std::stringstream ss; - cereal::BinaryOutputArchive ar(ss); - ar(blob_mdm_, bkt_mdm_); - std::string data = ss.str(); - *custom_ = data; - } - - /** Deserialize parameters */ - void Deserialize() { - std::string data = custom_->str(); - std::stringstream ss(data); - cereal::BinaryInputArchive ar(ss); - ar(blob_mdm_, bkt_mdm_); - } - - HSHM_ALWAYS_INLINE - ~ConstructTask() { - // Custom params - } -}; - -/** A task to destroy data_stager */ -using chi::Admin::DestroyTaskStateTask; -struct DestructTask : public DestroyTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(const hipc::CtxAllocator &alloc) - : DestroyTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - PoolId &pool_id) - : DestroyTaskStateTask(alloc, task_node, domain_id, pool_id) {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * Register a new stager - * */ -struct RegisterStagerTask : public Task, TaskFlags { - hermes::BucketId bkt_id_; - chi::string tag_name_; - chi::string params_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - RegisterStagerTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - RegisterStagerTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id, - hermes::BucketId bkt_id, - const chi::charbuf &tag_name, - const chi::charbuf ¶ms) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = bkt_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kRegisterStager; - task_flags_.SetBits(TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET); - domain_id_ = chi::DomainQuery::GetGlobalBcast(); - - // Custom params - bkt_id_ = bkt_id; - HSHM_MAKE_AR(tag_name_, alloc, tag_name); - HSHM_MAKE_AR(params_, alloc, params); - } - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, RegisterStagerTask &other) { - task_dup(other); - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, RegisterStagerTask &dup_task) { - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(bkt_id_, tag_name_, params_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Begin replication */ - void ReplicateStart(u32 count) {} - - /** Finalize replication */ - void ReplicateEnd() {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * Unregister a new stager - * */ -struct UnregisterStagerTask : public Task, TaskFlags { - IN hermes::BucketId bkt_id_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - UnregisterStagerTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - UnregisterStagerTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id, - const hermes::BucketId &bkt_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = bkt_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kUnregisterStager; - task_flags_.SetBits(TASK_FIRE_AND_FORGET); - domain_id_ = chi::DomainQuery::GetGlobalBcast(); - - // Custom params - bkt_id_ = bkt_id; - } - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, UnregisterStagerTask &other) { - task_dup(other); - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, UnregisterStagerTask &dup_task) { - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(bkt_id_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Begin replication */ - void ReplicateStart(u32 count) {} - - /** Finalize replication */ - void ReplicateEnd() {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * A task to stage in data from a remote source - * */ -struct StageInTask : public Task, TaskFlags { - IN hermes::BucketId bkt_id_; - IN chi::charbuf blob_name_; - IN float score_; - IN u32 node_id_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - StageInTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - StageInTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id, - const BucketId &bkt_id, - const chi::charbuf &blob_name, - float score, - u32 node_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = bkt_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kStageIn; - task_flags_.SetBits(TASK_COROUTINE | TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK); - domain_id_ = DomainId::GetLocal(); - - // Custom params - bkt_id_ = bkt_id; - HSHM_MAKE_AR(blob_name_, alloc, blob_name); - score_ = score; - node_id_ = node_id; - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { -// hipc::LocalSerialize srl(group); -// srl << bkt_id_.unique_; -// srl << bkt_id_.node_id_; -// return 0; - return TASK_UNORDERED; - } -}; - -/** - * A task to stage data out of a hermes to a remote source - * */ -struct StageOutTask : public Task, TaskFlags { - IN hermes::BucketId bkt_id_; - IN chi::charbuf blob_name_; - IN hipc::Pointer data_; - IN size_t data_size_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - StageOutTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - StageOutTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id, - const BucketId &bkt_id, - const chi::charbuf &blob_name, - const hipc::Pointer &data, - size_t data_size, - u32 task_flags): Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = bkt_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kStageOut; - task_flags_.SetBits(task_flags | TASK_COROUTINE | TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK); - domain_id_ = DomainId::GetLocal(); - - // Custom params - bkt_id_ = bkt_id; - HSHM_MAKE_AR(blob_name_, alloc, blob_name); - data_ = data; - data_size_ = data_size; - } - - /** Destructor */ - HSHM_ALWAYS_INLINE - ~StageOutTask() { - if (IsDataOwner()) { - CHI_CLIENT->FreeBuffer(data_); - } - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * A task to stage data out in a Hermes deployment - * */ -struct UpdateSizeTask : public Task, TaskFlags { - IN hermes::BucketId bkt_id_; - IN chi::charbuf blob_name_; - IN size_t blob_off_, data_size_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - UpdateSizeTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - UpdateSizeTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id, - const BucketId &bkt_id, - const chi::charbuf &blob_name, - size_t blob_off, - size_t data_size, - u32 task_flags): Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = bkt_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kUpdateSize; - task_flags_.SetBits(task_flags | TASK_FIRE_AND_FORGET | TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK); - domain_id_ = DomainId::GetLocal(); - - // Custom params - bkt_id_ = bkt_id; - HSHM_MAKE_AR(blob_name_, alloc, blob_name); - blob_off_ = blob_off; - data_size_ = data_size; - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -} // namespace chi::data_stager - -#endif // HRUN_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ diff --git a/tasks/old/data_stager/include/data_stager/factory/abstract_stager.h b/tasks/old/data_stager/include/data_stager/factory/abstract_stager.h deleted file mode 100644 index 80e000f20..000000000 --- a/tasks/old/data_stager/include/data_stager/factory/abstract_stager.h +++ /dev/null @@ -1,29 +0,0 @@ -// -// Created by lukemartinlogan on 9/30/23. -// - -#ifndef HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ -#define HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ - -#include "../data_stager.h" -#include "hermes_bucket_mdm/hermes_bucket_mdm.h" - -namespace hermes::data_stager { - -class AbstractStager { - public: - std::string path_; - std::string params_; - - AbstractStager() = default; - ~AbstractStager() = default; - - virtual void RegisterStager(RegisterStagerTask *task, RunContext &rctx) = 0; - virtual void StageIn(blob_mdm::Client &blob_mdm, StageInTask *task, RunContext &rctx) = 0; - virtual void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) = 0; - virtual void UpdateSize(bucket_mdm::Client &bkt_mdm, UpdateSizeTask *task, RunContext &rctx) = 0; -}; - -} // namespace hermes - -#endif // HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ diff --git a/tasks/old/data_stager/include/data_stager/factory/binary_stager.h b/tasks/old/data_stager/include/data_stager/factory/binary_stager.h deleted file mode 100644 index 3e94411b5..000000000 --- a/tasks/old/data_stager/include/data_stager/factory/binary_stager.h +++ /dev/null @@ -1,148 +0,0 @@ -// -// Created by lukemartinlogan on 9/30/23. -// - -#ifndef HERMES_TASKS_DATA_STAGER_SRC_BINARY_STAGER_H_ -#define HERMES_TASKS_DATA_STAGER_SRC_BINARY_STAGER_H_ - -#include "abstract_stager.h" -#include "hermes_adapters/mapper/abstract_mapper.h" - -namespace hermes::data_stager { - -class BinaryFileStager : public AbstractStager { - public: - size_t page_size_; - std::string path_; - bitfield32_t flags_; - - public: - /** Default constructor */ - BinaryFileStager() = default; - - /** Destructor */ - ~BinaryFileStager() {} - - /** Build context for staging */ - static Context BuildContext(size_t page_size, - u32 flags = 0, - size_t elmt_size = 1) { - Context ctx; - ctx.flags_.SetBits(HERMES_SHOULD_STAGE); - ctx.bkt_params_ = BuildFileParams(page_size, flags, elmt_size); - return ctx; - } - - /** Build serialized file parameter pack */ - static std::string BuildFileParams(size_t page_size, - u32 flags = 0, - size_t elmt_size = 1) { - chi::charbuf params(32); - page_size = (page_size / elmt_size) * elmt_size; - hipc::LocalSerialize srl(params); - srl << std::string("file"); - srl << flags; - srl << page_size; - return params.str(); - } - - /** Create the data stager payload */ - void RegisterStager(RegisterStagerTask *task, RunContext &rctx) override { - std::string params = task->params_->str(); - std::string protocol; - hipc::LocalDeserialize srl(params); - srl >> protocol; - srl >> flags_.bits_; - srl >> page_size_; - path_ = task->tag_name_->str(); - } - - /** Stage data in from remote source */ - void StageIn(blob_mdm::Client &blob_mdm, StageInTask *task, RunContext &rctx) override { - if (flags_.Any(HERMES_STAGE_NO_READ)) { - return; - } - adapter::BlobPlacement plcmnt; - plcmnt.DecodeBlobName(*task->blob_name_, page_size_); - HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", - page_size_, path_, plcmnt.bucket_off_); - LPointer blob = HRUN_CLIENT->AllocateBufferServer(page_size_); - int fd = HERMES_POSIX_API->open(path_.c_str(), O_CREAT | O_RDWR, 0666); - if (fd < 0) { - HELOG(kError, "Failed to open file {}", path_); - HRUN_CLIENT->FreeBuffer(blob); - return; - } - ssize_t real_size = HERMES_POSIX_API->pread(fd, - blob.ptr_, - page_size_, - (off_t)plcmnt.bucket_off_); - HERMES_POSIX_API->close(fd); - if (real_size < 0) { -// HELOG(kError, "Failed to stage in {} bytes from {}", -// page_size_, path_); - HRUN_CLIENT->FreeBuffer(blob); - return; - } else if (real_size == 0) { - HRUN_CLIENT->FreeBuffer(blob); - return; - } - HILOG(kDebug, "Staged {} bytes from the backend file {}", - real_size, path_); - HILOG(kDebug, "Submitting put blob {} ({}) to blob mdm ({})", - task->blob_name_->str(), task->bkt_id_, blob_mdm.id_) - hapi::Context ctx; - ctx.flags_.SetBits(HERMES_SHOULD_STAGE); - LPointer put_task = - blob_mdm.AsyncPutBlob(task->task_node_ + 1, - task->bkt_id_, - hshm::to_charbuf(*task->blob_name_), - hermes::BlobId::GetNull(), - 0, real_size, blob.shm_, task->score_, 0, - ctx, TASK_DATA_OWNER | TASK_LOW_LATENCY); - put_task->Wait(task); - HRUN_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, put_task); - } - - /** Stage data out to remote source */ - void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) override { - if (flags_.Any(HERMES_STAGE_NO_WRITE)) { - return; - } - adapter::BlobPlacement plcmnt; - plcmnt.DecodeBlobName(*task->blob_name_, page_size_); - HILOG(kDebug, "Attempting to stage {} bytes to the backend file {} at offset {}", - page_size_, path_, plcmnt.bucket_off_); - char *data = HRUN_CLIENT->GetDataPointer(task->data_); - int fd = HERMES_POSIX_API->open(path_.c_str(), O_CREAT | O_RDWR, 0666); - if (fd < 0) { - HELOG(kError, "Failed to open file {}", path_); - return; - } - ssize_t real_size = HERMES_POSIX_API->pwrite(fd, - data, - task->data_size_, - (off_t)plcmnt.bucket_off_); - HERMES_POSIX_API->close(fd); - if (real_size < 0) { - HELOG(kError, "Failed to stage out {} bytes from {}", - task->data_size_, path_); - } - HILOG(kDebug, "Staged out {} bytes to the backend file {}", - real_size, path_); - } - - void UpdateSize(bucket_mdm::Client &bkt_mdm, UpdateSizeTask *task, RunContext &rctx) override { - adapter::BlobPlacement p; - std::string blob_name_str = task->blob_name_->str(); - p.DecodeBlobName(blob_name_str, page_size_); - bkt_mdm.AsyncUpdateSize(task->task_node_ + 1, - task->bkt_id_, - p.bucket_off_ + task->blob_off_ + task->data_size_, - UpdateSizeMode::kCap); - } -}; - -} // namespace hermes::data_stager - -#endif // HERMES_TASKS_DATA_STAGER_SRC_BINARY_STAGER_H_ diff --git a/tasks/old/data_stager/include/data_stager/factory/stager_factory.h b/tasks/old/data_stager/include/data_stager/factory/stager_factory.h deleted file mode 100644 index d8a309e8d..000000000 --- a/tasks/old/data_stager/include/data_stager/factory/stager_factory.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by lukemartinlogan on 9/30/23. -// - -#ifndef HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ -#define HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ - -#include "../data_stager.h" -#include "abstract_stager.h" -#include "binary_stager.h" - -namespace hermes::data_stager { - -class StagerFactory { - public: - static std::unique_ptr Get(const std::string &path, - const std::string ¶ms) { - std::string protocol; - hipc::LocalDeserialize srl(params); - srl >> protocol; - - std::unique_ptr stager; - if (protocol == "file") { - stager = std::make_unique(); - } else if (protocol == "parquet") { - } else if (protocol == "hdf5") { - } else { - throw std::runtime_error("Unknown stager type"); - } - stager->path_ = path; - stager->params_ = params; - return stager; - } -}; - -} // namespace hermes - -#endif // HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ diff --git a/tasks/old/data_stager/src/CMakeLists.txt b/tasks/old/data_stager/src/CMakeLists.txt deleted file mode 100644 index 050d88a31..000000000 --- a/tasks/old/data_stager/src/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Small Message Task Library -#------------------------------------------------------------------------------ -add_library(data_stager SHARED - data_stager.cc) -# add_dependencies(data_stager ${Hermes_RUNTIME_DEPS}) -target_link_libraries(data_stager ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Install Small Message Task Library -#------------------------------------------------------------------------------ -install( - TARGETS - data_stager - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install for import into other projects -#----------------------------------------------------------------------------- -install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - data_stager - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#------------------------------------------------------------------------------ -# Coverage -#------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(data_stager) -endif() diff --git a/tasks/old/data_stager/src/data_stager.cc b/tasks/old/data_stager/src/data_stager.cc deleted file mode 100644 index e18cc240e..000000000 --- a/tasks/old/data_stager/src/data_stager.cc +++ /dev/null @@ -1,127 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/api/chimaera_runtime.h" -#include "data_stager/data_stager.h" -#include "hermes_adapters/mapper/abstract_mapper.h" -#include "hermes_adapters/posix/posix_api.h" -#include "hermes_blob_mdm/hermes_blob_mdm.h" -#include "hermes/staging/stager_factory.h" -#include "hermes_bucket_mdm/hermes_bucket_mdm.h" - -namespace hermes::data_stager { - -class Server : public Module { - public: - std::vector>> url_map_; - blob_mdm::Client blob_mdm_; - bucket_mdm::Client bkt_mdm_; - - public: - Server() = default; - - /** Construct data stager */ - void Construct(ConstructTask *task, RunContext &rctx) { - task->Deserialize(); - url_map_.resize(HRUN_QM_RUNTIME->max_lanes_); - blob_mdm_.Init(task->blob_mdm_, CHI_ADMIN->queue_id_); - bkt_mdm_.Init(task->bkt_mdm_, CHI_ADMIN->queue_id_); - HILOG(kInfo, "(node {}) BLOB MDM: {}", CHI_CLIENT->node_id_, blob_mdm_.id_); - task->SetModuleComplete(); - } - void MonitorConstruct(u32 mode, ConstructTask *task, RunContext &rctx) { - } - - /** Destroy data stager */ - void Destruct(DestructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - void MonitorDestruct(u32 mode, DestructTask *task, RunContext &rctx) { - } - - /** Register a stager */ - void RegisterStager(RegisterStagerTask *task, RunContext &rctx) { - std::string tag_name = task->tag_name_->str(); - std::string params = task->params_->str(); - HILOG(kDebug, "Registering stager {}: {}", task->bkt_id_, tag_name); - std::unique_ptr stager = StagerFactory::Get(tag_name, params); - stager->RegisterStager(task, rctx); - url_map_[rctx.lane_id_].emplace(task->bkt_id_, std::move(stager)); - task->SetModuleComplete(); - } - void MonitorRegisterStager(u32 mode, RegisterStagerTask *task, RunContext &rctx) { - } - - /** Unregister stager */ - void UnregisterStager(UnregisterStagerTask *task, RunContext &rctx) { - HILOG(kDebug, "Unregistering stager {}", task->bkt_id_); - if (url_map_[rctx.lane_id_].find(task->bkt_id_) == url_map_[rctx.lane_id_].end()) { - task->SetModuleComplete(); - return; - } - url_map_[rctx.lane_id_].erase(task->bkt_id_); - task->SetModuleComplete(); - } - void MonitorUnregisterStager(u32 mode, UnregisterStagerTask *task, RunContext &rctx) { - } - - /** Stage in data */ - void StageIn(StageInTask *task, RunContext &rctx) { - // HILOG(kDebug, "Beginning stage in"); - std::unordered_map>::iterator it = - url_map_[rctx.lane_id_].find(task->bkt_id_); - if (it == url_map_[rctx.lane_id_].end()) { - // - // HELOG(kError, "Could not find stager for bucket: {}", task->bkt_id_); - // TODO(llogan): Probably should add back... - // task->SetModuleComplete(); - return; - } - std::unique_ptr &stager = it->second; - stager->StageIn(blob_mdm_, task, rctx); - task->SetModuleComplete(); - } - void MonitorStageIn(u32 mode, StageInTask *task, RunContext &rctx) { - } - - /** Stage out data */ - void StageOut(StageOutTask *task, RunContext &rctx) { - std::unordered_map>::iterator it = - url_map_[rctx.lane_id_].find(task->bkt_id_); - if (it == url_map_[rctx.lane_id_].end()) { - HELOG(kError, "Could not find stager for bucket: {}", task->bkt_id_); - task->SetModuleComplete(); - return; - } - std::unique_ptr &stager = it->second; - stager->StageOut(blob_mdm_, task, rctx); - task->SetModuleComplete(); - } - void MonitorStageOut(u32 mode, StageOutTask *task, RunContext &rctx) { - } - - /** Update the size of the bucket */ - void UpdateSize(UpdateSizeTask *task, RunContext &rctx) { - std::unordered_map>::iterator it = - url_map_[rctx.lane_id_].find(task->bkt_id_); - if (it == url_map_[rctx.lane_id_].end()) { - HELOG(kError, "Could not find stager for bucket: {}", task->bkt_id_); - task->SetModuleComplete(); - return; - } - std::unique_ptr &stager = it->second; - stager->UpdateSize(bkt_mdm_, task, rctx); - task->SetModuleComplete(); - } - void MonitorUpdateSize(u32 mode, UpdateSizeTask *task, RunContext &rctx) { - } - - public: -#include "data_stager/data_stager_lib_exec.h" -}; - -} // namespace chi::data_stager - -HRUN_TASK_CC(hermes::data_stager::Server, "data_stager"); diff --git a/tasks/old/hermes_blob_mdm/CMakeLists.txt b/tasks/old/hermes_blob_mdm/CMakeLists.txt deleted file mode 100644 index 7cd7a5538..000000000 --- a/tasks/old/hermes_blob_mdm/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Hermes Blob MDM Task Library -#------------------------------------------------------------------------------ -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install HRUN Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h deleted file mode 100644 index b35578bce..000000000 --- a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ /dev/null @@ -1,109 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#ifndef HRUN_hermes_blob_mdm_H_ -#define HRUN_hermes_blob_mdm_H_ - -#include "hermes_blob_mdm_tasks.h" - -namespace hermes::blob_mdm { - -/** Create hermes_blob_mdm requests */ -class Client : public ModuleClient { - - public: - /** Default constructor */ - Client() = default; - - /** Destructor */ - ~Client() = default; - - /** Create a hermes_blob_mdm */ - HSHM_ALWAYS_INLINE - LPointer AsyncCreate(const TaskNode &task_node, - const DomainQuery &dom_query, - const std::string &state_name) { - id_ = PoolId::GetNull(); - QueueManagerInfo &qm = CHI_CLIENT->server_config_.queue_manager_; - std::vector queue_info; - return CHI_ADMIN->AsyncCreateTaskState( - task_node, domain_id, state_name, id_, queue_info); - } - void AsyncCreateComplete(ConstructTask *task) { - if (task->IsModuleComplete()) { - id_ = task->id_; - queue_id_ = QueueId(id_); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, task); - } - } - HRUN_TASK_NODE_ROOT(AsyncCreate); - template - HSHM_ALWAYS_INLINE - void Create(Args&& ...args) { - LPointer task = AsyncCreate(std::forward(args)...); - task->Wait(); - AsyncCreateComplete(task.ptr_); - } - - /** Destroy task state + queue */ - HSHM_ALWAYS_INLINE - void Destroy(const DomainQuery &dom_query) { - CHI_ADMIN->DestroyTaskState(domain_id, id_); - } - - /**==================================== - * Blob Operations - * ===================================*/ - - /** Initialize automatic flushing */ - void AsyncFlushDataConstruct(FlushDataTask *task, - const TaskNode &task_node, - size_t period_ms) { - CHI_CLIENT->ConstructTask( - task, task_node, id_, period_ms); - } - CHI_TASK_METHODS(FlushData); - - /** - * Get all blob metadata - * */ - void AsyncPollBlobMetadataConstruct(PollBlobMetadataTask *task, - const TaskNode &task_node) { - CHI_CLIENT->ConstructTask( - task, task_node, id_); - } - std::vector PollBlobMetadata() { - LPointer task = - AsyncPollBlobMetadata(); - task->Wait(); - std::vector blob_mdms = - task->DeserializeBlobMetadata(); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, task); - return blob_mdms; - } - CHI_TASK_METHODS(PollBlobMetadata); - - /** - * Get all target metadata - * */ - void AsyncPollTargetMetadataConstruct(PollTargetMetadataTask *task, - const TaskNode &task_node) { - CHI_CLIENT->ConstructTask( - task, task_node, id_); - } - std::vector PollTargetMetadata() { - LPointer task = - AsyncPollTargetMetadata(); - task->Wait(); - std::vector target_mdms = - task->DeserializeTargetMetadata(); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, task); - return target_mdms; - } - CHI_TASK_METHODS(PollTargetMetadata); -}; - -} // namespace hrun - -#endif // HRUN_hermes_blob_mdm_H_ \ No newline at end of file diff --git a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h deleted file mode 100644 index 123c5c174..000000000 --- a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ /dev/null @@ -1,1030 +0,0 @@ -#ifndef HRUN_HERMES_BLOB_MDM_LIB_EXEC_H_ -#define HRUN_HERMES_BLOB_MDM_LIB_EXEC_H_ - -/** Execute a task */ -void Run(u32 method, Task *task, RunContext &rctx) override { - switch (method) { - case Method::kConstruct: { - Construct(reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - Destruct(reinterpret_cast(task), rctx); - break; - } - case Method::kPutBlob: { - PutBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlob: { - GetBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kTruncateBlob: { - TruncateBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kDestroyBlob: { - DestroyBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kTagBlob: { - TagBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kBlobHasTag: { - BlobHasTag(reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobId: { - GetBlobId(reinterpret_cast(task), rctx); - break; - } - case Method::kGetOrCreateBlobId: { - GetOrCreateBlobId(reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobName: { - GetBlobName(reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobSize: { - GetBlobSize(reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobScore: { - GetBlobScore(reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobBuffers: { - GetBlobBuffers(reinterpret_cast(task), rctx); - break; - } - case Method::kRenameBlob: { - RenameBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kReorganizeBlob: { - ReorganizeBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kSetBucketMdm: { - SetBucketMdm(reinterpret_cast(task), rctx); - break; - } - case Method::kFlushData: { - FlushData(reinterpret_cast(task), rctx); - break; - } - case Method::kPollBlobMetadata: { - PollBlobMetadata(reinterpret_cast(task), rctx); - break; - } - case Method::kPollTargetMetadata: { - PollTargetMetadata(reinterpret_cast(task), rctx); - break; - } - } -} -/** Execute a task */ -void Monitor(u32 mode, Task *task, RunContext &rctx) override { - switch (task->method_) { - case Method::kConstruct: { - MonitorConstruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - MonitorDestruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kPutBlob: { - MonitorPutBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlob: { - MonitorGetBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kTruncateBlob: { - MonitorTruncateBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kDestroyBlob: { - MonitorDestroyBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kTagBlob: { - MonitorTagBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kBlobHasTag: { - MonitorBlobHasTag(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobId: { - MonitorGetBlobId(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetOrCreateBlobId: { - MonitorGetOrCreateBlobId(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobName: { - MonitorGetBlobName(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobSize: { - MonitorGetBlobSize(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobScore: { - MonitorGetBlobScore(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetBlobBuffers: { - MonitorGetBlobBuffers(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kRenameBlob: { - MonitorRenameBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kReorganizeBlob: { - MonitorReorganizeBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kSetBucketMdm: { - MonitorSetBucketMdm(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kFlushData: { - MonitorFlushData(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kPollBlobMetadata: { - MonitorPollBlobMetadata(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kPollTargetMetadata: { - MonitorPollTargetMetadata(mode, reinterpret_cast(task), rctx); - break; - } - } -} -/** Delete a task */ -void Del(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kPutBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kTruncateBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestroyBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kTagBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kBlobHasTag: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetBlobId: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetOrCreateBlobId: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetBlobName: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetBlobSize: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetBlobScore: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetBlobBuffers: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kRenameBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kReorganizeBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kSetBucketMdm: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kFlushData: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kPollBlobMetadata: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kPollTargetMetadata: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - } -} -/** Duplicate a task */ -void Dup(u32 method, Task *orig_task, std::vector> &dups) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kPutBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kTruncateBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestroyBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kTagBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kBlobHasTag: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetBlobId: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetOrCreateBlobId: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetBlobName: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetBlobSize: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetBlobScore: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetBlobBuffers: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kRenameBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kReorganizeBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kSetBucketMdm: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kFlushData: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kPollBlobMetadata: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kPollTargetMetadata: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - } -} -/** Register the duplicate output with the origin task */ -void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kPutBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kTruncateBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestroyBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kTagBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kBlobHasTag: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetBlobId: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetOrCreateBlobId: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetBlobName: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetBlobSize: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetBlobScore: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetBlobBuffers: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kRenameBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kReorganizeBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kSetBucketMdm: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kFlushData: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kPollBlobMetadata: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kPollTargetMetadata: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - } -} -/** Ensure there is space to store replicated outputs */ -void ReplicateStart(u32 method, u32 count, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kPutBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kTruncateBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestroyBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kTagBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kBlobHasTag: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetBlobId: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetOrCreateBlobId: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetBlobName: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetBlobSize: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetBlobScore: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetBlobBuffers: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kRenameBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kReorganizeBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kSetBucketMdm: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kFlushData: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kPollBlobMetadata: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kPollTargetMetadata: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - } -} -/** Determine success and handle failures */ -void ReplicateEnd(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kPutBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kTruncateBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestroyBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kTagBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kBlobHasTag: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetBlobId: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetOrCreateBlobId: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetBlobName: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetBlobSize: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetBlobScore: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetBlobBuffers: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kRenameBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kReorganizeBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kSetBucketMdm: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kFlushData: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kPollBlobMetadata: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kPollTargetMetadata: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - } -} -/** Serialize a task when initially pushing into remote */ -std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kPutBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTruncateBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestroyBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTagBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kBlobHasTag: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobId: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetOrCreateBlobId: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobName: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobSize: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobScore: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobBuffers: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRenameBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kReorganizeBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kSetBucketMdm: { - ar << *reinterpret_cast(task); - break; - } - case Method::kFlushData: { - ar << *reinterpret_cast(task); - break; - } - case Method::kPollBlobMetadata: { - ar << *reinterpret_cast(task); - break; - } - case Method::kPollTargetMetadata: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when popping from remote queue */ -TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { - TaskPointer task_ptr; - switch (method) { - case Method::kConstruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kPutBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kTruncateBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestroyBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kTagBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kBlobHasTag: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetBlobId: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetOrCreateBlobId: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetBlobName: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetBlobSize: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetBlobScore: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetBlobBuffers: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kRenameBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kReorganizeBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kSetBucketMdm: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kFlushData: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kPollBlobMetadata: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kPollTargetMetadata: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - } - return task_ptr; -} -/** Serialize a task when returning from remote queue */ -std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kPutBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTruncateBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestroyBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTagBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kBlobHasTag: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobId: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetOrCreateBlobId: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobName: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobSize: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobScore: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetBlobBuffers: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRenameBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kReorganizeBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kSetBucketMdm: { - ar << *reinterpret_cast(task); - break; - } - case Method::kFlushData: { - ar << *reinterpret_cast(task); - break; - } - case Method::kPollBlobMetadata: { - ar << *reinterpret_cast(task); - break; - } - case Method::kPollTargetMetadata: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when returning from remote queue */ -void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kPutBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kTruncateBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestroyBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kTagBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kBlobHasTag: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetBlobId: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetOrCreateBlobId: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetBlobName: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetBlobSize: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetBlobScore: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetBlobBuffers: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kRenameBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kReorganizeBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kSetBucketMdm: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kFlushData: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kPollBlobMetadata: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kPollTargetMetadata: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - } -} -/** Get the grouping of the task */ -u32 GetGroup(u32 method, Task *task, chi::charbuf &group) override { - switch (method) { - case Method::kConstruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kPutBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kTruncateBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestroyBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kTagBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kBlobHasTag: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetBlobId: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetOrCreateBlobId: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetBlobName: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetBlobSize: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetBlobScore: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetBlobBuffers: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kRenameBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kReorganizeBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kSetBucketMdm: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kFlushData: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kPollBlobMetadata: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kPollTargetMetadata: { - return reinterpret_cast(task)->GetGroup(group); - } - } - return -1; -} - -#endif // HRUN_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h b/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h deleted file mode 100644 index 7bd0cc751..000000000 --- a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef HRUN_HERMES_BLOB_MDM_METHODS_H_ -#define HRUN_HERMES_BLOB_MDM_METHODS_H_ - -/** The set of methods in the admin task */ -struct Method : public TaskMethod { - TASK_METHOD_T kPutBlob = kLast + 0; - TASK_METHOD_T kGetBlob = kLast + 1; - TASK_METHOD_T kTruncateBlob = kLast + 2; - TASK_METHOD_T kDestroyBlob = kLast + 3; - TASK_METHOD_T kTagBlob = kLast + 4; - TASK_METHOD_T kBlobHasTag = kLast + 6; - TASK_METHOD_T kGetBlobId = kLast + 8; - TASK_METHOD_T kGetOrCreateBlobId = kLast + 9; - TASK_METHOD_T kGetBlobName = kLast + 10; - TASK_METHOD_T kGetBlobSize = kLast + 11; - TASK_METHOD_T kGetBlobScore = kLast + 12; - TASK_METHOD_T kGetBlobBuffers = kLast + 13; - TASK_METHOD_T kRenameBlob = kLast + 14; - TASK_METHOD_T kReorganizeBlob = kLast + 15; - TASK_METHOD_T kSetBucketMdm = kLast + 16; - TASK_METHOD_T kFlushData = kLast + 17; - TASK_METHOD_T kPollBlobMetadata = kLast + 18; - TASK_METHOD_T kPollTargetMetadata = kLast + 19; -}; - -#endif // HRUN_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml b/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml deleted file mode 100644 index a886bf876..000000000 --- a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml +++ /dev/null @@ -1,20 +0,0 @@ -kPutBlob: 0 -kGetBlob: 1 -kTruncateBlob: 2 -kDestroyBlob: 3 -kTagBlob: 4 -#kUntagBlob: 5 -kBlobHasTag: 6 -#kGetBlobTags: 7 -kGetBlobId: 8 -kGetOrCreateBlobId: 9 -kGetBlobName: 10 -kGetBlobSize: 11 -kGetBlobScore: 12 -kGetBlobBuffers: 13 -kRenameBlob: 14 -kReorganizeBlob: 15 -kSetBucketMdm: 16 -kFlushData: 17 -kPollBlobMetadata: 18 -kPollTargetMetadata: 19 \ No newline at end of file diff --git a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h deleted file mode 100644 index 403051635..000000000 --- a/tasks/old/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ /dev/null @@ -1,1403 +0,0 @@ -// -// Created by lukemartinlogan on 8/14/23. -// - -#ifndef HRUN_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ -#define HRUN_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ - -#include "chimaera/api/chimaera_client.h" -#include "chimaera/module_registry/task_lib.h" -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/queue_manager/queue_manager_client.h" -#include "hermes/hermes_types.h" -#include "bdev/bdev.h" -#include "chimaera/api/chimaera_client.h" -#include "proc_queue/proc_queue.h" - -namespace hermes::blob_mdm { - -#include "hermes_blob_mdm_methods.h" -#include "chimaera/chimaera_namespace.h" - -using chi::Task; -using chi::TaskFlags; -using chi::DataTransfer; - -static inline u32 HashBlobName(const TagId &tag_id, const chi::charbuf &blob_name) { - u32 h2 = std::hash{}(tag_id); - u32 h1 = std::hash{}(blob_name); - return std::hash{}(h1 ^ h2); -} - -/** - * A task to create hermes_core - * */ -using chi::Admin::CreateTaskStateTask; -struct ConstructTask : public CreateTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc) : CreateTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const std::string &state_name, - const PoolId &id, - const std::vector &queue_info) - : CreateTaskStateTask(alloc, task_node, domain_id, state_name, - "hermes_blob_mdm", id, queue_info) { - } -}; - -/** A task to destroy hermes_core */ -using chi::Admin::DestroyTaskStateTask; -struct DestructTask : public DestroyTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(const hipc::CtxAllocator &alloc) : DestroyTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id) - : DestroyTaskStateTask(alloc, task_node, domain_id, pool_id) {} - -}; - -/** Set the BUCKET MDM ID */ -struct SetBucketMdmTask : public Task, TaskFlags { - IN PoolId bkt_mdm_; - IN PoolId stager_mdm_; - IN PoolId op_mdm_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - SetBucketMdmTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - SetBucketMdmTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const PoolId &bkt_mdm, - const PoolId &stager_mdm, - const PoolId &op_mdm) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = 0; - prio_ = TaskPrio::kAdmin; - pool_ = pool_id; - method_ = Method::kSetBucketMdm; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom params - bkt_mdm_ = bkt_mdm; - stager_mdm_ = stager_mdm; - op_mdm_ = op_mdm; - } - - /** Destructor */ - ~SetBucketMdmTask() {} - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, SetBucketMdmTask &other) { - task_dup(other); - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, SetBucketMdmTask &dup_task) { - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(bkt_mdm_, stager_mdm_, op_mdm_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - - /** Begin replication */ - void ReplicateStart(u32 count) {} - - /** Finalize replication */ - void ReplicateEnd() {} -}; - -/**==================================== - * Blob Operations - * ===================================*/ - -/** - * Get \a blob_name BLOB from \a bkt_id bucket - * */ -struct GetOrCreateBlobIdTask : public Task, TaskFlags { - IN TagId tag_id_; - IN chi::charbuf blob_name_; - OUT BlobId blob_id_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - GetOrCreateBlobIdTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - GetOrCreateBlobIdTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const chi::charbuf &blob_name) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = HashBlobName(tag_id, blob_name); - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kGetOrCreateBlobId; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom - tag_id_ = tag_id; - HSHM_MAKE_AR(blob_name_, alloc, blob_name) - } - - /** Destructor */ - ~GetOrCreateBlobIdTask() { - HSHM_DESTROY_AR(blob_name_) - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_name_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(blob_id_); - } - -}; - -/** A task to put data in a blob */ -struct PutBlobTask : public Task, TaskFlags { - IN TagId tag_id_; - IN chi::charbuf blob_name_; - IN size_t blob_off_; - IN size_t data_size_; - IN hipc::Pointer data_; - IN float score_; - IN bitfield32_t flags_; - IN BlobId blob_id_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - PutBlobTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - PutBlobTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const chi::charbuf &blob_name, - const BlobId &blob_id, - size_t blob_off, - size_t data_size, - const hipc::Pointer &data, - float score, - u32 flags, - const Context &ctx, - u32 task_flags) : Task(alloc) { - // Initialize task - task_node_ = task_node; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kPutBlob; - task_flags_ = bitfield32_t(task_flags); - task_flags_.SetBits(TASK_COROUTINE); - if (!blob_id.IsNull()) { - lane_hash_ = blob_id.hash_; - domain_id_ = domain_id; - } else { - lane_hash_ = HashBlobName(tag_id, blob_name); - domain_id_ = DomainId::GetNode(HASH_TO_NODE_ID(lane_hash_)); - } - - // Custom params - tag_id_ = tag_id; - HSHM_MAKE_AR(blob_name_, alloc, blob_name); - blob_id_ = blob_id; - blob_off_ = blob_off; - data_size_ = data_size; - data_ = data; - score_ = score; - flags_ = bitfield32_t(flags | ctx.flags_.bits_); - // HILOG(kInfo, "Creating PUT {} of size {}", task_node_, data_size_); - } - - /** Destructor */ - ~PutBlobTask() { - HSHM_DESTROY_AR(blob_name_); - if (IsDataOwner()) { - // HILOG(kInfo, "Actually freeing PUT {} of size {}", task_node_, data_size_); - CHI_CLIENT->FreeBuffer(data_); - } - } - - /** (De)serialize message call */ - template - void SaveStart(Ar &ar) { - DataTransfer xfer(DT_RECEIVER_READ, - HERMES_MEMORY_MANAGER->Convert(data_), - data_size_, domain_id_); - task_serialize(ar); - ar & xfer; - ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_); - } - - /** Deserialize message call */ - template - void LoadStart(Ar &ar) { - DataTransfer xfer; - task_serialize(ar); - ar & xfer; - data_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); - ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - if (flags_.Any(HERMES_GET_BLOB_ID)) { - ar(blob_id_); - } - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** Phases for the get task */ -class GetBlobPhase { - public: - TASK_METHOD_T kStart = 0; - TASK_METHOD_T kWait = 1; -}; - -/** A task to get data from a blob */ -struct GetBlobTask : public Task, TaskFlags { - IN TagId tag_id_; - IN chi::charbuf blob_name_; - INOUT BlobId blob_id_; - IN size_t blob_off_; - IN hipc::Pointer data_; - INOUT size_t data_size_; - IN bitfield32_t flags_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const chi::charbuf &blob_name, - const BlobId &blob_id, - size_t off, - size_t data_size, - hipc::Pointer &data, - const Context &ctx, - u32 flags) : Task(alloc) { - // Initialize task - task_node_ = task_node; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kGetBlob; - task_flags_.SetBits(TASK_LOW_LATENCY | TASK_COROUTINE); - if (!blob_id.IsNull()) { - lane_hash_ = blob_id.hash_; - domain_id_ = domain_id; - } else { - lane_hash_ = HashBlobName(tag_id, blob_name); - domain_id_ = DomainId::GetNode(HASH_TO_NODE_ID(lane_hash_)); - } - - // Custom params - tag_id_ = tag_id; - blob_id_ = blob_id; - blob_off_ = off; - data_size_ = data_size; - data_ = data; - flags_ = bitfield32_t(flags | ctx.flags_.bits_); - HSHM_MAKE_AR(blob_name_, alloc, blob_name); - } - - /** Convert data to a data structure */ - template - HSHM_ALWAYS_INLINE - void Get(T &obj) { - char *data = CHI_CLIENT->GetDataPointer(data_); - std::stringstream ss(std::string(data, data_size_)); - cereal::BinaryInputArchive ar(ss); - ar >> obj; - } - - /** Convert data to a data structure */ - template - HSHM_ALWAYS_INLINE - T Get() { - T obj; - return Get(obj); - } - - /** Destructor */ - ~GetBlobTask() { - HSHM_DESTROY_AR(blob_name_); - } - - /** (De)serialize message call */ - template - void SaveStart(Ar &ar) { - DataTransfer xfer(DT_RECEIVER_WRITE, - HERMES_MEMORY_MANAGER->Convert(data_), - data_size_, domain_id_); - task_serialize(ar); - ar & xfer; - ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, flags_); - } - - /** Deserialize message call */ - template - void LoadStart(Ar &ar) { - DataTransfer xfer; - task_serialize(ar); - ar & xfer; - data_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); - ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, flags_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - if (flags_.Any(HERMES_GET_BLOB_ID)) { - ar(blob_id_); - } - ar(data_size_); - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** A task to tag a blob */ -struct TagBlobTask : public Task, TaskFlags { - IN TagId tag_id_; - IN BlobId blob_id_; - IN TagId tag_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - TagBlobTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - TagBlobTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const BlobId &blob_id, - const TagId &tag) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kTagBlob; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom - tag_id_ = tag_id; - blob_id_ = blob_id; - tag_ = tag; - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(blob_id_, tag_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** - * Check if blob has a tag - * */ -struct BlobHasTagTask : public Task, TaskFlags { - IN TagId tag_id_; - IN BlobId blob_id_; - IN TagId tag_; - OUT bool has_tag_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - BlobHasTagTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - BlobHasTagTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const BlobId &blob_id, - const TagId &tag) : Task(alloc) { - // Initialize task - task_node_ = task_node; - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kBlobHasTag; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom - tag_id_ = tag_id; - blob_id_ = blob_id; - tag_ = tag; - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_id_, tag_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(has_tag_); - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** - * Get \a blob_name BLOB from \a bkt_id bucket - * */ -struct GetBlobIdTask : public Task, TaskFlags { - IN TagId tag_id_; - IN chi::charbuf blob_name_; - OUT BlobId blob_id_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobIdTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobIdTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const chi::charbuf &blob_name) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = HashBlobName(tag_id, blob_name); - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kGetBlobId; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom - tag_id_ = tag_id; - HSHM_MAKE_AR(blob_name_, alloc, blob_name) - } - - /** Destructor */ - ~GetBlobIdTask() { - HSHM_DESTROY_AR(blob_name_) - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_name_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(blob_id_); - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** - * Get \a blob_name BLOB name from \a blob_id BLOB id - * */ -struct GetBlobNameTask : public Task, TaskFlags { - IN TagId tag_id_; - IN BlobId blob_id_; - OUT chi::string blob_name_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobNameTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobNameTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const BlobId &blob_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kGetBlobName; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom - tag_id_ = tag_id; - blob_id_ = blob_id; - HSHM_MAKE_AR0(blob_name_, alloc) - } - - /** Destructor */ - ~GetBlobNameTask() { - HSHM_DESTROY_AR(blob_name_) - }; - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_id_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(blob_name_); - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** Get \a score from \a blob_id BLOB id */ -struct GetBlobSizeTask : public Task, TaskFlags { - IN TagId tag_id_; - IN chi::charbuf blob_name_; - IN BlobId blob_id_; - OUT size_t size_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobSizeTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobSizeTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const chi::charbuf &blob_name, - const BlobId &blob_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kGetBlobSize; - task_flags_.SetBits(TASK_LOW_LATENCY); - if (!blob_id.IsNull()) { - lane_hash_ = blob_id.hash_; - domain_id_ = domain_id; - } else { - lane_hash_ = HashBlobName(tag_id, blob_name); - domain_id_ = DomainId::GetNode(HASH_TO_NODE_ID(lane_hash_)); - } - - // Custom - tag_id_ = tag_id; - HSHM_MAKE_AR(blob_name_, alloc, blob_name) - blob_id_ = blob_id; - } - - /** Destructor */ - ~GetBlobSizeTask() { - HSHM_DESTROY_AR(blob_name_) - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_name_, blob_id_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(size_); - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** Get \a score from \a blob_id BLOB id */ -struct GetBlobScoreTask : public Task, TaskFlags { - IN TagId tag_id_; - IN BlobId blob_id_; - OUT float score_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobScoreTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobScoreTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const BlobId &blob_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kGetBlobScore; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom - tag_id_ = tag_id; - blob_id_ = blob_id; - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_id_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(score_); - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** Get \a blob_id blob's buffers */ -struct GetBlobBuffersTask : public Task, TaskFlags { - IN TagId tag_id_; - IN BlobId blob_id_; - OUT hipc::ShmArchive> buffers_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobBuffersTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - GetBlobBuffersTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const BlobId &blob_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kGetBlobBuffers; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom - tag_id_ = tag_id; - blob_id_ = blob_id; - HSHM_MAKE_AR0(buffers_, alloc) - } - - /** Destructor */ - ~GetBlobBuffersTask() { - HSHM_DESTROY_AR(buffers_) - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_id_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(buffers_); - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** - * Rename \a blob_id blob to \a new_blob_name new blob name - * in \a bkt_id bucket. - * */ -struct RenameBlobTask : public Task, TaskFlags { - IN TagId tag_id_; - IN BlobId blob_id_; - IN chi::charbuf new_blob_name_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - RenameBlobTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - RenameBlobTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const BlobId &blob_id, - const chi::charbuf &new_blob_name) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kRenameBlob; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom - tag_id_ = tag_id; - blob_id_ = blob_id; - HSHM_MAKE_AR(new_blob_name_, alloc, new_blob_name) - } - - /** Destructor */ - ~RenameBlobTask() { - HSHM_DESTROY_AR(new_blob_name_) - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_id_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(new_blob_name_); - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** A task to truncate a blob */ -struct TruncateBlobTask : public Task, TaskFlags { - IN TagId tag_id_; - IN BlobId blob_id_; - IN u64 size_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - TruncateBlobTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - TruncateBlobTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const BlobId &blob_id, - u64 size) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kTruncateBlob; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom params - tag_id_ = tag_id; - blob_id_ = blob_id; - size_ = size; - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_id_, size_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** Phases of the destroy blob task */ -struct DestroyBlobPhase { - TASK_METHOD_T kFreeBuffers = 0; - TASK_METHOD_T kWaitFreeBuffers = 1; -}; - -/** A task to destroy a blob */ -struct DestroyBlobTask : public Task, TaskFlags { - IN TagId tag_id_; - IN BlobId blob_id_; - IN bool update_size_; - TEMP int phase_ = DestroyBlobPhase::kFreeBuffers; - TEMP hipc::ShmArchive> free_tasks_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - DestroyBlobTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - DestroyBlobTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const BlobId &blob_id, - bool update_size = true) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kDestroyBlob; - task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; - - // Custom params - tag_id_ = tag_id; - blob_id_ = blob_id; - update_size_ = update_size; - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_id_, update_size_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** Phases of the destroy blob task */ -struct ReorganizeBlobPhase { - TASK_METHOD_T kGet = 0; - TASK_METHOD_T kWaitGet = 1; - TASK_METHOD_T kPut = 2; -}; - -/** A task to reorganize a blob's composition in the hierarchy */ -struct ReorganizeBlobTask : public Task, TaskFlags { - IN chi::charbuf blob_name_; - IN BlobId blob_id_; - IN float score_; - IN u32 node_id_; - IN bool is_user_score_; - TEMP int phase_ = ReorganizeBlobPhase::kGet; - TEMP hipc::Pointer data_; - TEMP size_t data_size_; - TEMP GetBlobTask *get_task_; - TEMP PutBlobTask *put_task_; - TEMP TagId tag_id_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - ReorganizeBlobTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - ReorganizeBlobTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - const chi::charbuf &blob_name, - const BlobId &blob_id, - float score, - bool is_user_score, - const Context &ctx, - u32 task_flags = TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = blob_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kReorganizeBlob; - task_flags_.SetBits(task_flags); - domain_id_ = domain_id; - - // Custom params - tag_id_ = tag_id; - HSHM_MAKE_AR(blob_name_, alloc, blob_name); - blob_id_ = blob_id; - score_ = score; - node_id_ = ctx.node_id_; - is_user_score_ = is_user_score; - } - - /** Destructor */ - ~ReorganizeBlobTask() { - HSHM_DESTROY_AR(blob_name_) - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(tag_id_, blob_name_, blob_id_, score_, node_id_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - hipc::LocalSerialize srl(group); - srl << std::string("blob_op"); - srl << tag_id_; - return 0; - } -}; - -/** A task to reorganize a blob's composition in the hierarchy */ -struct FlushDataTask : public Task, TaskFlags { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - FlushDataTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - FlushDataTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id, - size_t period_ms) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = 0; - prio_ = TaskPrio::kLongRunningTether; - pool_ = pool_id; - method_ = Method::kFlushData; - task_flags_.SetBits( - TASK_LANE_ALL | - TASK_FIRE_AND_FORGET | - TASK_LONG_RUNNING | - TASK_COROUTINE | - TASK_REMOTE_DEBUG_MARK); - SetPeriodMs((double)period_ms); - domain_id_ = DomainId::GetLocal(); - } - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, FlushDataTask &other) { - task_dup(other); - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, FlushDataTask &dup_task) { - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Begin replication */ - void ReplicateStart(u32 count) {} - - /** Finalize replication */ - void ReplicateEnd() {} - -}; - -/** A task to collect blob metadata */ -struct PollBlobMetadataTask : public Task, TaskFlags { - TEMP chi::string my_blob_mdm_; - TEMP hipc::ShmArchive> blob_mdms_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - PollBlobMetadataTask(const hipc::CtxAllocator &alloc) : Task(alloc) { - HSHM_MAKE_AR0(blob_mdms_, alloc) - } - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - PollBlobMetadataTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = 0; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kPollBlobMetadata; - task_flags_.SetBits(TASK_LANE_ALL); - domain_id_ = chi::DomainQuery::GetGlobalBcast(); - - // Custom params - HSHM_MAKE_AR0(blob_mdms_, alloc) - HSHM_MAKE_AR0(my_blob_mdm_, alloc) - } - - /** Serialize blob info */ - void SerializeBlobMetadata(const std::vector &blob_info) { - std::stringstream ss; - cereal::BinaryOutputArchive ar(ss); - ar << blob_info; - (*my_blob_mdm_) = ss.str(); - } - - /** Deserialize blob info */ - void DeserializeBlobMetadata(const std::string &srl, std::vector &blob_mdms) { - std::vector tmp_blob_mdms; - std::stringstream ss(srl); - cereal::BinaryInputArchive ar(ss); - ar >> tmp_blob_mdms; - for (BlobInfo &blob_info : tmp_blob_mdms) { - blob_mdms.emplace_back(blob_info); - } - } - - /** Get combined output of all replicas */ - std::vector MergeBlobMetadata() { - std::vector blob_mdms; - for (const chi::string &srl : *blob_mdms_) { - DeserializeBlobMetadata(srl.str(), blob_mdms); - } - return blob_mdms; - } - - /** Deserialize final query output */ - std::vector DeserializeBlobMetadata() { - std::vector blob_mdms; - DeserializeBlobMetadata((*my_blob_mdm_).str(), blob_mdms); - return blob_mdms; - } - - /** Destructor */ - ~PollBlobMetadataTask() { - HSHM_DESTROY_AR(blob_mdms_) - HSHM_DESTROY_AR(my_blob_mdm_) - } - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, PollBlobMetadataTask &other) { - task_dup(other); - HSHM_MAKE_AR(blob_mdms_, alloc, *other.blob_mdms_) - HSHM_MAKE_AR(my_blob_mdm_, alloc, *other.my_blob_mdm_) - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, PollBlobMetadataTask &dup_task) { - (*blob_mdms_)[replica] = (*dup_task.my_blob_mdm_); - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(my_blob_mdm_); - } - - /** (De)serialize message return */ - template - void SaveEnd(Ar &ar) { - ar(my_blob_mdm_); - } - - /** (De)serialize message return */ - template - void LoadEnd(u32 replica, Ar &ar) { - ar(my_blob_mdm_); - DupEnd(replica, *this); - } - - /** Begin replication */ - void ReplicateStart(u32 count) { - blob_mdms_->resize(count); - } - - /** Finalize replication */ - void ReplicateEnd() { - std::vector blob_mdms = MergeBlobMetadata(); - SerializeBlobMetadata(blob_mdms); - } - -}; - -/** A task to collect blob metadata */ -struct PollTargetMetadataTask : public Task, TaskFlags { - OUT chi::string my_target_mdms_; - TEMP hipc::ShmArchive> target_mdms_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - PollTargetMetadataTask(const hipc::CtxAllocator &alloc) : Task(alloc) { - HSHM_MAKE_AR0(target_mdms_, alloc) - } - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - PollTargetMetadataTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = 0; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kPollTargetMetadata; - task_flags_.SetBits(TASK_COROUTINE); - domain_id_ = chi::DomainQuery::GetGlobalBcast(); - - // Custom params - HSHM_MAKE_AR0(my_target_mdms_, alloc) - HSHM_MAKE_AR0(target_mdms_, alloc) - } - - /** Serialize target info */ - void SerializeTargetMetadata(const std::vector &target_info) { - std::stringstream ss; - cereal::BinaryOutputArchive ar(ss); - ar << target_info; - (*my_target_mdms_) = ss.str(); - } - - /** Deserialize target info */ - void DeserializeTargetMetadata(const std::string &srl, std::vector &target_mdms) { - std::vector tmp_target_mdms; - std::stringstream ss(srl); - cereal::BinaryInputArchive ar(ss); - ar >> tmp_target_mdms; - for (TargetStats &target_info : tmp_target_mdms) { - target_mdms.emplace_back(target_info); - } - } - - /** Get combined output of all replicas */ - std::vector MergeTargetMetadata() { - std::vector target_mdms; - for (const chi::string &srl : *target_mdms_) { - DeserializeTargetMetadata(srl.str(), target_mdms); - } - return target_mdms; - } - - /** Deserialize final query output */ - std::vector DeserializeTargetMetadata() { - std::vector target_mdms; - DeserializeTargetMetadata(my_target_mdms_->str(), target_mdms); - return target_mdms; - } - - /** Destructor */ - ~PollTargetMetadataTask() { - HSHM_DESTROY_AR(my_target_mdms_) - HSHM_DESTROY_AR(target_mdms_) - } - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, PollTargetMetadataTask &other) {} - - /** Process duplicate message output */ - void DupEnd(u32 replica, PollTargetMetadataTask &dup_task) { - (*target_mdms_)[replica] = (*dup_task.my_target_mdms_); - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(my_target_mdms_); - } - - /** (De)serialize message return */ - template - void SaveEnd(Ar &ar) { - ar(my_target_mdms_); - } - - /** (De)serialize message return */ - template - void LoadEnd(u32 replica, Ar &ar) { - ar(my_target_mdms_); - DupEnd(replica, *this); - } - - /** Begin replication */ - void ReplicateStart(u32 count) { - target_mdms_->resize(count); - } - - /** Finalize replication */ - void ReplicateEnd() { - std::vector target_mdms = MergeTargetMetadata(); - SerializeTargetMetadata(target_mdms); - } - -}; - -} // namespace hermes::blob_mdm - -#endif //HRUN_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ diff --git a/tasks/old/hermes_blob_mdm/src/CMakeLists.txt b/tasks/old/hermes_blob_mdm/src/CMakeLists.txt deleted file mode 100644 index da977028e..000000000 --- a/tasks/old/hermes_blob_mdm/src/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Small Message Task Library -#------------------------------------------------------------------------------ -add_library(hermes_blob_mdm SHARED - hermes_blob_mdm.cc) -# add_dependencies(hermes_blob_mdm ${Hermes_RUNTIME_DEPS}) -target_link_libraries(hermes_blob_mdm ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Install Small Message Task Library -#------------------------------------------------------------------------------ -install( - TARGETS - hermes_blob_mdm - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install for import into other projects -#----------------------------------------------------------------------------- -install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_blob_mdm - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#------------------------------------------------------------------------------ -# Coverage -#------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_blob_mdm) -endif() diff --git a/tasks/old/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/old/hermes_blob_mdm/src/hermes_blob_mdm.cc deleted file mode 100644 index cf20fa07c..000000000 --- a/tasks/old/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ /dev/null @@ -1,972 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/api/chimaera_runtime.h" -#include "hermes/config_server.h" -#include "hermes_blob_mdm/hermes_blob_mdm.h" -#include "hermes_adapters/mapper/mapper_factory.h" -#include "hermes/dpe/dpe_factory.h" -#include "hermes_adapters/posix/posix_api.h" -#include "bdev/bdev.h" -#include "data_stager/data_stager.h" -#include "hermes_data_op/hermes_data_op.h" -#include "hermes/score_histogram.h" - -namespace hermes::blob_mdm { - -/** Type name simplification for the various map types */ -typedef std::unordered_map BLOB_ID_MAP_T; -typedef std::unordered_map BLOB_MAP_T; -typedef hipc::mpsc_queue IO_PATTERN_LOG_T; - -class Server : public Module { - public: - /**==================================== - * Configuration - * ===================================*/ - u32 node_id_; - - /**==================================== - * Maps - * ===================================*/ - std::vector blob_id_map_; - std::vector blob_map_; - std::atomic id_alloc_; - - /**==================================== - * I/O pattern log - * ===================================*/ - IO_PATTERN_LOG_T *io_pattern_log_; - bool enable_io_tracing_; - bool is_mpi_; - - /**==================================== - * Targets + devices - * ===================================*/ - std::vector target_tasks_; - std::vector targets_; - bdev::Client *fallback_target_; - std::unordered_map target_map_; - Client blob_mdm_; - bucket_mdm::Client bkt_mdm_; - data_stager::Client stager_mdm_; - data_op::Client op_mdm_; - LPointer flush_task_; - - public: - Server() = default; - - /** Construct blob mdm and it targets */ - void Construct(ConstructTask *task, RunContext &rctx) { - id_alloc_ = 0; - node_id_ = CHI_CLIENT->node_id_; - // Initialize blob maps - blob_id_map_.resize(HRUN_QM_RUNTIME->max_lanes_); - blob_map_.resize(HRUN_QM_RUNTIME->max_lanes_); - // Initialize targets - target_tasks_.reserve(HERMES_SERVER_CONF.devices_.size()); - for (DeviceInfo &dev : HERMES_SERVER_CONF.devices_) { - std::string dev_type; - if (dev.mount_dir_.empty()) { - dev_type = "ram_bdev"; - dev.mount_point_ = - hshm::Formatter::format("{}/{}", dev.mount_dir_, dev.dev_name_); - } else { - dev_type = "posix_bdev"; - } - targets_.emplace_back(); - bdev::Client &client = targets_.back(); - bdev::ConstructTask *create_task = client.AsyncCreate( - task->task_node_ + 1, - DomainId::GetLocal(), - "hermes_" + dev.dev_name_ + "/" + std::to_string(CHI_CLIENT->node_id_), - dev_type, - dev).ptr_; - target_tasks_.emplace_back(create_task); - } - for (int i = 0; i < target_tasks_.size(); ++i) { - bdev::ConstructTask *tgt_task = target_tasks_[i]; - tgt_task->Wait(task); - bdev::Client &client = targets_[i]; - client.AsyncCreateComplete(tgt_task); - } - std::sort(targets_.begin(), targets_.end(), - [](const bdev::Client &a, const bdev::Client &b) { - return a.bandwidth_ > b.bandwidth_; - }); - float bw_max = targets_.front().bandwidth_; - float bw_min = targets_.back().bandwidth_; - for (bdev::Client &client : targets_) { - client.bw_score_ = (client.bandwidth_ - bw_min) / (bw_max - bw_min); - client.score_ = client.bw_score_; - } - for (bdev::Client &client : targets_) { - target_map_.emplace(client.id_, &client); - HILOG(kInfo, "(node {}) Target {} has bw {} and score {}", CHI_CLIENT->node_id_, - client.id_, client.bandwidth_, client.bw_score_); - } - fallback_target_ = &targets_.back(); - blob_mdm_.Init(id_, CHI_ADMIN->queue_id_); - HILOG(kInfo, "(node {}) Created Blob MDM", CHI_CLIENT->node_id_); - task->SetModuleComplete(); - } - void MonitorConstruct(u32 mode, ConstructTask *task, RunContext &rctx) { - } - - /** Destroy blob mdm */ - void Destruct(DestructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - void MonitorDestruct(u32 mode, DestructTask *task, RunContext &rctx) { - } - - private: - /** Get the globally unique blob name */ - const chi::charbuf GetBlobNameWithBucket(TagId tag_id, const chi::charbuf &blob_name) { - chi::charbuf new_name(sizeof(TagId) + blob_name.size()); - hipc::LocalSerialize srl(new_name); - srl << tag_id; - srl << blob_name; - return new_name; - } - - public: - /** - * Set the Bucket MDM - * */ - void SetBucketMdm(SetBucketMdmTask *task, RunContext &rctx) { - if (bkt_mdm_.id_.IsNull()) { - bkt_mdm_.Init(task->bkt_mdm_, CHI_ADMIN->queue_id_); - stager_mdm_.Init(task->stager_mdm_, CHI_ADMIN->queue_id_); - op_mdm_.Init(task->op_mdm_, CHI_ADMIN->queue_id_); - flush_task_ = blob_mdm_.AsyncFlushData( - task->task_node_ + 1, HERMES_SERVER_CONF.borg_.flush_period_); - } - task->SetModuleComplete(); - } - void MonitorSetBucketMdm(u32 mode, SetBucketMdmTask *task, RunContext &rctx) { - } - - /** New score */ - float NormalizeScore(float score) { - if (score > 1) { - return 1; - } - if (score < 0) { - return 0; - } - return score; - } - float MakeScore(BlobInfo &blob_info, hshm::Timepoint &now) { - ServerConfig &server = HERMES_CONF->server_config_; - // Frequency score: how many times blob accessed? - float freq_min = server.borg_.freq_min_; - float freq_diff = server.borg_.freq_max_ - freq_min; - float freq_score = NormalizeScore((blob_info.access_freq_ - freq_min) / freq_diff); - // Temporal score: how recently the blob was accessed? - float time_diff = blob_info.last_access_.GetSecFromStart(now); - float rec_min = server.borg_.recency_min_; - float rec_max = server.borg_.recency_max_; - float rec_diff = rec_max - rec_min; - float temporal_score = NormalizeScore((time_diff - rec_min) / rec_diff); - temporal_score = 1 - temporal_score; - // Access score: was the blob accessed recently or frequently? - float access_score = std::max(freq_score, temporal_score); - float user_score = blob_info.user_score_; - // Final scores - if (!blob_info.flags_.Any(HERMES_USER_SCORE_STATIONARY)) { - return user_score * access_score; - } else { - return std::max(access_score, user_score); - } - } - const bdev::Client& FindNearestTarget(float score) { - for (const bdev::Client &cmp_tgt: targets_) { - if (cmp_tgt.score_ > score + .05) { - continue; - } - return cmp_tgt; - } - return targets_.back(); - } - - /** Check if blob should be reorganized */ - template - bool ShouldReorganize(BlobInfo &blob_info, - float score, - TaskNode &task_node) { - ServerConfig &server = HERMES_CONF->server_config_; - for (BufferInfo &buf : blob_info.buffers_) { - TargetInfo &target = *target_map_[buf.tid_]; - Histogram &hist = target.monitor_task_->score_hist_; - u32 percentile = hist.GetPercentile(score); - u32 precentile_lt = hist.GetPercentileLT(score); - size_t rem_cap = target.monitor_task_->rem_cap_; - size_t max_cap = target.max_cap_; - float borg_cap_min = target.borg_min_thresh_; - float borg_cap_max = target.borg_max_thresh_; - // float min_score = hist.GetQuantile(0); - // Update the target score - target.score_ = target.bw_score_; - // Update blob score - if constexpr(UPDATE_SCORE) { - u32 bin_orig = hist.GetBin(blob_info.score_); - u32 bin_new = hist.GetBin(score); - if (bin_orig != bin_new) { - target.AsyncUpdateScore(task_node + 1, - blob_info.score_, score); - } - } - // Determine if the blob should be reorganized - // Get the target with minimum difference in score to this blob - if (abs(target.score_ - score) < .1) { - continue; - } - const bdev::Client &cmp_tgt = FindNearestTarget(score); - if (cmp_tgt.id_ == target.id_) { - continue; - } - if (cmp_tgt.score_ <= target.score_) { - // Demote if we have sufficiently low capacity - if (rem_cap < max_cap * borg_cap_min) { - HILOG(kInfo, "Demoting blob {} of score {} from tgt={} tgt_score={} to tgt={} tgt_score={}", - blob_info.blob_id_, blob_info.score_, - target.id_, target.score_, - cmp_tgt.id_, cmp_tgt.score_); - return true; - } - } else { - // Promote if the guy above us has sufficiently high capacity - float cmp_rem_cap = cmp_tgt.monitor_task_->rem_cap_; - if (cmp_rem_cap > blob_info.blob_size_) { - HILOG(kInfo, "Promoting blob {} of score {} from tgt={} tgt_score={} to tgt={} tgt_score={}", - blob_info.blob_id_, blob_info.score_, - target.id_, target.score_, - cmp_tgt.id_, cmp_tgt.score_); - return true; - } - } - } - return false; - } - - /** - * Long-running task to stage out data periodically and - * reorganize blobs - * */ - struct FlushInfo { - BlobInfo *blob_info_; - LPointer stage_task_; - size_t mod_count_; - }; - void FlushData(FlushDataTask *task, RunContext &rctx) { - hshm::Timepoint now; - now.Now(); - // Get the blob info data structure - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - std::vector stage_tasks; - stage_tasks.reserve(256); - for (auto &it : blob_map) { - BlobInfo &blob_info = it.second; - // Update blob scores - float new_score = MakeScore(blob_info, now); - blob_info.score_ = new_score; - if (ShouldReorganize(blob_info, new_score, task->task_node_)) { - Context ctx; - LPointer reorg_task = - blob_mdm_.AsyncReorganizeBlob(task->task_node_ + 1, - blob_info.tag_id_, - chi::charbuf(""), - blob_info.blob_id_, - new_score, false, ctx, - TASK_LOW_LATENCY); - reorg_task->Wait(task); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, reorg_task); - } - blob_info.access_freq_ = 0; - - // Flush data - FlushInfo flush_info; - flush_info.blob_info_ = &blob_info; - flush_info.mod_count_ = blob_info.mod_count_; - if (blob_info.last_flush_ > 0 && - flush_info.mod_count_ > blob_info.last_flush_) { - HILOG(kDebug, "Flushing blob {} (mod_count={}, last_flush={})", - blob_info.blob_id_, flush_info.mod_count_, blob_info.last_flush_); - LPointer data = CHI_CLIENT->AllocateBufferServer( - blob_info.blob_size_, task); - LPointer get_blob = - blob_mdm_.AsyncGetBlob(task->task_node_ + 1, - blob_info.tag_id_, - blob_info.name_, - blob_info.blob_id_, - 0, blob_info.blob_size_, - data.shm_); - get_blob->Wait(task); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, get_blob); - flush_info.stage_task_ = - stager_mdm_.AsyncStageOut(task->task_node_ + 1, - blob_info.tag_id_, - blob_info.name_, - data.shm_, blob_info.blob_size_, - TASK_DATA_OWNER); - stage_tasks.emplace_back(flush_info); - } - if (stage_tasks.size() == 256) { - break; - } - } - - for (FlushInfo &flush_info : stage_tasks) { - BlobInfo &blob_info = *flush_info.blob_info_; - flush_info.stage_task_->Wait(task); - blob_info.last_flush_ = flush_info.mod_count_; - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, flush_info.stage_task_); - } - } - void MonitorFlushData(u32 mode, FlushDataTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - for (auto &it : blob_map) { - BlobInfo &blob_info = it.second; - if (blob_info.last_flush_ > 0 && - blob_info.mod_count_ > blob_info.last_flush_) { - rctx.flush_->count_ += 1; - return; - } - } - } - - /** - * Create a blob's metadata - * */ - void PutBlob(PutBlobTask *task, RunContext &rctx) { - // Get the blob info data structure - chi::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - if (task->blob_id_.IsNull()) { - task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, - blob_name, rctx, task->flags_); - } - HILOG(kDebug, "Beginning PUT for (hash: {})", - std::hash{}(blob_name)); - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - BlobInfo &blob_info = blob_map[task->blob_id_]; - blob_info.score_ = task->score_; - blob_info.user_score_ = task->score_; - - // Stage Blob - if (task->flags_.Any(HERMES_SHOULD_STAGE) && blob_info.last_flush_ == 0) { - HILOG(kDebug, "This file has not yet been flushed"); - blob_info.last_flush_ = 1; - LPointer stage_task = - stager_mdm_.AsyncStageIn(task->task_node_ + 1, - task->tag_id_, - blob_info.name_, - task->score_, 0); - stage_task->Wait(task); - blob_info.mod_count_ = 1; - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, stage_task); - } - if (task->flags_.Any(HERMES_SHOULD_STAGE)) { - HILOG(kDebug, "This is marked as a file: {} {}", - blob_info.mod_count_, blob_info.last_flush_); - } - ssize_t bkt_size_diff = 0; - if (task->flags_.Any(HERMES_BLOB_REPLACE)) { - bkt_size_diff -= blob_info.blob_size_; - PutBlobFreeBuffersPhase(blob_info, task, rctx); - } - - // Determine amount of additional buffering space needed - size_t needed_space = task->blob_off_ + task->data_size_; - size_t size_diff = 0; - if (needed_space > blob_info.max_blob_size_) { - size_diff = needed_space - blob_info.max_blob_size_; - } - size_t min_blob_size = task->blob_off_ + task->data_size_; - if (min_blob_size > blob_info.blob_size_) { - blob_info.blob_size_ = task->blob_off_ + task->data_size_; - } - bkt_size_diff += (ssize_t)size_diff; - HILOG(kDebug, "The size diff is {} bytes (bkt diff {})", size_diff, bkt_size_diff) - - // Use DPE - std::vector schema_vec; - if (size_diff > 0) { - Context ctx; - auto *dpe = DpeFactory::Get(ctx.dpe_); - ctx.blob_score_ = task->score_; - dpe->Placement({size_diff}, targets_, ctx, schema_vec); - } - - // Allocate blob buffers - for (PlacementSchema &schema : schema_vec) { - schema.plcmnts_.emplace_back(0, fallback_target_->id_); - for (size_t sub_idx = 0; sub_idx < schema.plcmnts_.size(); ++sub_idx) { - SubPlacement &placement = schema.plcmnts_[sub_idx]; - TargetInfo &bdev = *target_map_[placement.tid_]; - LPointer alloc_task = - bdev.AsyncAllocate(task->task_node_ + 1, - blob_info.score_, - placement.size_, - blob_info.buffers_); - alloc_task->Wait(task); -// HILOG(kInfo, "(node {}) Placing {}/{} bytes in target {} of bw {}", -// CHI_CLIENT->node_id_, -// alloc_task->alloc_size_, task->data_size_, -// placement.tid_, bdev.bandwidth_) - if (alloc_task->alloc_size_ < alloc_task->size_) { - SubPlacement &next_placement = schema.plcmnts_[sub_idx + 1]; - size_t diff = alloc_task->size_ - alloc_task->alloc_size_; - next_placement.size_ += diff; - } - // bdev.monitor_task_->rem_cap_ -= alloc_task->alloc_size_; - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, alloc_task); - } - } - - // Place blob in buffers - std::vector> write_tasks; - write_tasks.reserve(blob_info.buffers_.size()); - size_t blob_off = task->blob_off_, buf_off = 0; - size_t buf_left = 0, buf_right = 0; - size_t blob_right = task->blob_off_ + task->data_size_; - char *blob_buf = CHI_CLIENT->GetDataPointer(task->data_); - HILOG(kDebug, "Number of buffers {}", blob_info.buffers_.size()); - bool found_left = false; - for (BufferInfo &buf : blob_info.buffers_) { - buf_right = buf_left + buf.t_size_; - if (blob_off >= blob_right) { - break; - } - if (buf_left <= blob_off && blob_off < buf_right) { - found_left = true; - } - if (found_left) { - size_t rel_off = blob_off - buf_left; - size_t tgt_off = buf.t_off_ + rel_off; - size_t buf_size = buf.t_size_ - rel_off; - if (buf_right > blob_right) { - buf_size = blob_right - (buf_left + rel_off); - } - HILOG(kDebug, "Writing {} bytes at off {} from target {}", buf_size, tgt_off, buf.tid_) - TargetInfo &target = *target_map_[buf.tid_]; - LPointer write_task = - target.AsyncWrite(task->task_node_ + 1, - blob_buf + buf_off, - tgt_off, buf_size); - write_tasks.emplace_back(write_task); - buf_off += buf_size; - blob_off = buf_right; - } - buf_left += buf.t_size_; - } - blob_info.max_blob_size_ = blob_off; - - // Wait for the placements to complete - for (LPointer &write_task : write_tasks) { - write_task->Wait(task); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, write_task); - } - - // Update information - if (task->flags_.Any(HERMES_SHOULD_STAGE)) { - stager_mdm_.AsyncUpdateSize(task->task_node_ + 1, - task->tag_id_, - blob_info.name_, - task->blob_off_, - task->data_size_, 0); - } else { - bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, - task->tag_id_, - bkt_size_diff, - UpdateSizeMode::kAdd); - } - if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { - bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, - task->tag_id_, - task->blob_id_); - } - if (task->flags_.Any(HERMES_HAS_DERIVED)) { - op_mdm_.AsyncRegisterData(task->task_node_ + 1, - task->tag_id_, - task->blob_name_->str(), - task->blob_id_, - task->blob_off_, - task->data_size_); - } - - // Free data - HILOG(kDebug, "Completing PUT for {}", blob_name.str()); - blob_info.UpdateWriteStats(); - task->SetModuleComplete(); - } - void MonitorPutBlob(u32 mode, PutBlobTask *task, RunContext &rctx) { - } - - /** Release buffers */ - void PutBlobFreeBuffersPhase(BlobInfo &blob_info, PutBlobTask *task, RunContext &rctx) { - for (BufferInfo &buf : blob_info.buffers_) { - TargetInfo &target = *target_map_[buf.tid_]; - std::vector buf_vec = {buf}; - target.AsyncFree(task->task_node_ + 1, - blob_info.score_, - std::move(buf_vec), true); - } - blob_info.buffers_.clear(); - blob_info.max_blob_size_ = 0; - blob_info.blob_size_ = 0; - } - - /** Get a blob's data */ - void GetBlob(GetBlobTask *task, RunContext &rctx) { - if (task->blob_id_.IsNull()) { - chi::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, - blob_name, rctx, task->flags_); - } - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - BlobInfo &blob_info = blob_map[task->blob_id_]; - - // Stage Blob - if (task->flags_.Any(HERMES_SHOULD_STAGE) && blob_info.last_flush_ == 0) { - // TODO(llogan): Don't hardcore score = 1 - blob_info.last_flush_ = 1; - LPointer stage_task = - stager_mdm_.AsyncStageIn(task->task_node_ + 1, - task->tag_id_, - blob_info.name_, - 1, 0); - stage_task->Wait(task); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, stage_task); - } - - // Read blob from buffers - std::vector read_tasks; - read_tasks.reserve(blob_info.buffers_.size()); - HILOG(kDebug, "Getting blob {} of size {} starting at offset {} (total_blob_size={}, buffers={})", - task->blob_id_, task->data_size_, task->blob_off_, blob_info.blob_size_, blob_info.buffers_.size()); - size_t blob_off = task->blob_off_; - size_t buf_left = 0, buf_right = 0; - size_t buf_off = 0; - size_t blob_right = task->blob_off_ + task->data_size_; - bool found_left = false; - char *blob_buf = CHI_CLIENT->GetDataPointer(task->data_); - for (BufferInfo &buf : blob_info.buffers_) { - buf_right = buf_left + buf.t_size_; - if (blob_off >= blob_right) { - break; - } - if (buf_left <= blob_off && blob_off < buf_right) { - found_left = true; - } - if (found_left) { - size_t rel_off = blob_off - buf_left; - size_t tgt_off = buf.t_off_ + rel_off; - size_t buf_size = buf.t_size_ - rel_off; - if (buf_right > blob_right) { - buf_size = blob_right - (buf_left + rel_off); - } - HILOG(kDebug, "Loading {} bytes at off {} from target {}", buf_size, tgt_off, buf.tid_) - TargetInfo &target = *target_map_[buf.tid_]; - bdev::ReadTask *read_task = target.AsyncRead(task->task_node_ + 1, - blob_buf + buf_off, - tgt_off, buf_size).ptr_; - read_tasks.emplace_back(read_task); - buf_off += buf_size; - blob_off = buf_right; - } - buf_left += buf.t_size_; - } - for (bdev::ReadTask *&read_task : read_tasks) { - read_task->Wait(task); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, read_task); - } - task->data_size_ = buf_off; - task->SetModuleComplete(); - } - void MonitorGetBlob(u32 mode, GetBlobTask *task, RunContext &rctx) { - } - - /** - * Tag a blob - * */ - void TagBlob(TagBlobTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BlobInfo &blob = it->second; - blob.tags_.push_back(task->tag_); - task->SetModuleComplete(); - } - void MonitorTagBlob(u32 mode, TagBlobTask *task, RunContext &rctx) { - } - - /** - * Check if blob has a tag - * */ - void BlobHasTag(BlobHasTagTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BlobInfo &blob = it->second; - task->has_tag_ = std::find(blob.tags_.begin(), - blob.tags_.end(), - task->tag_) != blob.tags_.end(); - task->SetModuleComplete(); - } - void MonitorBlobHasTag(u32 mode, BlobHasTagTask *task, RunContext &rctx) { - } - - /** - * Create \a blob_id BLOB ID - * */ - BlobId GetOrCreateBlobId(TagId &tag_id, u32 lane_hash, - const chi::charbuf &blob_name, RunContext &rctx, - bitfield32_t &flags) { - chi::charbuf blob_name_unique = GetBlobNameWithBucket(tag_id, blob_name); - BLOB_ID_MAP_T &blob_id_map = blob_id_map_[rctx.lane_id_]; - auto it = blob_id_map.find(blob_name_unique); - if (it == blob_id_map.end()) { - BlobId blob_id = BlobId(node_id_, lane_hash, id_alloc_.fetch_add(1)); - blob_id_map.emplace(blob_name_unique, blob_id); - flags.SetBits(HERMES_BLOB_DID_CREATE); - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - blob_map.emplace(blob_id, BlobInfo()); - BlobInfo &blob_info = blob_map[blob_id]; - blob_info.name_ = blob_name; - blob_info.blob_id_ = blob_id; - blob_info.tag_id_ = tag_id; - blob_info.blob_size_ = 0; - blob_info.max_blob_size_ = 0; - blob_info.score_ = 1; - blob_info.mod_count_ = 0; - blob_info.access_freq_ = 0; - blob_info.last_flush_ = 0; - return blob_id; - } - return it->second; - } - void GetOrCreateBlobId(GetOrCreateBlobIdTask *task, RunContext &rctx) { - chi::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - bitfield32_t flags; - task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, blob_name, rctx, flags); - task->SetModuleComplete(); - } - void MonitorGetOrCreateBlobId(u32 mode, GetOrCreateBlobIdTask *task, RunContext &rctx) { - } - - /** - * Get \a blob_name BLOB from \a bkt_id bucket - * */ - HSHM_ALWAYS_INLINE - void GetBlobId(GetBlobIdTask *task, RunContext &rctx) { - chi::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - chi::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); - BLOB_ID_MAP_T &blob_id_map = blob_id_map_[rctx.lane_id_]; - auto it = blob_id_map.find(blob_name_unique); - if (it == blob_id_map.end()) { - task->blob_id_ = BlobId::GetNull(); - task->SetModuleComplete(); - HILOG(kDebug, "Failed to find blob {} in {}", blob_name.str(), task->tag_id_); - return; - } - task->blob_id_ = it->second; - HILOG(kDebug, "Found blob {} / {} in {}", task->blob_id_, blob_name.str(), task->tag_id_); - task->SetModuleComplete(); - } - void MonitorGetBlobId(u32 mode, GetBlobIdTask *task, RunContext &rctx) { - } - - /** - * Get \a blob_name BLOB name from \a blob_id BLOB id - * */ - void GetBlobName(GetBlobNameTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BlobInfo &blob = it->second; - (*task->blob_name_) = blob.name_; - task->SetModuleComplete(); - } - void MonitorGetBlobName(u32 mode, GetBlobNameTask *task, RunContext &rctx) { - } - - /** - * Get \a score from \a blob_id BLOB id - * */ - void GetBlobSize(GetBlobSizeTask *task, RunContext &rctx) { - if (task->blob_id_.IsNull()) { - bitfield32_t flags; - task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, - hshm::to_charbuf(*task->blob_name_), - rctx, flags); - } - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->size_ = 0; - task->SetModuleComplete(); - return; - } - BlobInfo &blob = it->second; - task->size_ = blob.blob_size_; - task->SetModuleComplete(); - } - void MonitorGetBlobSize(u32 mode, GetBlobSizeTask *task, RunContext &rctx) { - } - - /** - * Get \a score from \a blob_id BLOB id - * */ - void GetBlobScore(GetBlobScoreTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BlobInfo &blob = it->second; - task->score_ = blob.score_; - task->SetModuleComplete(); - } - void MonitorGetBlobScore(u32 mode, GetBlobScoreTask *task, RunContext &rctx) { - } - - /** - * Get \a blob_id blob's buffers - * */ - void GetBlobBuffers(GetBlobBuffersTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BlobInfo &blob = it->second; - (*task->buffers_) = blob.buffers_; - task->SetModuleComplete(); - } - void MonitorGetBlobBuffers(u32 mode, GetBlobBuffersTask *task, RunContext &rctx) { - } - - /** - * Rename \a blob_id blob to \a new_blob_name new blob name - * in \a bkt_id bucket. - * */ - void RenameBlob(RenameBlobTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BLOB_ID_MAP_T &blob_id_map = blob_id_map_[rctx.lane_id_]; - BlobInfo &blob = it->second; - blob_id_map.erase(blob.name_); - blob_id_map[blob.name_] = task->blob_id_; - blob.name_ = hshm::to_charbuf(*task->new_blob_name_); - task->SetModuleComplete(); - } - void MonitorRenameBlob(u32 mode, RenameBlobTask *task, RunContext &rctx) { - } - - /** - * Truncate a blob to a new size - * */ - void TruncateBlob(TruncateBlobTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BlobInfo &blob_info = it->second; - // TODO(llogan): truncate blob - task->SetModuleComplete(); - } - void MonitorTruncateBlob(u32 mode, TruncateBlobTask *task, RunContext &rctx) { - } - - /** - * Destroy \a blob_id blob in \a bkt_id bucket - * */ - void DestroyBlob(DestroyBlobTask *task, RunContext &rctx) { - switch (task->phase_) { - case DestroyBlobPhase::kFreeBuffers: { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BLOB_ID_MAP_T &blob_id_map = blob_id_map_[rctx.lane_id_]; - BlobInfo &blob_info = it->second; - chi::charbuf unique_name = GetBlobNameWithBucket(blob_info.tag_id_, blob_info.name_); - blob_id_map.erase(unique_name); - HSHM_MAKE_AR0(task->free_tasks_, nullptr); - task->free_tasks_->reserve(blob_info.buffers_.size()); - for (BufferInfo &buf : blob_info.buffers_) { - TargetInfo &tgt_info = *target_map_[buf.tid_]; - std::vector buf_vec = {buf}; - bdev::FreeTask *free_task = tgt_info.AsyncFree( - task->task_node_ + 1, blob_info.score_, - std::move(buf_vec), false).ptr_; - task->free_tasks_->emplace_back(free_task); - } - task->phase_ = DestroyBlobPhase::kWaitFreeBuffers; - } - case DestroyBlobPhase::kWaitFreeBuffers: { - std::vector &free_tasks = *task->free_tasks_; - for (bdev::FreeTask *&free_task : free_tasks) { - if (!free_task->IsComplete()) { - return; - } - } - for (bdev::FreeTask *&free_task : free_tasks) { - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, free_task); - } - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - BlobInfo &blob_info = blob_map[task->blob_id_]; - if (task->update_size_) { - bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, - task->tag_id_, - -(ssize_t) blob_info.blob_size_, - UpdateSizeMode::kAdd); - } - HSHM_DESTROY_AR(task->free_tasks_); - blob_map.erase(task->blob_id_); - task->SetModuleComplete(); - } - } - } - void MonitorDestroyBlob(u32 mode, DestroyBlobTask *task, RunContext &rctx) { - } - - /** - * Reorganize \a blob_id blob in \a bkt_id bucket - * */ - void ReorganizeBlob(ReorganizeBlobTask *task, RunContext &rctx) { - switch (task->phase_) { - case ReorganizeBlobPhase::kGet: { - chi::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - if (task->blob_id_.IsNull()) { - bitfield32_t flags; - task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, - blob_name, rctx, flags); - } - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->SetModuleComplete(); - return; - } - BlobInfo &blob_info = it->second; - if (task->is_user_score_) { - blob_info.user_score_ = task->score_; - blob_info.score_ = blob_info.user_score_; - } else { - blob_info.score_ = task->score_; - } -// if (!ShouldReorganize(blob_info, task->score_, task->task_node_)) { -// task->SetModuleComplete(); -// return; -// } - task->data_ = CHI_CLIENT->AllocateBufferServer( - blob_info.blob_size_, task).shm_; - task->data_size_ = blob_info.blob_size_; - task->get_task_ = blob_mdm_.AsyncGetBlob(task->task_node_ + 1, - task->tag_id_, - chi::charbuf(""), - task->blob_id_, - 0, - task->data_size_, - task->data_).ptr_; - task->tag_id_ = blob_info.tag_id_; - task->phase_ = ReorganizeBlobPhase::kWaitGet; - } - case ReorganizeBlobPhase::kWaitGet: { - if (!task->get_task_->IsComplete()) { - return; - } - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, task->get_task_); - task->phase_ = ReorganizeBlobPhase::kPut; - } - case ReorganizeBlobPhase::kPut: { - task->put_task_ = blob_mdm_.AsyncPutBlob( - task->task_node_ + 1, - task->tag_id_, chi::charbuf(""), - task->blob_id_, 0, - task->data_size_, - task->data_, - task->score_, - HERMES_BLOB_REPLACE | TASK_FIRE_AND_FORGET | TASK_DATA_OWNER).ptr_; - task->SetModuleComplete(); - } - } - } - void MonitorReorganizeBlob(u32 mode, ReorganizeBlobTask *task, RunContext &rctx) { - } - - /** - * Get all metadata about a blob - * */ - HSHM_ALWAYS_INLINE - void PollBlobMetadata(PollBlobMetadataTask *task, RunContext &rctx) { - BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - std::vector blob_mdms; - blob_mdms.reserve(blob_map.size()); - for (const std::pair &blob_part : blob_map) { - const BlobInfo &blob_info = blob_part.second; - blob_mdms.emplace_back(blob_info); - } - task->SerializeBlobMetadata(blob_mdms); - task->SetModuleComplete(); - } - void MonitorPollBlobMetadata(u32 mode, PollBlobMetadataTask *task, RunContext &rctx) { - } - - /** - * Get all metadata about a blob - * */ - HSHM_ALWAYS_INLINE - void PollTargetMetadata(PollTargetMetadataTask *task, RunContext &rctx) { - std::vector target_mdms; - target_mdms.reserve(targets_.size()); - for (const bdev::Client &bdev_client : targets_) { - bool is_remote = bdev_client.domain_id_.IsRemote(HRUN_RPC->GetNumHosts(), CHI_CLIENT->node_id_); - if (is_remote) { - continue; - } - TargetStats stats; - stats.tgt_id_ = bdev_client.id_; - stats.node_id_ = CHI_CLIENT->node_id_; - stats.rem_cap_ = bdev_client.monitor_task_->rem_cap_; - stats.max_cap_ = bdev_client.max_cap_; - stats.bandwidth_ = bdev_client.bandwidth_; - stats.latency_ = bdev_client.latency_; - stats.score_ = bdev_client.score_; - target_mdms.emplace_back(stats); - } - task->SerializeTargetMetadata(target_mdms); - task->SetModuleComplete(); - } - void MonitorPollTargetMetadata(u32 mode, PollTargetMetadataTask *task, RunContext &rctx) { - } - - public: -#include "hermes_blob_mdm/hermes_blob_mdm_lib_exec.h" -}; - -} // namespace hrun - -HRUN_TASK_CC(hermes::blob_mdm::Server, "hermes_blob_mdm"); diff --git a/tasks/old/hermes_bucket_mdm/CMakeLists.txt b/tasks/old/hermes_bucket_mdm/CMakeLists.txt deleted file mode 100644 index be52b3c1e..000000000 --- a/tasks/old/hermes_bucket_mdm/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Hermes Bucket MDM Task Library -#------------------------------------------------------------------------------ -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install HRUN Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h deleted file mode 100644 index 353e24538..000000000 --- a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ /dev/null @@ -1,65 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#ifndef HRUN_hermes_bucket_mdm_H_ -#define HRUN_hermes_bucket_mdm_H_ - -#include "hermes_bucket_mdm_tasks.h" - -namespace hermes::bucket_mdm { - -/** Create hermes_bucket_mdm requests */ -class Client : public ModuleClient { - public: - /** Default constructor */ - Client() = default; - - /** Destructor */ - ~Client() = default; - - /** Create a hermes_bucket_mdm */ - HSHM_ALWAYS_INLINE - void Create(const DomainQuery &dom_query, - const std::string &state_name) { - id_ = PoolId::GetNull(); - QueueManagerInfo &qm = CHI_CLIENT->server_config_.queue_manager_; - std::vector queue_info; - id_ = CHI_ADMIN->CreateTaskState( - domain_id, state_name, id_, queue_info); - Init(id_, CHI_ADMIN->queue_id_); - } - - /** Destroy task state + queue */ - HSHM_ALWAYS_INLINE - void Destroy(const DomainQuery &dom_query) { - CHI_ADMIN->DestroyTaskState(domain_id, id_); - } - - /**==================================== - * Tag Operations - * ===================================*/ - - /** - * Get all tag metadata - * */ - void AsyncPollTagMetadataConstruct(PollTagMetadataTask *task, - const TaskNode &task_node) { - CHI_CLIENT->ConstructTask( - task, task_node, id_); - } - std::vector PollTagMetadata() { - LPointer task = - AsyncPollTagMetadata(); - task->Wait(); - std::vector target_mdms = - task->DeserializeTagMetadata(); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, task); - return target_mdms; - } - CHI_TASK_METHODS(PollTagMetadata); -}; - -} // namespace hrun - -#endif // HRUN_hermes_bucket_mdm_H_ diff --git a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h deleted file mode 100644 index 6687ea2e8..000000000 --- a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ /dev/null @@ -1,886 +0,0 @@ -#ifndef HRUN_HERMES_BUCKET_MDM_LIB_EXEC_H_ -#define HRUN_HERMES_BUCKET_MDM_LIB_EXEC_H_ - -/** Execute a task */ -void Run(u32 method, Task *task, RunContext &rctx) override { - switch (method) { - case Method::kConstruct: { - Construct(reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - Destruct(reinterpret_cast(task), rctx); - break; - } - case Method::kGetOrCreateTag: { - GetOrCreateTag(reinterpret_cast(task), rctx); - break; - } - case Method::kGetTagId: { - GetTagId(reinterpret_cast(task), rctx); - break; - } - case Method::kGetTagName: { - GetTagName(reinterpret_cast(task), rctx); - break; - } - case Method::kRenameTag: { - RenameTag(reinterpret_cast(task), rctx); - break; - } - case Method::kDestroyTag: { - DestroyTag(reinterpret_cast(task), rctx); - break; - } - case Method::kTagAddBlob: { - TagAddBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kTagRemoveBlob: { - TagRemoveBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kTagClearBlobs: { - TagClearBlobs(reinterpret_cast(task), rctx); - break; - } - case Method::kUpdateSize: { - UpdateSize(reinterpret_cast(task), rctx); - break; - } - case Method::kAppendBlobSchema: { - AppendBlobSchema(reinterpret_cast(task), rctx); - break; - } - case Method::kAppendBlob: { - AppendBlob(reinterpret_cast(task), rctx); - break; - } - case Method::kGetSize: { - GetSize(reinterpret_cast(task), rctx); - break; - } - case Method::kSetBlobMdm: { - SetBlobMdm(reinterpret_cast(task), rctx); - break; - } - case Method::kGetContainedBlobIds: { - GetContainedBlobIds(reinterpret_cast(task), rctx); - break; - } - case Method::kPollTagMetadata: { - PollTagMetadata(reinterpret_cast(task), rctx); - break; - } - } -} -/** Execute a task */ -void Monitor(u32 mode, Task *task, RunContext &rctx) override { - switch (task->method_) { - case Method::kConstruct: { - MonitorConstruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - MonitorDestruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetOrCreateTag: { - MonitorGetOrCreateTag(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetTagId: { - MonitorGetTagId(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetTagName: { - MonitorGetTagName(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kRenameTag: { - MonitorRenameTag(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kDestroyTag: { - MonitorDestroyTag(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kTagAddBlob: { - MonitorTagAddBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kTagRemoveBlob: { - MonitorTagRemoveBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kTagClearBlobs: { - MonitorTagClearBlobs(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kUpdateSize: { - MonitorUpdateSize(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kAppendBlobSchema: { - MonitorAppendBlobSchema(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kAppendBlob: { - MonitorAppendBlob(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetSize: { - MonitorGetSize(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kSetBlobMdm: { - MonitorSetBlobMdm(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kGetContainedBlobIds: { - MonitorGetContainedBlobIds(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kPollTagMetadata: { - MonitorPollTagMetadata(mode, reinterpret_cast(task), rctx); - break; - } - } -} -/** Delete a task */ -void Del(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetOrCreateTag: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetTagId: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetTagName: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kRenameTag: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestroyTag: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kTagAddBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kTagRemoveBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kTagClearBlobs: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kUpdateSize: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kAppendBlobSchema: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kAppendBlob: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetSize: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kSetBlobMdm: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kGetContainedBlobIds: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kPollTagMetadata: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - } -} -/** Duplicate a task */ -void Dup(u32 method, Task *orig_task, std::vector> &dups) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetOrCreateTag: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetTagId: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetTagName: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kRenameTag: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestroyTag: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kTagAddBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kTagRemoveBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kTagClearBlobs: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kUpdateSize: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kAppendBlobSchema: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kAppendBlob: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetSize: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kSetBlobMdm: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kGetContainedBlobIds: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kPollTagMetadata: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - } -} -/** Register the duplicate output with the origin task */ -void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetOrCreateTag: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetTagId: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetTagName: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kRenameTag: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestroyTag: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kTagAddBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kTagRemoveBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kTagClearBlobs: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kUpdateSize: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kAppendBlobSchema: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kAppendBlob: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetSize: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kSetBlobMdm: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kGetContainedBlobIds: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kPollTagMetadata: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - } -} -/** Ensure there is space to store replicated outputs */ -void ReplicateStart(u32 method, u32 count, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetOrCreateTag: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetTagId: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetTagName: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kRenameTag: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestroyTag: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kTagAddBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kTagRemoveBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kTagClearBlobs: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kUpdateSize: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kAppendBlobSchema: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kAppendBlob: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetSize: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kSetBlobMdm: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kGetContainedBlobIds: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kPollTagMetadata: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - } -} -/** Determine success and handle failures */ -void ReplicateEnd(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetOrCreateTag: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetTagId: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetTagName: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kRenameTag: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestroyTag: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kTagAddBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kTagRemoveBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kTagClearBlobs: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kUpdateSize: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kAppendBlobSchema: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kAppendBlob: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetSize: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kSetBlobMdm: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kGetContainedBlobIds: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kPollTagMetadata: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - } -} -/** Serialize a task when initially pushing into remote */ -std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetOrCreateTag: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetTagId: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetTagName: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRenameTag: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestroyTag: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTagAddBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTagRemoveBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTagClearBlobs: { - ar << *reinterpret_cast(task); - break; - } - case Method::kUpdateSize: { - ar << *reinterpret_cast(task); - break; - } - case Method::kAppendBlobSchema: { - ar << *reinterpret_cast(task); - break; - } - case Method::kAppendBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetSize: { - ar << *reinterpret_cast(task); - break; - } - case Method::kSetBlobMdm: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetContainedBlobIds: { - ar << *reinterpret_cast(task); - break; - } - case Method::kPollTagMetadata: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when popping from remote queue */ -TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { - TaskPointer task_ptr; - switch (method) { - case Method::kConstruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetOrCreateTag: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetTagId: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetTagName: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kRenameTag: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestroyTag: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kTagAddBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kTagRemoveBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kTagClearBlobs: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kUpdateSize: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kAppendBlobSchema: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kAppendBlob: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetSize: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kSetBlobMdm: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kGetContainedBlobIds: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kPollTagMetadata: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - } - return task_ptr; -} -/** Serialize a task when returning from remote queue */ -std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetOrCreateTag: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetTagId: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetTagName: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRenameTag: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestroyTag: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTagAddBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTagRemoveBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kTagClearBlobs: { - ar << *reinterpret_cast(task); - break; - } - case Method::kUpdateSize: { - ar << *reinterpret_cast(task); - break; - } - case Method::kAppendBlobSchema: { - ar << *reinterpret_cast(task); - break; - } - case Method::kAppendBlob: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetSize: { - ar << *reinterpret_cast(task); - break; - } - case Method::kSetBlobMdm: { - ar << *reinterpret_cast(task); - break; - } - case Method::kGetContainedBlobIds: { - ar << *reinterpret_cast(task); - break; - } - case Method::kPollTagMetadata: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when returning from remote queue */ -void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetOrCreateTag: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetTagId: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetTagName: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kRenameTag: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestroyTag: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kTagAddBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kTagRemoveBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kTagClearBlobs: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kUpdateSize: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kAppendBlobSchema: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kAppendBlob: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetSize: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kSetBlobMdm: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kGetContainedBlobIds: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kPollTagMetadata: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - } -} -/** Get the grouping of the task */ -u32 GetGroup(u32 method, Task *task, chi::charbuf &group) override { - switch (method) { - case Method::kConstruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetOrCreateTag: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetTagId: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetTagName: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kRenameTag: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestroyTag: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kTagAddBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kTagRemoveBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kTagClearBlobs: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kUpdateSize: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kAppendBlobSchema: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kAppendBlob: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetSize: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kSetBlobMdm: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kGetContainedBlobIds: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kPollTagMetadata: { - return reinterpret_cast(task)->GetGroup(group); - } - } - return -1; -} - -#endif // HRUN_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h b/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h deleted file mode 100644 index 0146a45ce..000000000 --- a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef HRUN_HERMES_BUCKET_MDM_METHODS_H_ -#define HRUN_HERMES_BUCKET_MDM_METHODS_H_ - -/** The set of methods in the admin task */ -struct Method : public TaskMethod { - TASK_METHOD_T kGetOrCreateTag = kLast + 0; - TASK_METHOD_T kGetTagId = kLast + 1; - TASK_METHOD_T kGetTagName = kLast + 2; - TASK_METHOD_T kRenameTag = kLast + 3; - TASK_METHOD_T kDestroyTag = kLast + 4; - TASK_METHOD_T kTagAddBlob = kLast + 5; - TASK_METHOD_T kTagRemoveBlob = kLast + 6; - TASK_METHOD_T kTagClearBlobs = kLast + 10; - TASK_METHOD_T kUpdateSize = kLast + 11; - TASK_METHOD_T kAppendBlobSchema = kLast + 12; - TASK_METHOD_T kAppendBlob = kLast + 13; - TASK_METHOD_T kGetSize = kLast + 14; - TASK_METHOD_T kSetBlobMdm = kLast + 15; - TASK_METHOD_T kGetContainedBlobIds = kLast + 16; - TASK_METHOD_T kPollTagMetadata = kLast + 17; -}; - -#endif // HRUN_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml b/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml deleted file mode 100644 index 527f10a58..000000000 --- a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kGetOrCreateTag: 0 -kGetTagId: 1 -kGetTagName: 2 -kRenameTag: 3 -kDestroyTag: 4 -kTagAddBlob: 5 -kTagRemoveBlob: 6 -# kTagGroupBy: 7 -# kTagAddTrait: 8 -# kTagRemoveTrait: 9 -kTagClearBlobs: 10 -kUpdateSize: 11 -kAppendBlobSchema: 12 -kAppendBlob: 13 -kGetSize: 14 -kSetBlobMdm: 15 -kGetContainedBlobIds: 16 -kPollTagMetadata: 17 \ No newline at end of file diff --git a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h deleted file mode 100644 index 73453cae3..000000000 --- a/tasks/old/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ /dev/null @@ -1,322 +0,0 @@ -// -// Created by lukemartinlogan on 8/14/23. -// - -#ifndef HRUN_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ -#define HRUN_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ - -#include "chimaera/api/chimaera_client.h" -#include "chimaera/module_registry/task_lib.h" -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/queue_manager/queue_manager_client.h" -#include "hermes/hermes_types.h" -#include "bdev/bdev.h" -#include "hermes_blob_mdm/hermes_blob_mdm.h" -#include "chimaera/api/chimaera_client.h" -#include "chimaera/chimaera_namespace.h" -#include "proc_queue/proc_queue.h" -#include "data_stager/data_stager.h" - -namespace hermes::bucket_mdm { - -#include "hermes_bucket_mdm_methods.h" -#include "chimaera/chimaera_namespace.h" - - -/** - * A task to create hermes_bucket_mdm - * */ -using chi::Admin::CreateTaskStateTask; -struct ConstructTask : public CreateTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc) : CreateTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const std::string &state_name, - const PoolId &id, - const std::vector &queue_info) - : CreateTaskStateTask(alloc, task_node, domain_id, state_name, - "hermes_bucket_mdm", id, queue_info) { - } -}; - -/** Update bucket size */ -struct UpdateSizeTask : public Task, TaskFlags { - IN TagId tag_id_; - IN ssize_t update_; - IN int mode_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - UpdateSizeTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - UpdateSizeTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - ssize_t update, - int mode) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = tag_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kUpdateSize; - task_flags_.SetBits(0 | TASK_FIRE_AND_FORGET); - dom_query_ = dom_query; - - // Custom params - tag_id_ = tag_id; - update_ = update; - mode_ = mode; - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - ar(tag_id_, update_, mode_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) {} -}; - -/** Phases for the append task */ -class AppendBlobPhase { - public: - TASK_METHOD_T kGetBlobIds = 0; - TASK_METHOD_T kWaitBlobIds = 1; - TASK_METHOD_T kWaitPutBlobs = 2; -}; - -/** A struct to store the */ -struct AppendInfo { - size_t blob_off_; - size_t data_size_; - chi::charbuf blob_name_; - BlobId blob_id_; - blob_mdm::GetOrCreateBlobIdTask *blob_id_task_; - blob_mdm::PutBlobTask *put_task_; - - template - void serialize(Ar &ar) { - ar(blob_off_, data_size_, blob_name_, blob_id_); - } -}; - -/** A task to append data to a bucket */ -struct AppendBlobSchemaTask : public Task, TaskFlags { - IN TagId tag_id_; - IN size_t data_size_; - IN size_t page_size_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - AppendBlobSchemaTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - AppendBlobSchemaTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - size_t data_size, - size_t page_size) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = tag_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kAppendBlobSchema; - task_flags_.SetBits(0); - dom_query_ = dom_query; - - // Custom params - tag_id_ = tag_id; - data_size_ = data_size; - page_size_ = page_size; - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - ar(tag_id_, data_size_, page_size_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - ar(append_info_); - } -}; - -/** A task to append data to a bucket */ -struct AppendBlobTask : public Task, TaskFlags { - IN TagId tag_id_; - IN size_t data_size_; - IN hipc::Pointer data_; - IN size_t page_size_; - IN u32 node_id_; - IN float score_; - TEMP int phase_ = AppendBlobPhase::kGetBlobIds; - TEMP AppendBlobSchemaTask *schema_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - AppendBlobTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - AppendBlobTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const TagId &tag_id, - size_t data_size, - const hipc::Pointer &data, - size_t page_size, - float score, - u32 node_id, - const Context &ctx) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = tag_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kAppendBlob; - task_flags_.SetBits(0 | TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_UNORDERED | TASK_REMOTE_DEBUG_MARK); - dom_query_ = dom_query; - - // Custom params - tag_id_ = tag_id; - data_size_ = data_size; - data_ = data; - score_ = score; - page_size_ = page_size; - node_id_ = node_id; - } - - /** Destructor */ - ~AppendBlobTask() { - if (IsDataOwner()) { - CHI_CLIENT->FreeBuffer(data_); - } - } -}; - -/** A task to collect blob metadata */ -struct PollTagMetadataTask : public Task, TaskFlags { - OUT chi::string my_tag_mdms_; - TEMP hipc::vector tag_mdms_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - PollTagMetadataTask(const hipc::CtxAllocator &alloc) : Task(alloc) { - } - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - PollTagMetadataTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = 0; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kPollTagMetadata; - task_flags_.SetBits(TASK_LANE_ALL); - domain_id_ = chi::DomainQuery::GetGlobalBcast(); - - // Custom params - } - - /** Serialize tag info */ - void SerializeTagMetadata(const std::vector &tag_info) { - std::stringstream ss; - cereal::BinaryOutputArchive ar(ss); - ar << tag_info; - (*my_tag_mdms_) = ss.str(); - } - - /** Deserialize tag info */ - void DeserializeTagMetadata(const std::string &srl, std::vector &tag_mdms) { - std::vector tmp_tag_mdms; - std::stringstream ss(srl); - cereal::BinaryInputArchive ar(ss); - ar >> tmp_tag_mdms; - for (TagInfo &tag_info : tmp_tag_mdms) { - tag_mdms.emplace_back(tag_info); - } - } - - /** Get combined output of all replicas */ - std::vector MergeTagMetadata() { - std::vector tag_mdms; - for (const chi::string &srl : *tag_mdms_) { - DeserializeTagMetadata(srl.str(), tag_mdms); - } - return tag_mdms; - } - - /** Deserialize final query output */ - std::vector DeserializeTagMetadata() { - std::vector tag_mdms; - DeserializeTagMetadata(my_tag_mdms_->str(), tag_mdms); - return tag_mdms; - } - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, PollTagMetadataTask &other) { - task_dup(other); - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, PollTagMetadataTask &dup_task) { - (*tag_mdms_)[replica] = (*dup_task.my_tag_mdms_); - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - ar(my_tag_mdms_); - } - - /** (De)serialize message return */ - template - void SaveEnd(Ar &ar) { - ar(my_tag_mdms_); - } - - /** (De)serialize message return */ - template - void LoadEnd(u32 replica, Ar &ar) { - ar(my_tag_mdms_); - DupEnd(replica, *this); - } - - /** Begin replication */ - void ReplicateStart(u32 count) { - tag_mdms_->resize(count); - } - - /** Finalize replication */ - void ReplicateEnd() { - std::vector tag_mdms = MergeTagMetadata(); - SerializeTagMetadata(tag_mdms); - } -}; - -} // namespace hermes::bucket_mdm - -#endif // HRUN_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ diff --git a/tasks/old/hermes_bucket_mdm/src/CMakeLists.txt b/tasks/old/hermes_bucket_mdm/src/CMakeLists.txt deleted file mode 100644 index 3c2f9d8b1..000000000 --- a/tasks/old/hermes_bucket_mdm/src/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Small Message Task Library -#------------------------------------------------------------------------------ -add_library(hermes_bucket_mdm SHARED - hermes_bucket_mdm.cc) -# add_dependencies(hermes_bucket_mdm ${Hermes_RUNTIME_DEPS}) -target_link_libraries(hermes_bucket_mdm ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Install Small Message Task Library -#------------------------------------------------------------------------------ -install( - TARGETS - hermes_bucket_mdm - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install for import into other projects -#----------------------------------------------------------------------------- -install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_bucket_mdm - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#------------------------------------------------------------------------------ -# Coverage -#------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_bucket_mdm) -endif() diff --git a/tasks/old/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/old/hermes_bucket_mdm/src/hermes_bucket_mdm.cc deleted file mode 100644 index 3e8f00d6e..000000000 --- a/tasks/old/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ /dev/null @@ -1,462 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/api/chimaera_runtime.h" -#include "hermes/config_server.h" -#include "hermes_bucket_mdm/hermes_bucket_mdm.h" -#include "hermes_adapters/mapper/abstract_mapper.h" -#include "hermes/dpe/dpe_factory.h" -#include "bdev/bdev.h" -#include "data_stager/data_stager.h" - -namespace hermes::bucket_mdm { - -typedef std::unordered_map TAG_ID_MAP_T; -typedef std::unordered_map TAG_MAP_T; - -class Server : public Module { - public: - std::vector tag_id_map_; - std::vector tag_map_; - u32 node_id_; - std::atomic id_alloc_; - Client bkt_mdm_; - blob_mdm::Client blob_mdm_; - data_stager::Client stager_mdm_; - - public: - Server() = default; - - /** Construct bucket mdm */ - void Construct(ConstructTask *task, RunContext &rctx) { - id_alloc_ = 0; - node_id_ = CHI_CLIENT->node_id_; - bkt_mdm_.Init(id_, CHI_ADMIN->queue_id_); - tag_id_map_.resize(HRUN_QM_RUNTIME->max_lanes_); - tag_map_.resize(HRUN_QM_RUNTIME->max_lanes_); - task->SetModuleComplete(); - } - void MonitorConstruct(u32 mode, ConstructTask *task, RunContext &rctx) { - } - - /** Destroy bucket mdm */ - void Destruct(DestructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - void MonitorDestruct(u32 mode, DestructTask *task, RunContext &rctx) { - } - - /** - * Set the Blob MDM - * */ - void SetBlobMdm(SetBlobMdmTask *task, RunContext &rctx) { - blob_mdm_.Init(task->blob_mdm_, CHI_ADMIN->queue_id_); - stager_mdm_.Init(task->stager_mdm_, CHI_ADMIN->queue_id_); - task->SetModuleComplete(); - } - void MonitorSetBlobMdm(u32 mode, SetBlobMdmTask *task, RunContext &rctx) { - } - - /** Update the size of the bucket */ - void UpdateSize(UpdateSizeTask *task, RunContext &rctx) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - TagInfo &tag_info = tag_map[task->tag_id_]; - ssize_t internal_size = (ssize_t) tag_info.internal_size_; - if (task->mode_ == UpdateSizeMode::kAdd) { - internal_size += task->update_; - } else { - internal_size = std::max(task->update_, internal_size); - } - HILOG(kDebug, "Updating size of tag {} from {} to {} with update {} (mode={})", - task->tag_id_, tag_info.internal_size_, internal_size, task->update_, task->mode_) - tag_info.internal_size_ = (size_t) internal_size; - task->SetModuleComplete(); - } - void MonitorUpdateSize(u32 mode, UpdateSizeTask *task, RunContext &rctx) { - } - - /** - * Create the PartialPuts for append operations. - * */ - void AppendBlobSchema(AppendBlobSchemaTask *task, RunContext &rctx) { - switch (task->phase_) { - case AppendBlobPhase::kGetBlobIds: { - HILOG(kDebug, "(node {}) Getting blob IDs for tag {} (task_node={})", - CHI_CLIENT->node_id_, task->tag_id_, task->task_node_) - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - TagInfo &tag_info = tag_map[task->tag_id_]; - size_t bucket_size = tag_info.internal_size_; - size_t cur_page = bucket_size / task->page_size_; - size_t cur_page_off = bucket_size % task->page_size_; - size_t update_size = task->page_size_ - cur_page_off; - size_t max_pages = task->data_size_ / task->page_size_ + 1; - size_t cur_size = 0; - HILOG(kDebug, "(node {}) Bucket size {}, page_size {}, cur_page {} (task_node={})", - CHI_CLIENT->node_id_, bucket_size, task->page_size_, cur_page, task->task_node_) - HSHM_MAKE_AR0(task->append_info_, nullptr); - std::vector &append_info = *task->append_info_; - append_info.reserve(max_pages); - while (cur_size < task->data_size_) { - if (update_size > task->data_size_) { - update_size = task->data_size_; - } - append_info.emplace_back(); - AppendInfo &append = append_info.back(); - append.blob_name_ = adapter::BlobPlacement::CreateBlobName(cur_page); - append.data_size_ = update_size; - append.blob_off_ = cur_page_off; - append.blob_id_task_ = blob_mdm_.AsyncGetOrCreateBlobId(task->task_node_ + 1, - task->tag_id_, - append.blob_name_).ptr_; - cur_size += update_size; - cur_page_off = 0; - ++cur_page; - update_size = task->page_size_; - } - task->phase_ = AppendBlobPhase::kWaitBlobIds; - } - case AppendBlobPhase::kWaitBlobIds: { - std::vector &append_info = *task->append_info_; - for (AppendInfo &append : append_info) { - if (!append.blob_id_task_->IsComplete()) { - return; - } - } - HILOG(kDebug, "(node {}) Finished blob IDs for tag {} (task_node={})", - CHI_CLIENT->node_id_, task->tag_id_, task->task_node_) - for (AppendInfo &append : append_info) { - append.blob_id_ = append.blob_id_task_->blob_id_; - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, append.blob_id_task_); - } - task->SetModuleComplete(); - } - } - } - void MonitorAppendBlobSchema(u32 mode, AppendBlobSchemaTask *task, RunContext &rctx) { - } - - /** - * Append data to a bucket. Assumes that the blobs in the bucket - * are named 0 ... N. Each blob is assumed to have a certain - * fixed page size. - * */ - void AppendBlob(AppendBlobTask *task, RunContext &rctx) { - switch (task->phase_) { - case AppendBlobPhase::kGetBlobIds: { - HILOG(kDebug, "(node {}) Appending {} bytes to bucket {} (task_node={})", - CHI_CLIENT->node_id_, task->data_size_, task->tag_id_, task->task_node_); - task->schema_ = bkt_mdm_.AsyncAppendBlobSchema(task->task_node_ + 1, - task->tag_id_, - task->data_size_, - task->page_size_).ptr_; - task->phase_ = AppendBlobPhase::kWaitBlobIds; - } - case AppendBlobPhase::kWaitBlobIds: { - if (!task->schema_->IsComplete()) { - return; - } - std::vector &append_info = *task->schema_->append_info_; - size_t buf_off = 0; - HILOG(kDebug, "(node {}) Got blob schema of size {} for tag {} (task_node={})", - CHI_CLIENT->node_id_, append_info.size(), task->tag_id_, task->task_node_) - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - TagInfo &tag_info = tag_map[task->tag_id_]; - for (AppendInfo &append : append_info) { - HILOG(kDebug, "(node {}) Spawning blob {} of size {} for tag {} (task_node={} blob_mdm={})", - CHI_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, - task->tag_id_, task->task_node_, blob_mdm_.id_); - Context ctx; - if (tag_info.flags_.Any(HERMES_SHOULD_STAGE)) { - ctx.flags_.SetBits(HERMES_SHOULD_STAGE); - } - append.put_task_ = blob_mdm_.AsyncPutBlob(task->task_node_ + 1, - task->tag_id_, - append.blob_name_, - append.blob_id_, - append.blob_off_, - append.data_size_, - task->data_ + buf_off, - task->score_, 0, - ctx, 0).ptr_; - HILOG(kDebug, "(node {}) Finished spawning blob {} of size {} for tag {} (task_node={} blob_mdm={})", - CHI_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, - task->tag_id_, task->task_node_, blob_mdm_.id_); - buf_off += append.data_size_; - } - task->phase_ = AppendBlobPhase::kWaitPutBlobs; - } - case AppendBlobPhase::kWaitPutBlobs: { - std::vector &append_info = *task->schema_->append_info_; - for (AppendInfo &append : append_info) { - if (!append.put_task_->IsComplete()) { - return; - } - } - HILOG(kDebug, "(node {}) PUT blobs for tag {} (task_node={})", - CHI_CLIENT->node_id_, task->tag_id_, task->task_node_) - for (AppendInfo &append : append_info) { - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, append.put_task_); - } - HSHM_DESTROY_AR(task->schema_->append_info_); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, task->schema_); - task->SetModuleComplete(); - } - } - } - void MonitorAppendBlob(u32 mode, AppendBlobTask *task, RunContext &rctx) { - } - - /** Get or create a tag */ - void GetOrCreateTag(GetOrCreateTagTask *task, RunContext &rctx) { - TagId tag_id; - - // Check if the tag exists - TAG_ID_MAP_T &tag_id_map = tag_id_map_[rctx.lane_id_]; - hshm::string url = hshm::to_charbuf(*task->tag_name_); - chi::charbuf tag_name = data_stager::Client::GetTagNameFromUrl(url); - bool did_create = false; - if (tag_name.size() > 0) { - did_create = tag_id_map.find(tag_name) == tag_id_map.end(); - } - HILOG(kDebug, "Creating a tag {} on lane {}", tag_name.str(), rctx.lane_id_); - - // Emplace bucket if it does not already exist - if (did_create) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - tag_id.unique_ = id_alloc_.fetch_add(1); - tag_id.hash_ = task->lane_hash_; - tag_id.node_id_ = HRUN_RUNTIME->rpc_.node_id_; - HILOG(kDebug, "Creating tag for the first time: {} {}", tag_name.str(), tag_id) - tag_id_map.emplace(tag_name, tag_id); - tag_map.emplace(tag_id, TagInfo()); - TagInfo &tag_info = tag_map[tag_id]; - tag_info.name_ = tag_name; - tag_info.tag_id_ = tag_id; - tag_info.owner_ = task->blob_owner_; - tag_info.internal_size_ = task->backend_size_; - if (task->flags_.Any(HERMES_SHOULD_STAGE)) { - stager_mdm_.AsyncRegisterStager(task->task_node_ + 1, - tag_id, - chi::charbuf(task->tag_name_->str()), - chi::charbuf(task->params_->str())); - tag_info.flags_.SetBits(HERMES_SHOULD_STAGE); - } - } else { - if (tag_name.size()) { - HILOG(kDebug, "Found existing tag: {}", tag_name.str()) - tag_id = tag_id_map[tag_name]; - } else { - HILOG(kDebug, "Found existing tag: {}", task->tag_id_) - tag_id = task->tag_id_; - } - } - - task->tag_id_ = tag_id; - // task->did_create_ = did_create; - task->SetModuleComplete(); - } - void MonitorGetOrCreateTag(u32 mode, GetOrCreateTagTask *task, RunContext &rctx) { - } - - /** Get tag ID */ - void GetTagId(GetTagIdTask *task, RunContext &rctx) { - TAG_ID_MAP_T &tag_id_map = tag_id_map_[rctx.lane_id_]; - chi::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); - auto it = tag_id_map.find(tag_name); - if (it == tag_id_map.end()) { - task->tag_id_ = TagId::GetNull(); - task->SetModuleComplete(); - return; - } - task->tag_id_ = it->second; - task->SetModuleComplete(); - } - void MonitorGetTagId(u32 mode, GetTagIdTask *task, RunContext &rctx) { - } - - /** Get tag name */ - void GetTagName(GetTagNameTask *task, RunContext &rctx) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - auto it = tag_map.find(task->tag_id_); - if (it == tag_map.end()) { - task->SetModuleComplete(); - return; - } - (*task->tag_name_) = it->second.name_; - task->SetModuleComplete(); - } - void MonitorGetTagName(u32 mode, GetTagNameTask *task, RunContext &rctx) { - } - - /** Rename tag */ - void RenameTag(RenameTagTask *task, RunContext &rctx) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - auto it = tag_map.find(task->tag_id_); - if (it == tag_map.end()) { - task->SetModuleComplete(); - return; - } - (*task->tag_name_) = (*task->tag_name_); - task->SetModuleComplete(); - } - void MonitorRenameTag(u32 mode, RenameTagTask *task, RunContext &rctx) { - } - - /** Destroy tag */ - void DestroyTag(DestroyTagTask *task, RunContext &rctx) { - switch (task->phase_) { - case DestroyTagPhase::kDestroyBlobs: { - TAG_ID_MAP_T &tag_id_map = tag_id_map_[rctx.lane_id_]; - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - TagInfo &tag = tag_map[task->tag_id_]; - tag_id_map.erase(tag.name_); - HSHM_MAKE_AR0(task->destroy_blob_tasks_, nullptr); - std::vector blob_tasks = *task->destroy_blob_tasks_; - blob_tasks.reserve(tag.blobs_.size()); - for (BlobId &blob_id : tag.blobs_) { - blob_mdm::DestroyBlobTask *blob_task = - blob_mdm_.AsyncDestroyBlob(task->task_node_ + 1, - task->tag_id_, blob_id, false).ptr_; - blob_tasks.emplace_back(blob_task); - } - stager_mdm_.AsyncUnregisterStager(task->task_node_ + 1, - task->tag_id_); - HILOG(kDebug, "Destroying the tag: {}", tag.name_.str()); - task->phase_ = DestroyTagPhase::kWaitDestroyBlobs; - return; - } - case DestroyTagPhase::kWaitDestroyBlobs: { - std::vector blob_tasks = *task->destroy_blob_tasks_; - for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { - if (!blob_task->IsComplete()) { - return; - } - } - for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, blob_task); - } - HSHM_DESTROY_AR(task->destroy_blob_tasks_); - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - tag_map.erase(task->tag_id_); - HILOG(kDebug, "Finished destroying the tag"); - task->SetModuleComplete(); - } - } - } - void MonitorDestroyTag(u32 mode, DestroyTagTask *task, RunContext &rctx) { - } - - /** Add a blob to a tag */ - void TagAddBlob(TagAddBlobTask *task, RunContext &rctx) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - auto it = tag_map.find(task->tag_id_); - if (it == tag_map.end()) { - task->SetModuleComplete(); - return; - } - TagInfo &tag = it->second; - tag.blobs_.emplace_back(task->blob_id_); - task->SetModuleComplete(); - } - void MonitorTagAddBlob(u32 mode, TagAddBlobTask *task, RunContext &rctx) { - } - - /** Remove a blob from a tag */ - void TagRemoveBlob(TagRemoveBlobTask *task, RunContext &rctx) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - auto it = tag_map.find(task->tag_id_); - if (it == tag_map.end()) { - task->SetModuleComplete(); - return; - } - TagInfo &tag = it->second; - auto blob_it = std::find(tag.blobs_.begin(), tag.blobs_.end(), task->blob_id_); - tag.blobs_.erase(blob_it); - task->SetModuleComplete(); - } - void MonitorTagRemoveBlob(u32 mode, TagRemoveBlobTask *task, RunContext &rctx) { - } - - /** Clear blobs from a tag */ - void TagClearBlobs(TagClearBlobsTask *task, RunContext &rctx) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - auto it = tag_map.find(task->tag_id_); - if (it == tag_map.end()) { - task->SetModuleComplete(); - return; - } - TagInfo &tag = it->second; - if (tag.owner_) { - for (BlobId &blob_id : tag.blobs_) { - blob_mdm_.AsyncDestroyBlob(task->task_node_ + 1, task->tag_id_, blob_id); - } - } - tag.blobs_.clear(); - tag.internal_size_ = 0; - task->SetModuleComplete(); - } - void MonitorTagClearBlobs(u32 mode, TagClearBlobsTask *task, RunContext &rctx) { - } - - /** Get size of the bucket */ - void GetSize(GetSizeTask *task, RunContext &rctx) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - auto it = tag_map.find(task->tag_id_); - if (it == tag_map.end()) { - task->size_ = 0; - task->SetModuleComplete(); - return; - } - TagInfo &tag = it->second; - task->size_ = tag.internal_size_; - task->SetModuleComplete(); - } - void MonitorGetSize(u32 mode, GetSizeTask *task, RunContext &rctx) { - } - - /** Get contained blob IDs */ - void GetContainedBlobIds(GetContainedBlobIdsTask *task, RunContext &rctx) { - TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; - auto it = tag_map.find(task->tag_id_); - if (it == tag_map.end()) { - task->SetModuleComplete(); - return; - } - TagInfo &tag = it->second; - hipc::vector &blobs = (*task->blob_ids_); - blobs.reserve(tag.blobs_.size()); - for (BlobId &blob_id : tag.blobs_) { - blobs.emplace_back(blob_id); - } - task->SetModuleComplete(); - } - void MonitorGetContainedBlobIds(u32 mode, GetContainedBlobIdsTask *task, RunContext &rctx) { - } - - /** - * Get all metadata about a blob - * */ - HSHM_ALWAYS_INLINE - void PollTagMetadata(PollTagMetadataTask *task, RunContext &rctx) { - TAG_MAP_T &blob_map = tag_map_[rctx.lane_id_]; - std::vector tag_mdms; - tag_mdms.reserve(blob_map.size()); - for (const std::pair &tag_part : blob_map) { - const TagInfo &tag_info = tag_part.second; - tag_mdms.emplace_back(tag_info); - } - task->SerializeTagMetadata(tag_mdms); - task->SetModuleComplete(); - } - void MonitorPollTagMetadata(u32 mode, PollTagMetadataTask *task, RunContext &rctx) { - } - - public: -#include "hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h" -}; - -} // namespace hrun - -HRUN_TASK_CC(hermes::bucket_mdm::Server, "hermes_bucket_mdm"); diff --git a/tasks/old/hermes_data_op/CMakeLists.txt b/tasks/old/hermes_data_op/CMakeLists.txt deleted file mode 100644 index d48611782..000000000 --- a/tasks/old/hermes_data_op/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Hrun Admin Task Library -#------------------------------------------------------------------------------ -include_directories(include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install HRUN Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op.h b/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op.h deleted file mode 100644 index f6c2d3e94..000000000 --- a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op.h +++ /dev/null @@ -1,97 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#ifndef HRUN_hermes_data_op_H_ -#define HRUN_hermes_data_op_H_ - -#include "hermes_data_op_tasks.h" - -namespace hermes::data_op { - -/** Create hermes_data_op requests */ -class Client : public ModuleClient { - - public: - /** Default constructor */ - Client() = default; - - /** Destructor */ - ~Client() = default; - - /** Async create a task state */ - HSHM_ALWAYS_INLINE - LPointer AsyncCreate(const TaskNode &task_node, - const DomainQuery &dom_query, - const std::string &state_name, - PoolId &bkt_mdm_id, - PoolId &blob_mdm_id) { - id_ = PoolId::GetNull(); - QueueManagerInfo &qm = CHI_CLIENT->server_config_.queue_manager_; - std::vector queue_info; - return CHI_ADMIN->AsyncCreateTaskState( - task_node, domain_id, state_name, id_, queue_info, - bkt_mdm_id, blob_mdm_id); - } - HRUN_TASK_NODE_ROOT(AsyncCreate) - template - HSHM_ALWAYS_INLINE - void Create(Args&& ...args) { - LPointer task = - AsyncCreate(std::forward(args)...); - task->Wait(); - id_ = task->id_; - Init(id_, CHI_ADMIN->queue_id_); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, task); - } - - /** Destroy task state + queue */ - HSHM_ALWAYS_INLINE - void Destroy(const DomainQuery &dom_query) { - CHI_ADMIN->DestroyTaskState(domain_id, id_); - } - - /** Register the OpGraph to perform on data */ - HSHM_ALWAYS_INLINE - void AsyncRegisterOpConstruct(RegisterOpTask *task, - const TaskNode &task_node, - const OpGraph &op_graph) { - CHI_CLIENT->ConstructTask( - task, task_node, chi::DomainQuery::GetGlobalBcast(), id_, op_graph); - } - HSHM_ALWAYS_INLINE - void RegisterOp(const OpGraph &op_graph) { - LPointer task = - AsyncRegisterOp(op_graph); - task.ptr_->Wait(); - } - CHI_TASK_METHODS(RegisterOp); - - /** Register data as ready for operations to be performed */ - HSHM_ALWAYS_INLINE - void AsyncRegisterDataConstruct(RegisterDataTask *task, - const TaskNode &task_node, - const BucketId &bkt_id, - const std::string &blob_name, - const BlobId &blob_id, - size_t off, - size_t size) { - CHI_CLIENT->ConstructTask( - task, task_node, id_, bkt_id, - blob_name, blob_id, off, size); - } - CHI_TASK_METHODS(RegisterData); - - /** Async task to run operators */ - HSHM_ALWAYS_INLINE - void AsyncRunOpConstruct(RunOpTask *task, - const TaskNode &task_node) { - CHI_CLIENT->ConstructTask( - task, task_node, id_); - } - CHI_TASK_METHODS(RunOp); -}; - -} // namespace hrun - -#endif // HRUN_hermes_data_op_H_ diff --git a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h b/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h deleted file mode 100644 index db6c07c0d..000000000 --- a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h +++ /dev/null @@ -1,310 +0,0 @@ -#ifndef HRUN_HERMES_DATA_OP_LIB_EXEC_H_ -#define HRUN_HERMES_DATA_OP_LIB_EXEC_H_ - -/** Execute a task */ -void Run(u32 method, Task *task, RunContext &rctx) override { - switch (method) { - case Method::kConstruct: { - Construct(reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - Destruct(reinterpret_cast(task), rctx); - break; - } - case Method::kRegisterOp: { - RegisterOp(reinterpret_cast(task), rctx); - break; - } - case Method::kRegisterData: { - RegisterData(reinterpret_cast(task), rctx); - break; - } - case Method::kRunOp: { - RunOp(reinterpret_cast(task), rctx); - break; - } - } -} -/** Execute a task */ -void Monitor(u32 mode, Task *task, RunContext &rctx) override { - switch (task->method_) { - case Method::kConstruct: { - MonitorConstruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - MonitorDestruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kRegisterOp: { - MonitorRegisterOp(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kRegisterData: { - MonitorRegisterData(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kRunOp: { - MonitorRunOp(mode, reinterpret_cast(task), rctx); - break; - } - } -} -/** Delete a task */ -void Del(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kRegisterOp: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kRegisterData: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kRunOp: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - } -} -/** Duplicate a task */ -void Dup(u32 method, Task *orig_task, std::vector> &dups) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kRegisterOp: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kRegisterData: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kRunOp: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - } -} -/** Register the duplicate output with the origin task */ -void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kRegisterOp: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kRegisterData: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kRunOp: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - } -} -/** Ensure there is space to store replicated outputs */ -void ReplicateStart(u32 method, u32 count, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kRegisterOp: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kRegisterData: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kRunOp: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - } -} -/** Determine success and handle failures */ -void ReplicateEnd(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kRegisterOp: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kRegisterData: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kRunOp: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - } -} -/** Serialize a task when initially pushing into remote */ -std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRegisterOp: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRegisterData: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRunOp: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when popping from remote queue */ -TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { - TaskPointer task_ptr; - switch (method) { - case Method::kConstruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kRegisterOp: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kRegisterData: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kRunOp: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - } - return task_ptr; -} -/** Serialize a task when returning from remote queue */ -std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRegisterOp: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRegisterData: { - ar << *reinterpret_cast(task); - break; - } - case Method::kRunOp: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when returning from remote queue */ -void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kRegisterOp: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kRegisterData: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kRunOp: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - } -} -/** Get the grouping of the task */ -u32 GetGroup(u32 method, Task *task, chi::charbuf &group) override { - switch (method) { - case Method::kConstruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kRegisterOp: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kRegisterData: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kRunOp: { - return reinterpret_cast(task)->GetGroup(group); - } - } - return -1; -} - -#endif // HRUN_HERMES_DATA_OP_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h b/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h deleted file mode 100644 index 06cc34c2b..000000000 --- a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef HRUN_HERMES_DATA_OP_METHODS_H_ -#define HRUN_HERMES_DATA_OP_METHODS_H_ - -/** The set of methods in the admin task */ -struct Method : public TaskMethod { - TASK_METHOD_T kRegisterOp = kLast + 0; - TASK_METHOD_T kRegisterData = kLast + 1; - TASK_METHOD_T kRunOp = kLast + 2; -}; - -#endif // HRUN_HERMES_DATA_OP_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.yaml b/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.yaml deleted file mode 100644 index 372caa08e..000000000 --- a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kRegisterOp: 0 -kRegisterData: 1 -kRunOp: 2 \ No newline at end of file diff --git a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h b/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h deleted file mode 100644 index 313fe0f8d..000000000 --- a/tasks/old/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h +++ /dev/null @@ -1,326 +0,0 @@ -// -// Created by lukemartinlogan on 8/11/23. -// - -#ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ -#define HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ - -#include "chimaera/api/chimaera_client.h" -#include "chimaera/module_registry/task_lib.h" -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/queue_manager/queue_manager_client.h" -#include "proc_queue/proc_queue.h" -#include "hermes/hermes_types.h" -#include "hermes_data_op/hermes_data_op.h" -#include "hermes_bucket_mdm/hermes_bucket_mdm.h" - -namespace hermes::data_op { - -#include "hermes_data_op_methods.h" -#include "chimaera/chimaera_namespace.h" -using chi::proc_queue::TypedPushTask; -using chi::proc_queue::PushTask; - -/** The bucket to use for op data */ -struct OpBucketName { - std::string url_; // URL of bucket - BucketId bkt_id_; // Bucket ID (internal) - LPointer - bkt_id_task_; // Task to get bucket ID (internal) - - /** Default constructor */ - OpBucketName() = default; - - /** Emplace constructor */ - OpBucketName(const std::string &url) : url_(url) {} - - template - void serialize(Ar &ar) { - ar(url_); - } -}; - -/** An operation to perform on a set of inputs */ -struct Op { - std::vector in_; // Input URLs, indicates data format as well - OpBucketName var_name_; // Output URL - std::string op_name_; // Operation name - u32 op_id_; // Operation ID (internal) - - template - void serialize(Ar &ar) { - ar(in_, var_name_, op_name_); - } -}; - -/** The graph of operations to perform on data */ -struct OpGraph { - std::vector ops_; - - template - void serialize(Ar &ar) { - ar(ops_); - } -}; - -/** - * A task to create hermes_data_op - * */ -using chi::Admin::CreateTaskStateTask; -struct ConstructTask : public CreateTaskStateTask { - IN PoolId bkt_mdm_; - IN PoolId blob_mdm_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc) - : CreateTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const std::string &state_name, - const PoolId &id, - const std::vector &queue_info, - const PoolId &bkt_mdm_id, - const PoolId &blob_mdm_id) - : CreateTaskStateTask(alloc, task_node, domain_id, state_name, - "hermes_data_op", id, queue_info) { - // Custom params - bkt_mdm_ = bkt_mdm_id; - blob_mdm_ = blob_mdm_id; - std::stringstream ss; - cereal::BinaryOutputArchive ar(ss); - ar(bkt_mdm_, blob_mdm_); - std::string data = ss.str(); - *custom_ = data; - } - - /** Deserialize parameters */ - void Deserialize() { - std::string data = custom_->str(); - std::stringstream ss(data); - cereal::BinaryInputArchive ar(ss); - ar(bkt_mdm_, blob_mdm_); - } - - HSHM_ALWAYS_INLINE - ~ConstructTask() { - // Custom params - } -}; - -/** A task to destroy hermes_data_op */ -using chi::Admin::DestroyTaskStateTask; -struct DestructTask : public DestroyTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(const hipc::CtxAllocator &alloc) - : DestroyTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - PoolId &pool_id) - : DestroyTaskStateTask(alloc, task_node, domain_id, pool_id) {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * Register an operation to perform on data - * */ -struct RegisterOpTask : public Task, TaskFlags { - IN chi::string op_graph_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - RegisterOpTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - RegisterOpTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id, - const OpGraph &graph) : Task(alloc) { - // Store OpGraph - std::stringstream ss; - cereal::BinaryOutputArchive ar(ss); - ar << graph; - std::string op_graph_str = ss.str(); - HSHM_MAKE_AR(op_graph_, alloc, op_graph_str); - - // Initialize task - task_node_ = task_node; - lane_hash_ = std::hash{}(op_graph_str); - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kRegisterOp; - task_flags_.SetBits(TASK_COROUTINE); - domain_id_ = domain_id; - } - - /** Get opgraph */ - OpGraph GetOpGraph() { - OpGraph graph; - std::stringstream ss(op_graph_->str()); - cereal::BinaryInputArchive ar(ss); - ar >> graph; - return graph; - } - - /** Destructor */ - ~RegisterOpTask() { - HSHM_DESTROY_AR(op_graph_); - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - ar(op_graph_); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, RegisterOpTask &other) { - task_dup(other); - HSHM_MAKE_AR(op_graph_, alloc, *other.op_graph_); - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, RegisterOpTask &dup_task) { - } - - /** Begin replication */ - void ReplicateStart(u32 count) {} - - /** Finalize replication */ - void ReplicateEnd() {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * Register some data as being available for derived - * quantity calculation - * */ -struct OpData { - BucketId bkt_id_; - BlobId blob_id_; - size_t off_; - size_t size_; - u64 data_id_; - std::string blob_name_; - int refcnt_; - - OpData() : refcnt_(0) {} -}; -struct RegisterDataTask : public Task, TaskFlags { - IN OpData data_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - RegisterDataTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - RegisterDataTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id, - const BucketId &bkt_id, - const std::string &blob_name, - const BlobId &blob_id, - size_t off, - size_t size) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = bkt_id.hash_; - prio_ = TaskPrio::kLowLatency; - pool_ = pool_id; - method_ = Method::kRegisterData; - task_flags_.SetBits(TASK_COROUTINE | TASK_FIRE_AND_FORGET); - domain_id_ = DomainId::GetLocal(); - - // Custom params - data_.bkt_id_ = bkt_id; - data_.blob_name_ = blob_name; - data_.blob_id_ = blob_id; - data_.off_ = off; - data_.size_ = size; - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * Run an operation over a piece of data - * */ -struct RunOpTask : public Task, TaskFlags { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - RunOpTask(const hipc::CtxAllocator &alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - RunOpTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const PoolId &pool_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = 0; - prio_ = TaskPrio::kLongRunning; - pool_ = pool_id; - method_ = Method::kRunOp; - task_flags_.SetBits( - TASK_LONG_RUNNING | TASK_COROUTINE | TASK_LANE_ALL | - TASK_REMOTE_DEBUG_MARK); - SetPeriodSec(1); // TODO(llogan): don't hardcode this - domain_id_ = DomainId::GetLocal(); - } - - /** Duplicate message */ - void Dup(const hipc::CtxAllocator &alloc, RunOpTask &other) { - task_dup(other); - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, RunOpTask &dup_task) {} - - /** Begin replication */ - void ReplicateStart(u32 count) {} - - /** Finalize replication */ - void ReplicateEnd() {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -} // namespace hermes::data_op - -#endif // HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ diff --git a/tasks/old/hermes_data_op/src/CMakeLists.txt b/tasks/old/hermes_data_op/src/CMakeLists.txt deleted file mode 100644 index 0d2a587ba..000000000 --- a/tasks/old/hermes_data_op/src/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Small Message Task Library -#------------------------------------------------------------------------------ -add_library(hermes_data_op SHARED - hermes_data_op.cc) -# add_dependencies(hermes_data_op ${Hermes_RUNTIME_DEPS}) -target_link_libraries(hermes_data_op ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Install Small Message Task Library -#------------------------------------------------------------------------------ -install( - TARGETS - hermes_data_op - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install for import into other projects -#----------------------------------------------------------------------------- -install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_data_op - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#------------------------------------------------------------------------------ -# Coverage -#------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_data_op) -endif() diff --git a/tasks/old/hermes_data_op/src/hermes_data_op.cc b/tasks/old/hermes_data_op/src/hermes_data_op.cc deleted file mode 100644 index a65ae4ede..000000000 --- a/tasks/old/hermes_data_op/src/hermes_data_op.cc +++ /dev/null @@ -1,236 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/api/chimaera_runtime.h" -#include "hermes_data_op/hermes_data_op.h" -#include "hermes_bucket_mdm/hermes_bucket_mdm.h" - -namespace hermes::data_op { - -struct OpPendingData { - std::list pending_; - u32 num_refs_; - u64 data_id_; - - OpPendingData() : num_refs_(0) {} -}; - -class Server : public Module { - public: - std::vector> op_graphs_; - std::unordered_map op_id_map_; - Mutex op_data_lock_; - std::unordered_map op_data_map_; - hermes::bucket_mdm::Client bkt_mdm_; - hermes::blob_mdm::Client blob_mdm_; - Client client_; - LPointer run_task_; - - public: - Server() = default; - - /** Construct data operator table */ - void Construct(ConstructTask *task, RunContext &rctx) { - task->Deserialize(); - bkt_mdm_.Init(task->bkt_mdm_, CHI_ADMIN->queue_id_); - blob_mdm_.Init(task->blob_mdm_, CHI_ADMIN->queue_id_); - client_.Init(id_, CHI_ADMIN->queue_id_); - op_id_map_["min"] = 0; - op_id_map_["max"] = 1; - op_graphs_.resize(HRUN_QM_RUNTIME->max_lanes_); - run_task_ = client_.AsyncRunOp(task->task_node_ + 1); - task->SetModuleComplete(); - } - void MonitorConstruct(u32 mode, ConstructTask *task, RunContext &rctx) { - } - - /** Destroy data operators */ - void Destruct(DestructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - void MonitorDestruct(u32 mode, DestructTask *task, RunContext &rctx) { - } - - /** Registor operators */ - void RegisterOp(RegisterOpTask *task, RunContext &rctx) { - // Load OpGraph - op_graphs_[rctx.lane_id_].push_back(task->GetOpGraph()); - OpGraph &op_graph = op_graphs_[rctx.lane_id_].back(); - - // Get or create all needed bucket IDs - std::vector traits; - for (Op &op : op_graph.ops_) { - // Spawn bucket ID task for each input - for (OpBucketName &bkt_name : op.in_) { - bkt_name.bkt_id_task_ = - bkt_mdm_.AsyncGetOrCreateTag(task->task_node_ + 1, - chi::charbuf(bkt_name.url_), - true, - traits, 0, 0); - } - // Spawn bucket ID task for the output - op.var_name_.bkt_id_task_ = - bkt_mdm_.AsyncGetOrCreateTag(task->task_node_ + 1, - chi::charbuf(op.var_name_.url_), - true, - traits, 0, 0); - // Set the op ID - op.op_id_ = op_id_map_[op.op_name_]; - } - - // Complete all bucket ID tasks - for (Op &op : op_graph.ops_) { - // Spawn bucket ID task for each input - for (OpBucketName &bkt_name : op.in_) { - bkt_name.bkt_id_task_->Wait(task); - bkt_name.bkt_id_ = bkt_name.bkt_id_task_->tag_id_; - op_data_map_.emplace(bkt_name.bkt_id_, OpPendingData()); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, bkt_name.bkt_id_task_); - } - // Spawn bucket ID task for the output - op.var_name_.bkt_id_task_->Wait(task); - op.var_name_.bkt_id_ = op.var_name_.bkt_id_task_->tag_id_; - op_data_map_.emplace(op.var_name_.bkt_id_, OpPendingData()); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, op.var_name_.bkt_id_task_); - } - - // Get number of operations that depend on each data object - for (Op &op : op_graph.ops_) { - for (OpBucketName &bkt_name : op.in_) { - op_data_map_[bkt_name.bkt_id_].num_refs_++; - } - } - - // Store the operator to perform - task->SetModuleComplete(); - } - void MonitorRegisterOp(u32 mode, RegisterOpTask *task, RunContext &rctx) { - } - - /** Inform that data is ready for operators */ - void RegisterData(RegisterDataTask *task, RunContext &rctx) { - if (!op_data_lock_.TryLock(0)) { - return; - } - OpPendingData &op_data = op_data_map_[task->data_.bkt_id_]; - task->data_.data_id_ = op_data.data_id_++; - op_data.pending_.emplace_back(task->data_); - op_data_lock_.Unlock(); - task->SetModuleComplete(); - } - void MonitorRegisterData(u32 mode, RegisterDataTask *task, RunContext &rctx) { - } - - /** Run the operator */ - void RunOp(RunOpTask *task, RunContext &rctx) { - for (OpGraph &op_graph : op_graphs_[rctx.lane_id_]) { - for (Op &op : op_graph.ops_) { - switch(op.op_id_) { - case 0: - RunMin(task, op); - break; - case 1: - RunMax(task, op); - break; - } - } - } - } - void MonitorRunOp(u32 mode, RunOpTask *task, RunContext &rctx) { - } - - /** Get pending data needed for the operator */ - std::list GetPendingData(Op &op) { - std::list pending; - if (!op_data_lock_.TryLock(0)) { - return pending; - } - for (OpBucketName &bkt_name : op.in_) { - if (op_data_map_.find(bkt_name.bkt_id_) == op_data_map_.end()) { - continue; - } - OpPendingData &op_data = op_data_map_[bkt_name.bkt_id_]; - pending = op_data.pending_; - std::list pruned; - for (auto iter = pending.begin(); iter != pending.end(); ++iter) { - OpData &data = *iter; - ++data.refcnt_; - if (data.refcnt_ < op_data.num_refs_) { - pruned.push_back(data); - } - op_data.pending_ = pruned; - } - } - op_data_lock_.Unlock(); - return pending; - } - - /** Get the minimum of data */ - void RunMin(RunOpTask *task, Op &op) { - // Get the pending data from Hermes - std::list op_data = GetPendingData(op); - if (op_data.empty()) { - return; - } - - // Get the input data from Hermes - typedef std::tuple, - LPointer> DataPair; - std::vector in_tasks; - for (OpData &data : op_data) { - // Get the input data - LPointer data_ptr = - CHI_CLIENT->AllocateBufferServer(data.size_, task); - LPointer in_task = - blob_mdm_.AsyncGetBlob(task->task_node_ + 1, - data.bkt_id_, - chi::charbuf(""), data.blob_id_, - data.off_, data.size_, - data_ptr.shm_); - in_tasks.emplace_back(data, data_ptr, in_task); - } - - // Calculate the min of input data - for (DataPair &data_pair : in_tasks) { - // Wait for data to be available - OpData &data = std::get<0>(data_pair); - LPointer &data_ptr = std::get<1>(data_pair); - LPointer &in_task = std::get<2>(data_pair); - in_task->Wait(task); - - // Calaculate the minimum - LPointer min_lptr = - CHI_CLIENT->AllocateBufferServer(sizeof(float), task); - float *min_ptr = (float*)min_lptr.ptr_; - *min_ptr = std::numeric_limits::max(); - for (size_t i = 0; i < in_task->data_size_; i += sizeof(float)) { - *min_ptr = std::min(*min_ptr, *(float*)(data_ptr.ptr_ + i)); - } - - // Store the minimum in Hermes - std::string min_blob_name = data.blob_name_; - blob_mdm_.AsyncPutBlob(task->task_node_ + 1, - op.var_name_.bkt_id_, - chi::charbuf(min_blob_name), - BlobId::GetNull(), - 0, sizeof(float), - min_lptr.shm_, 0, 0); - CHI_CLIENT->FreeBuffer(in_task->data_); - CHI_CLIENT->DelTask(HSHM_DEFAULT_MEM_CTX, in_task); - } - } - - /** Get the maximum of data */ - void RunMax(RunOpTask *task, Op &op) { - } - - public: -#include "hermes_data_op/hermes_data_op_lib_exec.h" -}; - -} // namespace hermes::data_op - -HRUN_TASK_CC(hermes::data_op::Server, "hermes_data_op"); diff --git a/tasks/old/hermes_mdm/CMakeLists.txt b/tasks/old/hermes_mdm/CMakeLists.txt deleted file mode 100644 index d48611782..000000000 --- a/tasks/old/hermes_mdm/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Hrun Admin Task Library -#------------------------------------------------------------------------------ -include_directories(include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install HRUN Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm.h b/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm.h deleted file mode 100644 index 7f4e7a4ae..000000000 --- a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm.h +++ /dev/null @@ -1,41 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#ifndef HRUN_hermes_core_H_ -#define HRUN_hermes_core_H_ - -#include "hermes_core_tasks.h" - -namespace hermes::mdm { - -/** Create requests */ -class Client : public ModuleClient { - public: - /** Default constructor */ - Client() = default; - - /** Destructor */ - ~Client() = default; - - /** Create a hermes_core */ - HSHM_ALWAYS_INLINE - void Create(const DomainQuery &dom_query, - const std::string &state_name) { - id_ = PoolId::GetNull(); - std::vector queue_info; - id_ = CHI_ADMIN->CreateTaskState( - domain_id, state_name, id_, queue_info); - Init(id_, CHI_ADMIN->queue_id_); - } - - /** Destroy task state + queue */ - HSHM_ALWAYS_INLINE - void Destroy(const DomainQuery &dom_query) { - CHI_ADMIN->DestroyTaskState(domain_id, id_); - } -}; - -} // namespace hrun - -#endif // HRUN_hermes_core_H_ diff --git a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h b/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h deleted file mode 100644 index 5e3e4534d..000000000 --- a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h +++ /dev/null @@ -1,166 +0,0 @@ -#ifndef HRUN_HERMES_MDM_LIB_EXEC_H_ -#define HRUN_HERMES_MDM_LIB_EXEC_H_ - -/** Execute a task */ -void Run(u32 method, Task *task, RunContext &rctx) override { - switch (method) { - case Method::kConstruct: { - Construct(reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - Destruct(reinterpret_cast(task), rctx); - break; - } - } -} -/** Execute a task */ -void Monitor(u32 mode, Task *task, RunContext &rctx) override { - switch (task->method_) { - case Method::kConstruct: { - MonitorConstruct(mode, reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - MonitorDestruct(mode, reinterpret_cast(task), rctx); - break; - } - } -} -/** Delete a task */ -void Del(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - CHI_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - } -} -/** Duplicate a task */ -void Dup(u32 method, Task *orig_task, std::vector> &dups) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - } -} -/** Register the duplicate output with the origin task */ -void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestruct: { - chi::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - } -} -/** Ensure there is space to store replicated outputs */ -void ReplicateStart(u32 method, u32 count, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - } -} -/** Determine success and handle failures */ -void ReplicateEnd(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - chi::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - } -} -/** Serialize a task when initially pushing into remote */ -std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when popping from remote queue */ -TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { - TaskPointer task_ptr; - switch (method) { - case Method::kConstruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestruct: { - task_ptr.ptr_ = CHI_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - } - return task_ptr; -} -/** Serialize a task when returning from remote queue */ -std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when returning from remote queue */ -void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - } -} -/** Get the grouping of the task */ -u32 GetGroup(u32 method, Task *task, chi::charbuf &group) override { - switch (method) { - case Method::kConstruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestruct: { - return reinterpret_cast(task)->GetGroup(group); - } - } - return -1; -} - -#endif // HRUN_HERMES_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h b/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h deleted file mode 100644 index 2dc9001ad..000000000 --- a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef HRUN_HERMES_MDM_METHODS_H_ -#define HRUN_HERMES_MDM_METHODS_H_ - -/** The set of methods in the admin task */ -struct Method : public TaskMethod { -}; - -#endif // HRUN_HERMES_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.yaml b/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h b/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h deleted file mode 100644 index 1249ebef4..000000000 --- a/tasks/old/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h +++ /dev/null @@ -1,84 +0,0 @@ -// -// Created by lukemartinlogan on 8/14/23. -// - -#ifndef HRUN_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ -#define HRUN_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ - -#include "chimaera/api/chimaera_client.h" -#include "chimaera/module_registry/task_lib.h" -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/queue_manager/queue_manager_client.h" -#include "hermes/hermes_types.h" -#include "bdev/bdev.h" -#include "proc_queue/proc_queue.h" - -namespace hermes::mdm { - -#include "hermes_core_methods.h" -#include "chimaera/chimaera_namespace.h" - -/** - * A task to create hermes_core - * */ -using chi::Admin::CreateTaskStateTask; -struct ConstructTask : public CreateTaskStateTask { - IN chi::string server_config_path_; - - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc) : CreateTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const std::string &state_name, - const PoolId &id, - const std::vector &queue_info, - const std::string &server_config_path = "") - : CreateTaskStateTask(alloc, task_node, domain_id, state_name, - "hermes_core", id, queue_info) { - // Custom params - HSHM_MAKE_AR(server_config_path_, alloc, server_config_path); - std::stringstream ss; - cereal::BinaryOutputArchive ar(ss); - ar(server_config_path_); - std::string data = ss.str(); - *custom_ = data; - } - - void Deserialize() { - std::string data = custom_->str(); - std::stringstream ss(data); - cereal::BinaryInputArchive ar(ss); - ar(server_config_path_); - } -}; - -/** A task to destroy hermes_core */ -using chi::Admin::DestroyTaskStateTask; -struct DestructTask : public DestroyTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(const hipc::CtxAllocator &alloc) : DestroyTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(const hipc::CtxAllocator &alloc, - const TaskNode &task_node, - const DomainQuery &dom_query, - const PoolId &pool_id) - : DestroyTaskStateTask(alloc, task_node, domain_id, pool_id) {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(chi::charbuf &group) { - return TASK_UNORDERED; - } -}; - -} // namespace hermes::mdm - -#endif // HRUN_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ diff --git a/tasks/old/hermes_mdm/src/CMakeLists.txt b/tasks/old/hermes_mdm/src/CMakeLists.txt deleted file mode 100644 index 655b82fa4..000000000 --- a/tasks/old/hermes_mdm/src/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Small Message Task Library -#------------------------------------------------------------------------------ -add_library(hermes_core SHARED - hermes_core.cc) -# add_dependencies(hermes_core ${Hermes_RUNTIME_DEPS} hermes) -target_link_libraries(hermes_core ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Install Small Message Task Library -#------------------------------------------------------------------------------ -install( - TARGETS - hermes_core - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install for import into other projects -#----------------------------------------------------------------------------- -install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_core - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#------------------------------------------------------------------------------ -# Coverage -#------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_core) -endif() diff --git a/tasks/old/hermes_mdm/src/hermes_mdm.cc b/tasks/old/hermes_mdm/src/hermes_mdm.cc deleted file mode 100644 index 751bb5562..000000000 --- a/tasks/old/hermes_mdm/src/hermes_mdm.cc +++ /dev/null @@ -1,52 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#include "chimaera_admin/chimaera_admin.h" -#include "chimaera/api/chimaera_runtime.h" -#include "hermes/config_server.h" -#include "hermes_core/hermes_core.h" -#include "hermes_blob_mdm/hermes_blob_mdm.h" -#include "hermes_bucket_mdm/hermes_bucket_mdm.h" -#include "hermes/dpe/dpe_factory.h" -#include "bdev/bdev.h" - -namespace hermes::mdm { - -class Server : public Module { - public: - /**==================================== - * Configuration - * ===================================*/ - u32 node_id_; - blob_mdm::Client blob_mdm_; - bucket_mdm::Client bkt_mdm_; - - public: - Server() = default; - - /** Construct hermes MDM */ - void Construct(ConstructTask *task, RunContext &rctx) { - HILOG(kDebug, "ConstructTaskPhase::kLoadConfig") - std::string config_path = task->server_config_path_->str(); - HERMES_CONF->LoadServerConfig(config_path); - node_id_ = CHI_CLIENT->node_id_; - task->SetModuleComplete(); - } - void MonitorConstruct(u32 mode, ConstructTask *task, RunContext &rctx) { - } - - /** Destory hermes MDM */ - void Destruct(DestructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - void MonitorDestruct(u32 mode, DestructTask *task, RunContext &rctx) { - } - - public: -#include "hermes_core/hermes_core_lib_exec.h" -}; - -} // namespace hrun - -HRUN_TASK_CC(hermes::mdm::Server, "hermes_core"); diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 1437cb8e9..9b66be74f 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -499,7 +499,7 @@ TEST_CASE("TestHermesDataStager") { // Put a blob hermes::Blob blob(page_size); memset(blob.data(), i % 256, blob.size()); - chi::charbuf blob_name = hermes::adapter::BlobPlacement::CreateBlobName(i); + chi::string blob_name = hermes::adapter::BlobPlacement::CreateBlobName(i); bkt.Put(blob_name.str(), blob, ctx); hermes::Blob blob2; bkt.Get(blob_name.str(), blob2, ctx); @@ -507,7 +507,7 @@ TEST_CASE("TestHermesDataStager") { REQUIRE(blob2 == blob); } for (size_t i = off; i < proc_count; ++i) { - chi::charbuf blob_name = hermes::adapter::BlobPlacement::CreateBlobName(i); + chi::string blob_name = hermes::adapter::BlobPlacement::CreateBlobName(i); HILOG(kInfo, "ContainsBlob Iteration: {}", i); REQUIRE(bkt.ContainsBlob(blob_name.str())); }