From 39cae0de214c31f90a21d9388f7e49465a33cbc9 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Thu, 17 Oct 2024 09:36:35 -0500 Subject: [PATCH 01/27] integrate lockrequest_gk into gokv tutorial --- tutorial/lockservice/1_lock_rpc.gb.go | 5 +- .../grackle/coq/.lockrequest_proof.aux | 1 + .../grackle/coq/lockrequest_proof.glob | 4 + .../grackle/coq/lockrequest_proof.v | 78 +++++++++++++++++++ .../lockrequest_gk/lockrequest_gk.go | 26 +++++++ tutorial/lockservice/lockservice.proto | 5 ++ 6 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 tutorial/lockservice/grackle/coq/.lockrequest_proof.aux create mode 100644 tutorial/lockservice/grackle/coq/lockrequest_proof.glob create mode 100644 tutorial/lockservice/grackle/coq/lockrequest_proof.v create mode 100644 tutorial/lockservice/lockrequest_gk/lockrequest_gk.go create mode 100644 tutorial/lockservice/lockservice.proto diff --git a/tutorial/lockservice/1_lock_rpc.gb.go b/tutorial/lockservice/1_lock_rpc.gb.go index 2e43797..720aacd 100644 --- a/tutorial/lockservice/1_lock_rpc.gb.go +++ b/tutorial/lockservice/1_lock_rpc.gb.go @@ -2,6 +2,7 @@ package lockservice import ( "github.com/mit-pdos/gokv/grove_ffi" + "github.com/mit-pdos/gokv/tutorial/lockservice/lockrequest_gk" "github.com/mit-pdos/gokv/urpc" ) @@ -29,7 +30,7 @@ func (cl *Client) getFreshNum() (uint64, Error) { func (cl *Client) tryAcquire(id uint64) (uint64, Error) { var reply []byte - args := EncodeUint64(id) + args := lockrequest_gk.Marshal(&lockrequest_gk.S{Id: id}, []byte{}) err := cl.cl.Call(RPC_TRY_ACQUIRE, args, &reply, 100) if err == urpc.ErrNone { return DecodeUint64(reply), err @@ -39,7 +40,7 @@ func (cl *Client) tryAcquire(id uint64) (uint64, Error) { func (cl *Client) release(id uint64) Error { var reply []byte - args := EncodeUint64(id) + args := lockrequest_gk.Marshal(&lockrequest_gk.S{Id: id}, []byte{}) return cl.cl.Call(RPC_RELEASE, args, &reply, 100) } diff --git a/tutorial/lockservice/grackle/coq/.lockrequest_proof.aux b/tutorial/lockservice/grackle/coq/.lockrequest_proof.aux new file mode 100644 index 0000000..c3e1422 --- /dev/null +++ b/tutorial/lockservice/grackle/coq/.lockrequest_proof.aux @@ -0,0 +1 @@ +COQAUX1 f4c24c6d98dc6fdef1208dd5dc379a0b /home/mjs/workspace/grackle/../gokv/tutorial/lockservice/grackle/coq/lockrequest_proof.v diff --git a/tutorial/lockservice/grackle/coq/lockrequest_proof.glob b/tutorial/lockservice/grackle/coq/lockrequest_proof.glob new file mode 100644 index 0000000..792202e --- /dev/null +++ b/tutorial/lockservice/grackle/coq/lockrequest_proof.glob @@ -0,0 +1,4 @@ +DIGEST f4c24c6d98dc6fdef1208dd5dc379a0b +Flockrequest_proof +R44:56 Perennial.program_proof.grove_prelude <> <> lib +R103:125 Perennial.program_proof.marshal_stateless_proof <> <> lib diff --git a/tutorial/lockservice/grackle/coq/lockrequest_proof.v b/tutorial/lockservice/grackle/coq/lockrequest_proof.v new file mode 100644 index 0000000..a3f859e --- /dev/null +++ b/tutorial/lockservice/grackle/coq/lockrequest_proof.v @@ -0,0 +1,78 @@ +From Perennial.program_proof Require Import grove_prelude. +From Perennial.program_proof Require Import marshal_stateless_proof. +From Goose Require Import github_com.mit-pdos.gokv.tutorial.lockservice.grackle.go.lockrequest_gk. + +Module LockRequest. +Section LockRequest. + +Typeclasses Opaque app. + +Context `{!heapGS Σ}. + +Record C := + mkC { + id : u64; + }. + +Definition has_encoding (encoded:list u8) (args:C) : Prop := + encoded = (u64_le args.(id)). + +Definition own (args_ptr:loc) (args:C) (q:dfrac) : iProp Σ := + "Hargs_id" ∷ args_ptr ↦[lockrequest_gk.S :: "id"]{q} #args.(id). + +Lemma wp_Encode (args_ptr:loc) (args:C) (pre_sl:Slice.t) (prefix:list u8) : + {{{ + own args_ptr args (DfracDiscarded) ∗ + own_slice pre_sl byteT (DfracOwn 1) prefix + }}} + lockrequest_gk.Marshal #args_ptr (slice_val pre_sl) + {{{ + enc enc_sl, RET (slice_val enc_sl); + ⌜has_encoding enc args⌝ ∗ + own_slice enc_sl byteT (DfracOwn 1) (prefix ++ enc) + }}}. + +Proof. + iIntros (?) "H HΦ". iDestruct "H" as "[Hown Hsl]". iNamed "Hown". + wp_rec. wp_pures. + wp_apply (wp_ref_to); first by val_ty. + iIntros (?) "Hptr". wp_pures. + wp_loadField. wp_load. wp_apply (wp_WriteInt with "[$Hsl]"). + iIntros (?) "Hsl". wp_store. + + wp_load. iApply "HΦ". iModIntro. rewrite -?app_assoc. + iFrame. iPureIntro. + + done. +Qed. + +Lemma wp_Decode enc enc_sl (args:C) (suffix:list u8) (q:dfrac): + {{{ + ⌜has_encoding enc args⌝ ∗ + own_slice_small enc_sl byteT q (enc ++ suffix) + }}} + lockrequest_gk.Unmarshal (slice_val enc_sl) + {{{ + args_ptr suff_sl, RET (#args_ptr, suff_sl); own args_ptr args (DfracOwn 1) ∗ + own_slice_small suff_sl byteT q suffix + }}}. + +Proof. + iIntros (?) "[%Henc Hsl] HΦ". wp_rec. + wp_apply wp_allocStruct; first by val_ty. + iIntros (?) "Hstruct". wp_pures. + wp_apply wp_ref_to; first done. + iIntros (?) "Hptr". wp_pures. + iDestruct (struct_fields_split with "Hstruct") as "HH". + iNamed "HH". + + rewrite Henc. rewrite -?app_assoc. + + wp_load. wp_apply (wp_ReadInt with "[$Hsl]"). iIntros (?) "Hsl". + wp_pures. wp_storeField. wp_store. + wp_load. wp_pures. iApply "HΦ". iModIntro. iFrame. +Qed. + +End LockRequest. +End LockRequest. + diff --git a/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go b/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go new file mode 100644 index 0000000..0511f51 --- /dev/null +++ b/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go @@ -0,0 +1,26 @@ +package lockrequest_gk + +import ( + "github.com/tchajed/marshal" +) + +type S struct { + Id uint64 +} + +func (l *S) approxSize() uint64 { + return 0 +} + +func Marshal(l *S, prefix []byte) []byte { + var enc = prefix + enc = marshal.WriteInt(enc, l.id) + return enc +} + +func Unmarshal(s []byte) (*S, []byte) { + l := new(S) + var enc = s // Needed for goose compatibility + l.id, enc = marshal.ReadInt(enc) + return l, enc +} diff --git a/tutorial/lockservice/lockservice.proto b/tutorial/lockservice/lockservice.proto new file mode 100644 index 0000000..7ff8a14 --- /dev/null +++ b/tutorial/lockservice/lockservice.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +message LockRequest { + uint64 id = 1; +} From 4e2602cdf4f948df1041b21078a92a3f263912e8 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Thu, 17 Oct 2024 09:37:25 -0500 Subject: [PATCH 02/27] remove extra coq files --- tutorial/lockservice/grackle/coq/.lockrequest_proof.aux | 1 - tutorial/lockservice/grackle/coq/lockrequest_proof.glob | 4 ---- 2 files changed, 5 deletions(-) delete mode 100644 tutorial/lockservice/grackle/coq/.lockrequest_proof.aux delete mode 100644 tutorial/lockservice/grackle/coq/lockrequest_proof.glob diff --git a/tutorial/lockservice/grackle/coq/.lockrequest_proof.aux b/tutorial/lockservice/grackle/coq/.lockrequest_proof.aux deleted file mode 100644 index c3e1422..0000000 --- a/tutorial/lockservice/grackle/coq/.lockrequest_proof.aux +++ /dev/null @@ -1 +0,0 @@ -COQAUX1 f4c24c6d98dc6fdef1208dd5dc379a0b /home/mjs/workspace/grackle/../gokv/tutorial/lockservice/grackle/coq/lockrequest_proof.v diff --git a/tutorial/lockservice/grackle/coq/lockrequest_proof.glob b/tutorial/lockservice/grackle/coq/lockrequest_proof.glob deleted file mode 100644 index 792202e..0000000 --- a/tutorial/lockservice/grackle/coq/lockrequest_proof.glob +++ /dev/null @@ -1,4 +0,0 @@ -DIGEST f4c24c6d98dc6fdef1208dd5dc379a0b -Flockrequest_proof -R44:56 Perennial.program_proof.grove_prelude <> <> lib -R103:125 Perennial.program_proof.marshal_stateless_proof <> <> lib From 862ce70fa05f2a2139787667afb47aa9063247b6 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 5 Nov 2024 13:58:45 -0600 Subject: [PATCH 03/27] config gokv for development with nix --- .envrc | 1 + .gitignore | 1 + flake.lock | 24 +++++++++++++++++++++++ flake.nix | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+) create mode 100644 .envrc create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore index bdee72a..66e07dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ kvdur kvdur_log *.prof +.direnv diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..7973da7 --- /dev/null +++ b/flake.lock @@ -0,0 +1,24 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1730531603, + "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=", + "path": "/nix/store/zq2axpgzd5kykk1v446rkffj3bxa2m2h-source", + "rev": "7ffd9ae656aec493492b44d0ddfb28e79a1ea25d", + "type": "path" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..52e0f23 --- /dev/null +++ b/flake.nix @@ -0,0 +1,57 @@ +{ + description = "A Flake for Applying Grackle to gokv"; + + inputs = { + nixpkgs.url = "nixpkgs"; + }; + + outputs = {nixpkgs, ...}: let + system = "x86_64-linux"; + in { + devShells."${system}".default = let + pkgs = import nixpkgs { + inherit system; + }; + goose = pkgs.buildGoModule { + name = "goose"; + src = pkgs.fetchFromGitHub { + owner = "goose-lang"; + repo = "goose"; + rev = "585abc3cfef50dd466e112d7c535dbdfccd3c0ca"; + hash = "sha256-M4zaZ1DdecYXeDugrL+TV7HWPMLuj1P25G6mf+fgljg="; + }; + vendorHash = "sha256-HCJ8v3TSv4UrkOsRuENWVz5Z7zQ1UsOygx0Mo7MELzY="; + }; + grackle = pkgs.buildGoModule { + name = "grackle"; + src = pkgs.fetchFromGitHub { + owner = "mjschwenne"; + repo = "grackle"; + rev = "2691072f22299d54e8b9e1cd6d2514e5bdea7f95"; + hash = "sha256-c+a3qRmqUiTaoo8p9GoCe2ZFBw5aJfj7Bhq/EJ4ayfI="; + }; + vendorHash = "sha256-Wk2v0HSAkrzxHJvCfbw6xOn0OQ1xukvYjDxk3c2LmH8="; + checkPhase = false; + }; + in + pkgs.mkShell { + # create an environment with the required coq libraries + packages = with pkgs; [ + # Go deps + go + gopls + goose + grackle + + # Protobuf deps + protobuf + protoc-gen-go + proto-contrib + protoscope + ]; + + shellHook = '' + ''; + }; + }; +} From c45937f5a6abba09c0e82360755e2b6d15a1ba94 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 5 Nov 2024 14:09:32 -0600 Subject: [PATCH 04/27] remove coq file from gokv --- .../grackle/coq/lockrequest_proof.v | 78 ------------------- 1 file changed, 78 deletions(-) delete mode 100644 tutorial/lockservice/grackle/coq/lockrequest_proof.v diff --git a/tutorial/lockservice/grackle/coq/lockrequest_proof.v b/tutorial/lockservice/grackle/coq/lockrequest_proof.v deleted file mode 100644 index a3f859e..0000000 --- a/tutorial/lockservice/grackle/coq/lockrequest_proof.v +++ /dev/null @@ -1,78 +0,0 @@ -From Perennial.program_proof Require Import grove_prelude. -From Perennial.program_proof Require Import marshal_stateless_proof. -From Goose Require Import github_com.mit-pdos.gokv.tutorial.lockservice.grackle.go.lockrequest_gk. - -Module LockRequest. -Section LockRequest. - -Typeclasses Opaque app. - -Context `{!heapGS Σ}. - -Record C := - mkC { - id : u64; - }. - -Definition has_encoding (encoded:list u8) (args:C) : Prop := - encoded = (u64_le args.(id)). - -Definition own (args_ptr:loc) (args:C) (q:dfrac) : iProp Σ := - "Hargs_id" ∷ args_ptr ↦[lockrequest_gk.S :: "id"]{q} #args.(id). - -Lemma wp_Encode (args_ptr:loc) (args:C) (pre_sl:Slice.t) (prefix:list u8) : - {{{ - own args_ptr args (DfracDiscarded) ∗ - own_slice pre_sl byteT (DfracOwn 1) prefix - }}} - lockrequest_gk.Marshal #args_ptr (slice_val pre_sl) - {{{ - enc enc_sl, RET (slice_val enc_sl); - ⌜has_encoding enc args⌝ ∗ - own_slice enc_sl byteT (DfracOwn 1) (prefix ++ enc) - }}}. - -Proof. - iIntros (?) "H HΦ". iDestruct "H" as "[Hown Hsl]". iNamed "Hown". - wp_rec. wp_pures. - wp_apply (wp_ref_to); first by val_ty. - iIntros (?) "Hptr". wp_pures. - wp_loadField. wp_load. wp_apply (wp_WriteInt with "[$Hsl]"). - iIntros (?) "Hsl". wp_store. - - wp_load. iApply "HΦ". iModIntro. rewrite -?app_assoc. - iFrame. iPureIntro. - - done. -Qed. - -Lemma wp_Decode enc enc_sl (args:C) (suffix:list u8) (q:dfrac): - {{{ - ⌜has_encoding enc args⌝ ∗ - own_slice_small enc_sl byteT q (enc ++ suffix) - }}} - lockrequest_gk.Unmarshal (slice_val enc_sl) - {{{ - args_ptr suff_sl, RET (#args_ptr, suff_sl); own args_ptr args (DfracOwn 1) ∗ - own_slice_small suff_sl byteT q suffix - }}}. - -Proof. - iIntros (?) "[%Henc Hsl] HΦ". wp_rec. - wp_apply wp_allocStruct; first by val_ty. - iIntros (?) "Hstruct". wp_pures. - wp_apply wp_ref_to; first done. - iIntros (?) "Hptr". wp_pures. - iDestruct (struct_fields_split with "Hstruct") as "HH". - iNamed "HH". - - rewrite Henc. rewrite -?app_assoc. - - wp_load. wp_apply (wp_ReadInt with "[$Hsl]"). iIntros (?) "Hsl". - wp_pures. wp_storeField. wp_store. - wp_load. wp_pures. iApply "HΦ". iModIntro. iFrame. -Qed. - -End LockRequest. -End LockRequest. - From 45ac296d928b64af7ef440692bc0dd8c686f10a2 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 5 Nov 2024 15:48:03 -0600 Subject: [PATCH 05/27] Create proto files for gokv tutorial --- tutorial/kvservice/kvservice.proto | 19 ++++++++++++ tutorial/objectstore/chunk/chunk.proto | 8 +++++ tutorial/objectstore/dir/objectstore.proto | 34 ++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 tutorial/kvservice/kvservice.proto create mode 100644 tutorial/objectstore/chunk/chunk.proto create mode 100644 tutorial/objectstore/dir/objectstore.proto diff --git a/tutorial/kvservice/kvservice.proto b/tutorial/kvservice/kvservice.proto new file mode 100644 index 0000000..c1e12cc --- /dev/null +++ b/tutorial/kvservice/kvservice.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +message put { + uint64 opId = 1; + string key = 2; + string val = 3; +} + +message connditionalPut { + uint64 opId = 1; + string key = 2; + string expectedVal = 3; + string newVal = 4; +} + +message get { + uint64 opId = 1; + string key = 2; +} diff --git a/tutorial/objectstore/chunk/chunk.proto b/tutorial/objectstore/chunk/chunk.proto new file mode 100644 index 0000000..f1a4572 --- /dev/null +++ b/tutorial/objectstore/chunk/chunk.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +message writeChunk { + // De-aliasing WriteID to uint64 + uint64 writeId = 1; + bytes chunk = 2; + uint64 index = 3; +} diff --git a/tutorial/objectstore/dir/objectstore.proto b/tutorial/objectstore/dir/objectstore.proto new file mode 100644 index 0000000..f18bc49 --- /dev/null +++ b/tutorial/objectstore/dir/objectstore.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +message preparedWrite { + // De-asliasing WriteID to uint64 + uint64 id = 1; + // De-asliasing grove_ffi.Address to uint64 + repeated uint64 checkAddrs = 2; +} + +message recordChucks { + // De-asliasing WriteID to uint64 + uint64 writeId = 1; + // De-asliasing grove_ffi.Address to uint64 + uint64 server = 2; + string contentHash = 3; + uint64 index = 4; +} + +message finishWrite { + // De-asliasing WriteID to uint64 + uint64 writeId = 1; + string keyname = 2; +} + +message chunkHandle { + // De-asliasing grove_ffi.Address to uint64 + uint64 addr = 1; + string contentHash = 2; +} + +message preparedRead { + // Slice of nested structs + repeated chunkHandle handles = 1; +} From 69b75d34a8d2f63c23563574b302fe1f02eb0d48 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 5 Nov 2024 15:49:12 -0600 Subject: [PATCH 06/27] Setup commit emails --- .github/commit-emails.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/commit-emails.toml diff --git a/.github/commit-emails.toml b/.github/commit-emails.toml new file mode 100644 index 0000000..0bc2546 --- /dev/null +++ b/.github/commit-emails.toml @@ -0,0 +1,2 @@ +# commit-emails.xyz config +to = "chajed@wisc.edu,schwennesen@cs.wisc.edu" From 0415483d71c1f49f75c00573adb6e398f1204640 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Wed, 6 Nov 2024 16:15:59 -0600 Subject: [PATCH 07/27] grackle go generation --- .../connditionalput_gk/connditionalput_gk.go | 57 +++++++++++++++++++ tutorial/kvservice/get_gk/get_gk.go | 39 +++++++++++++ tutorial/kvservice/put_gk/put_gk.go | 48 ++++++++++++++++ .../chunk/writechunk_gk/writechunk_gk.go | 41 +++++++++++++ .../dir/chunkhandle_gk/chunkhandle_gk.go | 39 +++++++++++++ .../dir/finishwrite_gk/finishwrite_gk.go | 39 +++++++++++++ tutorial/objectstore/dir/objectstore.proto | 20 +++---- .../dir/recordchucks_gk/recordchucks_gk.go | 45 +++++++++++++++ 8 files changed, 318 insertions(+), 10 deletions(-) create mode 100644 tutorial/kvservice/connditionalput_gk/connditionalput_gk.go create mode 100644 tutorial/kvservice/get_gk/get_gk.go create mode 100644 tutorial/kvservice/put_gk/put_gk.go create mode 100644 tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go create mode 100644 tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go create mode 100644 tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go create mode 100644 tutorial/objectstore/dir/recordchucks_gk/recordchucks_gk.go diff --git a/tutorial/kvservice/connditionalput_gk/connditionalput_gk.go b/tutorial/kvservice/connditionalput_gk/connditionalput_gk.go new file mode 100644 index 0000000..31da54e --- /dev/null +++ b/tutorial/kvservice/connditionalput_gk/connditionalput_gk.go @@ -0,0 +1,57 @@ +package connditionalput_gk + +import ( + "github.com/tchajed/marshal" +) + +type S struct { + OpId uint64 + Key string + ExpectedVal string + NewVal string +} + +func (c *S) approxSize() uint64 { + return 0 +} + +func Marshal(c *S, prefix []byte) []byte { + var enc = prefix + + enc = marshal.WriteInt(enc, c.OpId) + keyBytes := []byte(c.Key) + enc = marshal.WriteInt(enc, uint64(len(keyBytes))) + enc = marshal.WriteBytes(enc, keyBytes) + expectedvalBytes := []byte(c.ExpectedVal) + enc = marshal.WriteInt(enc, uint64(len(expectedvalBytes))) + enc = marshal.WriteBytes(enc, expectedvalBytes) + newvalBytes := []byte(c.NewVal) + enc = marshal.WriteInt(enc, uint64(len(newvalBytes))) + enc = marshal.WriteBytes(enc, newvalBytes) + + return enc +} + +func Unmarshal(s []byte) (*S, []byte) { + c := new(S) + var enc = s // Needed for goose compatibility + + c.OpId, enc = marshal.ReadInt(enc) + var keyLen uint64 + var keyBytes []byte + keyLen, enc = marshal.ReadInt(enc) + keyBytes, enc = marshal.ReadBytes(enc, keyLen) + c.Key = string(keyBytes) + var expectedValLen uint64 + var expectedValBytes []byte + expectedValLen, enc = marshal.ReadInt(enc) + expectedValBytes, enc = marshal.ReadBytes(enc, expectedValLen) + c.ExpectedVal = string(expectedValBytes) + var newValLen uint64 + var newValBytes []byte + newValLen, enc = marshal.ReadInt(enc) + newValBytes, enc = marshal.ReadBytes(enc, newValLen) + c.NewVal = string(newValBytes) + + return c, enc +} diff --git a/tutorial/kvservice/get_gk/get_gk.go b/tutorial/kvservice/get_gk/get_gk.go new file mode 100644 index 0000000..f63ad3c --- /dev/null +++ b/tutorial/kvservice/get_gk/get_gk.go @@ -0,0 +1,39 @@ +package get_gk + +import ( + "github.com/tchajed/marshal" +) + +type S struct { + OpId uint64 + Key string +} + +func (g *S) approxSize() uint64 { + return 0 +} + +func Marshal(g *S, prefix []byte) []byte { + var enc = prefix + + enc = marshal.WriteInt(enc, g.OpId) + keyBytes := []byte(g.Key) + enc = marshal.WriteInt(enc, uint64(len(keyBytes))) + enc = marshal.WriteBytes(enc, keyBytes) + + return enc +} + +func Unmarshal(s []byte) (*S, []byte) { + g := new(S) + var enc = s // Needed for goose compatibility + + g.OpId, enc = marshal.ReadInt(enc) + var keyLen uint64 + var keyBytes []byte + keyLen, enc = marshal.ReadInt(enc) + keyBytes, enc = marshal.ReadBytes(enc, keyLen) + g.Key = string(keyBytes) + + return g, enc +} diff --git a/tutorial/kvservice/put_gk/put_gk.go b/tutorial/kvservice/put_gk/put_gk.go new file mode 100644 index 0000000..e60a2a9 --- /dev/null +++ b/tutorial/kvservice/put_gk/put_gk.go @@ -0,0 +1,48 @@ +package put_gk + +import ( + "github.com/tchajed/marshal" +) + +type S struct { + OpId uint64 + Key string + Val string +} + +func (p *S) approxSize() uint64 { + return 0 +} + +func Marshal(p *S, prefix []byte) []byte { + var enc = prefix + + enc = marshal.WriteInt(enc, p.OpId) + keyBytes := []byte(p.Key) + enc = marshal.WriteInt(enc, uint64(len(keyBytes))) + enc = marshal.WriteBytes(enc, keyBytes) + valBytes := []byte(p.Val) + enc = marshal.WriteInt(enc, uint64(len(valBytes))) + enc = marshal.WriteBytes(enc, valBytes) + + return enc +} + +func Unmarshal(s []byte) (*S, []byte) { + p := new(S) + var enc = s // Needed for goose compatibility + + p.OpId, enc = marshal.ReadInt(enc) + var keyLen uint64 + var keyBytes []byte + keyLen, enc = marshal.ReadInt(enc) + keyBytes, enc = marshal.ReadBytes(enc, keyLen) + p.Key = string(keyBytes) + var valLen uint64 + var valBytes []byte + valLen, enc = marshal.ReadInt(enc) + valBytes, enc = marshal.ReadBytes(enc, valLen) + p.Val = string(valBytes) + + return p, enc +} diff --git a/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go b/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go new file mode 100644 index 0000000..7705a54 --- /dev/null +++ b/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go @@ -0,0 +1,41 @@ +package writechunk_gk + +import ( + "github.com/tchajed/marshal" +) + +type S struct { + WriteId uint64 + Chunk []byte + Index uint64 +} + +func (w *S) approxSize() uint64 { + return 0 +} + +func Marshal(w *S, prefix []byte) []byte { + var enc = prefix + + enc = marshal.WriteInt(enc, w.WriteId) + enc = marshal.WriteInt(enc, uint64(len(w.Chunk))) + enc = marshal.WriteBytes(enc, w.Chunk) + enc = marshal.WriteInt(enc, w.Index) + + return enc +} + +func Unmarshal(s []byte) (*S, []byte) { + w := new(S) + var enc = s // Needed for goose compatibility + + w.WriteId, enc = marshal.ReadInt(enc) + var chunkLen uint64 + var chunkBytes []byte + chunkLen, enc = marshal.ReadInt(enc) + chunkBytes, enc = marshal.ReadBytes(enc, chunkLen) + w.Chunk = chunkBytes + w.Index, enc = marshal.ReadInt(enc) + + return w, enc +} diff --git a/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go b/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go new file mode 100644 index 0000000..99d683c --- /dev/null +++ b/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go @@ -0,0 +1,39 @@ +package chunkhandle_gk + +import ( + "github.com/tchajed/marshal" +) + +type S struct { + Addr uint64 + ContentHash string +} + +func (c *S) approxSize() uint64 { + return 0 +} + +func Marshal(c *S, prefix []byte) []byte { + var enc = prefix + + enc = marshal.WriteInt(enc, c.Addr) + contenthashBytes := []byte(c.ContentHash) + enc = marshal.WriteInt(enc, uint64(len(contenthashBytes))) + enc = marshal.WriteBytes(enc, contenthashBytes) + + return enc +} + +func Unmarshal(s []byte) (*S, []byte) { + c := new(S) + var enc = s // Needed for goose compatibility + + c.Addr, enc = marshal.ReadInt(enc) + var contentHashLen uint64 + var contentHashBytes []byte + contentHashLen, enc = marshal.ReadInt(enc) + contentHashBytes, enc = marshal.ReadBytes(enc, contentHashLen) + c.ContentHash = string(contentHashBytes) + + return c, enc +} diff --git a/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go b/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go new file mode 100644 index 0000000..9f97b73 --- /dev/null +++ b/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go @@ -0,0 +1,39 @@ +package finishwrite_gk + +import ( + "github.com/tchajed/marshal" +) + +type S struct { + WriteId uint64 + Keyname string +} + +func (f *S) approxSize() uint64 { + return 0 +} + +func Marshal(f *S, prefix []byte) []byte { + var enc = prefix + + enc = marshal.WriteInt(enc, f.WriteId) + keynameBytes := []byte(f.Keyname) + enc = marshal.WriteInt(enc, uint64(len(keynameBytes))) + enc = marshal.WriteBytes(enc, keynameBytes) + + return enc +} + +func Unmarshal(s []byte) (*S, []byte) { + f := new(S) + var enc = s // Needed for goose compatibility + + f.WriteId, enc = marshal.ReadInt(enc) + var keynameLen uint64 + var keynameBytes []byte + keynameLen, enc = marshal.ReadInt(enc) + keynameBytes, enc = marshal.ReadBytes(enc, keynameLen) + f.Keyname = string(keynameBytes) + + return f, enc +} diff --git a/tutorial/objectstore/dir/objectstore.proto b/tutorial/objectstore/dir/objectstore.proto index f18bc49..7676b85 100644 --- a/tutorial/objectstore/dir/objectstore.proto +++ b/tutorial/objectstore/dir/objectstore.proto @@ -1,11 +1,11 @@ syntax = "proto3"; -message preparedWrite { - // De-asliasing WriteID to uint64 - uint64 id = 1; - // De-asliasing grove_ffi.Address to uint64 - repeated uint64 checkAddrs = 2; -} +// message preparedWrite { +// // De-asliasing WriteID to uint64 +// uint64 id = 1; +// // De-asliasing grove_ffi.Address to uint64 +// repeated uint64 checkAddrs = 2; +// } message recordChucks { // De-asliasing WriteID to uint64 @@ -28,7 +28,7 @@ message chunkHandle { string contentHash = 2; } -message preparedRead { - // Slice of nested structs - repeated chunkHandle handles = 1; -} +// message preparedRead { +// // Slice of nested structs +// repeated chunkHandle handles = 1; +// } diff --git a/tutorial/objectstore/dir/recordchucks_gk/recordchucks_gk.go b/tutorial/objectstore/dir/recordchucks_gk/recordchucks_gk.go new file mode 100644 index 0000000..9005b9e --- /dev/null +++ b/tutorial/objectstore/dir/recordchucks_gk/recordchucks_gk.go @@ -0,0 +1,45 @@ +package recordchucks_gk + +import ( + "github.com/tchajed/marshal" +) + +type S struct { + WriteId uint64 + Server uint64 + ContentHash string + Index uint64 +} + +func (r *S) approxSize() uint64 { + return 0 +} + +func Marshal(r *S, prefix []byte) []byte { + var enc = prefix + + enc = marshal.WriteInt(enc, r.WriteId) + enc = marshal.WriteInt(enc, r.Server) + contenthashBytes := []byte(r.ContentHash) + enc = marshal.WriteInt(enc, uint64(len(contenthashBytes))) + enc = marshal.WriteBytes(enc, contenthashBytes) + enc = marshal.WriteInt(enc, r.Index) + + return enc +} + +func Unmarshal(s []byte) (*S, []byte) { + r := new(S) + var enc = s // Needed for goose compatibility + + r.WriteId, enc = marshal.ReadInt(enc) + r.Server, enc = marshal.ReadInt(enc) + var contentHashLen uint64 + var contentHashBytes []byte + contentHashLen, enc = marshal.ReadInt(enc) + contentHashBytes, enc = marshal.ReadBytes(enc, contentHashLen) + r.ContentHash = string(contentHashBytes) + r.Index, enc = marshal.ReadInt(enc) + + return r, enc +} From 3ae75b45276c6f11d552cc3c104f4f72caf22ada Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Fri, 8 Nov 2024 10:28:27 -0600 Subject: [PATCH 08/27] Integrate grackle code into tutorial/kvservice --- tutorial/kvservice/client.go | 27 +++++++++-------- .../conditionalput_gk.go} | 2 +- tutorial/kvservice/kvservice.proto | 2 +- tutorial/kvservice/kvservice_rpc.gb.go | 15 ++++++---- tutorial/kvservice/kvservice_rpc_server.gb.go | 12 ++++++-- tutorial/kvservice/server.go | 29 ++++++++++--------- 6 files changed, 51 insertions(+), 36 deletions(-) rename tutorial/kvservice/{connditionalput_gk/connditionalput_gk.go => conditionalput_gk/conditionalput_gk.go} (98%) diff --git a/tutorial/kvservice/client.go b/tutorial/kvservice/client.go index 4093a44..0d1a58e 100644 --- a/tutorial/kvservice/client.go +++ b/tutorial/kvservice/client.go @@ -2,6 +2,9 @@ package kvservice import ( "github.com/mit-pdos/gokv/grove_ffi" + "github.com/mit-pdos/gokv/tutorial/kvservice/conditionalput_gk" + "github.com/mit-pdos/gokv/tutorial/kvservice/get_gk" + "github.com/mit-pdos/gokv/tutorial/kvservice/put_gk" "github.com/mit-pdos/gokv/urpc" ) @@ -34,10 +37,10 @@ func (ck *Clerk) Put(key string, val string) { for { // TODO: allocate these once, outside the loop. Waiting to do this until // heap has dfrac for convenience. - args := &putArgs{ - opId: opId, - key: key, - val: val, + args := &put_gk.S{ + OpId: opId, + Key: key, + Val: val, } if ck.rpcCl.putRpc(args) == urpc.ErrNone { break @@ -60,11 +63,11 @@ func (ck *Clerk) ConditionalPut(key string, expectedVal string, newVal string) b var ret bool for { - args := &conditionalPutArgs{ - opId: opId, - key: key, - expectedVal: expectedVal, - newVal: newVal, + args := &conditionalput_gk.S{ + OpId: opId, + Key: key, + ExpectedVal: expectedVal, + NewVal: newVal, } reply, err := ck.rpcCl.conditionalPutRpc(args) @@ -91,9 +94,9 @@ func (ck *Clerk) Get(key string) string { var ret string for { - args := &getArgs{ - opId: opId, - key: key, + args := &get_gk.S{ + OpId: opId, + Key: key, } reply, err := ck.rpcCl.getRpc(args) diff --git a/tutorial/kvservice/connditionalput_gk/connditionalput_gk.go b/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go similarity index 98% rename from tutorial/kvservice/connditionalput_gk/connditionalput_gk.go rename to tutorial/kvservice/conditionalput_gk/conditionalput_gk.go index 31da54e..b0d8c4d 100644 --- a/tutorial/kvservice/connditionalput_gk/connditionalput_gk.go +++ b/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go @@ -1,4 +1,4 @@ -package connditionalput_gk +package conditionalput_gk import ( "github.com/tchajed/marshal" diff --git a/tutorial/kvservice/kvservice.proto b/tutorial/kvservice/kvservice.proto index c1e12cc..4f78714 100644 --- a/tutorial/kvservice/kvservice.proto +++ b/tutorial/kvservice/kvservice.proto @@ -6,7 +6,7 @@ message put { string val = 3; } -message connditionalPut { +message conditionalPut { uint64 opId = 1; string key = 2; string expectedVal = 3; diff --git a/tutorial/kvservice/kvservice_rpc.gb.go b/tutorial/kvservice/kvservice_rpc.gb.go index c9b5779..9327da7 100644 --- a/tutorial/kvservice/kvservice_rpc.gb.go +++ b/tutorial/kvservice/kvservice_rpc.gb.go @@ -2,6 +2,9 @@ package kvservice import ( "github.com/mit-pdos/gokv/grove_ffi" + "github.com/mit-pdos/gokv/tutorial/kvservice/conditionalput_gk" + "github.com/mit-pdos/gokv/tutorial/kvservice/get_gk" + "github.com/mit-pdos/gokv/tutorial/kvservice/put_gk" "github.com/mit-pdos/gokv/urpc" ) @@ -27,27 +30,27 @@ func (cl *Client) getFreshNumRpc() (uint64, Error) { return 0, err } -func (cl *Client) putRpc(args *putArgs) Error { +func (cl *Client) putRpc(args *put_gk.S) Error { var reply []byte - err := cl.cl.Call(rpcIdPut, encodePutArgs(args), &reply, 100) + err := cl.cl.Call(rpcIdPut, put_gk.Marshal(args, make([]byte, 0)), &reply, 100) if err == urpc.ErrNone { return err } return err } -func (cl *Client) conditionalPutRpc(args *conditionalPutArgs) (string, Error) { +func (cl *Client) conditionalPutRpc(args *conditionalput_gk.S) (string, Error) { var reply []byte - err := cl.cl.Call(rpcIdConditionalPut, encodeConditionalPutArgs(args), &reply, 100) + err := cl.cl.Call(rpcIdConditionalPut, conditionalput_gk.Marshal(args, make([]byte, 0)), &reply, 100) if err == urpc.ErrNone { return string(reply), err } return "", err } -func (cl *Client) getRpc(args *getArgs) (string, Error) { +func (cl *Client) getRpc(args *get_gk.S) (string, Error) { var reply []byte - err := cl.cl.Call(rpcIdGet, encodeGetArgs(args), &reply, 100) + err := cl.cl.Call(rpcIdGet, get_gk.Marshal(args, make([]byte, 0)), &reply, 100) if err == urpc.ErrNone { return string(reply), err } diff --git a/tutorial/kvservice/kvservice_rpc_server.gb.go b/tutorial/kvservice/kvservice_rpc_server.gb.go index 532b08b..27674a0 100644 --- a/tutorial/kvservice/kvservice_rpc_server.gb.go +++ b/tutorial/kvservice/kvservice_rpc_server.gb.go @@ -2,6 +2,9 @@ package kvservice import ( "github.com/mit-pdos/gokv/grove_ffi" + "github.com/mit-pdos/gokv/tutorial/kvservice/conditionalput_gk" + "github.com/mit-pdos/gokv/tutorial/kvservice/get_gk" + "github.com/mit-pdos/gokv/tutorial/kvservice/put_gk" "github.com/mit-pdos/gokv/urpc" ) @@ -15,17 +18,20 @@ func (s *Server) Start(me grove_ffi.Address) { handlers[rpcIdPut] = func(enc_args []byte, enc_reply *[]byte) { - s.put(decodePutArgs(enc_args)) + args, _ := put_gk.Unmarshal(enc_args) + s.put(args) } handlers[rpcIdConditionalPut] = func(enc_args []byte, enc_reply *[]byte) { - *enc_reply = []byte(s.conditionalPut(decodeConditionalPutArgs(enc_args))) + args, _ := conditionalput_gk.Unmarshal(enc_args) + *enc_reply = []byte(s.conditionalPut(args)) } handlers[rpcIdGet] = func(enc_args []byte, enc_reply *[]byte) { - *enc_reply = []byte(s.get(decodeGetArgs(enc_args))) + args, _ := get_gk.Unmarshal(enc_args) + *enc_reply = []byte(s.get(args)) } urpc.MakeServer(handlers).Serve(me) diff --git a/tutorial/kvservice/server.go b/tutorial/kvservice/server.go index e629f55..d0cf375 100644 --- a/tutorial/kvservice/server.go +++ b/tutorial/kvservice/server.go @@ -2,6 +2,9 @@ package kvservice import ( "github.com/goose-lang/std" + "github.com/mit-pdos/gokv/tutorial/kvservice/conditionalput_gk" + "github.com/mit-pdos/gokv/tutorial/kvservice/get_gk" + "github.com/mit-pdos/gokv/tutorial/kvservice/put_gk" "sync" ) @@ -21,45 +24,45 @@ func (s *Server) getFreshNum() uint64 { return n } -func (s *Server) put(args *putArgs) { +func (s *Server) put(args *put_gk.S) { s.mu.Lock() - _, ok := s.lastReplies[args.opId] + _, ok := s.lastReplies[args.OpId] if ok { s.mu.Unlock() return } - s.kvs[args.key] = args.val - s.lastReplies[args.opId] = "" + s.kvs[args.Key] = args.Val + s.lastReplies[args.OpId] = "" s.mu.Unlock() } -func (s *Server) conditionalPut(args *conditionalPutArgs) string { +func (s *Server) conditionalPut(args *conditionalput_gk.S) string { s.mu.Lock() - ret, ok := s.lastReplies[args.opId] + ret, ok := s.lastReplies[args.OpId] if ok { s.mu.Unlock() return ret } var ret2 string = "" - if s.kvs[args.key] == args.expectedVal { - s.kvs[args.key] = args.newVal + if s.kvs[args.Key] == args.ExpectedVal { + s.kvs[args.Key] = args.NewVal ret2 = "ok" } - s.lastReplies[args.opId] = ret2 + s.lastReplies[args.OpId] = ret2 s.mu.Unlock() return ret2 } -func (s *Server) get(args *getArgs) string { +func (s *Server) get(args *get_gk.S) string { s.mu.Lock() - ret, ok := s.lastReplies[args.opId] + ret, ok := s.lastReplies[args.OpId] if ok { s.mu.Unlock() return ret } - ret2 := s.kvs[args.key] - s.lastReplies[args.opId] = ret2 + ret2 := s.kvs[args.Key] + s.lastReplies[args.OpId] = ret2 s.mu.Unlock() return ret2 } From 5393385f20cc87e707417ce2a6618d259f40030e Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Fri, 8 Nov 2024 13:25:25 -0600 Subject: [PATCH 09/27] Integrate grackle code into objectstore --- tutorial/objectstore/chunk/client.go | 5 +++-- tutorial/objectstore/chunk/server.go | 10 ++++++---- tutorial/objectstore/client/client.go | 6 ++++-- tutorial/objectstore/dir/client.go | 10 ++++++---- tutorial/objectstore/dir/objectstore.proto | 2 +- .../recordchunk_gk.go} | 2 +- tutorial/objectstore/dir/server.go | 14 ++++++++------ 7 files changed, 29 insertions(+), 20 deletions(-) rename tutorial/objectstore/dir/{recordchucks_gk/recordchucks_gk.go => recordchunk_gk/recordchunk_gk.go} (97%) diff --git a/tutorial/objectstore/chunk/client.go b/tutorial/objectstore/chunk/client.go index 935cdc1..ff289da 100644 --- a/tutorial/objectstore/chunk/client.go +++ b/tutorial/objectstore/chunk/client.go @@ -3,6 +3,7 @@ package chunk import ( "github.com/mit-pdos/gokv/connman" "github.com/mit-pdos/gokv/grove_ffi" + "github.com/mit-pdos/gokv/tutorial/objectstore/chunk/writechunk_gk" ) type WriteID = uint64 @@ -18,8 +19,8 @@ type ClerkPool struct { cm *connman.ConnMan } -func (ck *ClerkPool) WriteChunk(addr grove_ffi.Address, args WriteChunkArgs) { - req := MarshalWriteChunkArgs(args) +func (ck *ClerkPool) WriteChunk(addr grove_ffi.Address, args writechunk_gk.S) { + req := writechunk_gk.Marshal(&args, make([]byte, 0)) reply := new([]byte) ck.cm.CallAtLeastOnce(addr, WriteChunkId, req, reply, 100 /*ms*/) } diff --git a/tutorial/objectstore/chunk/server.go b/tutorial/objectstore/chunk/server.go index e39ba69..0ec4e1b 100644 --- a/tutorial/objectstore/chunk/server.go +++ b/tutorial/objectstore/chunk/server.go @@ -5,7 +5,9 @@ import ( "github.com/mit-pdos/gokv/grove_ffi" "github.com/mit-pdos/gokv/trusted_hash" + "github.com/mit-pdos/gokv/tutorial/objectstore/chunk/writechunk_gk" "github.com/mit-pdos/gokv/tutorial/objectstore/dir" + "github.com/mit-pdos/gokv/tutorial/objectstore/dir/recordchunk_gk" "github.com/mit-pdos/gokv/urpc" ) @@ -16,12 +18,12 @@ type Server struct { me grove_ffi.Address } -func (s *Server) WriteChunk(args WriteChunkArgs) { +func (s *Server) WriteChunk(args writechunk_gk.S) { content_hash := trusted_hash.Hash(args.Chunk) s.m.Lock() s.chunks[content_hash] = args.Chunk s.m.Unlock() - s.dir.RecordChunk(dir.RecordChunkArgs{ + s.dir.RecordChunk(recordchunk_gk.S{ WriteId: args.WriteId, Server: s.me, ContentHash: content_hash, @@ -46,8 +48,8 @@ func StartServer(me grove_ffi.Address, dir_addr grove_ffi.Address) { } handlers := make(map[uint64]func([]byte, *[]byte)) handlers[WriteChunkId] = func(req []byte, reply *[]byte) { - args := ParseWriteChunkArgs(req) - s.WriteChunk(args) + args, _ := writechunk_gk.Unmarshal(req) + s.WriteChunk(*args) *reply = make([]byte, 0) // TODO: is this needed? } handlers[GetChunkId] = func(req []byte, reply *[]byte) { diff --git a/tutorial/objectstore/client/client.go b/tutorial/objectstore/client/client.go index 3cefb8f..210a65d 100644 --- a/tutorial/objectstore/client/client.go +++ b/tutorial/objectstore/client/client.go @@ -5,7 +5,9 @@ import ( "github.com/mit-pdos/gokv/grove_ffi" "github.com/mit-pdos/gokv/tutorial/objectstore/chunk" + "github.com/mit-pdos/gokv/tutorial/objectstore/chunk/writechunk_gk" "github.com/mit-pdos/gokv/tutorial/objectstore/dir" + "github.com/mit-pdos/gokv/tutorial/objectstore/dir/finishwrite_gk" ) type Clerk struct { @@ -38,7 +40,7 @@ func (w *Writer) AppendChunk(data []byte) { w.index = w.index + 1 go func() { addr := w.chunkAddrs[index%uint64(len(w.chunkAddrs))] - args := chunk.WriteChunkArgs{ + args := writechunk_gk.S{ WriteId: w.writeId, Chunk: data, Index: index, @@ -51,7 +53,7 @@ func (w *Writer) AppendChunk(data []byte) { func (w *Writer) Done() { w.wg.Wait() - w.ck.dCk.FinishWrite(dir.FinishWriteArgs{ + w.ck.dCk.FinishWrite(finishwrite_gk.S{ WriteId: w.writeId, Keyname: w.keyname, }) diff --git a/tutorial/objectstore/dir/client.go b/tutorial/objectstore/dir/client.go index b42c3dd..99ec258 100644 --- a/tutorial/objectstore/dir/client.go +++ b/tutorial/objectstore/dir/client.go @@ -3,6 +3,8 @@ package dir import ( "github.com/mit-pdos/gokv/grove_ffi" "github.com/mit-pdos/gokv/reconnectclient" + "github.com/mit-pdos/gokv/tutorial/objectstore/dir/finishwrite_gk" + "github.com/mit-pdos/gokv/tutorial/objectstore/dir/recordchunk_gk" ) type WriteID = uint64 @@ -33,15 +35,15 @@ func (ck *Clerk) PrepareWrite() PreparedWrite { } // From chunk -func (ck *Clerk) RecordChunk(args RecordChunkArgs) { - req := MarshalRecordChunkArgs(args) +func (ck *Clerk) RecordChunk(args recordchunk_gk.S) { + req := recordchunk_gk.Marshal(&args, make([]byte, 0)) reply := new([]byte) ck.client.Call(RecordChunkId, req, reply, 100 /*ms*/) } // From chunk -func (ck *Clerk) FinishWrite(args FinishWriteArgs) { - req := MarshalFinishWriteArgs(args) +func (ck *Clerk) FinishWrite(args finishwrite_gk.S) { + req := finishwrite_gk.Marshal(&args, make([]byte, 0)) reply := new([]byte) ck.client.Call(FinishWriteId, req, reply, 100 /*ms*/) } diff --git a/tutorial/objectstore/dir/objectstore.proto b/tutorial/objectstore/dir/objectstore.proto index 7676b85..02764c1 100644 --- a/tutorial/objectstore/dir/objectstore.proto +++ b/tutorial/objectstore/dir/objectstore.proto @@ -7,7 +7,7 @@ syntax = "proto3"; // repeated uint64 checkAddrs = 2; // } -message recordChucks { +message recordChunk { // De-asliasing WriteID to uint64 uint64 writeId = 1; // De-asliasing grove_ffi.Address to uint64 diff --git a/tutorial/objectstore/dir/recordchucks_gk/recordchucks_gk.go b/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go similarity index 97% rename from tutorial/objectstore/dir/recordchucks_gk/recordchucks_gk.go rename to tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go index 9005b9e..75269ff 100644 --- a/tutorial/objectstore/dir/recordchucks_gk/recordchucks_gk.go +++ b/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go @@ -1,4 +1,4 @@ -package recordchucks_gk +package recordchunk_gk import ( "github.com/tchajed/marshal" diff --git a/tutorial/objectstore/dir/server.go b/tutorial/objectstore/dir/server.go index b09b99d..d73a996 100644 --- a/tutorial/objectstore/dir/server.go +++ b/tutorial/objectstore/dir/server.go @@ -4,6 +4,8 @@ import ( "sync" "github.com/mit-pdos/gokv/grove_ffi" + "github.com/mit-pdos/gokv/tutorial/objectstore/dir/finishwrite_gk" + "github.com/mit-pdos/gokv/tutorial/objectstore/dir/recordchunk_gk" "github.com/mit-pdos/gokv/urpc" ) @@ -42,7 +44,7 @@ func (s *Server) PrepareWrite() PreparedWrite { } // From chunk -func (s *Server) RecordChunk(args RecordChunkArgs) { +func (s *Server) RecordChunk(args recordchunk_gk.S) { s.m.Lock() // TODO: check if this write is still ongoing s.ongoing[args.WriteId].servers[args.Index] = ChunkHandle{ @@ -53,7 +55,7 @@ func (s *Server) RecordChunk(args RecordChunkArgs) { } // From chunk -func (s *Server) FinishWrite(args FinishWriteArgs) { +func (s *Server) FinishWrite(args finishwrite_gk.S) { s.m.Lock() v := s.ongoing[args.WriteId].servers // TODO: do we want to forget ongoing writes? @@ -90,13 +92,13 @@ func StartServer(me grove_ffi.Address) { *reply = MarshalPreparedWrite(ret) } handlers[RecordChunkId] = func(req []byte, reply *[]byte) { - args := ParseRecordChunkArgs(req) - s.RecordChunk(args) + args, _ := recordchunk_gk.Unmarshal(req) + s.RecordChunk(*args) *reply = make([]byte, 0) } handlers[FinishWriteId] = func(req []byte, reply *[]byte) { - args := ParseFinishWriteArgs(req) - s.FinishWrite(args) + args, _ := finishwrite_gk.Unmarshal(req) + s.FinishWrite(*args) *reply = make([]byte, 0) } handlers[PrepareReadId] = func(req []byte, reply *[]byte) { From c69372b1bddc25447714f59ba1e4432b8e9363e9 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Mon, 11 Nov 2024 15:09:14 -0600 Subject: [PATCH 10/27] scripting grackle code generation --- flake.nix | 8 +-- .../lockrequest_gk/lockrequest_gk.go | 8 ++- update-grackle.sh | 63 +++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) create mode 100755 update-grackle.sh diff --git a/flake.nix b/flake.nix index 52e0f23..fbd1faf 100644 --- a/flake.nix +++ b/flake.nix @@ -17,8 +17,8 @@ src = pkgs.fetchFromGitHub { owner = "goose-lang"; repo = "goose"; - rev = "585abc3cfef50dd466e112d7c535dbdfccd3c0ca"; - hash = "sha256-M4zaZ1DdecYXeDugrL+TV7HWPMLuj1P25G6mf+fgljg="; + rev = "8352f2a82040a814b60e0dd9ac75f09cec3dd619"; + hash = "sha256-jrcV/J70yzkmbXUiBsYYswqIph1mNQCC2LwvsxFnSSs="; }; vendorHash = "sha256-HCJ8v3TSv4UrkOsRuENWVz5Z7zQ1UsOygx0Mo7MELzY="; }; @@ -27,8 +27,8 @@ src = pkgs.fetchFromGitHub { owner = "mjschwenne"; repo = "grackle"; - rev = "2691072f22299d54e8b9e1cd6d2514e5bdea7f95"; - hash = "sha256-c+a3qRmqUiTaoo8p9GoCe2ZFBw5aJfj7Bhq/EJ4ayfI="; + rev = "18dbbd313c299d74895f566aae028eeba62a699a"; + hash = "sha256-GDo3c3VbRl0FR7sJ2Cvmdau+rBRcxwb/sGAEwV5seFQ="; }; vendorHash = "sha256-Wk2v0HSAkrzxHJvCfbw6xOn0OQ1xukvYjDxk3c2LmH8="; checkPhase = false; diff --git a/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go b/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go index 0511f51..9aa2c0b 100644 --- a/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go +++ b/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go @@ -14,13 +14,17 @@ func (l *S) approxSize() uint64 { func Marshal(l *S, prefix []byte) []byte { var enc = prefix - enc = marshal.WriteInt(enc, l.id) + + enc = marshal.WriteInt(enc, l.Id) + return enc } func Unmarshal(s []byte) (*S, []byte) { l := new(S) var enc = s // Needed for goose compatibility - l.id, enc = marshal.ReadInt(enc) + + l.Id, enc = marshal.ReadInt(enc) + return l, enc } diff --git a/update-grackle.sh b/update-grackle.sh new file mode 100755 index 0000000..82596f0 --- /dev/null +++ b/update-grackle.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env sh + +# Have I mentioned that I dislike bash? Just look at this arcane +# and archaic syntax +function compile_grackle () { + CWD=$(pwd) + cd $1 + go install ./cmd/grackle + cd $CWD +} + +# Run grackle on the input go package. +# +# We will assume that: +# 1. The proto file is in this directory +# 2. We only want to output go code +# 3. The go code should be output into this directory +# 4. The desired go package matches the directory structure +# 5. Grackle is on your $PATH +function run_grackle () { + grackle --go-output-path $1 --go-package "github.com/mit-pdos/gokv/$1" $1 +} + +ARGS=$(getopt -o "c:g:h" --long "compile-goose:,compile-grackle:,help" -- "$@") + +eval set -- "$ARGS" +while [ $# -ge 1 ]; do + case "$1" in + -g | --compile-grackle) + echo "compiling grackle $2" + compile_grackle $2 + shift + ;; + -h | --help) + cat < | -c ] [--help] + +Calls grackle on all go modules known to have proto files for grackle usage. + +--compile-grackle [-g] : Takes the path to the grackle repository, recompiles and installs grackle +EOF + shift + exit 0 + ;; + --) + shift + break + ;; + esac + shift +done + +grackle_packages=( + "tutorial/kvservice" + "tutorial/lockservice" + "tutorial/objectstore/chunk" + "tutorial/objectstore/dir" +) + +for gopkg in ${grackle_packages[@]}; do + echo $gopkg + run_grackle $gopkg +done From fa600ed984de33c1b25327530f8a0b86169f0f46 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 12 Nov 2024 11:24:39 -0600 Subject: [PATCH 11/27] Clean up update-grackle.sh thanks to LSP --- update-grackle.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/update-grackle.sh b/update-grackle.sh index 82596f0..49f2dcb 100755 --- a/update-grackle.sh +++ b/update-grackle.sh @@ -1,12 +1,12 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash # Have I mentioned that I dislike bash? Just look at this arcane # and archaic syntax -function compile_grackle () { +compile_grackle () { CWD=$(pwd) - cd $1 + cd "$1" || return go install ./cmd/grackle - cd $CWD + cd "$CWD" || exit } # Run grackle on the input go package. @@ -17,8 +17,8 @@ function compile_grackle () { # 3. The go code should be output into this directory # 4. The desired go package matches the directory structure # 5. Grackle is on your $PATH -function run_grackle () { - grackle --go-output-path $1 --go-package "github.com/mit-pdos/gokv/$1" $1 +run_grackle () { + grackle --go-output-path "$1" --go-package "github.com/mit-pdos/gokv/$1" "$1" } ARGS=$(getopt -o "c:g:h" --long "compile-goose:,compile-grackle:,help" -- "$@") @@ -28,12 +28,12 @@ while [ $# -ge 1 ]; do case "$1" in -g | --compile-grackle) echo "compiling grackle $2" - compile_grackle $2 + compile_grackle "$2" shift ;; -h | --help) cat < | -c ] [--help] +usage: update-grackle.sh [--compile-grackle | -c ] [--help | -h] Calls grackle on all go modules known to have proto files for grackle usage. @@ -57,7 +57,7 @@ grackle_packages=( "tutorial/objectstore/dir" ) -for gopkg in ${grackle_packages[@]}; do - echo $gopkg - run_grackle $gopkg +for gopkg in "${grackle_packages[@]}"; do + echo "$gopkg" + run_grackle "$gopkg" done From 28d6fc7d3b22c3f62189ea9739cf3237e68e8db8 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 12 Nov 2024 11:41:14 -0600 Subject: [PATCH 12/27] Fix flag mismatch and remove old goose compile flag --- update-grackle.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/update-grackle.sh b/update-grackle.sh index 49f2dcb..ab43a1f 100755 --- a/update-grackle.sh +++ b/update-grackle.sh @@ -21,12 +21,12 @@ run_grackle () { grackle --go-output-path "$1" --go-package "github.com/mit-pdos/gokv/$1" "$1" } -ARGS=$(getopt -o "c:g:h" --long "compile-goose:,compile-grackle:,help" -- "$@") +ARGS=$(getopt -o "c:h" --long "compile-grackle:,help" -- "$@") eval set -- "$ARGS" while [ $# -ge 1 ]; do case "$1" in - -g | --compile-grackle) + -c | --compile-grackle) echo "compiling grackle $2" compile_grackle "$2" shift @@ -37,7 +37,7 @@ usage: update-grackle.sh [--compile-grackle | -c ] Calls grackle on all go modules known to have proto files for grackle usage. ---compile-grackle [-g] : Takes the path to the grackle repository, recompiles and installs grackle +--compile-grackle [-c] : Takes the path to the grackle repository, recompiles and installs grackle EOF shift exit 0 @@ -58,6 +58,5 @@ grackle_packages=( ) for gopkg in "${grackle_packages[@]}"; do - echo "$gopkg" run_grackle "$gopkg" done From 9889c62c48961eb46cf784a96a16799dcfc249d6 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 19 Nov 2024 17:08:31 -0600 Subject: [PATCH 13/27] proto files for the rest of gokv --- cachekv/cachekv.proto | 6 +++ fencing/ctr/ctr.proto | 20 +++++++++ map_marshal/map.proto | 19 ++++++++ map_string_marshal/map_string.proto | 10 +++++ memkv/memkv.proto | 62 ++++++++++++++++++++++++++ paxi/comulti/comulti.proto | 15 +++++++ paxi/reconf/reconf.proto | 31 +++++++++++++ reconfig/replica/replica.proto | 33 ++++++++++++++ reconfig/util/membership.proto | 6 +++ vrsm/apps/vkv/vkv.proto | 16 +++++++ vrsm/configservice/configservice.proto | 14 ++++++ vrsm/e/error.proto | 17 +++++++ vrsm/paxos/paxos.proto | 43 ++++++++++++++++++ vrsm/replica/replica.proto | 42 +++++++++++++++++ 14 files changed, 334 insertions(+) create mode 100644 cachekv/cachekv.proto create mode 100644 fencing/ctr/ctr.proto create mode 100644 map_marshal/map.proto create mode 100644 map_string_marshal/map_string.proto create mode 100644 memkv/memkv.proto create mode 100644 paxi/comulti/comulti.proto create mode 100644 paxi/reconf/reconf.proto create mode 100644 reconfig/replica/replica.proto create mode 100644 reconfig/util/membership.proto create mode 100644 vrsm/apps/vkv/vkv.proto create mode 100644 vrsm/configservice/configservice.proto create mode 100644 vrsm/e/error.proto create mode 100644 vrsm/paxos/paxos.proto create mode 100644 vrsm/replica/replica.proto diff --git a/cachekv/cachekv.proto b/cachekv/cachekv.proto new file mode 100644 index 0000000..fa23bda --- /dev/null +++ b/cachekv/cachekv.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message cacheValue { + string v = 1; + uint64 l = 2; +} diff --git a/fencing/ctr/ctr.proto b/fencing/ctr/ctr.proto new file mode 100644 index 0000000..2820840 --- /dev/null +++ b/fencing/ctr/ctr.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +enum Error { + ENone = 0; + Estale = 1; +} + +message putArgs { + uint64 epoch = 1; + uint64 v = 2; +} + +message getArgs { + uint64 epoch = 1; +} + +message getReply { + Error err = 1; + uint64 val = 2; +} diff --git a/map_marshal/map.proto b/map_marshal/map.proto new file mode 100644 index 0000000..ab11a48 --- /dev/null +++ b/map_marshal/map.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +message u64ToU64 { + uint64 k = 1; + uint64 v = 2; +} + +message mapU64ToU64 { + repeated u64ToU64 entries = 1; +} + +message u64ToBytes { + uint64 k = 1; + bytes v = 2; +} + +message mapU64ToBytes { + repeated u64ToBytes entries = 1; +} diff --git a/map_string_marshal/map_string.proto b/map_string_marshal/map_string.proto new file mode 100644 index 0000000..a503ee9 --- /dev/null +++ b/map_string_marshal/map_string.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +message u64ToString { + uint64 k = 1; + string v = 2; +} + +message mapU64ToString { + repeated u64ToString entries = 1; +} diff --git a/memkv/memkv.proto b/memkv/memkv.proto new file mode 100644 index 0000000..e1f6660 --- /dev/null +++ b/memkv/memkv.proto @@ -0,0 +1,62 @@ +syntax = "proto3"; + +import "../map_marshal/map.proto"; + +enum Error { + ENone = 0; + EDontHaveShard = 1; +} + +enum KvOp { + KV_FreshCID = 0; + KV_Put = 1; + KV_Get = 2; + KV_Conditional_Put = 3; + KV_Ins_Shard = 4; + KV_Mov_Shard = 5; +} + +message putRequest { + uint64 key = 1; + bytes value = 2; +} + +message putReply { + Error err = 1; +} + +message getRequest { + uint64 key = 1; +} + +message getReply { + Error err = 1; + bytes value = 2; +} + +message conditionalPutRequest { + uint64 key = 1; + bytes expectedValue = 2; + bytes newValue = 3; +} + +message conditionalPutReply { + Error err = 1; + bool success = 2; +} + +message installShardRequest { + uint64 sid = 1; + // This might be tricky since now the proto files aren't standalone + mapU64ToBytes kvs = 2; +} + +message moveShardRequest { + uint64 sid = 1; + // De-alias HostName to uint64 + uint64 hostName = 2; +} + +message shardMap { + repeated uint64 shards = 1; +} diff --git a/paxi/comulti/comulti.proto b/paxi/comulti/comulti.proto new file mode 100644 index 0000000..8c6b59e --- /dev/null +++ b/paxi/comulti/comulti.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +message prepareReply { + bool success = 1; + // De-alias Entry to uint64 + repeated uint64 log = 2; + uint64 pn = 3; +} + +message proposeArgs { + uint64 pn = 1; + uint64 commiteIndex = 2; + // De-alias Entry to uint64 + repeated uint64 log = 3; +} diff --git a/paxi/reconf/reconf.proto b/paxi/reconf/reconf.proto new file mode 100644 index 0000000..a9f2ac4 --- /dev/null +++ b/paxi/reconf/reconf.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +message config { + // De-alias grove_ffi.Address to uint64 + repeated uint64 memebers = 1; + // De-alias grove_ffi.Address to uint64 + repeated uint64 nextMembers = 2; +} + +message monotonicValue { + uint64 version = 1; + bytes val = 2; + config conf = 3; +} + +message prepareReply { + // Maybe should be enum? + uint64 err = 1; + uint64 term = 2; + monotonicValue val = 3; +} + + message proposeArgs { + uint64 term = 1; + monotonicValue val = 2; + } + +message tryCommitReply { + uint64 err = 1; + uint64 version = 2; +} diff --git a/reconfig/replica/replica.proto b/reconfig/replica/replica.proto new file mode 100644 index 0000000..4496961 --- /dev/null +++ b/reconfig/replica/replica.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +message appendArgs { + uint64 epoch = 1; + // De-aliasing LogEntry to []byte + bytes entry = 2; + uint64 index = 3; +} + +message configuration { + // De-aliasing grove_ffi.Address to uint64 + repeated uint64 replicas = 1; +} + +message becomeReplicaArgs { + uint64 epoch = 1; + uint64 startIndex = 2; + // De-aliasing LogEntry to []byte + repeated bytes log = 3; +} + +message becomePrimaryArgs { + uint64 epoch = 1; + configuration conf = 2; +} + +message getLogReply { + // De-aliasing Error to uint64. Should this be an enum? + uint64 err = 1; + // De-aliasing LogEntry to []byte + repeated bytes log = 2; + uint64 startIndex = 3; +} diff --git a/reconfig/util/membership.proto b/reconfig/util/membership.proto new file mode 100644 index 0000000..fff5e57 --- /dev/null +++ b/reconfig/util/membership.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message configuration { + // De-alias grove_ffi.Address to uint64 + repeated uint64 addrs = 1; +} diff --git a/vrsm/apps/vkv/vkv.proto b/vrsm/apps/vkv/vkv.proto new file mode 100644 index 0000000..3b7b083 --- /dev/null +++ b/vrsm/apps/vkv/vkv.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +message putArgs { + string key = 1; + string val = 2; +} + +message getArgs { + string get = 1; +} + +message condPutArgs { + string key = 1; + string expect = 2; + string val = 3; +} diff --git a/vrsm/configservice/configservice.proto b/vrsm/configservice/configservice.proto new file mode 100644 index 0000000..ff58e6e --- /dev/null +++ b/vrsm/configservice/configservice.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +message config { + repeated uint64 addrs = 1; +} + +message state { + uint64 epoch = 1; + uint64 reservedEpoch = 2; + uint64 leaseExpiration = 3; + bool wantLeaseToExpire = 4; + // De-alias grove_ffi.Address to uint64 + repeated uint64 config = 5; +} diff --git a/vrsm/e/error.proto b/vrsm/e/error.proto new file mode 100644 index 0000000..0b00b99 --- /dev/null +++ b/vrsm/e/error.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +enum Err { + None = 0; + Stale = 1; + OutOfOrder = 2; + Timeout = 3; + EmptyConfig = 4; + NotLeader = 5; + Sealed = 6; + LeaseExpired = 7; + Leased = 8; +} + +message Error { + Err error = 1; +} diff --git a/vrsm/paxos/paxos.proto b/vrsm/paxos/paxos.proto new file mode 100644 index 0000000..f80ed49 --- /dev/null +++ b/vrsm/paxos/paxos.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; + +enum Error { + ENone = 0; + EEpochStale = 1; + EOutOfOrder = 2; + ETimeout = 3; + ENotLeader = 4; +} + +message applyAsFollowerArgs { + uint64 epoch = 1; + uint64 nextIndex = 2; + bytes state = 3; +} + +message applyAsFollowerReply { + Error err = 1; +} + +message enterNewEpochArgs { + uint64 epoch = 1; +} + +message enterNewEpochReply { + Error err = 1; + uint64 accpetedEpoch = 2; + uint64 nextIndex = 3; + bytes state = 4; +} + +message applyReply { + Error err = 1; + bytes ret = 2; +} + +message paxosState { + uint64 epoch = 1; + uint64 acceptedEpoch = 2; + uint64 nextIndex = 3; + bytes state = 4; + bool isLeader = 5; +} diff --git a/vrsm/replica/replica.proto b/vrsm/replica/replica.proto new file mode 100644 index 0000000..f32cf61 --- /dev/null +++ b/vrsm/replica/replica.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +import "../e/error.proto"; + +message applyAsBackupArgs { + uint64 epoch = 1; + uint64 index = 2; + bytes op = 3; +} + +message setStateArgs { + uint64 epoch = 1; + uint64 nextIndex = 2; + uint64 committedNextIndex = 3; + bytes state = 4; +} + +message getStateArgs { + uint64 epoch = 1; +} + +message getStateReply { + Error err = 1; + uint64 nextIndex = 2; + uint64 committedNextIndex = 3; + bytes state = 4; +} + +message becomePrimaryArgs { + uint64 epoch = 1; + // De-alias grove_ffi.Address to uint64 + repeated uint64 replicas = 2; +} + +message applyReply { + Error err = 1; + bytes reply = 2; +} + +message IncreaseCommitArgs { + uint64 v = 1; +} From b6ada5efeb8751b311833b9ab6de59334413a40f Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 19 Nov 2024 17:13:53 -0600 Subject: [PATCH 14/27] update grackle and goose --- flake.nix | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index fbd1faf..f18ae86 100644 --- a/flake.nix +++ b/flake.nix @@ -17,8 +17,8 @@ src = pkgs.fetchFromGitHub { owner = "goose-lang"; repo = "goose"; - rev = "8352f2a82040a814b60e0dd9ac75f09cec3dd619"; - hash = "sha256-jrcV/J70yzkmbXUiBsYYswqIph1mNQCC2LwvsxFnSSs="; + rev = "8d13c771b9a80957089f7c5b0ee2ccf58e5eb06f"; + sha256 = "1fbqs75ya4as3my2knkaq4m0crdh3n004grw5g5iczvb5h5k06lz"; }; vendorHash = "sha256-HCJ8v3TSv4UrkOsRuENWVz5Z7zQ1UsOygx0Mo7MELzY="; }; @@ -27,8 +27,8 @@ src = pkgs.fetchFromGitHub { owner = "mjschwenne"; repo = "grackle"; - rev = "18dbbd313c299d74895f566aae028eeba62a699a"; - hash = "sha256-GDo3c3VbRl0FR7sJ2Cvmdau+rBRcxwb/sGAEwV5seFQ="; + rev = "d961e849c7737781b6a09a18c9f3cc669178f98a"; + sha256 = "1385sinsmm1k58blvv2sf951fi7kihka5g7zhg0m8r5ijzy9ac91"; }; vendorHash = "sha256-Wk2v0HSAkrzxHJvCfbw6xOn0OQ1xukvYjDxk3c2LmH8="; checkPhase = false; @@ -48,6 +48,11 @@ protoc-gen-go proto-contrib protoscope + + # nix tools + nix-prefetch-git + nix-prefetch + update-nix-fetchgit ]; shellHook = '' From 84272bfd7211f57dc9e377af612ae5fc4531438b Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Mon, 2 Dec 2024 20:59:13 -0600 Subject: [PATCH 15/27] update update-grackle.sh to load grackle directly --- tutorial/kvservice/conditionalput_gk/conditionalput_gk.go | 6 +++--- tutorial/kvservice/get_gk/get_gk.go | 2 +- tutorial/kvservice/put_gk/put_gk.go | 4 ++-- tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go | 2 +- tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go | 2 +- tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go | 2 +- tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go | 2 +- update-grackle.sh | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go b/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go index b0d8c4d..091b7d2 100644 --- a/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go +++ b/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go @@ -40,17 +40,17 @@ func Unmarshal(s []byte) (*S, []byte) { var keyLen uint64 var keyBytes []byte keyLen, enc = marshal.ReadInt(enc) - keyBytes, enc = marshal.ReadBytes(enc, keyLen) + keyBytes, enc = marshal.ReadBytesCopy(enc, keyLen) c.Key = string(keyBytes) var expectedValLen uint64 var expectedValBytes []byte expectedValLen, enc = marshal.ReadInt(enc) - expectedValBytes, enc = marshal.ReadBytes(enc, expectedValLen) + expectedValBytes, enc = marshal.ReadBytesCopy(enc, expectedValLen) c.ExpectedVal = string(expectedValBytes) var newValLen uint64 var newValBytes []byte newValLen, enc = marshal.ReadInt(enc) - newValBytes, enc = marshal.ReadBytes(enc, newValLen) + newValBytes, enc = marshal.ReadBytesCopy(enc, newValLen) c.NewVal = string(newValBytes) return c, enc diff --git a/tutorial/kvservice/get_gk/get_gk.go b/tutorial/kvservice/get_gk/get_gk.go index f63ad3c..3de9e8a 100644 --- a/tutorial/kvservice/get_gk/get_gk.go +++ b/tutorial/kvservice/get_gk/get_gk.go @@ -32,7 +32,7 @@ func Unmarshal(s []byte) (*S, []byte) { var keyLen uint64 var keyBytes []byte keyLen, enc = marshal.ReadInt(enc) - keyBytes, enc = marshal.ReadBytes(enc, keyLen) + keyBytes, enc = marshal.ReadBytesCopy(enc, keyLen) g.Key = string(keyBytes) return g, enc diff --git a/tutorial/kvservice/put_gk/put_gk.go b/tutorial/kvservice/put_gk/put_gk.go index e60a2a9..b48d1d5 100644 --- a/tutorial/kvservice/put_gk/put_gk.go +++ b/tutorial/kvservice/put_gk/put_gk.go @@ -36,12 +36,12 @@ func Unmarshal(s []byte) (*S, []byte) { var keyLen uint64 var keyBytes []byte keyLen, enc = marshal.ReadInt(enc) - keyBytes, enc = marshal.ReadBytes(enc, keyLen) + keyBytes, enc = marshal.ReadBytesCopy(enc, keyLen) p.Key = string(keyBytes) var valLen uint64 var valBytes []byte valLen, enc = marshal.ReadInt(enc) - valBytes, enc = marshal.ReadBytes(enc, valLen) + valBytes, enc = marshal.ReadBytesCopy(enc, valLen) p.Val = string(valBytes) return p, enc diff --git a/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go b/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go index 7705a54..a336d7b 100644 --- a/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go +++ b/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go @@ -33,7 +33,7 @@ func Unmarshal(s []byte) (*S, []byte) { var chunkLen uint64 var chunkBytes []byte chunkLen, enc = marshal.ReadInt(enc) - chunkBytes, enc = marshal.ReadBytes(enc, chunkLen) + chunkBytes, enc = marshal.ReadBytesCopy(enc, chunkLen) w.Chunk = chunkBytes w.Index, enc = marshal.ReadInt(enc) diff --git a/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go b/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go index 99d683c..2e3b823 100644 --- a/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go +++ b/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go @@ -32,7 +32,7 @@ func Unmarshal(s []byte) (*S, []byte) { var contentHashLen uint64 var contentHashBytes []byte contentHashLen, enc = marshal.ReadInt(enc) - contentHashBytes, enc = marshal.ReadBytes(enc, contentHashLen) + contentHashBytes, enc = marshal.ReadBytesCopy(enc, contentHashLen) c.ContentHash = string(contentHashBytes) return c, enc diff --git a/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go b/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go index 9f97b73..68a69dd 100644 --- a/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go +++ b/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go @@ -32,7 +32,7 @@ func Unmarshal(s []byte) (*S, []byte) { var keynameLen uint64 var keynameBytes []byte keynameLen, enc = marshal.ReadInt(enc) - keynameBytes, enc = marshal.ReadBytes(enc, keynameLen) + keynameBytes, enc = marshal.ReadBytesCopy(enc, keynameLen) f.Keyname = string(keynameBytes) return f, enc diff --git a/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go b/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go index 75269ff..6ccb578 100644 --- a/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go +++ b/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go @@ -37,7 +37,7 @@ func Unmarshal(s []byte) (*S, []byte) { var contentHashLen uint64 var contentHashBytes []byte contentHashLen, enc = marshal.ReadInt(enc) - contentHashBytes, enc = marshal.ReadBytes(enc, contentHashLen) + contentHashBytes, enc = marshal.ReadBytesCopy(enc, contentHashLen) r.ContentHash = string(contentHashBytes) r.Index, enc = marshal.ReadInt(enc) diff --git a/update-grackle.sh b/update-grackle.sh index ab43a1f..e922935 100755 --- a/update-grackle.sh +++ b/update-grackle.sh @@ -18,7 +18,7 @@ compile_grackle () { # 4. The desired go package matches the directory structure # 5. Grackle is on your $PATH run_grackle () { - grackle --go-output-path "$1" --go-package "github.com/mit-pdos/gokv/$1" "$1" + go run github.com/mjschwenne/grackle/cmd/grackle@latest --go-output-path "$1" --go-package "github.com/mit-pdos/gokv/$1" "$1" } ARGS=$(getopt -o "c:h" --long "compile-grackle:,help" -- "$@") From ff04c8e6b83c50937f5a2e421c94052e710bfecb Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Mon, 2 Dec 2024 21:11:53 -0600 Subject: [PATCH 16/27] setup grackle ci workflow --- .github/workflows/ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c7985ab --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI + +on: + push: + branches: + - main + - grackle + pull_request: + +jobs: + update-grackle: + strategy: + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + - name: Install protoc + run: sudo apt-get install protobuf-compiler -y + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + - name: Check Grackle + run: | + ./update-grackle.sh + git diff --exit-code From f3d406659b3c30b5cb3c42eea7c39bb90880cc84 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 3 Dec 2024 09:20:03 -0600 Subject: [PATCH 17/27] update grackle and goose, check for nix shells in update-grackle --- flake.nix | 4 ++-- update-grackle.sh | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index f18ae86..88cbd6b 100644 --- a/flake.nix +++ b/flake.nix @@ -27,8 +27,8 @@ src = pkgs.fetchFromGitHub { owner = "mjschwenne"; repo = "grackle"; - rev = "d961e849c7737781b6a09a18c9f3cc669178f98a"; - sha256 = "1385sinsmm1k58blvv2sf951fi7kihka5g7zhg0m8r5ijzy9ac91"; + rev = "275fce5626d662527ff987ad61aad61044019fa6"; + sha256 = "16iw5hqg2hd12mc62wqv5hxqb8jwymbnhyy16zsza0limhmr0aya"; }; vendorHash = "sha256-Wk2v0HSAkrzxHJvCfbw6xOn0OQ1xukvYjDxk3c2LmH8="; checkPhase = false; diff --git a/update-grackle.sh b/update-grackle.sh index e922935..4d762ef 100755 --- a/update-grackle.sh +++ b/update-grackle.sh @@ -1,5 +1,13 @@ #!/usr/bin/env bash +if [[ -z "$IN_NIX_SHELL" ]]; then + # Not in nix shell, use the default go installs + GRACKLE="go run github.com/mjschwenne/grackle/cmd/grackle@latest" +else + # In a nix shell, expect grackle to be on the PATH and use that + GRACKLE="grackle" +fi + # Have I mentioned that I dislike bash? Just look at this arcane # and archaic syntax compile_grackle () { @@ -16,9 +24,8 @@ compile_grackle () { # 2. We only want to output go code # 3. The go code should be output into this directory # 4. The desired go package matches the directory structure -# 5. Grackle is on your $PATH run_grackle () { - go run github.com/mjschwenne/grackle/cmd/grackle@latest --go-output-path "$1" --go-package "github.com/mit-pdos/gokv/$1" "$1" + $GRACKLE --go-output-path "$1" --go-package "github.com/mit-pdos/gokv/$1" "$1" } ARGS=$(getopt -o "c:h" --long "compile-grackle:,help" -- "$@") From f42d63d8858051b85c2111494d9b1924a6343ddc Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 3 Dec 2024 11:51:30 -0600 Subject: [PATCH 18/27] grackle fencing/ctr --- fencing/ctr/client.go | 25 ++++++++++++++----------- fencing/ctr/ctr.proto | 2 +- fencing/ctr/server.go | 36 ++++++++++++++++++------------------ update-grackle.sh | 1 + 4 files changed, 34 insertions(+), 30 deletions(-) diff --git a/fencing/ctr/client.go b/fencing/ctr/client.go index f33fa45..b1bcaeb 100644 --- a/fencing/ctr/client.go +++ b/fencing/ctr/client.go @@ -7,6 +7,9 @@ import ( "github.com/mit-pdos/gokv/urpc" "github.com/tchajed/marshal" "log" + + "github.com/mit-pdos/gokv/fencing/ctr/getreply_gk" + "github.com/mit-pdos/gokv/fencing/ctr/putargs_gk" ) const ( @@ -32,22 +35,22 @@ func (c *Clerk) Get(epoch uint64) uint64 { log.Println("ctr: urpc get call failed/timed out") primitive.Exit(1) } - r := DecGetReply(*reply_ptr) + r, _ := getreply_gk.Unmarshal(*reply_ptr) - if r.err != ENone { + if r.Err != ENone { log.Println("ctr: get() stale epoch number") primitive.Exit(1) } - valProph.ResolveU64(r.val) - return r.val + valProph.ResolveU64(r.Val) + return r.Val } func (c *Clerk) Put(v uint64, epoch uint64) { - args := &PutArgs{ - v: v, - epoch: epoch, + args := &putargs_gk.S{ + V: v, + Epoch: epoch, } - req := c.e.NewRequest(EncPutArgs(args)) + req := c.e.NewRequest(putargs_gk.Marshal(args, make([]byte, 0))) reply_ptr := new([]byte) err := c.cl.Call(RPC_PUT, req, reply_ptr, 100 /* ms */) @@ -56,8 +59,7 @@ func (c *Clerk) Put(v uint64, epoch uint64) { primitive.Exit(1) } - dec := marshal.NewDec(*reply_ptr) - epochErr := dec.GetInt() + epochErr, _ := marshal.ReadInt(*reply_ptr) if epochErr != ENone { log.Println("ctr: get() stale epoch number") @@ -77,7 +79,8 @@ func MakeClerk(host grove_ffi.Address) *Clerk { log.Println("ctr: urpc getcid call failed/timed out") primitive.Exit(1) } - ck.e = erpc.MakeClient(marshal.NewDec(*reply_ptr).GetInt()) + reply, _ := marshal.ReadInt(*reply_ptr) + ck.e = erpc.MakeClient(reply) return ck } diff --git a/fencing/ctr/ctr.proto b/fencing/ctr/ctr.proto index 2820840..8ae2810 100644 --- a/fencing/ctr/ctr.proto +++ b/fencing/ctr/ctr.proto @@ -15,6 +15,6 @@ message getArgs { } message getReply { - Error err = 1; + uint64 err = 1; uint64 val = 2; } diff --git a/fencing/ctr/server.go b/fencing/ctr/server.go index 9d6fa04..dca7c97 100644 --- a/fencing/ctr/server.go +++ b/fencing/ctr/server.go @@ -1,12 +1,16 @@ package ctr import ( + "sync" + "github.com/goose-lang/primitive" "github.com/mit-pdos/gokv/erpc" "github.com/mit-pdos/gokv/grove_ffi" "github.com/mit-pdos/gokv/urpc" "github.com/tchajed/marshal" - "sync" + + "github.com/mit-pdos/gokv/fencing/ctr/getreply_gk" + "github.com/mit-pdos/gokv/fencing/ctr/putargs_gk" ) type Server struct { @@ -17,33 +21,33 @@ type Server struct { lastEpoch uint64 } -func (s *Server) Put(args *PutArgs) uint64 { +func (s *Server) Put(args *putargs_gk.S) uint64 { s.mu.Lock() // check if epoch is stale - if args.epoch < s.lastEpoch { + if args.Epoch < s.lastEpoch { s.mu.Unlock() return EStale } - s.lastEpoch = args.epoch - s.v = args.v + s.lastEpoch = args.Epoch + s.v = args.V s.mu.Unlock() return ENone } -func (s *Server) Get(epoch uint64, reply *GetReply) { +func (s *Server) Get(epoch uint64, reply *getreply_gk.S) { s.mu.Lock() - reply.err = ENone + reply.Err = ENone // check if epoch is stale if epoch < s.lastEpoch { s.mu.Unlock() - reply.err = EStale + reply.Err = EStale return } s.lastEpoch = epoch - reply.val = s.v + reply.Val = s.v primitive.Linearize() s.mu.Unlock() return @@ -59,23 +63,19 @@ func StartServer(me grove_ffi.Address) { handlers[RPC_GET] = func(raw_args []byte, raw_reply *[]byte) { dec := marshal.NewDec(raw_args) epoch := dec.GetInt() - reply := new(GetReply) + reply := new(getreply_gk.S) s.Get(epoch, reply) - *raw_reply = EncGetReply(reply) + *raw_reply = getreply_gk.Marshal(reply, make([]byte, 0)) } handlers[RPC_PUT] = s.e.HandleRequest(func(raw_args []byte, reply *[]byte) { - args := DecPutArgs(raw_args) + args, _ := putargs_gk.Unmarshal(raw_args) err := s.Put(args) - enc := marshal.NewEnc(8) - enc.PutInt(err) - *reply = enc.Finish() + *reply = marshal.WriteInt(make([]byte, 0), err) }) handlers[RPC_FRESHCID] = func(raw_args []byte, reply *[]byte) { - enc := marshal.NewEnc(8) - enc.PutInt(s.e.GetFreshCID()) - *reply = enc.Finish() + *reply = marshal.WriteInt(make([]byte, 0), s.e.GetFreshCID()) } r := urpc.MakeServer(handlers) diff --git a/update-grackle.sh b/update-grackle.sh index 4d762ef..064f190 100755 --- a/update-grackle.sh +++ b/update-grackle.sh @@ -62,6 +62,7 @@ grackle_packages=( "tutorial/lockservice" "tutorial/objectstore/chunk" "tutorial/objectstore/dir" + "fencing/ctr" ) for gopkg in "${grackle_packages[@]}"; do From d863f4c2b40ba6aa8faf024769ecc6098ce97800 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Tue, 3 Dec 2024 15:48:06 -0600 Subject: [PATCH 19/27] Revert "grackle fencing/ctr" This reverts commit f42d63d8858051b85c2111494d9b1924a6343ddc. I was unable to get the perennial proofs to do through. --- fencing/ctr/client.go | 25 +++++++++++-------------- fencing/ctr/ctr.proto | 2 +- fencing/ctr/server.go | 36 ++++++++++++++++++------------------ update-grackle.sh | 1 - 4 files changed, 30 insertions(+), 34 deletions(-) diff --git a/fencing/ctr/client.go b/fencing/ctr/client.go index b1bcaeb..f33fa45 100644 --- a/fencing/ctr/client.go +++ b/fencing/ctr/client.go @@ -7,9 +7,6 @@ import ( "github.com/mit-pdos/gokv/urpc" "github.com/tchajed/marshal" "log" - - "github.com/mit-pdos/gokv/fencing/ctr/getreply_gk" - "github.com/mit-pdos/gokv/fencing/ctr/putargs_gk" ) const ( @@ -35,22 +32,22 @@ func (c *Clerk) Get(epoch uint64) uint64 { log.Println("ctr: urpc get call failed/timed out") primitive.Exit(1) } - r, _ := getreply_gk.Unmarshal(*reply_ptr) + r := DecGetReply(*reply_ptr) - if r.Err != ENone { + if r.err != ENone { log.Println("ctr: get() stale epoch number") primitive.Exit(1) } - valProph.ResolveU64(r.Val) - return r.Val + valProph.ResolveU64(r.val) + return r.val } func (c *Clerk) Put(v uint64, epoch uint64) { - args := &putargs_gk.S{ - V: v, - Epoch: epoch, + args := &PutArgs{ + v: v, + epoch: epoch, } - req := c.e.NewRequest(putargs_gk.Marshal(args, make([]byte, 0))) + req := c.e.NewRequest(EncPutArgs(args)) reply_ptr := new([]byte) err := c.cl.Call(RPC_PUT, req, reply_ptr, 100 /* ms */) @@ -59,7 +56,8 @@ func (c *Clerk) Put(v uint64, epoch uint64) { primitive.Exit(1) } - epochErr, _ := marshal.ReadInt(*reply_ptr) + dec := marshal.NewDec(*reply_ptr) + epochErr := dec.GetInt() if epochErr != ENone { log.Println("ctr: get() stale epoch number") @@ -79,8 +77,7 @@ func MakeClerk(host grove_ffi.Address) *Clerk { log.Println("ctr: urpc getcid call failed/timed out") primitive.Exit(1) } - reply, _ := marshal.ReadInt(*reply_ptr) - ck.e = erpc.MakeClient(reply) + ck.e = erpc.MakeClient(marshal.NewDec(*reply_ptr).GetInt()) return ck } diff --git a/fencing/ctr/ctr.proto b/fencing/ctr/ctr.proto index 8ae2810..2820840 100644 --- a/fencing/ctr/ctr.proto +++ b/fencing/ctr/ctr.proto @@ -15,6 +15,6 @@ message getArgs { } message getReply { - uint64 err = 1; + Error err = 1; uint64 val = 2; } diff --git a/fencing/ctr/server.go b/fencing/ctr/server.go index dca7c97..9d6fa04 100644 --- a/fencing/ctr/server.go +++ b/fencing/ctr/server.go @@ -1,16 +1,12 @@ package ctr import ( - "sync" - "github.com/goose-lang/primitive" "github.com/mit-pdos/gokv/erpc" "github.com/mit-pdos/gokv/grove_ffi" "github.com/mit-pdos/gokv/urpc" "github.com/tchajed/marshal" - - "github.com/mit-pdos/gokv/fencing/ctr/getreply_gk" - "github.com/mit-pdos/gokv/fencing/ctr/putargs_gk" + "sync" ) type Server struct { @@ -21,33 +17,33 @@ type Server struct { lastEpoch uint64 } -func (s *Server) Put(args *putargs_gk.S) uint64 { +func (s *Server) Put(args *PutArgs) uint64 { s.mu.Lock() // check if epoch is stale - if args.Epoch < s.lastEpoch { + if args.epoch < s.lastEpoch { s.mu.Unlock() return EStale } - s.lastEpoch = args.Epoch - s.v = args.V + s.lastEpoch = args.epoch + s.v = args.v s.mu.Unlock() return ENone } -func (s *Server) Get(epoch uint64, reply *getreply_gk.S) { +func (s *Server) Get(epoch uint64, reply *GetReply) { s.mu.Lock() - reply.Err = ENone + reply.err = ENone // check if epoch is stale if epoch < s.lastEpoch { s.mu.Unlock() - reply.Err = EStale + reply.err = EStale return } s.lastEpoch = epoch - reply.Val = s.v + reply.val = s.v primitive.Linearize() s.mu.Unlock() return @@ -63,19 +59,23 @@ func StartServer(me grove_ffi.Address) { handlers[RPC_GET] = func(raw_args []byte, raw_reply *[]byte) { dec := marshal.NewDec(raw_args) epoch := dec.GetInt() - reply := new(getreply_gk.S) + reply := new(GetReply) s.Get(epoch, reply) - *raw_reply = getreply_gk.Marshal(reply, make([]byte, 0)) + *raw_reply = EncGetReply(reply) } handlers[RPC_PUT] = s.e.HandleRequest(func(raw_args []byte, reply *[]byte) { - args, _ := putargs_gk.Unmarshal(raw_args) + args := DecPutArgs(raw_args) err := s.Put(args) - *reply = marshal.WriteInt(make([]byte, 0), err) + enc := marshal.NewEnc(8) + enc.PutInt(err) + *reply = enc.Finish() }) handlers[RPC_FRESHCID] = func(raw_args []byte, reply *[]byte) { - *reply = marshal.WriteInt(make([]byte, 0), s.e.GetFreshCID()) + enc := marshal.NewEnc(8) + enc.PutInt(s.e.GetFreshCID()) + *reply = enc.Finish() } r := urpc.MakeServer(handlers) diff --git a/update-grackle.sh b/update-grackle.sh index 064f190..4d762ef 100755 --- a/update-grackle.sh +++ b/update-grackle.sh @@ -62,7 +62,6 @@ grackle_packages=( "tutorial/lockservice" "tutorial/objectstore/chunk" "tutorial/objectstore/dir" - "fencing/ctr" ) for gopkg in "${grackle_packages[@]}"; do From 92f8eeb22785f6a74d5866a76bd8a120b6ca5b81 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Fri, 6 Dec 2024 15:56:37 -0600 Subject: [PATCH 20/27] rewrite update-grackle in python --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 88cbd6b..6b9ae2f 100644 --- a/flake.nix +++ b/flake.nix @@ -27,8 +27,8 @@ src = pkgs.fetchFromGitHub { owner = "mjschwenne"; repo = "grackle"; - rev = "275fce5626d662527ff987ad61aad61044019fa6"; - sha256 = "16iw5hqg2hd12mc62wqv5hxqb8jwymbnhyy16zsza0limhmr0aya"; + rev = "ee8a2fbea1c4cef22336a2a1760de5c0ba4a9c72"; + sha256 = "08drdzjgj3006l3hxyiqfvm7y2i8m6hmsc108rj6i1w22kd0pv4g"; }; vendorHash = "sha256-Wk2v0HSAkrzxHJvCfbw6xOn0OQ1xukvYjDxk3c2LmH8="; checkPhase = false; From a89feeba88657beb28ca746d05ee09a0d875c8f6 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Fri, 6 Dec 2024 15:57:00 -0600 Subject: [PATCH 21/27] remove update-grackle.sh --- update-grackle.sh | 69 ----------------------------------------------- 1 file changed, 69 deletions(-) delete mode 100755 update-grackle.sh diff --git a/update-grackle.sh b/update-grackle.sh deleted file mode 100755 index 4d762ef..0000000 --- a/update-grackle.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -if [[ -z "$IN_NIX_SHELL" ]]; then - # Not in nix shell, use the default go installs - GRACKLE="go run github.com/mjschwenne/grackle/cmd/grackle@latest" -else - # In a nix shell, expect grackle to be on the PATH and use that - GRACKLE="grackle" -fi - -# Have I mentioned that I dislike bash? Just look at this arcane -# and archaic syntax -compile_grackle () { - CWD=$(pwd) - cd "$1" || return - go install ./cmd/grackle - cd "$CWD" || exit -} - -# Run grackle on the input go package. -# -# We will assume that: -# 1. The proto file is in this directory -# 2. We only want to output go code -# 3. The go code should be output into this directory -# 4. The desired go package matches the directory structure -run_grackle () { - $GRACKLE --go-output-path "$1" --go-package "github.com/mit-pdos/gokv/$1" "$1" -} - -ARGS=$(getopt -o "c:h" --long "compile-grackle:,help" -- "$@") - -eval set -- "$ARGS" -while [ $# -ge 1 ]; do - case "$1" in - -c | --compile-grackle) - echo "compiling grackle $2" - compile_grackle "$2" - shift - ;; - -h | --help) - cat < | -c ] [--help | -h] - -Calls grackle on all go modules known to have proto files for grackle usage. - ---compile-grackle [-c] : Takes the path to the grackle repository, recompiles and installs grackle -EOF - shift - exit 0 - ;; - --) - shift - break - ;; - esac - shift -done - -grackle_packages=( - "tutorial/kvservice" - "tutorial/lockservice" - "tutorial/objectstore/chunk" - "tutorial/objectstore/dir" -) - -for gopkg in "${grackle_packages[@]}"; do - run_grackle "$gopkg" -done From badeef1b08301a59dad7225f65cc829dddd407ab Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Fri, 6 Dec 2024 15:57:29 -0600 Subject: [PATCH 22/27] add update-grackle to make fix rule --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 3129c42..24651ae 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,5 @@ check: go test ./... fix: + ./update-grackle.py gofmt -w -s . From ceca6846337307d077d2663433628c76a26551c5 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Fri, 6 Dec 2024 16:54:20 -0600 Subject: [PATCH 23/27] add missing update-grackle.py and update ci workflow --- .github/workflows/ci.yml | 7 +++- update-grackle.py | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100755 update-grackle.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7985ab..5ad9726 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,12 @@ jobs: uses: actions/setup-go@v5 with: go-version: "1.22" + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" - name: Check Grackle run: | - ./update-grackle.sh + ./update-grackle.py git diff --exit-code diff --git a/update-grackle.py b/update-grackle.py new file mode 100755 index 0000000..4bb0699 --- /dev/null +++ b/update-grackle.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess as sp + +grackle_packages = [ + "tutorial/kvservice", + "tutorial/lockservice", + "tutorial/objectstore/chunk", + "tutorial/objectstore/dir", +] + +# Use $PATH grackle if in a nix shell, otherwise the standard go install +if "IN_NIX_SHELL" in os.environ: + grackle_cmd = ["grackle"] +else: + grackle_cmd = ["go", "run", "github.com/mjschwenne/grackle/cmd/grackle@latest"] + + +def compile_grackle(grackle_repo): + original_working_directory = os.getcwd() + os.chdir(grackle_repo) + sp.run(["go", "install", "./cmd/grackle"]) + os.chdir(original_working_directory) + + +def run_grackle(pkg, debug=False): + opts = [ + "--go-output-path", + pkg, + "--go-package", + f"github.com/mit-pdos/gokv/{pkg}", + pkg, + ] + if debug: + opts.insert(4, "--debug") + print(opts) + try: + sp.run( + grackle_cmd + opts, + timeout=1, + ) + except sp.TimeoutExpired: + print(f"Grackle timed out on package: {pkg}") + + +def main(): + parser = argparse.ArgumentParser( + prog="update-grackle", + description="Update the grackled files in gokv", + ) + parser.add_argument( + "-c", + "--compile", + help="Path to the grackle repository. If set, compile and install grackle from this repository", + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + help="Print grackle output to stdout rather than update the files", + ) + + args = parser.parse_args() + if args.compile is not None: + compile_grackle(args.compile) + + for pkg in grackle_packages: + run_grackle(pkg, args.debug) + + +if __name__ == "__main__": + main() From fa07c3eb976b468062b30c27c4b3bb3d3f475540 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Fri, 6 Dec 2024 16:57:32 -0600 Subject: [PATCH 24/27] add minimal pyproject.toml --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d837dce --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "gokv" +version = "2024.0.0" +dependencies = [ + "ruff==0.8.1", +] From 80fb0f22a0ff2700acc7a3c4107806e590cceb48 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Sat, 7 Dec 2024 12:24:51 -0600 Subject: [PATCH 25/27] attempt to fix python post install --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ad9726..b89f33b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" - cache: "pip" - name: Check Grackle run: | ./update-grackle.py From 4115e2622bc67b4a1bb0586ef226bd1ebdca065f Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Wed, 8 Jan 2025 21:02:34 -0600 Subject: [PATCH 26/27] update to use value-based grackle --- flake.nix | 8 +-- tutorial/kvservice/client.go | 8 +-- .../conditionalput_gk/conditionalput_gk.go | 33 +++++++----- tutorial/kvservice/get_gk/get_gk.go | 25 +++++---- tutorial/kvservice/kvservice.proto | 2 +- tutorial/kvservice/kvservice_rpc.gb.go | 6 +-- tutorial/kvservice/put_gk/put_gk.go | 51 +++++++++++-------- tutorial/kvservice/server.go | 8 +-- tutorial/lockservice/1_lock_rpc.gb.go | 4 +- .../lockrequest_gk/lockrequest_gk.go | 21 ++++---- tutorial/objectstore/chunk/client.go | 2 +- tutorial/objectstore/chunk/server.go | 2 +- .../chunk/writechunk_gk/writechunk_gk.go | 31 ++++++----- .../dir/chunkhandle_gk/chunkhandle_gk.go | 25 +++++---- tutorial/objectstore/dir/client.go | 4 +- .../dir/finishwrite_gk/finishwrite_gk.go | 25 +++++---- .../dir/recordchunk_gk/recordchunk_gk.go | 35 ++++++++----- tutorial/objectstore/dir/server.go | 4 +- 18 files changed, 172 insertions(+), 122 deletions(-) diff --git a/flake.nix b/flake.nix index 6b9ae2f..d15486b 100644 --- a/flake.nix +++ b/flake.nix @@ -17,8 +17,8 @@ src = pkgs.fetchFromGitHub { owner = "goose-lang"; repo = "goose"; - rev = "8d13c771b9a80957089f7c5b0ee2ccf58e5eb06f"; - sha256 = "1fbqs75ya4as3my2knkaq4m0crdh3n004grw5g5iczvb5h5k06lz"; + rev = "a4f2f84193d34f56dd84fc623adc43a6441da1eb"; + sha256 = "1b1dfa1qsv2h7hy5x20zhic2npr5gz1zp76m1lab4v490adxj2rx"; }; vendorHash = "sha256-HCJ8v3TSv4UrkOsRuENWVz5Z7zQ1UsOygx0Mo7MELzY="; }; @@ -27,8 +27,8 @@ src = pkgs.fetchFromGitHub { owner = "mjschwenne"; repo = "grackle"; - rev = "ee8a2fbea1c4cef22336a2a1760de5c0ba4a9c72"; - sha256 = "08drdzjgj3006l3hxyiqfvm7y2i8m6hmsc108rj6i1w22kd0pv4g"; + rev = "101412356cdfbcad78f8aaa724101312928c4978"; + sha256 = "06zf2bvrbbjhgrd6994h3wcaml7m83m6f9r61pj7y09xq9nw10br"; }; vendorHash = "sha256-Wk2v0HSAkrzxHJvCfbw6xOn0OQ1xukvYjDxk3c2LmH8="; checkPhase = false; diff --git a/tutorial/kvservice/client.go b/tutorial/kvservice/client.go index 0d1a58e..45b8c38 100644 --- a/tutorial/kvservice/client.go +++ b/tutorial/kvservice/client.go @@ -37,10 +37,10 @@ func (ck *Clerk) Put(key string, val string) { for { // TODO: allocate these once, outside the loop. Waiting to do this until // heap has dfrac for convenience. - args := &put_gk.S{ + args := put_gk.S{ OpId: opId, Key: key, - Val: val, + Value: val, } if ck.rpcCl.putRpc(args) == urpc.ErrNone { break @@ -63,7 +63,7 @@ func (ck *Clerk) ConditionalPut(key string, expectedVal string, newVal string) b var ret bool for { - args := &conditionalput_gk.S{ + args := conditionalput_gk.S{ OpId: opId, Key: key, ExpectedVal: expectedVal, @@ -94,7 +94,7 @@ func (ck *Clerk) Get(key string) string { var ret string for { - args := &get_gk.S{ + args := get_gk.S{ OpId: opId, Key: key, } diff --git a/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go b/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go index 091b7d2..45ca40a 100644 --- a/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go +++ b/tutorial/kvservice/conditionalput_gk/conditionalput_gk.go @@ -1,3 +1,8 @@ +//-------------------------------------- +// This file is autogenerated by grackle +// DO NOT MANUALLY EDIT THIS FILE +//-------------------------------------- + package conditionalput_gk import ( @@ -11,11 +16,7 @@ type S struct { NewVal string } -func (c *S) approxSize() uint64 { - return 0 -} - -func Marshal(c *S, prefix []byte) []byte { +func Marshal(c S, prefix []byte) []byte { var enc = prefix enc = marshal.WriteInt(enc, c.OpId) @@ -32,26 +33,34 @@ func Marshal(c *S, prefix []byte) []byte { return enc } -func Unmarshal(s []byte) (*S, []byte) { - c := new(S) +func Unmarshal(s []byte) (S, []byte) { var enc = s // Needed for goose compatibility + var opId uint64 + var key string + var expectedVal string + var newVal string - c.OpId, enc = marshal.ReadInt(enc) + opId, enc = marshal.ReadInt(enc) var keyLen uint64 var keyBytes []byte keyLen, enc = marshal.ReadInt(enc) keyBytes, enc = marshal.ReadBytesCopy(enc, keyLen) - c.Key = string(keyBytes) + key = string(keyBytes) var expectedValLen uint64 var expectedValBytes []byte expectedValLen, enc = marshal.ReadInt(enc) expectedValBytes, enc = marshal.ReadBytesCopy(enc, expectedValLen) - c.ExpectedVal = string(expectedValBytes) + expectedVal = string(expectedValBytes) var newValLen uint64 var newValBytes []byte newValLen, enc = marshal.ReadInt(enc) newValBytes, enc = marshal.ReadBytesCopy(enc, newValLen) - c.NewVal = string(newValBytes) + newVal = string(newValBytes) - return c, enc + return S{ + OpId: opId, + Key: key, + ExpectedVal: expectedVal, + NewVal: newVal, + }, enc } diff --git a/tutorial/kvservice/get_gk/get_gk.go b/tutorial/kvservice/get_gk/get_gk.go index 3de9e8a..d8430fa 100644 --- a/tutorial/kvservice/get_gk/get_gk.go +++ b/tutorial/kvservice/get_gk/get_gk.go @@ -1,3 +1,8 @@ +//-------------------------------------- +// This file is autogenerated by grackle +// DO NOT MANUALLY EDIT THIS FILE +//-------------------------------------- + package get_gk import ( @@ -9,11 +14,7 @@ type S struct { Key string } -func (g *S) approxSize() uint64 { - return 0 -} - -func Marshal(g *S, prefix []byte) []byte { +func Marshal(g S, prefix []byte) []byte { var enc = prefix enc = marshal.WriteInt(enc, g.OpId) @@ -24,16 +25,20 @@ func Marshal(g *S, prefix []byte) []byte { return enc } -func Unmarshal(s []byte) (*S, []byte) { - g := new(S) +func Unmarshal(s []byte) (S, []byte) { var enc = s // Needed for goose compatibility + var opId uint64 + var key string - g.OpId, enc = marshal.ReadInt(enc) + opId, enc = marshal.ReadInt(enc) var keyLen uint64 var keyBytes []byte keyLen, enc = marshal.ReadInt(enc) keyBytes, enc = marshal.ReadBytesCopy(enc, keyLen) - g.Key = string(keyBytes) + key = string(keyBytes) - return g, enc + return S{ + OpId: opId, + Key: key, + }, enc } diff --git a/tutorial/kvservice/kvservice.proto b/tutorial/kvservice/kvservice.proto index 4f78714..b7e15f0 100644 --- a/tutorial/kvservice/kvservice.proto +++ b/tutorial/kvservice/kvservice.proto @@ -3,7 +3,7 @@ syntax = "proto3"; message put { uint64 opId = 1; string key = 2; - string val = 3; + string value = 3; } message conditionalPut { diff --git a/tutorial/kvservice/kvservice_rpc.gb.go b/tutorial/kvservice/kvservice_rpc.gb.go index 9327da7..189d495 100644 --- a/tutorial/kvservice/kvservice_rpc.gb.go +++ b/tutorial/kvservice/kvservice_rpc.gb.go @@ -30,7 +30,7 @@ func (cl *Client) getFreshNumRpc() (uint64, Error) { return 0, err } -func (cl *Client) putRpc(args *put_gk.S) Error { +func (cl *Client) putRpc(args put_gk.S) Error { var reply []byte err := cl.cl.Call(rpcIdPut, put_gk.Marshal(args, make([]byte, 0)), &reply, 100) if err == urpc.ErrNone { @@ -39,7 +39,7 @@ func (cl *Client) putRpc(args *put_gk.S) Error { return err } -func (cl *Client) conditionalPutRpc(args *conditionalput_gk.S) (string, Error) { +func (cl *Client) conditionalPutRpc(args conditionalput_gk.S) (string, Error) { var reply []byte err := cl.cl.Call(rpcIdConditionalPut, conditionalput_gk.Marshal(args, make([]byte, 0)), &reply, 100) if err == urpc.ErrNone { @@ -48,7 +48,7 @@ func (cl *Client) conditionalPutRpc(args *conditionalput_gk.S) (string, Error) { return "", err } -func (cl *Client) getRpc(args *get_gk.S) (string, Error) { +func (cl *Client) getRpc(args get_gk.S) (string, Error) { var reply []byte err := cl.cl.Call(rpcIdGet, get_gk.Marshal(args, make([]byte, 0)), &reply, 100) if err == urpc.ErrNone { diff --git a/tutorial/kvservice/put_gk/put_gk.go b/tutorial/kvservice/put_gk/put_gk.go index b48d1d5..fadd543 100644 --- a/tutorial/kvservice/put_gk/put_gk.go +++ b/tutorial/kvservice/put_gk/put_gk.go @@ -1,3 +1,8 @@ +//-------------------------------------- +// This file is autogenerated by grackle +// DO NOT MANUALLY EDIT THIS FILE +//-------------------------------------- + package put_gk import ( @@ -5,44 +10,46 @@ import ( ) type S struct { - OpId uint64 - Key string - Val string -} - -func (p *S) approxSize() uint64 { - return 0 + OpId uint64 + Key string + Value string } -func Marshal(p *S, prefix []byte) []byte { +func Marshal(p S, prefix []byte) []byte { var enc = prefix enc = marshal.WriteInt(enc, p.OpId) keyBytes := []byte(p.Key) enc = marshal.WriteInt(enc, uint64(len(keyBytes))) enc = marshal.WriteBytes(enc, keyBytes) - valBytes := []byte(p.Val) - enc = marshal.WriteInt(enc, uint64(len(valBytes))) - enc = marshal.WriteBytes(enc, valBytes) + valueBytes := []byte(p.Value) + enc = marshal.WriteInt(enc, uint64(len(valueBytes))) + enc = marshal.WriteBytes(enc, valueBytes) return enc } -func Unmarshal(s []byte) (*S, []byte) { - p := new(S) +func Unmarshal(s []byte) (S, []byte) { var enc = s // Needed for goose compatibility + var opId uint64 + var key string + var value string - p.OpId, enc = marshal.ReadInt(enc) + opId, enc = marshal.ReadInt(enc) var keyLen uint64 var keyBytes []byte keyLen, enc = marshal.ReadInt(enc) keyBytes, enc = marshal.ReadBytesCopy(enc, keyLen) - p.Key = string(keyBytes) - var valLen uint64 - var valBytes []byte - valLen, enc = marshal.ReadInt(enc) - valBytes, enc = marshal.ReadBytesCopy(enc, valLen) - p.Val = string(valBytes) - - return p, enc + key = string(keyBytes) + var valueLen uint64 + var valueBytes []byte + valueLen, enc = marshal.ReadInt(enc) + valueBytes, enc = marshal.ReadBytesCopy(enc, valueLen) + value = string(valueBytes) + + return S{ + OpId: opId, + Key: key, + Value: value, + }, enc } diff --git a/tutorial/kvservice/server.go b/tutorial/kvservice/server.go index d0cf375..a697bc3 100644 --- a/tutorial/kvservice/server.go +++ b/tutorial/kvservice/server.go @@ -24,19 +24,19 @@ func (s *Server) getFreshNum() uint64 { return n } -func (s *Server) put(args *put_gk.S) { +func (s *Server) put(args put_gk.S) { s.mu.Lock() _, ok := s.lastReplies[args.OpId] if ok { s.mu.Unlock() return } - s.kvs[args.Key] = args.Val + s.kvs[args.Key] = args.Value s.lastReplies[args.OpId] = "" s.mu.Unlock() } -func (s *Server) conditionalPut(args *conditionalput_gk.S) string { +func (s *Server) conditionalPut(args conditionalput_gk.S) string { s.mu.Lock() ret, ok := s.lastReplies[args.OpId] if ok { @@ -54,7 +54,7 @@ func (s *Server) conditionalPut(args *conditionalput_gk.S) string { return ret2 } -func (s *Server) get(args *get_gk.S) string { +func (s *Server) get(args get_gk.S) string { s.mu.Lock() ret, ok := s.lastReplies[args.OpId] if ok { diff --git a/tutorial/lockservice/1_lock_rpc.gb.go b/tutorial/lockservice/1_lock_rpc.gb.go index 720aacd..f3bf980 100644 --- a/tutorial/lockservice/1_lock_rpc.gb.go +++ b/tutorial/lockservice/1_lock_rpc.gb.go @@ -30,7 +30,7 @@ func (cl *Client) getFreshNum() (uint64, Error) { func (cl *Client) tryAcquire(id uint64) (uint64, Error) { var reply []byte - args := lockrequest_gk.Marshal(&lockrequest_gk.S{Id: id}, []byte{}) + args := lockrequest_gk.Marshal(lockrequest_gk.S{Id: id}, []byte{}) err := cl.cl.Call(RPC_TRY_ACQUIRE, args, &reply, 100) if err == urpc.ErrNone { return DecodeUint64(reply), err @@ -40,7 +40,7 @@ func (cl *Client) tryAcquire(id uint64) (uint64, Error) { func (cl *Client) release(id uint64) Error { var reply []byte - args := lockrequest_gk.Marshal(&lockrequest_gk.S{Id: id}, []byte{}) + args := lockrequest_gk.Marshal(lockrequest_gk.S{Id: id}, []byte{}) return cl.cl.Call(RPC_RELEASE, args, &reply, 100) } diff --git a/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go b/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go index 9aa2c0b..547d38d 100644 --- a/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go +++ b/tutorial/lockservice/lockrequest_gk/lockrequest_gk.go @@ -1,3 +1,8 @@ +//-------------------------------------- +// This file is autogenerated by grackle +// DO NOT MANUALLY EDIT THIS FILE +//-------------------------------------- + package lockrequest_gk import ( @@ -8,11 +13,7 @@ type S struct { Id uint64 } -func (l *S) approxSize() uint64 { - return 0 -} - -func Marshal(l *S, prefix []byte) []byte { +func Marshal(l S, prefix []byte) []byte { var enc = prefix enc = marshal.WriteInt(enc, l.Id) @@ -20,11 +21,13 @@ func Marshal(l *S, prefix []byte) []byte { return enc } -func Unmarshal(s []byte) (*S, []byte) { - l := new(S) +func Unmarshal(s []byte) (S, []byte) { var enc = s // Needed for goose compatibility + var id uint64 - l.Id, enc = marshal.ReadInt(enc) + id, enc = marshal.ReadInt(enc) - return l, enc + return S{ + Id: id, + }, enc } diff --git a/tutorial/objectstore/chunk/client.go b/tutorial/objectstore/chunk/client.go index ff289da..c446162 100644 --- a/tutorial/objectstore/chunk/client.go +++ b/tutorial/objectstore/chunk/client.go @@ -20,7 +20,7 @@ type ClerkPool struct { } func (ck *ClerkPool) WriteChunk(addr grove_ffi.Address, args writechunk_gk.S) { - req := writechunk_gk.Marshal(&args, make([]byte, 0)) + req := writechunk_gk.Marshal(args, make([]byte, 0)) reply := new([]byte) ck.cm.CallAtLeastOnce(addr, WriteChunkId, req, reply, 100 /*ms*/) } diff --git a/tutorial/objectstore/chunk/server.go b/tutorial/objectstore/chunk/server.go index 0ec4e1b..5405166 100644 --- a/tutorial/objectstore/chunk/server.go +++ b/tutorial/objectstore/chunk/server.go @@ -49,7 +49,7 @@ func StartServer(me grove_ffi.Address, dir_addr grove_ffi.Address) { handlers := make(map[uint64]func([]byte, *[]byte)) handlers[WriteChunkId] = func(req []byte, reply *[]byte) { args, _ := writechunk_gk.Unmarshal(req) - s.WriteChunk(*args) + s.WriteChunk(args) *reply = make([]byte, 0) // TODO: is this needed? } handlers[GetChunkId] = func(req []byte, reply *[]byte) { diff --git a/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go b/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go index a336d7b..ff75936 100644 --- a/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go +++ b/tutorial/objectstore/chunk/writechunk_gk/writechunk_gk.go @@ -1,3 +1,8 @@ +//-------------------------------------- +// This file is autogenerated by grackle +// DO NOT MANUALLY EDIT THIS FILE +//-------------------------------------- + package writechunk_gk import ( @@ -10,11 +15,7 @@ type S struct { Index uint64 } -func (w *S) approxSize() uint64 { - return 0 -} - -func Marshal(w *S, prefix []byte) []byte { +func Marshal(w S, prefix []byte) []byte { var enc = prefix enc = marshal.WriteInt(enc, w.WriteId) @@ -25,17 +26,23 @@ func Marshal(w *S, prefix []byte) []byte { return enc } -func Unmarshal(s []byte) (*S, []byte) { - w := new(S) +func Unmarshal(s []byte) (S, []byte) { var enc = s // Needed for goose compatibility + var writeId uint64 + var chunk []byte + var index uint64 - w.WriteId, enc = marshal.ReadInt(enc) + writeId, enc = marshal.ReadInt(enc) var chunkLen uint64 var chunkBytes []byte chunkLen, enc = marshal.ReadInt(enc) chunkBytes, enc = marshal.ReadBytesCopy(enc, chunkLen) - w.Chunk = chunkBytes - w.Index, enc = marshal.ReadInt(enc) - - return w, enc + chunk = chunkBytes + index, enc = marshal.ReadInt(enc) + + return S{ + WriteId: writeId, + Chunk: chunk, + Index: index, + }, enc } diff --git a/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go b/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go index 2e3b823..dc25aad 100644 --- a/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go +++ b/tutorial/objectstore/dir/chunkhandle_gk/chunkhandle_gk.go @@ -1,3 +1,8 @@ +//-------------------------------------- +// This file is autogenerated by grackle +// DO NOT MANUALLY EDIT THIS FILE +//-------------------------------------- + package chunkhandle_gk import ( @@ -9,11 +14,7 @@ type S struct { ContentHash string } -func (c *S) approxSize() uint64 { - return 0 -} - -func Marshal(c *S, prefix []byte) []byte { +func Marshal(c S, prefix []byte) []byte { var enc = prefix enc = marshal.WriteInt(enc, c.Addr) @@ -24,16 +25,20 @@ func Marshal(c *S, prefix []byte) []byte { return enc } -func Unmarshal(s []byte) (*S, []byte) { - c := new(S) +func Unmarshal(s []byte) (S, []byte) { var enc = s // Needed for goose compatibility + var addr uint64 + var contentHash string - c.Addr, enc = marshal.ReadInt(enc) + addr, enc = marshal.ReadInt(enc) var contentHashLen uint64 var contentHashBytes []byte contentHashLen, enc = marshal.ReadInt(enc) contentHashBytes, enc = marshal.ReadBytesCopy(enc, contentHashLen) - c.ContentHash = string(contentHashBytes) + contentHash = string(contentHashBytes) - return c, enc + return S{ + Addr: addr, + ContentHash: contentHash, + }, enc } diff --git a/tutorial/objectstore/dir/client.go b/tutorial/objectstore/dir/client.go index 99ec258..b5dfd9b 100644 --- a/tutorial/objectstore/dir/client.go +++ b/tutorial/objectstore/dir/client.go @@ -36,14 +36,14 @@ func (ck *Clerk) PrepareWrite() PreparedWrite { // From chunk func (ck *Clerk) RecordChunk(args recordchunk_gk.S) { - req := recordchunk_gk.Marshal(&args, make([]byte, 0)) + req := recordchunk_gk.Marshal(args, make([]byte, 0)) reply := new([]byte) ck.client.Call(RecordChunkId, req, reply, 100 /*ms*/) } // From chunk func (ck *Clerk) FinishWrite(args finishwrite_gk.S) { - req := finishwrite_gk.Marshal(&args, make([]byte, 0)) + req := finishwrite_gk.Marshal(args, make([]byte, 0)) reply := new([]byte) ck.client.Call(FinishWriteId, req, reply, 100 /*ms*/) } diff --git a/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go b/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go index 68a69dd..e446c49 100644 --- a/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go +++ b/tutorial/objectstore/dir/finishwrite_gk/finishwrite_gk.go @@ -1,3 +1,8 @@ +//-------------------------------------- +// This file is autogenerated by grackle +// DO NOT MANUALLY EDIT THIS FILE +//-------------------------------------- + package finishwrite_gk import ( @@ -9,11 +14,7 @@ type S struct { Keyname string } -func (f *S) approxSize() uint64 { - return 0 -} - -func Marshal(f *S, prefix []byte) []byte { +func Marshal(f S, prefix []byte) []byte { var enc = prefix enc = marshal.WriteInt(enc, f.WriteId) @@ -24,16 +25,20 @@ func Marshal(f *S, prefix []byte) []byte { return enc } -func Unmarshal(s []byte) (*S, []byte) { - f := new(S) +func Unmarshal(s []byte) (S, []byte) { var enc = s // Needed for goose compatibility + var writeId uint64 + var keyname string - f.WriteId, enc = marshal.ReadInt(enc) + writeId, enc = marshal.ReadInt(enc) var keynameLen uint64 var keynameBytes []byte keynameLen, enc = marshal.ReadInt(enc) keynameBytes, enc = marshal.ReadBytesCopy(enc, keynameLen) - f.Keyname = string(keynameBytes) + keyname = string(keynameBytes) - return f, enc + return S{ + WriteId: writeId, + Keyname: keyname, + }, enc } diff --git a/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go b/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go index 6ccb578..485e709 100644 --- a/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go +++ b/tutorial/objectstore/dir/recordchunk_gk/recordchunk_gk.go @@ -1,3 +1,8 @@ +//-------------------------------------- +// This file is autogenerated by grackle +// DO NOT MANUALLY EDIT THIS FILE +//-------------------------------------- + package recordchunk_gk import ( @@ -11,11 +16,7 @@ type S struct { Index uint64 } -func (r *S) approxSize() uint64 { - return 0 -} - -func Marshal(r *S, prefix []byte) []byte { +func Marshal(r S, prefix []byte) []byte { var enc = prefix enc = marshal.WriteInt(enc, r.WriteId) @@ -28,18 +29,26 @@ func Marshal(r *S, prefix []byte) []byte { return enc } -func Unmarshal(s []byte) (*S, []byte) { - r := new(S) +func Unmarshal(s []byte) (S, []byte) { var enc = s // Needed for goose compatibility + var writeId uint64 + var server uint64 + var contentHash string + var index uint64 - r.WriteId, enc = marshal.ReadInt(enc) - r.Server, enc = marshal.ReadInt(enc) + writeId, enc = marshal.ReadInt(enc) + server, enc = marshal.ReadInt(enc) var contentHashLen uint64 var contentHashBytes []byte contentHashLen, enc = marshal.ReadInt(enc) contentHashBytes, enc = marshal.ReadBytesCopy(enc, contentHashLen) - r.ContentHash = string(contentHashBytes) - r.Index, enc = marshal.ReadInt(enc) - - return r, enc + contentHash = string(contentHashBytes) + index, enc = marshal.ReadInt(enc) + + return S{ + WriteId: writeId, + Server: server, + ContentHash: contentHash, + Index: index, + }, enc } diff --git a/tutorial/objectstore/dir/server.go b/tutorial/objectstore/dir/server.go index d73a996..22e23d9 100644 --- a/tutorial/objectstore/dir/server.go +++ b/tutorial/objectstore/dir/server.go @@ -93,12 +93,12 @@ func StartServer(me grove_ffi.Address) { } handlers[RecordChunkId] = func(req []byte, reply *[]byte) { args, _ := recordchunk_gk.Unmarshal(req) - s.RecordChunk(*args) + s.RecordChunk(args) *reply = make([]byte, 0) } handlers[FinishWriteId] = func(req []byte, reply *[]byte) { args, _ := finishwrite_gk.Unmarshal(req) - s.FinishWrite(*args) + s.FinishWrite(args) *reply = make([]byte, 0) } handlers[PrepareReadId] = func(req []byte, reply *[]byte) { From 8b270bfad2f926e92c3ca943ca8f81cd181d23e5 Mon Sep 17 00:00:00 2001 From: Matt Schwennesen Date: Sat, 11 Jan 2025 20:26:00 -0600 Subject: [PATCH 27/27] correct formatting --- tutorial/kvservice/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorial/kvservice/client.go b/tutorial/kvservice/client.go index 45b8c38..8237eb2 100644 --- a/tutorial/kvservice/client.go +++ b/tutorial/kvservice/client.go @@ -38,9 +38,9 @@ func (ck *Clerk) Put(key string, val string) { // TODO: allocate these once, outside the loop. Waiting to do this until // heap has dfrac for convenience. args := put_gk.S{ - OpId: opId, - Key: key, - Value: val, + OpId: opId, + Key: key, + Value: val, } if ck.rpcCl.putRpc(args) == urpc.ErrNone { break