From f0e1beea800c0dbe86120789cf8b3e4c9f7a303b Mon Sep 17 00:00:00 2001 From: Bruno Calza Date: Thu, 14 Dec 2023 18:38:49 -0300 Subject: [PATCH 01/16] renaming things to vaults terminology Signed-off-by: Bruno Calza --- .gitignore | 2 +- Makefile | 2 +- README.md | 84 +++---- cmd/basin/publication_test.go | 124 ---------- cmd/basin/wallet.go | 69 ------ .../publication.go => vaults/commands.go} | 218 ++++++++++-------- cmd/{basin => vaults}/config.go | 2 +- cmd/{basin => vaults}/main.go | 11 +- goreleaser.yml | 36 +-- internal/app/basin_provider.go | 4 +- internal/app/db.go | 4 +- internal/app/streamer.go | 14 +- internal/app/streamer_test.go | 28 +-- internal/app/uploader.go | 18 +- .../provider.go | 22 +- scripts/run.sh | 2 +- test/postgres.go | 4 +- 17 files changed, 244 insertions(+), 400 deletions(-) delete mode 100644 cmd/basin/publication_test.go delete mode 100644 cmd/basin/wallet.go rename cmd/{basin/publication.go => vaults/commands.go} (77%) rename cmd/{basin => vaults}/config.go (96%) rename cmd/{basin => vaults}/main.go (51%) rename pkg/{basinprovider => vaultsprovider}/provider.go (86%) diff --git a/.gitignore b/.gitignore index 41b70c9..de58d41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -/basin +/vaults /cover.out .vscode/* diff --git a/Makefile b/Makefile index 6e96c4d..f45d4ab 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ lint: # Build build: - go build -o basin cmd/basin/* + go build -o vaults cmd/vaults/* .PHONY: build # Test diff --git a/README.md b/README.md index 0ade378..e35aeab 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# basin-cli +# vaults-cli [![License](https://img.shields.io/github/license/tablelandnetwork/basin-cli.svg)](./LICENSE) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg)](https://github.com/RichardLitt/standard-readme) @@ -12,20 +12,20 @@ - [Self-hosted](#self-hosted) - [Amazon RDS](#amazon-rds) - [Supabase](#supabase) -- [Create a publication](#create-a-publication) -- [Start replicating a publication](#start-replicating-a-publication) -- [Upload a Parquet file](#upload-a-parquet-file) -- [Listing Publications](#listing-publications) -- [Listing Deals](#listing-deals) +- [Create a vault](#create-a-vault) +- [Start replicating a database](#start-replicating-a-database) +- [Write a Parquet file](#write-a-parquet-file) +- [Listing Vaults](#listing-vaults) +- [Listing Events](#listing-events) - [Running](#running) - [Run tests](#run-tests) - [Retrieving](#retrieving) # Background -Tableland Basin is a secure and verifiable open data platform. The Basin CLI is a tool that allows you to continuously replicate a table or view from your database to the network. Currently, only PostgreSQL is supported. +Textile Vaults is a secure and verifiable open data platform. The Vaults CLI is a tool that allows you to continuously replicate a table or view from your database to the network. Currently, only PostgreSQL is supported. -> 🚧 Basin is currently not in a production-ready state. Any data that is pushed to the network may be subject to deletion. 🚧 +> 🚧 Vaults is currently not in a production-ready state. Any data that is pushed to the network may be subject to deletion. 🚧 # Usage @@ -34,14 +34,14 @@ Tableland Basin is a secure and verifiable open data platform. The Basin CLI is ```bash git clone https://github.com/tablelandnetwork/basin-cli.git cd basin-cli -go install ./cmd/basin +go install ./cmd/vaults ``` ## Postgres Setup ### Self-hosted -- Make sure you have access to a superuser role. For example, you can create a new role such as `CREATE ROLE basin WITH PASSWORD NULL LOGIN SUPERUSER;`. +- Make sure you have access to a superuser role. For example, you can create a new role such as `CREATE ROLE vaults WITH PASSWORD NULL LOGIN SUPERUSER;`. - Check that your Postgres installation has the [wal2json](https://github.com/eulerto/wal2json) plugin installed. - Check if logical replication is enabled: @@ -69,7 +69,7 @@ go install ./cmd/basin WHERE name = 'rds.logical_replication'; ``` -- If it's on, go to [Create a publication](#create-a-publication) +- If it's on, go to [Create a vault](#create-a-vault) - If it's off, follow the next steps: - [Create a custom RDS parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithDBInstanceParamGroups.html#USER_WorkingWithParamGroups.Creating) - After creation, edit it and set the `rds.logical_replication` parameter to `1` @@ -83,92 +83,92 @@ go install ./cmd/basin - Log into the [Supabase](https://supabase.io/) dashboard and go to your project, or create a new one. - Check if logical replication is enabled. This should be the default setting, so you shouldn't have to change anything. You can do this in the `SQL Editor` section on the left hand side of the Supabase dashboard by running `SHOW wal_level;` query, which should log `logical`. - You can find the database connection information on the left hand side under `Project Settings` > `Database`. You will need the `Host`, `Port`, `Database`, `Username`, and `Password` to connect to your database. - - When you create a publication, the `--dburi` should follow this format: + - When you create a vault, the `--dburi` should follow this format: ```sh postgresql://postgres:[PASSWORD]@db.[PROJECT_ID].supabase.co:5432/postgres ``` -## Create a publication +## Create a vault -_Publications_ define the data you are pushing to Basin. +_Vaults_ define the place you push data into. -Basin uses public key authentication, so you will need an Ethereum style (ECDSA, secp256k1) wallet to create a new publication. You can use an existing wallet or set up a new one with `basin wallet create`. Your private key is only used locally for signing. +Vaults uses public key authentication, so you will need an Ethereum style (ECDSA, secp256k1) wallet to create a new vault. You can use an existing wallet or set up a new one with `vaults wallet create`. Your private key is only used locally for signing. ```bash -basin wallet create [FILENAME] +vaults wallet create [FILENAME] ``` A new private key will be written to `FILENAME`. -The name of a publication contains a `namespace` (e.g. `my_company`) and the name of an existing database relation (e.g. `my_table`), separated by a period (`.`). Use `basin publication create` to create a new publication. See `basin publication create --help` for more info. +The name of a vault contains a `namespace` (e.g. `my_company`) and the name of an existing database relation (e.g. `my_table`), separated by a period (`.`). Use `vaults create` to create a new vault. See `vaults create --help` for more info. ```bash -basin publication create --dburi [DBURI] --address [WALLET_ADDRESS] namespace.relation_name +vaults create --dburi [DBURI] --account [WALLET_ADDRESS] namespace.relation_name ``` -🚧 Basin currently only replicates `INSERT` statements, which means that it only replicates append-only data (e.g., log-style data). Row updates and deletes will be ignored. 🚧 +🚧 Vaults currently only replicates `INSERT` statements, which means that it only replicates append-only data (e.g., log-style data). Row updates and deletes will be ignored. 🚧 -## Start replicating a publication +## Start replicating a database -Use `basin publication start` to start a daemon that will continuously push changes to the underlying table/view to the network. See `basin publication start --help` for more info. +Use `vaults stream` to start a daemon that will continuously push changes to the underlying table/view to the network. See `vaults stream --help` for more info. ```bash -basin publication start --private-key [PRIVATE_KEY] namespace.relation_name +vaults stream --private-key [PRIVATE_KEY] namespace.relation_name ``` -## Upload a Parquet file +## Write a Parquet file -Before uploading a Parquet file, you need to [Create a publication](#create-a-publication), if not already created. You can omit the `--dburi` flag, in this case. +Before writing a Parquet file, you need to [Create a vault](#create-a-vault), if not already created. You can omit the `--dburi` flag, in this case. -Then, use `basin publication upload` to upload a Parquet file. +Then, use `vaults write` to write a Parquet file. ```bash -basin publication upload --name [namespace.relation_name] --private-key [PRIVATE_KEY] filepath +vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] filepath ``` -You can attach a timestamp to that file upload, e.g. +You can attach a timestamp to that file write, e.g. ```bash -basin publication upload --name [namespace.relation_name] --private-key [PRIVATE_KEY] --timestamp 1699984703 filepath +vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] --timestamp 1699984703 filepath # or use data format -basin publication upload --name [namespace.relation_name] --private-key [PRIVATE_KEY] --timestamp 2006-01-02 filepath +vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] --timestamp 2006-01-02 filepath # or use RFC3339 format -basin publication upload --name [namespace.relation_name] --private-key [PRIVATE_KEY] --timestamp 2006-01-02T15:04:05Z07:00 filepath +vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] --timestamp 2006-01-02T15:04:05Z07:00 filepath ``` If a timestamp is not provided, the CLI will assume the timestamp is the current client epoch in UTC. -## Listing Publications +## Listing Vaults -You can list the publications from an address by running: +You can list the vaults from an account by running: ```bash -basin publication list --address [ETH_ADDRESS] +vaults list --account [ETH_ADDRESS] ``` -## Listing Deals +## Listing Events -You can list deals of a given publication by running: +You can list events of a given vault by running: ```bash -basin publication deals --publication [PUBLICATION] --latest 5 +vaults events --vault [VAULT_NAME] --latest 5 ``` -Deals command accept `--before`,`--after` , and `--at` flags to filter deals by timestamp +Events command accept `--before`,`--after` , and `--at` flags to filter events by timestamp ```bash # examples -basin publication deals --publication demotest.data --at 1699569502 -basin publication deals --publication demotest.data --before 2023-11-09T19:38:23-03:00 -basin publication deals --publication demotest.data --after 2023-11-09 +vaults events --vault demotest.data --at 1699569502 +vaults events --vault demotest.data --before 2023-11-09T19:38:23-03:00 +vaults events --vault demotest.data --after 2023-11-09 ``` ## Retrieving ```bash -basin publication retrieve bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq +vaults retrieve bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq ``` # Development @@ -185,7 +185,7 @@ PORT=8888 ./scripts/server.sh ./scripts/run.sh wallet create pk.out # Start replicating -./scripts/run.sh publication start --private-key [PRIVATE_KEY] namespace.relation_name +./scripts/run.sh vaults stream --private-key [PRIVATE_KEY] namespace.relation_name ``` ## Run tests diff --git a/cmd/basin/publication_test.go b/cmd/basin/publication_test.go deleted file mode 100644 index 655cdbf..0000000 --- a/cmd/basin/publication_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "context" - "crypto/ecdsa" - "database/sql" - "encoding/hex" - "fmt" - "testing" - "time" - - "github.com/ethereum/go-ethereum/crypto" - _ "github.com/lib/pq" - "github.com/stretchr/testify/require" - "github.com/tablelandnetwork/basin-cli/test" - "github.com/urfave/cli/v2" -) - -func TestPublication(t *testing.T) { - // This is supposed to be the closest to an e2E test we get - // skipping this for now until I think of a way to assert the data that went to Basin Provider - t.Skip() - - t.Parallel() - - pool := test.GetDockerPool() - - db, resource, dburi := pool.RunPostgres() - - cliApp := &cli.App{ - Name: "basin", - Usage: "basin replicates your database as logs and store them in Filecoin", - Commands: []*cli.Command{ - newPublicationCommand(), - }, - } - - h, err := newHelper(cliApp, db, dburi, t.TempDir()) - require.NoError(t, err) - - h.CreateTable(t) - h.CreatePublication(t) - - go func() { - h.StartReplication(t) - }() - - time.Sleep(1 * time.Second) - - h.AddNewRecord(t) - h.AddNewRecord(t) - - pool.Purge(resource) -} - -type helper struct { - app *cli.App - db *sql.DB - dburi string - pk *ecdsa.PrivateKey - ns string - rel string - dir string -} - -func newHelper(app *cli.App, db *sql.DB, dburi string, dir string) (*helper, error) { - pk, err := crypto.GenerateKey() - if err != nil { - return nil, err - } - - return &helper{ - pk: pk, - ns: "n", - rel: "t", - dir: dir, - - app: app, - db: db, - dburi: dburi, - }, nil -} - -func (h *helper) CreateTable(t *testing.T) { - _, err := h.db.ExecContext( - context.Background(), - fmt.Sprintf("create table %s(id serial primary key, name text);", h.rel), - ) - require.NoError(t, err) -} - -func (h *helper) CreatePublication(t *testing.T) { - name := fmt.Sprintf("%s.%s", h.ns, h.rel) - err := h.app.Run([]string{ - "basin", - "publication", - "--dir", h.dir, - "create", - "--dburi", h.dburi, - "--address", crypto.PubkeyToAddress(h.pk.PublicKey).Hex(), - "--provider", "mock", - name, - }) - require.NoError(t, err) -} - -func (h *helper) StartReplication(t *testing.T) { - ctx := context.Background() - name := fmt.Sprintf("%s.%s", h.ns, h.rel) - err := h.app.RunContext(ctx, []string{ - "basin", - "publication", - "--dir", h.dir, - "start", - "--private-key", hex.EncodeToString(crypto.FromECDSA(h.pk)), - name, - }) - require.NoError(t, err) -} - -func (h *helper) AddNewRecord(t *testing.T) { - _, err := h.db.ExecContext(context.Background(), fmt.Sprintf("insert into %s (name) values ('foo');", h.rel)) - require.NoError(t, err) -} diff --git a/cmd/basin/wallet.go b/cmd/basin/wallet.go deleted file mode 100644 index da70df1..0000000 --- a/cmd/basin/wallet.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "crypto/ecdsa" - "errors" - "fmt" - "os" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/crypto" - "github.com/urfave/cli/v2" -) - -func newWalletCommand() *cli.Command { - return &cli.Command{ - Name: "wallet", - Usage: "wallet commands", - Subcommands: []*cli.Command{ - { - Name: "create", - Usage: "creates a new wallet", - Action: func(cCtx *cli.Context) error { - filename := cCtx.Args().Get(0) - if filename == "" { - return errors.New("filename is empty") - } - - privateKey, err := crypto.GenerateKey() - if err != nil { - return fmt.Errorf("generate key: %s", err) - } - privateKeyBytes := crypto.FromECDSA(privateKey) - - if err := os.WriteFile(filename, []byte(hexutil.Encode(privateKeyBytes)[2:]), 0o644); err != nil { - return fmt.Errorf("writing to file %s: %s", filename, err) - } - pubk, _ := privateKey.Public().(*ecdsa.PublicKey) - publicKey := common.HexToAddress(crypto.PubkeyToAddress(*pubk).Hex()) - - fmt.Printf("Wallet address %s created\n", publicKey) - fmt.Printf("Private key saved in %s\n", filename) - return nil - }, - }, - { - Name: "pubkey", - Usage: "print the public key for a private key", - Action: func(cCtx *cli.Context) error { - filename := cCtx.Args().Get(0) - if filename == "" { - return errors.New("filename is empty") - } - - privateKey, err := crypto.LoadECDSA(filename) - if err != nil { - return fmt.Errorf("loading key: %s", err) - } - - pubk, _ := privateKey.Public().(*ecdsa.PublicKey) - publicKey := common.HexToAddress(crypto.PubkeyToAddress(*pubk).Hex()) - - fmt.Println(publicKey) - return nil - }, - }, - }, - } -} diff --git a/cmd/basin/publication.go b/cmd/vaults/commands.go similarity index 77% rename from cmd/basin/publication.go rename to cmd/vaults/commands.go index 6cfacfc..2a32372 100644 --- a/cmd/basin/publication.go +++ b/cmd/vaults/commands.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/ecdsa" "encoding/json" "errors" "fmt" @@ -12,6 +13,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/filecoin-project/lassie/pkg/lassie" "github.com/filecoin-project/lassie/pkg/storage" @@ -25,47 +27,26 @@ import ( "github.com/olekukonko/tablewriter" "github.com/schollz/progressbar/v3" "github.com/tablelandnetwork/basin-cli/internal/app" - "github.com/tablelandnetwork/basin-cli/pkg/basinprovider" "github.com/tablelandnetwork/basin-cli/pkg/pgrepl" + "github.com/tablelandnetwork/basin-cli/pkg/vaultsprovider" "github.com/urfave/cli/v2" "gopkg.in/yaml.v3" ) var pubNameRx = regexp.MustCompile(`^([a-zA-Z_][a-zA-Z0-9_]*)[.]([a-zA-Z_][a-zA-Z0-9_]*$)`) -func newPublicationCommand() *cli.Command { - return &cli.Command{ - Name: "publication", - Usage: "publication commands", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "dir", - Usage: "The directory where config will be stored (default: $HOME)", - }, - }, - Subcommands: []*cli.Command{ - newPublicationCreateCommand(), - newPublicationStartCommand(), - newPublicationUploadCommand(), - newPublicationListCommand(), - newPublicationDealsCommand(), - newPublicationRetrieveCommand(), - }, - } -} - -func newPublicationCreateCommand() *cli.Command { - var owner, dburi, provider string +func newVaultCreateCommand() *cli.Command { + var address, dburi, provider string var winSize, cache int64 return &cli.Command{ Name: "create", - Usage: "create a new publication", + Usage: "create a new vault", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "address", + Name: "account", Usage: "Ethereum wallet address", - Destination: &owner, + Destination: &address, Required: true, }, &cli.StringFlag{ @@ -98,15 +79,15 @@ func newPublicationCreateCommand() *cli.Command { } pub := cCtx.Args().First() - ns, rel, err := parsePublicationName(pub) + ns, rel, err := parseVaultName(pub) if err != nil { return err } - if !common.IsHexAddress(owner) { - return fmt.Errorf("%s is not a valid Ethereum wallet address", owner) + account, err := app.NewAccount(address) + if err != nil { + return fmt.Errorf("not a valid account: %s", err) } - pgConfig, err := pgconn.ParseConfig(dburi) if err != nil { return fmt.Errorf("parse config: %s", err) @@ -144,13 +125,13 @@ func newPublicationCreateCommand() *cli.Command { return fmt.Errorf("encode: %s", err) } - exists, err := createPublication(cCtx.Context, dburi, ns, rel, provider, owner, cache) + exists, err := createVault(cCtx.Context, dburi, ns, rel, provider, account, cache) if err != nil { - return fmt.Errorf("failed to create publication: %s", err) + return fmt.Errorf("failed to create vault: %s", err) } if exists { - fmt.Printf("Publication %s.%s already exists.\n\n", ns, rel) + fmt.Printf("Vault %s.%s already exists.\n\n", ns, rel) return nil } @@ -158,18 +139,18 @@ func newPublicationCreateCommand() *cli.Command { return fmt.Errorf("mk db dir: %s", err) } - fmt.Printf("\033[32mPublication %s.%s created.\033[0m\n\n", ns, rel) + fmt.Printf("\033[32mVault %s.%s created.\033[0m\n\n", ns, rel) return nil }, } } -func newPublicationStartCommand() *cli.Command { +func newStreamCommand() *cli.Command { var privateKey string return &cli.Command{ - Name: "start", - Usage: "start a daemon process that replicates Postgres changes to Basin server", + Name: "stream", + Usage: "starts a daemon process that streams Postgres changes to a vault", Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", @@ -183,8 +164,8 @@ func newPublicationStartCommand() *cli.Command { return errors.New("one argument should be provided") } - publication := cCtx.Args().First() - ns, rel, err := parsePublicationName(publication) + vault := cCtx.Args().First() + ns, rel, err := parseVaultName(vault) if err != nil { return err } @@ -200,11 +181,11 @@ func newPublicationStartCommand() *cli.Command { } connString := fmt.Sprintf("postgres://%s:%s@%s:%d/%s", - cfg.Publications[publication].User, - cfg.Publications[publication].Password, - cfg.Publications[publication].Host, - cfg.Publications[publication].Port, - cfg.Publications[publication].Database, + cfg.Publications[vault].User, + cfg.Publications[vault].Password, + cfg.Publications[vault].Host, + cfg.Publications[vault].Port, + cfg.Publications[vault].Database, ) r, err := pgrepl.New(connString, pgrepl.Publication(rel)) @@ -217,7 +198,7 @@ func newPublicationStartCommand() *cli.Command { return err } - bp := basinprovider.New(cfg.Publications[publication].ProviderHost) + bp := vaultsprovider.New(cfg.Publications[vault].ProviderHost) pgxConn, err := pgx.Connect(cCtx.Context, connString) if err != nil { @@ -243,9 +224,9 @@ func newPublicationStartCommand() *cli.Command { } // Creates a new db manager when replication starts - dbDir := path.Join(dir, publication) - winSize := time.Duration(cfg.Publications[publication].WindowSize) * time.Second - uploader := app.NewBasinUploader(ns, rel, bp, privateKey) + dbDir := path.Join(dir, vault) + winSize := time.Duration(cfg.Publications[vault].WindowSize) * time.Second + uploader := app.NewVaultsUploader(ns, rel, bp, privateKey) dbm := app.NewDBManager(dbDir, rel, cols, winSize, uploader) // Before starting replication, upload the remaining data @@ -253,8 +234,8 @@ func newPublicationStartCommand() *cli.Command { return fmt.Errorf("upload all: %s", err) } - basinStreamer := app.NewBasinStreamer(ns, r, dbm) - if err := basinStreamer.Run(cCtx.Context); err != nil { + vaultsStreamer := app.NewVaultsStreamer(ns, r, dbm) + if err := vaultsStreamer.Run(cCtx.Context); err != nil { return fmt.Errorf("run: %s", err) } @@ -263,13 +244,13 @@ func newPublicationStartCommand() *cli.Command { } } -func newPublicationUploadCommand() *cli.Command { - var privateKey, publicationName string +func newWriteCommand() *cli.Command { + var privateKey, vaultName string var timestamp string return &cli.Command{ - Name: "upload", - Usage: "upload a Parquet file", + Name: "write", + Usage: "write a Parquet file", Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", @@ -278,9 +259,9 @@ func newPublicationUploadCommand() *cli.Command { Required: true, }, &cli.StringFlag{ - Name: "name", - Usage: "Publication name", - Destination: &publicationName, + Name: "vault", + Usage: "Vault name", + Destination: &vaultName, Required: true, }, &cli.StringFlag{ @@ -293,7 +274,7 @@ func newPublicationUploadCommand() *cli.Command { if cCtx.NArg() != 1 { return errors.New("one argument should be provided") } - ns, rel, err := parsePublicationName(publicationName) + ns, rel, err := parseVaultName(vaultName) if err != nil { return err } @@ -313,7 +294,7 @@ func newPublicationUploadCommand() *cli.Command { return fmt.Errorf("load config: %s", err) } - bp := basinprovider.New(cfg.Publications[publicationName].ProviderHost) + bp := vaultsprovider.New(cfg.Publications[vaultName].ProviderHost) filepath := cCtx.Args().First() @@ -332,7 +313,7 @@ func newPublicationUploadCommand() *cli.Command { bar := progressbar.DefaultBytes( fi.Size(), - "Uploading file...", + "Writing...", ) if timestamp == "" { @@ -344,8 +325,8 @@ func newPublicationUploadCommand() *cli.Command { return err } - basinStreamer := app.NewBasinUploader(ns, rel, bp, privateKey) - if err := basinStreamer.Upload(cCtx.Context, filepath, bar, ts, fi.Size()); err != nil { + vaultsStreamer := app.NewVaultsUploader(ns, rel, bp, privateKey) + if err := vaultsStreamer.Upload(cCtx.Context, filepath, bar, ts, fi.Size()); err != nil { return fmt.Errorf("upload: %s", err) } @@ -354,17 +335,17 @@ func newPublicationUploadCommand() *cli.Command { } } -func newPublicationListCommand() *cli.Command { - var owner, provider string +func newListCommand() *cli.Command { + var address, provider string return &cli.Command{ Name: "list", - Usage: "list publications of a given address", + Usage: "list vaults of a given account", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "address", + Name: "account", Usage: "Ethereum wallet address", - Destination: &owner, + Destination: &address, Required: true, }, &cli.StringFlag{ @@ -375,15 +356,15 @@ func newPublicationListCommand() *cli.Command { }, }, Action: func(cCtx *cli.Context) error { - account, err := app.NewAccount(owner) + account, err := app.NewAccount(address) if err != nil { - return fmt.Errorf("%s is not a valid Ethereum wallet address", owner) + return fmt.Errorf("%s is not a valid Ethereum wallet address", address) } - bp := basinprovider.New(provider) + bp := vaultsprovider.New(provider) vaults, err := bp.ListVaults(cCtx.Context, app.ListVaultsParams{Account: account}) if err != nil { - return fmt.Errorf("failed to list publications: %s", err) + return fmt.Errorf("failed to list vaults: %s", err) } for _, vault := range vaults { @@ -395,18 +376,18 @@ func newPublicationListCommand() *cli.Command { } } -func newPublicationDealsCommand() *cli.Command { - var publication, provider, before, after, at, format string +func newListEventsCommand() *cli.Command { + var vault, provider, before, after, at, format string var limit, offset, latest int return &cli.Command{ - Name: "deals", - Usage: "list deals of a given publications", + Name: "events", + Usage: "list events of a given vault", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "publication", - Usage: "Publication name", - Destination: &publication, + Name: "vault", + Usage: "vault name", + Destination: &vault, Required: true, }, &cli.StringFlag{ @@ -458,12 +439,12 @@ func newPublicationDealsCommand() *cli.Command { }, }, Action: func(cCtx *cli.Context) error { - ns, rel, err := parsePublicationName(publication) + ns, rel, err := parseVaultName(vault) if err != nil { return err } - bp := basinprovider.New(provider) + bp := vaultsprovider.New(provider) b, a, err := validateBeforeAndAfter(before, after, at) if err != nil { @@ -534,10 +515,10 @@ func newPublicationDealsCommand() *cli.Command { } } -func newPublicationRetrieveCommand() *cli.Command { +func newRetrieveCommand() *cli.Command { return &cli.Command{ Name: "retrieve", - Usage: "Retrieve files by CID", + Usage: "Retrieve an event by CID", Action: func(cCtx *cli.Context) error { arg := cCtx.Args().Get(0) if arg == "" { @@ -584,10 +565,66 @@ func newPublicationRetrieveCommand() *cli.Command { } } -func parsePublicationName(name string) (ns string, rel string, err error) { +func newWalletCommand() *cli.Command { + return &cli.Command{ + Name: "wallet", + Usage: "wallet commands", + Subcommands: []*cli.Command{ + { + Name: "create", + Usage: "creates a new wallet", + Action: func(cCtx *cli.Context) error { + filename := cCtx.Args().Get(0) + if filename == "" { + return errors.New("filename is empty") + } + + privateKey, err := crypto.GenerateKey() + if err != nil { + return fmt.Errorf("generate key: %s", err) + } + privateKeyBytes := crypto.FromECDSA(privateKey) + + if err := os.WriteFile(filename, []byte(hexutil.Encode(privateKeyBytes)[2:]), 0o644); err != nil { + return fmt.Errorf("writing to file %s: %s", filename, err) + } + pubk, _ := privateKey.Public().(*ecdsa.PublicKey) + publicKey := common.HexToAddress(crypto.PubkeyToAddress(*pubk).Hex()) + + fmt.Printf("Wallet address %s created\n", publicKey) + fmt.Printf("Private key saved in %s\n", filename) + return nil + }, + }, + { + Name: "pubkey", + Usage: "print the public key for a private key", + Action: func(cCtx *cli.Context) error { + filename := cCtx.Args().Get(0) + if filename == "" { + return errors.New("filename is empty") + } + + privateKey, err := crypto.LoadECDSA(filename) + if err != nil { + return fmt.Errorf("loading key: %s", err) + } + + pubk, _ := privateKey.Public().(*ecdsa.PublicKey) + publicKey := common.HexToAddress(crypto.PubkeyToAddress(*pubk).Hex()) + + fmt.Println(publicKey) + return nil + }, + }, + }, + } +} + +func parseVaultName(name string) (ns string, rel string, err error) { match := pubNameRx.FindStringSubmatch(name) if len(match) != 3 { - return "", "", errors.New("publication name must be of the form `namespace.relation_name` using only letters, numbers, and underscores (_), where `namespace` and `relation` do not start with a number") // nolint + return "", "", errors.New("vault name must be of the form `namespace.relation_name` using only letters, numbers, and underscores (_), where `namespace` and `relation` do not start with a number") // nolint } ns = match[1] rel = match[2] @@ -651,21 +688,16 @@ func inspectTable(ctx context.Context, tx pgx.Tx, rel string) ([]app.Column, err return columns, nil } -func createPublication( +func createVault( ctx context.Context, dburi string, ns string, rel string, provider string, - owner string, + account *app.Account, cacheDuration int64, ) (exists bool, err error) { - account, err := app.NewAccount(owner) - if err != nil { - return false, fmt.Errorf("not a valid account: %s", err) - } - - bp := basinprovider.New(provider) + bp := vaultsprovider.New(provider) req := app.CreateVaultParams{ Account: account, Vault: app.Vault(fmt.Sprintf("%s.%s", ns, rel)), diff --git a/cmd/basin/config.go b/cmd/vaults/config.go similarity index 96% rename from cmd/basin/config.go rename to cmd/vaults/config.go index 63b4e61..0c7ef80 100644 --- a/cmd/basin/config.go +++ b/cmd/vaults/config.go @@ -9,7 +9,7 @@ import ( "gopkg.in/yaml.v3" ) -// DefaultProviderHost is the address of Basin Provider. +// DefaultProviderHost is the address of Vaults Provider. const DefaultProviderHost = "https://basin.tableland.xyz" // DefaultWindowSize is the number of seconds for which WAL updates diff --git a/cmd/basin/main.go b/cmd/vaults/main.go similarity index 51% rename from cmd/basin/main.go rename to cmd/vaults/main.go index e00abe4..eafb151 100644 --- a/cmd/basin/main.go +++ b/cmd/vaults/main.go @@ -9,10 +9,15 @@ import ( func main() { cliApp := &cli.App{ - Name: "basin", - Usage: "Continuously publish data from your database to the Tableland network.", + Name: "vaults", + Usage: "Continuously publish data from your database to the Textile Vaults network.", Commands: []*cli.Command{ - newPublicationCommand(), + newVaultCreateCommand(), + newStreamCommand(), + newWriteCommand(), + newListCommand(), + newListEventsCommand(), + newRetrieveCommand(), newWalletCommand(), }, } diff --git a/goreleaser.yml b/goreleaser.yml index 8131285..67f57b7 100644 --- a/goreleaser.yml +++ b/goreleaser.yml @@ -6,12 +6,12 @@ before: env: - CGO_ENABLED=1 -project_name: basin +project_name: vaults builds: -- id: basin-darwin-amd64 - binary: basin - main: ./cmd/basin +- id: vaults-darwin-amd64 + binary: vaults + main: ./cmd/vaults goarch: - amd64 goos: @@ -23,9 +23,9 @@ builds: - -trimpath ldflags: - -s -w -X main.version={{.Version}} -X main.commit={{.FullCommit}} -X main.date={{.CommitDate}} -- id: basin-darwin-arm64 - binary: basin - main: ./cmd/basin +- id: vaults-darwin-arm64 + binary: vaults + main: ./cmd/vaults goarch: - arm64 goos: @@ -37,9 +37,9 @@ builds: - -trimpath ldflags: - -s -w -X main.version={{.Version}} -X main.commit={{.FullCommit}} -X main.date={{.CommitDate}} -- id: basin-linux-amd64 - binary: basin - main: ./cmd/basin +- id: vaults-linux-amd64 + binary: vaults + main: ./cmd/vaults goarch: - amd64 goos: @@ -51,9 +51,9 @@ builds: - -trimpath ldflags: - -s -w -X main.version={{.Version}} -X main.commit={{.FullCommit}} -X main.date={{.CommitDate}} -- id: basin-linux-arm64 - binary: basin - main: ./cmd/basin +- id: vaults-linux-arm64 + binary: vaults + main: ./cmd/vaults goarch: - arm64 goos: @@ -67,15 +67,15 @@ builds: - -s -w -X main.version={{.Version}} -X main.commit={{.FullCommit}} -X main.date={{.CommitDate}} archives: - - id: basin-archive + - id: vaults-archive format: tar.gz files: - none* builds: - - basin-darwin-amd64 - - basin-darwin-arm64 - - basin-linux-amd64 - - basin-linux-arm64 + - vaults-darwin-amd64 + - vaults-darwin-arm64 + - vaults-linux-amd64 + - vaults-linux-arm64 name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}" checksum: diff --git a/internal/app/basin_provider.go b/internal/app/basin_provider.go index 8dec2b4..dd6b6f5 100644 --- a/internal/app/basin_provider.go +++ b/internal/app/basin_provider.go @@ -5,8 +5,8 @@ import ( "io" ) -// BasinProvider ... -type BasinProvider interface { +// VaultsProvider defines Vaults API. +type VaultsProvider interface { CreateVault(context.Context, CreateVaultParams) error ListVaults(context.Context, ListVaultsParams) ([]Vault, error) ListVaultEvents(context.Context, ListVaultEventsParams) ([]EventInfo, error) diff --git a/internal/app/db.go b/internal/app/db.go index 88242cf..504509f 100644 --- a/internal/app/db.go +++ b/internal/app/db.go @@ -32,11 +32,11 @@ type DBManager struct { createdAT time.Time cols []Column winSize time.Duration - uploader *BasinUploader + uploader *VaultsUploader } // NewDBManager creates a new DBManager. -func NewDBManager(dbDir, table string, cols []Column, winSize time.Duration, uploader *BasinUploader) *DBManager { +func NewDBManager(dbDir, table string, cols []Column, winSize time.Duration, uploader *VaultsUploader) *DBManager { return &DBManager{ dbDir: dbDir, table: table, diff --git a/internal/app/streamer.go b/internal/app/streamer.go index ef8fbab..b7bdf13 100644 --- a/internal/app/streamer.go +++ b/internal/app/streamer.go @@ -19,24 +19,24 @@ type Replicator interface { Shutdown() } -// BasinStreamer contains logic of streaming Postgres changes to Basin Provider. -type BasinStreamer struct { +// VaultsStreamer contains logic of streaming Postgres changes to Vaults Provider. +type VaultsStreamer struct { namespace string replicator Replicator dbMngr *DBManager } -// NewBasinStreamer creates new streamer. -func NewBasinStreamer(ns string, r Replicator, dbm *DBManager) *BasinStreamer { - return &BasinStreamer{ +// NewVaultsStreamer creates new streamer. +func NewVaultsStreamer(ns string, r Replicator, dbm *DBManager) *VaultsStreamer { + return &VaultsStreamer{ namespace: ns, replicator: r, dbMngr: dbm, } } -// Run runs the BasinStreamer logic. -func (b *BasinStreamer) Run(ctx context.Context) error { +// Run runs the VaultsStreamer logic. +func (b *VaultsStreamer) Run(ctx context.Context) error { // Open a local DB for replaying txs if err := b.dbMngr.NewDB(ctx); err != nil { return err diff --git a/internal/app/streamer_test.go b/internal/app/streamer_test.go index 281180e..fa09b5c 100644 --- a/internal/app/streamer_test.go +++ b/internal/app/streamer_test.go @@ -29,7 +29,7 @@ var cols = []Column{ // Test when window threshold is crossed before // second Tx is received: . -func TestBasinStreamerOne(t *testing.T) { +func TestVaultsStreamerOne(t *testing.T) { // used for testing privateKey, err := crypto.HexToECDSA(pk) require.NoError(t, err) @@ -38,15 +38,15 @@ func TestBasinStreamerOne(t *testing.T) { feed := make(chan *pgrepl.Tx) testDBDir := t.TempDir() winSize := 3 * time.Second - providerMock := &basinProviderMock{ + providerMock := &vaultsProviderMock{ owner: make(map[string]string), uploaderInputs: make(chan *os.File), } - uploader := NewBasinUploader(testNS, testTable, providerMock, privateKey) + uploader := NewVaultsUploader(testNS, testTable, providerMock, privateKey) dbm := NewDBManager( testDBDir, testTable, cols, winSize, uploader) - streamer := NewBasinStreamer(testNS, &replicatorMock{feed: feed}, dbm) + streamer := NewVaultsStreamer(testNS, &replicatorMock{feed: feed}, dbm) go func() { // start listening to WAL records in a separate goroutine err = streamer.Run(context.Background()) @@ -105,7 +105,7 @@ func TestBasinStreamerOne(t *testing.T) { dbm.db = nil dbm.dbFname = "" dbm.createdAT = time.Time{} - uploader.provider = &basinProviderMock{ + uploader.provider = &vaultsProviderMock{ owner: make(map[string]string), uploaderInputs: ch2, } @@ -132,7 +132,7 @@ func TestBasinStreamerOne(t *testing.T) { // Test when window threshold is crossed after // second Tx is received: . -func TestBasinStreamerTwo(t *testing.T) { +func TestVaultsStreamerTwo(t *testing.T) { privateKey, err := crypto.HexToECDSA(pk) require.NoError(t, err) @@ -140,14 +140,14 @@ func TestBasinStreamerTwo(t *testing.T) { feed := make(chan *pgrepl.Tx) testDBDir := t.TempDir() winSize := 3 * time.Second - providerMock := &basinProviderMock{ + providerMock := &vaultsProviderMock{ owner: make(map[string]string), uploaderInputs: make(chan *os.File), } - uploader := NewBasinUploader(testNS, testTable, providerMock, privateKey) + uploader := NewVaultsUploader(testNS, testTable, providerMock, privateKey) dbm := NewDBManager( testDBDir, testTable, cols, winSize, uploader) - streamer := NewBasinStreamer(testNS, &replicatorMock{feed: feed}, dbm) + streamer := NewVaultsStreamer(testNS, &replicatorMock{feed: feed}, dbm) go func() { // start listening to WAL records in a separate goroutine err = streamer.Run(context.Background()) @@ -234,29 +234,29 @@ func (rm *replicatorMock) Shutdown() { close(rm.feed) } -type basinProviderMock struct { +type vaultsProviderMock struct { owner map[string]string uploaderInputs chan *os.File } -func (bp *basinProviderMock) CreateVault( +func (bp *vaultsProviderMock) CreateVault( _ context.Context, params CreateVaultParams, ) error { bp.owner[string(params.Vault)] = params.Account.Hex() return nil } -func (bp *basinProviderMock) ListVaults(_ context.Context, _ ListVaultsParams) ([]Vault, error) { +func (bp *vaultsProviderMock) ListVaults(_ context.Context, _ ListVaultsParams) ([]Vault, error) { return []Vault{}, nil } -func (bp *basinProviderMock) ListVaultEvents( +func (bp *vaultsProviderMock) ListVaultEvents( context.Context, ListVaultEventsParams, ) ([]EventInfo, error) { return []EventInfo{}, nil } -func (bp *basinProviderMock) WriteVaultEvent( +func (bp *vaultsProviderMock) WriteVaultEvent( _ context.Context, params WriteVaultEventParams, ) error { file := params.Content.(*os.File) diff --git a/internal/app/uploader.go b/internal/app/uploader.go index 8b573c4..13facc0 100644 --- a/internal/app/uploader.go +++ b/internal/app/uploader.go @@ -15,19 +15,19 @@ import ( "golang.org/x/crypto/sha3" ) -// BasinUploader contains logic of uploading Parquet files to Basin Provider. -type BasinUploader struct { +// VaultsUploader contains logic of uploading Parquet files to Vaults Provider. +type VaultsUploader struct { namespace string relation string privateKey *ecdsa.PrivateKey - provider BasinProvider + provider VaultsProvider } -// NewBasinUploader creates new uploader. -func NewBasinUploader( - ns string, rel string, bp BasinProvider, pk *ecdsa.PrivateKey, -) *BasinUploader { - return &BasinUploader{ +// NewVaultsUploader creates new uploader. +func NewVaultsUploader( + ns string, rel string, bp VaultsProvider, pk *ecdsa.PrivateKey, +) *VaultsUploader { + return &VaultsUploader{ namespace: ns, relation: rel, provider: bp, @@ -36,7 +36,7 @@ func NewBasinUploader( } // Upload sends file to provider for upload. -func (bu *BasinUploader) Upload( +func (bu *VaultsUploader) Upload( ctx context.Context, filepath string, progress io.Writer, ts Timestamp, sz int64, ) error { f, err := os.Open(filepath) diff --git a/pkg/basinprovider/provider.go b/pkg/vaultsprovider/provider.go similarity index 86% rename from pkg/basinprovider/provider.go rename to pkg/vaultsprovider/provider.go index 79034fa..4d50bb3 100644 --- a/pkg/basinprovider/provider.go +++ b/pkg/vaultsprovider/provider.go @@ -1,4 +1,4 @@ -package basinprovider +package vaultsprovider import ( "context" @@ -14,28 +14,28 @@ import ( "github.com/tablelandnetwork/basin-cli/internal/app" ) -// BasinProvider implements the app.BasinProvider interface. -type BasinProvider struct { +// VaultsProvider implements the app.VaultsProvider interface. +type VaultsProvider struct { provider string client *http.Client } -var _ app.BasinProvider = (*BasinProvider)(nil) +var _ app.VaultsProvider = (*VaultsProvider)(nil) -// New creates a new BasinProvider. -func New(provider string) *BasinProvider { +// New creates a new VaultsProvider. +func New(provider string) *VaultsProvider { client := &http.Client{ Timeout: 10 * time.Second, } - return &BasinProvider{ + return &VaultsProvider{ provider: provider, client: client, } } // CreateVault creates a vault. -func (bp *BasinProvider) CreateVault(ctx context.Context, params app.CreateVaultParams) error { +func (bp *VaultsProvider) CreateVault(ctx context.Context, params app.CreateVaultParams) error { form := url.Values{} form.Add("account", params.Account.Hex()) form.Add("cache", fmt.Sprint(params.CacheDuration)) @@ -63,7 +63,7 @@ func (bp *BasinProvider) CreateVault(ctx context.Context, params app.CreateVault } // ListVaults lists all vaults from a given account. -func (bp *BasinProvider) ListVaults( +func (bp *VaultsProvider) ListVaults( ctx context.Context, params app.ListVaultsParams, ) ([]app.Vault, error) { req, err := http.NewRequestWithContext( @@ -88,7 +88,7 @@ func (bp *BasinProvider) ListVaults( } // ListVaultEvents lists all events from a given vault. -func (bp *BasinProvider) ListVaultEvents( +func (bp *VaultsProvider) ListVaultEvents( ctx context.Context, params app.ListVaultEventsParams, ) ([]app.EventInfo, error) { req, err := http.NewRequestWithContext( @@ -120,7 +120,7 @@ func (bp *BasinProvider) ListVaultEvents( } // WriteVaultEvent write an event. -func (bp *BasinProvider) WriteVaultEvent(ctx context.Context, params app.WriteVaultEventParams) error { +func (bp *VaultsProvider) WriteVaultEvent(ctx context.Context, params app.WriteVaultEventParams) error { req, err := http.NewRequestWithContext( ctx, http.MethodPost, diff --git a/scripts/run.sh b/scripts/run.sh index afa075a..1653a64 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,3 +1,3 @@ #!/bin/sh -go run $( ls -1 cmd/basin/*.go | grep -v _test.go) $@ \ No newline at end of file +go run $( ls -1 cmd/vaults/*.go | grep -v _test.go) $@ \ No newline at end of file diff --git a/test/postgres.go b/test/postgres.go index 08f4a5b..0039537 100644 --- a/test/postgres.go +++ b/test/postgres.go @@ -55,7 +55,7 @@ func (dp *DockerPool) RunPostgres() (db *sql.DB, resource *dockertest.Resource, Env: []string{ "POSTGRES_PASSWORD=secret", "POSTGRES_USER=admin", - "POSTGRES_DB=basin", + "POSTGRES_DB=vaults", "listen_addresses = '*'", }, }, func(config *docker.HostConfig) { @@ -68,7 +68,7 @@ func (dp *DockerPool) RunPostgres() (db *sql.DB, resource *dockertest.Resource, _ = resource.Expire(120) // Tell docker to hard kill the container in 120 seconds - uri = fmt.Sprintf("postgres://admin:secret@%s/basin?sslmode=disable", resource.GetHostPort("5432/tcp")) + uri = fmt.Sprintf("postgres://admin:secret@%s/vaults?sslmode=disable", resource.GetHostPort("5432/tcp")) db, err = sql.Open("postgres", uri) if err != nil { log.Fatalf("Could not open the database: %s", err) From 711e06c14e08d11b6f947eac4aef3110ebaa6ee3 Mon Sep 17 00:00:00 2001 From: Bruno Calza Date: Fri, 15 Dec 2023 12:43:50 -0300 Subject: [PATCH 02/16] implements config migration Signed-off-by: Bruno Calza --- cmd/vaults/commands.go | 30 +++++------ cmd/vaults/config.go | 115 +++++++++++++++++++++++++++++++++++++++++ cmd/vaults/main.go | 3 ++ 3 files changed, 133 insertions(+), 15 deletions(-) diff --git a/cmd/vaults/commands.go b/cmd/vaults/commands.go index 2a32372..bc87d2a 100644 --- a/cmd/vaults/commands.go +++ b/cmd/vaults/commands.go @@ -93,7 +93,7 @@ func newVaultCreateCommand() *cli.Command { return fmt.Errorf("parse config: %s", err) } - dir, err := defaultConfigLocation(cCtx.String("dir")) + dir, _, err := defaultConfigLocationV2(cCtx.String("dir")) if err != nil { return fmt.Errorf("default config location: %s", err) } @@ -106,12 +106,12 @@ func newVaultCreateCommand() *cli.Command { _ = f.Close() }() - cfg, err := loadConfig(path.Join(dir, "config.yaml")) + cfg, err := loadConfigV2(path.Join(dir, "config.yaml")) if err != nil { return fmt.Errorf("load config: %s", err) } - cfg.Publications[pub] = publication{ + cfg.Vaults[pub] = vault{ Host: pgConfig.Host, Port: int(pgConfig.Port), User: pgConfig.User, @@ -170,22 +170,22 @@ func newStreamCommand() *cli.Command { return err } - dir, err := defaultConfigLocation(cCtx.String("dir")) + dir, _, err := defaultConfigLocationV2(cCtx.String("dir")) if err != nil { return fmt.Errorf("default config location: %s", err) } - cfg, err := loadConfig(path.Join(dir, "config.yaml")) + cfg, err := loadConfigV2(path.Join(dir, "config.yaml")) if err != nil { return fmt.Errorf("load config: %s", err) } connString := fmt.Sprintf("postgres://%s:%s@%s:%d/%s", - cfg.Publications[vault].User, - cfg.Publications[vault].Password, - cfg.Publications[vault].Host, - cfg.Publications[vault].Port, - cfg.Publications[vault].Database, + cfg.Vaults[vault].User, + cfg.Vaults[vault].Password, + cfg.Vaults[vault].Host, + cfg.Vaults[vault].Port, + cfg.Vaults[vault].Database, ) r, err := pgrepl.New(connString, pgrepl.Publication(rel)) @@ -198,7 +198,7 @@ func newStreamCommand() *cli.Command { return err } - bp := vaultsprovider.New(cfg.Publications[vault].ProviderHost) + bp := vaultsprovider.New(cfg.Vaults[vault].ProviderHost) pgxConn, err := pgx.Connect(cCtx.Context, connString) if err != nil { @@ -225,7 +225,7 @@ func newStreamCommand() *cli.Command { // Creates a new db manager when replication starts dbDir := path.Join(dir, vault) - winSize := time.Duration(cfg.Publications[vault].WindowSize) * time.Second + winSize := time.Duration(cfg.Vaults[vault].WindowSize) * time.Second uploader := app.NewVaultsUploader(ns, rel, bp, privateKey) dbm := app.NewDBManager(dbDir, rel, cols, winSize, uploader) @@ -284,17 +284,17 @@ func newWriteCommand() *cli.Command { return err } - dir, err := defaultConfigLocation(cCtx.String("dir")) + dir, _, err := defaultConfigLocationV2(cCtx.String("dir")) if err != nil { return fmt.Errorf("default config location: %s", err) } - cfg, err := loadConfig(path.Join(dir, "config.yaml")) + cfg, err := loadConfigV2(path.Join(dir, "config.yaml")) if err != nil { return fmt.Errorf("load config: %s", err) } - bp := vaultsprovider.New(cfg.Publications[vaultName].ProviderHost) + bp := vaultsprovider.New(cfg.Vaults[vaultName].ProviderHost) filepath := cCtx.Args().First() diff --git a/cmd/vaults/config.go b/cmd/vaults/config.go index 0c7ef80..5c481a2 100644 --- a/cmd/vaults/config.go +++ b/cmd/vaults/config.go @@ -1,11 +1,13 @@ package main import ( + "errors" "fmt" "os" "path" "github.com/mitchellh/go-homedir" + "golang.org/x/exp/slog" "gopkg.in/yaml.v3" ) @@ -20,6 +22,10 @@ type config struct { Publications map[string]publication `yaml:"publications"` } +type configV2 struct { + Vaults map[string]vault `yaml:"vaults"` +} + type publication struct { User string `yaml:"user"` Password string `yaml:"password"` @@ -30,12 +36,28 @@ type publication struct { WindowSize int64 `yaml:"window_size"` } +type vault struct { + User string `yaml:"user"` + Password string `yaml:"password"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Database string `yaml:"database"` + ProviderHost string `yaml:"provider_host"` + WindowSize int64 `yaml:"window_size"` +} + func newConfig() *config { return &config{ Publications: make(map[string]publication), } } +func newConfigV2() *configV2 { + return &configV2{ + Vaults: make(map[string]vault), + } +} + func loadConfig(path string) (*config, error) { buf, err := os.ReadFile(path) if err != nil { @@ -50,6 +72,20 @@ func loadConfig(path string) (*config, error) { return conf, nil } +func loadConfigV2(path string) (*configV2, error) { + buf, err := os.ReadFile(path) + if err != nil { + return &configV2{}, err + } + + conf := newConfigV2() + if err := yaml.Unmarshal(buf, conf); err != nil { + return &configV2{}, err + } + + return conf, nil +} + func defaultConfigLocation(dir string) (string, error) { if dir == "" { // the default directory is home @@ -73,3 +109,82 @@ func defaultConfigLocation(dir string) (string, error) { return dir, nil } + +func defaultConfigLocationV2(dir string) (string, bool, error) { + if dir == "" { + // the default directory is home + var err error + dir, err = homedir.Dir() + if err != nil { + return "", false, fmt.Errorf("home dir: %s", err) + } + + dir = path.Join(dir, ".vaults") + } + + _, err := os.Stat(dir) + doesNotExist := os.IsNotExist(err) + if doesNotExist { + if err := os.Mkdir(dir, 0o755); err != nil { + return "", doesNotExist, fmt.Errorf("mkdir: %s", err) + } + } else if err != nil { + return "", doesNotExist, fmt.Errorf("is not exist: %s", err) + } + + return dir, !doesNotExist, nil +} + +func migrateConfigV1ToV2() { + dirV1, err := defaultConfigLocation("") + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + dirV2, exists, err := defaultConfigLocationV2("") + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + // create v2 config.yaml if necessary + f, err := os.OpenFile(path.Join(dirV2, "config.yaml"), os.O_RDWR|os.O_CREATE, 0o666) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + defer func() { + _ = f.Close() + }() + + if exists { + return + } + + // .basin/config.yaml does not exist, there's nothing to migrate + if _, err := os.Stat(path.Join(dirV1, "config.yaml")); errors.Is(err, os.ErrNotExist) { + return + } + + cfgV1, err := loadConfig(path.Join(dirV1, "config.yaml")) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + cfgV2, err := loadConfigV2(path.Join(dirV2, "config.yaml")) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for name, item := range cfgV1.Publications { + cfgV2.Vaults[name] = vault(item) + } + + if err := yaml.NewEncoder(f).Encode(cfgV2); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } +} diff --git a/cmd/vaults/main.go b/cmd/vaults/main.go index eafb151..300bd4e 100644 --- a/cmd/vaults/main.go +++ b/cmd/vaults/main.go @@ -8,6 +8,9 @@ import ( ) func main() { + // migrate v1 config to v2 config + migrateConfigV1ToV2() + cliApp := &cli.App{ Name: "vaults", Usage: "Continuously publish data from your database to the Textile Vaults network.", From 764e83e123ed5c6e946d262b8e0a914a6a980011 Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Sun, 17 Dec 2023 12:01:32 -0800 Subject: [PATCH 03/16] feat: version flag & GH tag/version action --- .github/workflows/update-version.yml | 38 ++++++++++++++++++++++++++++ cmd/vaults/main.go | 6 +++-- cmd/vaults/utils.go | 23 +++++++++++++++++ version.json | 3 +++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/update-version.yml create mode 100644 cmd/vaults/utils.go create mode 100644 version.json diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml new file mode 100644 index 0000000..292d82f --- /dev/null +++ b/.github/workflows/update-version.yml @@ -0,0 +1,38 @@ +name: Update Version + +on: + push: + branches: + - main + - dtb/cli-positionals # tmp + +jobs: + update-version: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Git + run: | + git config --global user.name 'GitHub Actions' + git config --global user.email 'actions@github.com' + + - name: Get latest tag + id: get-latest-tag + run: echo ::set-output name=TAG::$(git describe --tags --abbrev=0) + + - name: Update version.json + run: | + echo "{\"version\": \"${{ steps.get-latest-tag.outputs.TAG }}\"}" > version.json + + - name: Commit and push if changed + run: | + git diff + if [ -n "$(git diff --name-only)" ]; then + git add version.json + git commit -m "Update version to ${{ steps.get-latest-tag.outputs.TAG }}" + git push + else + echo "No changes in version.json" diff --git a/cmd/vaults/main.go b/cmd/vaults/main.go index 300bd4e..621e69d 100644 --- a/cmd/vaults/main.go +++ b/cmd/vaults/main.go @@ -10,10 +10,12 @@ import ( func main() { // migrate v1 config to v2 config migrateConfigV1ToV2() + var version = getVersion() cliApp := &cli.App{ - Name: "vaults", - Usage: "Continuously publish data from your database to the Textile Vaults network.", + Name: "vaults", + Usage: "Continuously publish data from your database to the Textile Vaults network.", + Version: version, Commands: []*cli.Command{ newVaultCreateCommand(), newStreamCommand(), diff --git a/cmd/vaults/utils.go b/cmd/vaults/utils.go new file mode 100644 index 0000000..b63c94e --- /dev/null +++ b/cmd/vaults/utils.go @@ -0,0 +1,23 @@ +package main + +import ( + "encoding/json" + "os" +) + +type version struct { + Version string `json:"version"` +} + +func getVersion() string { + var v version + data, err := os.ReadFile("version.json") + if err != nil { + return "unknown" + } + err = json.Unmarshal(data, &v) + if err != nil { + return "unknown" + } + return v.Version +} diff --git a/version.json b/version.json new file mode 100644 index 0000000..1be1b18 --- /dev/null +++ b/version.json @@ -0,0 +1,3 @@ +{ + "version": "0.0.0" +} From 945ea7dc392fdb2aeb47a67c55e03cbb823c9cac Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Sun, 17 Dec 2023 14:10:22 -0800 Subject: [PATCH 04/16] feat: add cmd aliases, descriptive error strings, & docs improvements --- cmd/vaults/commands.go | 118 ++++++++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 36 deletions(-) diff --git a/cmd/vaults/commands.go b/cmd/vaults/commands.go index bc87d2a..05f9b64 100644 --- a/cmd/vaults/commands.go +++ b/cmd/vaults/commands.go @@ -40,42 +40,53 @@ func newVaultCreateCommand() *cli.Command { var winSize, cache int64 return &cli.Command{ - Name: "create", - Usage: "create a new vault", + Name: "create", + Usage: "Create a new vault", + UsageText: "vaults create [command options]", + Description: "Create a vault for a given account's address as either database streaming or file uploading. Optionally, also set a cache duration for the data.", Flags: []cli.Flag{ &cli.StringFlag{ Name: "account", + Aliases: []string{"a"}, + Category: "REQUIRED", Usage: "Ethereum wallet address", Destination: &address, Required: true, }, - &cli.StringFlag{ - Name: "dburi", - Usage: "PostgreSQL connection string", - Destination: &dburi, - }, &cli.StringFlag{ Name: "provider", - Usage: "The provider's address and port (e.g. localhost:8080)", + Category: "OPTIONAL", + Aliases: []string{"p"}, + Usage: "The provider's address and port (e.g., localhost:8080)", + DefaultText: DefaultProviderHost, Destination: &provider, Value: DefaultProviderHost, }, - &cli.Int64Flag{ - Name: "window-size", - Usage: "Number of seconds for which WAL updates are buffered before being sent to the provider", - Destination: &winSize, - Value: DefaultWindowSize, - }, &cli.Int64Flag{ Name: "cache", + Category: "OPTIONAL", Usage: "Time duration (in minutes) that the data will be available in the cache", Destination: &cache, Value: 0, }, + &cli.StringFlag{ + Name: "dburi", + Category: "DATABASE", + Usage: "PostgreSQL connection string (e.g., postgresql://postgres:[PASSWORD]@[HOST]:[PORT]/postgres)", + Destination: &dburi, + }, + &cli.Int64Flag{ + Name: "window-size", + Category: "DATABASE", + Usage: "Number of seconds for which WAL updates are buffered before being sent to the provider", + DefaultText: fmt.Sprintf("%d", DefaultWindowSize), + Destination: &winSize, + Value: DefaultWindowSize, + }, }, Action: func(cCtx *cli.Context) error { if cCtx.NArg() != 1 { - return errors.New("one argument should be provided") + return errors.New("must provide a vault name") } pub := cCtx.Args().First() @@ -149,11 +160,14 @@ func newStreamCommand() *cli.Command { var privateKey string return &cli.Command{ - Name: "stream", - Usage: "starts a daemon process that streams Postgres changes to a vault", + Name: "stream", + Usage: "Starts a daemon process that streams Postgres changes to a vault", + UsageText: "vaults stream [command options]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", + Aliases: []string{"k"}, + Category: "REQUIRED", Usage: "Ethereum wallet private key", Destination: &privateKey, Required: true, @@ -161,7 +175,7 @@ func newStreamCommand() *cli.Command { }, Action: func(cCtx *cli.Context) error { if cCtx.NArg() != 1 { - return errors.New("one argument should be provided") + return errors.New("must provide a vault name") } vault := cCtx.Args().First() @@ -249,30 +263,36 @@ func newWriteCommand() *cli.Command { var timestamp string return &cli.Command{ - Name: "write", - Usage: "write a Parquet file", + Name: "write", + Usage: "Write a Parquet file", + UsageText: "vaults write [command options]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", + Aliases: []string{"k"}, + Category: "REQUIRED", Usage: "Ethereum wallet private key", Destination: &privateKey, Required: true, }, &cli.StringFlag{ Name: "vault", + Category: "REQUIRED", Usage: "Vault name", Destination: &vaultName, Required: true, }, &cli.StringFlag{ Name: "timestamp", - Usage: "The time the file was created (default: current epoch in UTC)", + Category: "OPTIONAL", + Usage: "The time the file was created", + DefaultText: "current epoch in UTC", Destination: ×tamp, }, }, Action: func(cCtx *cli.Context) error { if cCtx.NArg() != 1 { - return errors.New("one argument should be provided") + return errors.New("must provide a file path") } ns, rel, err := parseVaultName(vaultName) if err != nil { @@ -339,18 +359,24 @@ func newListCommand() *cli.Command { var address, provider string return &cli.Command{ - Name: "list", - Usage: "list vaults of a given account", + Name: "list", + Usage: "List vaults of a given account", + UsageText: "vaults list [command options]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "account", + Aliases: []string{"a"}, + Category: "REQUIRED", Usage: "Ethereum wallet address", Destination: &address, Required: true, }, &cli.StringFlag{ Name: "provider", - Usage: "The provider's address and port (e.g. localhost:8080)", + Aliases: []string{"p"}, + Category: "OPTIONAL", + Usage: "The provider's address and port (e.g., localhost:8080)", + DefaultText: DefaultProviderHost, Destination: &provider, Value: DefaultProviderHost, }, @@ -381,59 +407,74 @@ func newListEventsCommand() *cli.Command { var limit, offset, latest int return &cli.Command{ - Name: "events", - Usage: "list events of a given vault", + Name: "events", + Usage: "List events of a given vault", + UsageText: "vaults events [command options]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "vault", + Category: "REQUIRED", Usage: "vault name", Destination: &vault, Required: true, }, &cli.StringFlag{ Name: "provider", - Usage: "The provider's address and port (e.g. localhost:8080)", + Category: "OPTIONAL", + Aliases: []string{"p"}, + Usage: "The provider's address and port (e.g., localhost:8080)", + DefaultText: DefaultProviderHost, Destination: &provider, Value: DefaultProviderHost, }, &cli.IntFlag{ Name: "limit", + Category: "OPTIONAL", Usage: "The number of deals to fetch", + DefaultText: "10", Destination: &limit, Value: 10, }, &cli.IntFlag{ Name: "latest", + Category: "OPTIONAL", Usage: "The latest N deals to fetch", Destination: &latest, }, &cli.IntFlag{ Name: "offset", + Category: "OPTIONAL", Usage: "The epoch to start from", + DefaultText: "0", Destination: &offset, Value: 0, }, &cli.StringFlag{ Name: "before", + Category: "OPTIONAL", Usage: "Filter deals created before this timestamp", Destination: &before, Value: "", }, &cli.StringFlag{ Name: "after", + Category: "OPTIONAL", Usage: "Filter deals created after this timestamp", Destination: &after, Value: "", }, &cli.StringFlag{ Name: "at", + Category: "OPTIONAL", Usage: "Filter deals created at this timestamp", Destination: &at, Value: "", }, &cli.StringFlag{ Name: "format", + Category: "OPTIONAL", Usage: "The output format (table or json)", + DefaultText: "table", Destination: &format, Value: "table", }, @@ -517,8 +558,10 @@ func newListEventsCommand() *cli.Command { func newRetrieveCommand() *cli.Command { return &cli.Command{ - Name: "retrieve", - Usage: "Retrieve an event by CID", + Name: "retrieve", + Usage: "Retrieve an event by CID", + UsageText: "vaults retrieve ", + Description: "Retrieving an event will download the event's CAR file into the current directory.", Action: func(cCtx *cli.Context) error { arg := cCtx.Args().Get(0) if arg == "" { @@ -567,12 +610,14 @@ func newRetrieveCommand() *cli.Command { func newWalletCommand() *cli.Command { return &cli.Command{ - Name: "wallet", - Usage: "wallet commands", + Name: "account", + Usage: "Account management for an Ethereum-style wallet", Subcommands: []*cli.Command{ { - Name: "create", - Usage: "creates a new wallet", + Name: "create", + Usage: "Creates a new account", + UsageText: "vaults account create ", + Description: "Create an Ethereum-style wallet (secp256k1 key pair) at a provided file path.", Action: func(cCtx *cli.Context) error { filename := cCtx.Args().Get(0) if filename == "" { @@ -597,8 +642,9 @@ func newWalletCommand() *cli.Command { }, }, { - Name: "pubkey", - Usage: "print the public key for a private key", + Name: "address", + Usage: "Print the public key for an account's private key", + UsageText: "vaults account address ", Action: func(cCtx *cli.Context) error { filename := cCtx.Args().Get(0) if filename == "" { From a7b0ce6b5a3aa51a9679298f588235c32cfafe09 Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Sun, 17 Dec 2023 14:15:04 -0800 Subject: [PATCH 05/16] chore: add install to makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index f45d4ab..f3c09d6 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,11 @@ build: go build -o vaults cmd/vaults/* .PHONY: build +# Install +install: + go install ./cmd/vaults +.PHONY: install + # Test test: go test ./... -short -race -timeout 1m From a1cf149758368d022c2a220d8937959e9bfbb9b0 Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Sun, 17 Dec 2023 20:04:06 -0800 Subject: [PATCH 06/16] feat: add version command, list cmd json format, more docs --- cmd/vaults/commands.go | 82 +++++++++++++++++++++++++++--------------- cmd/vaults/main.go | 10 +++++- 2 files changed, 62 insertions(+), 30 deletions(-) diff --git a/cmd/vaults/commands.go b/cmd/vaults/commands.go index 05f9b64..78f9d5a 100644 --- a/cmd/vaults/commands.go +++ b/cmd/vaults/commands.go @@ -42,8 +42,8 @@ func newVaultCreateCommand() *cli.Command { return &cli.Command{ Name: "create", Usage: "Create a new vault", - UsageText: "vaults create [command options]", - Description: "Create a vault for a given account's address as either database streaming or file uploading. Optionally, also set a cache duration for the data.", + ArgsUsage: "", + Description: "Create a vault for a given account's address as either database streaming or file uploading. Optionally, also set a cache duration for the data.\n\nExample:\n\nvaults create --account 0x1234abcd --cache 10 my.vault", Flags: []cli.Flag{ &cli.StringFlag{ Name: "account", @@ -55,8 +55,8 @@ func newVaultCreateCommand() *cli.Command { }, &cli.StringFlag{ Name: "provider", - Category: "OPTIONAL", Aliases: []string{"p"}, + Category: "OPTIONAL", Usage: "The provider's address and port (e.g., localhost:8080)", DefaultText: DefaultProviderHost, Destination: &provider, @@ -66,18 +66,19 @@ func newVaultCreateCommand() *cli.Command { Name: "cache", Category: "OPTIONAL", Usage: "Time duration (in minutes) that the data will be available in the cache", + DefaultText: "0", Destination: &cache, Value: 0, }, &cli.StringFlag{ Name: "dburi", - Category: "DATABASE", + Category: "OPTIONAL", Usage: "PostgreSQL connection string (e.g., postgresql://postgres:[PASSWORD]@[HOST]:[PORT]/postgres)", Destination: &dburi, }, &cli.Int64Flag{ Name: "window-size", - Category: "DATABASE", + Category: "OPTIONAL", Usage: "Number of seconds for which WAL updates are buffered before being sent to the provider", DefaultText: fmt.Sprintf("%d", DefaultWindowSize), Destination: &winSize, @@ -160,9 +161,10 @@ func newStreamCommand() *cli.Command { var privateKey string return &cli.Command{ - Name: "stream", - Usage: "Starts a daemon process that streams Postgres changes to a vault", - UsageText: "vaults stream [command options]", + Name: "stream", + Usage: "Starts a daemon process that streams Postgres changes to a vault", + ArgsUsage: "", + Description: "The daemon will continuously stream database changes (except deletions) to the vault, as long as the daemon is actively running.\n\nExample:\n\nvaults stream --vault my.vault --private-key 0x1234abcd", Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", @@ -263,9 +265,10 @@ func newWriteCommand() *cli.Command { var timestamp string return &cli.Command{ - Name: "write", - Usage: "Write a Parquet file", - UsageText: "vaults write [command options]", + Name: "write", + Usage: "Write a Parquet file", + ArgsUsage: "", + Description: "A Parquet file can be pushed directly to the vault, as an alternative to continuous Postgres data streaming.\n\nExample:\n\nvaults write --vault my.vault --private-key 0x1234abcd /path/to/file.parquet", Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", @@ -356,12 +359,12 @@ func newWriteCommand() *cli.Command { } func newListCommand() *cli.Command { - var address, provider string + var address, provider, format string return &cli.Command{ - Name: "list", - Usage: "List vaults of a given account", - UsageText: "vaults list [command options]", + Name: "list", + Usage: "List vaults of a given account", + Description: "Listing vaults will show all vaults that have been created by the provided account's address and logged as either line delimited text or a json array.\n\nExample:\n\nvaults list --account 0x1234abcd --format json", Flags: []cli.Flag{ &cli.StringFlag{ Name: "account", @@ -380,6 +383,14 @@ func newListCommand() *cli.Command { Destination: &provider, Value: DefaultProviderHost, }, + &cli.StringFlag{ + Name: "format", + Category: "OPTIONAL", + Usage: "The output format (text or json)", + DefaultText: "text", + Destination: &format, + Value: "text", + }, }, Action: func(cCtx *cli.Context) error { account, err := app.NewAccount(address) @@ -393,8 +404,18 @@ func newListCommand() *cli.Command { return fmt.Errorf("failed to list vaults: %s", err) } - for _, vault := range vaults { - fmt.Printf("%s\n", vault) + if format == "text" { + for _, vault := range vaults { + fmt.Printf("%s\n", vault) + } + } else if format == "json" { + jsonData, err := json.Marshal(vaults) + if err != nil { + return fmt.Errorf("error serializing events to JSON") + } + fmt.Println(string(jsonData)) + } else { + return fmt.Errorf("invalid format: %s", format) } return nil @@ -407,21 +428,22 @@ func newListEventsCommand() *cli.Command { var limit, offset, latest int return &cli.Command{ - Name: "events", - Usage: "List events of a given vault", - UsageText: "vaults events [command options]", + Name: "events", + Usage: "List events of a given vault", + UsageText: "vaults events [command options]", + Description: "Vault events can be filtered by date ranges (unix, ISO 8601 date, or ISO 8601 date & time), returning the event metadata and corresponding CID.\n\nExample:\n\nvaults events --vault my.vault \\\n--limit 10 --offset 3 \\\n--after 2023-09-01 --before 2023-12-01 \\\n--format json", Flags: []cli.Flag{ &cli.StringFlag{ Name: "vault", Category: "REQUIRED", - Usage: "vault name", + Usage: "Vault name", Destination: &vault, Required: true, }, &cli.StringFlag{ Name: "provider", - Category: "OPTIONAL", Aliases: []string{"p"}, + Category: "OPTIONAL", Usage: "The provider's address and port (e.g., localhost:8080)", DefaultText: DefaultProviderHost, Destination: &provider, @@ -610,14 +632,15 @@ func newRetrieveCommand() *cli.Command { func newWalletCommand() *cli.Command { return &cli.Command{ - Name: "account", - Usage: "Account management for an Ethereum-style wallet", + Name: "account", + Usage: "Account management for an Ethereum-style wallet", + UsageText: "vaults account [arguments...]", Subcommands: []*cli.Command{ { Name: "create", Usage: "Creates a new account", - UsageText: "vaults account create ", - Description: "Create an Ethereum-style wallet (secp256k1 key pair) at a provided file path.", + UsageText: "vaults account create ", + Description: "Create an Ethereum-style wallet (secp256k1 key pair) at a provided file path.\n\nExample:\n\nvaults account create /path/to/file", Action: func(cCtx *cli.Context) error { filename := cCtx.Args().Get(0) if filename == "" { @@ -642,9 +665,10 @@ func newWalletCommand() *cli.Command { }, }, { - Name: "address", - Usage: "Print the public key for an account's private key", - UsageText: "vaults account address ", + Name: "address", + Usage: "Print the public key for an account's private key", + UsageText: "vaults account address ", + Description: "The result of the `vaults account create` command will write a private key to a file, and this lets you retrieve that value for use in other commands.\n\nExample:\n\nvaults account address /path/to/file", Action: func(cCtx *cli.Context) error { filename := cCtx.Args().Get(0) if filename == "" { diff --git a/cmd/vaults/main.go b/cmd/vaults/main.go index 621e69d..0c6d0ca 100644 --- a/cmd/vaults/main.go +++ b/cmd/vaults/main.go @@ -1,12 +1,20 @@ package main import ( + "fmt" "os" "github.com/urfave/cli/v2" "golang.org/x/exp/slog" ) +func init() { + cli.VersionFlag = &cli.BoolFlag{Name: "version", Aliases: []string{"V"}, Usage: "show version"} // Enforce uppercase + cli.VersionPrinter = func(c *cli.Context) { + fmt.Printf("v%s\n", c.App.Version) + } +} + func main() { // migrate v1 config to v2 config migrateConfigV1ToV2() @@ -14,7 +22,7 @@ func main() { cliApp := &cli.App{ Name: "vaults", - Usage: "Continuously publish data from your database to the Textile Vaults network.", + Usage: "Continuously publish data from your database or file uploads to the Textile Vaults network.", Version: version, Commands: []*cli.Command{ newVaultCreateCommand(), From f02b0aba75dfd4f0c4443ca705c23039b3496d4c Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Sun, 17 Dec 2023 20:23:10 -0800 Subject: [PATCH 07/16] docs: update readme --- README.md | 90 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index e35aeab..7d599fd 100644 --- a/README.md +++ b/README.md @@ -3,43 +3,54 @@ [![License](https://img.shields.io/github/license/tablelandnetwork/basin-cli.svg)](./LICENSE) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg)](https://github.com/RichardLitt/standard-readme) -> Continuously publish data from your database to the Tableland network. +> Continuously publish data from your database or file uploads to the Tableland Vaults network. -# Table of Contents +## Table of Contents -- [Install](#install) -- [Postgres Setup](#postgres-setup) - - [Self-hosted](#self-hosted) - - [Amazon RDS](#amazon-rds) +- [Background](#background) +- [Usage](#usage) + - [Install](#install) + - [Postgres Setup](#postgres-setup) - [Supabase](#supabase) -- [Create a vault](#create-a-vault) -- [Start replicating a database](#start-replicating-a-database) -- [Write a Parquet file](#write-a-parquet-file) -- [Listing Vaults](#listing-vaults) -- [Listing Events](#listing-events) -- [Running](#running) -- [Run tests](#run-tests) -- [Retrieving](#retrieving) + - [Create a vault](#create-a-vault) + - [Start replicating a database](#start-replicating-a-database) + - [Write a Parquet file](#write-a-parquet-file) + - [Listing Vaults](#listing-vaults) + - [Listing Events](#listing-events) + - [Retrieving](#retrieving) +- [Development](#development) + - [Running](#running) + - [Run tests](#run-tests) +- [Contributing](#contributing) +- [License](#license) + +## Background + +Textile Vaults is a secure and verifiable open data platform. The Vaults CLI is a tool that allows you to continuously replicate a table or view from your database to the network (currently, only PostgreSQL is supported). Or, you can directly upload files to the vault (currently, parquet is only supported) -# Background +> 🚧 Vaults is currently not in a production-ready state. Any data that is pushed to the network may be subject to deletion. 🚧 -Textile Vaults is a secure and verifiable open data platform. The Vaults CLI is a tool that allows you to continuously replicate a table or view from your database to the network. Currently, only PostgreSQL is supported. +## Usage -> 🚧 Vaults is currently not in a production-ready state. Any data that is pushed to the network may be subject to deletion. 🚧 +### Install + +You can either install the CLI from the remote source: -# Usage +```bash +go install github.com/tablelandnetwork/basin-cli/cmd/basin@latest +``` -## Install +Or clone from source and run the Makefile `install` command: ```bash git clone https://github.com/tablelandnetwork/basin-cli.git cd basin-cli -go install ./cmd/vaults +make install ``` -## Postgres Setup +### Postgres Setup -### Self-hosted +#### Self-hosted - Make sure you have access to a superuser role. For example, you can create a new role such as `CREATE ROLE vaults WITH PASSWORD NULL LOGIN SUPERUSER;`. - Check that your Postgres installation has the [wal2json](https://github.com/eulerto/wal2json) plugin installed. @@ -53,7 +64,7 @@ go install ./cmd/vaults - Restart the database in order for the new `wal_level` to take effect (be careful!). -### Amazon RDS +#### Amazon RDS - Make sure you have a user with the `rds_superuser` role, and use `psql` to connect to your database. @@ -88,14 +99,14 @@ go install ./cmd/vaults postgresql://postgres:[PASSWORD]@db.[PROJECT_ID].supabase.co:5432/postgres ``` -## Create a vault +### Create a vault _Vaults_ define the place you push data into. Vaults uses public key authentication, so you will need an Ethereum style (ECDSA, secp256k1) wallet to create a new vault. You can use an existing wallet or set up a new one with `vaults wallet create`. Your private key is only used locally for signing. ```bash -vaults wallet create [FILENAME] +vaults account create [FILENAME] ``` A new private key will be written to `FILENAME`. @@ -108,7 +119,7 @@ vaults create --dburi [DBURI] --account [WALLET_ADDRESS] namespace.relation_nam 🚧 Vaults currently only replicates `INSERT` statements, which means that it only replicates append-only data (e.g., log-style data). Row updates and deletes will be ignored. 🚧 -## Start replicating a database +### Start replicating a database Use `vaults stream` to start a daemon that will continuously push changes to the underlying table/view to the network. See `vaults stream --help` for more info. @@ -116,7 +127,7 @@ Use `vaults stream` to start a daemon that will continuously push changes to the vaults stream --private-key [PRIVATE_KEY] namespace.relation_name ``` -## Write a Parquet file +### Write a Parquet file Before writing a Parquet file, you need to [Create a vault](#create-a-vault), if not already created. You can omit the `--dburi` flag, in this case. @@ -126,7 +137,7 @@ Then, use `vaults write` to write a Parquet file. vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] filepath ``` -You can attach a timestamp to that file write, e.g. +You can attach a timestamp to that file write, e.g. ```bash vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] --timestamp 1699984703 filepath @@ -140,7 +151,7 @@ vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] --tim If a timestamp is not provided, the CLI will assume the timestamp is the current client epoch in UTC. -## Listing Vaults +### Listing Vaults You can list the vaults from an account by running: @@ -148,7 +159,7 @@ You can list the vaults from an account by running: vaults list --account [ETH_ADDRESS] ``` -## Listing Events +### Listing Events You can list events of a given vault by running: @@ -162,18 +173,18 @@ Events command accept `--before`,`--after` , and `--at` flags to filter events b # examples vaults events --vault demotest.data --at 1699569502 vaults events --vault demotest.data --before 2023-11-09T19:38:23-03:00 -vaults events --vault demotest.data --after 2023-11-09 +vaults events --vault demotest.data --after 2023-11-09 ``` -## Retrieving +### Retrieving ```bash vaults retrieve bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq ``` -# Development +## Development -## Running +### Running You can make use of the scripts inside `scripts` to facilitate running the CLI locally without building. @@ -181,14 +192,14 @@ You can make use of the scripts inside `scripts` to facilitate running the CLI l # Starting the Provider Server PORT=8888 ./scripts/server.sh -# Create a wallet -./scripts/run.sh wallet create pk.out +# Create an account +./scripts/run.sh account create pk.out # Start replicating ./scripts/run.sh vaults stream --private-key [PRIVATE_KEY] namespace.relation_name ``` -## Run tests +### Run tests ```bash make test @@ -196,14 +207,13 @@ make test Note: One of the tests requires Docker Engine to be running. - -# Contributing +## Contributing PRs accepted. Small note: If editing the README, please conform to the [standard-readme](https://github.com/RichardLitt/standard-readme) specification. -# License +## License MIT AND Apache-2.0, © 2021-2023 Tableland Network Contributors From c270b94d58fdf165fdb1b86da390235534be8698 Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Sun, 17 Dec 2023 21:19:15 -0800 Subject: [PATCH 08/16] fix: alter gh version action logic, version logging --- .github/workflows/update-version.yml | 38 ++++++++++++---------------- cmd/vaults/main.go | 2 +- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml index 292d82f..642c1b1 100644 --- a/.github/workflows/update-version.yml +++ b/.github/workflows/update-version.yml @@ -1,38 +1,32 @@ -name: Update Version +name: Update version file on: push: branches: - main - dtb/cli-positionals # tmp + workflow_dispatch: jobs: - update-version: + update_version: runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Git - run: | - git config --global user.name 'GitHub Actions' - git config --global user.email 'actions@github.com' + - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get latest tag - id: get-latest-tag - run: echo ::set-output name=TAG::$(git describe --tags --abbrev=0) + id: get-tag + run: | + echo "LATEST_TAG=$(git describe --tags $(git rev-list --tags --max-count=1))" >> $GITHUB_OUTPUT - name: Update version.json run: | - echo "{\"version\": \"${{ steps.get-latest-tag.outputs.TAG }}\"}" > version.json + echo "{ + \"version\": \"${{steps.get-tag.outputs.LATEST_TAG}}\" + }" > version.json - - name: Commit and push if changed - run: | - git diff - if [ -n "$(git diff --name-only)" ]; then - git add version.json - git commit -m "Update version to ${{ steps.get-latest-tag.outputs.TAG }}" - git push - else - echo "No changes in version.json" + - name: Commit changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: "chore: update version to ${{steps.get-tag.outputs.LATEST_TAG}}" diff --git a/cmd/vaults/main.go b/cmd/vaults/main.go index 0c6d0ca..d4dfd9b 100644 --- a/cmd/vaults/main.go +++ b/cmd/vaults/main.go @@ -11,7 +11,7 @@ import ( func init() { cli.VersionFlag = &cli.BoolFlag{Name: "version", Aliases: []string{"V"}, Usage: "show version"} // Enforce uppercase cli.VersionPrinter = func(c *cli.Context) { - fmt.Printf("v%s\n", c.App.Version) + fmt.Printf("%s\n", c.App.Version) } } From dea664ea17d0009a4e2d1406d0c70dd41f144ddf Mon Sep 17 00:00:00 2001 From: dtbuchholz Date: Mon, 18 Dec 2023 05:19:32 +0000 Subject: [PATCH 09/16] chore: update version to v0.0.6 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 1be1b18..0c7dfa8 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "version": "0.0.0" + "version": "v0.0.6" } From b7a45aa1ef0359487fd128ca172a233ac7b47125 Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Sun, 17 Dec 2023 21:25:08 -0800 Subject: [PATCH 10/16] feat: add vault alias flag '-v' --- cmd/vaults/commands.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/vaults/commands.go b/cmd/vaults/commands.go index 78f9d5a..12df9b6 100644 --- a/cmd/vaults/commands.go +++ b/cmd/vaults/commands.go @@ -280,6 +280,7 @@ func newWriteCommand() *cli.Command { }, &cli.StringFlag{ Name: "vault", + Aliases: []string{"v"}, Category: "REQUIRED", Usage: "Vault name", Destination: &vaultName, @@ -435,6 +436,7 @@ func newListEventsCommand() *cli.Command { Flags: []cli.Flag{ &cli.StringFlag{ Name: "vault", + Aliases: []string{"v"}, Category: "REQUIRED", Usage: "Vault name", Destination: &vault, From d71ceac10b540e2e902af95dedfd8dc30eb0b495 Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Sun, 17 Dec 2023 22:46:02 -0800 Subject: [PATCH 11/16] feat: retrieve to custom output dir or stdout --- .github/workflows/update-version.yml | 3 +- README.md | 16 +++++++- cmd/vaults/commands.go | 60 +++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml index 642c1b1..209fd09 100644 --- a/.github/workflows/update-version.yml +++ b/.github/workflows/update-version.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - dtb/cli-positionals # tmp workflow_dispatch: jobs: @@ -23,7 +22,7 @@ jobs: - name: Update version.json run: | echo "{ - \"version\": \"${{steps.get-tag.outputs.LATEST_TAG}}\" + \"version\": \"${{steps.get-tag.outputs.LATEST_TAG}}\" }" > version.json - name: Commit changes diff --git a/README.md b/README.md index 7d599fd..93608a9 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ A new private key will be written to `FILENAME`. The name of a vault contains a `namespace` (e.g. `my_company`) and the name of an existing database relation (e.g. `my_table`), separated by a period (`.`). Use `vaults create` to create a new vault. See `vaults create --help` for more info. ```bash -vaults create --dburi [DBURI] --account [WALLET_ADDRESS] namespace.relation_name +vaults create --dburi [DBURI] --account [WALLET_ADDRESS] namespace.relation_name ``` 🚧 Vaults currently only replicates `INSERT` statements, which means that it only replicates append-only data (e.g., log-style data). Row updates and deletes will be ignored. 🚧 @@ -178,10 +178,24 @@ vaults events --vault demotest.data --after 2023-11-09 ### Retrieving +You can retrieve a file from a vault by running: + ```bash vaults retrieve bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq ``` +You can also specify where to save the file: + +```bash +vaults retrieve --output /path/to/dir bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq +``` + +Or stream the file to stdout the `-` value (note: the short form `-o` is for `--output`), and then pipe it to something like [`car extract`](https://github.com/ipld/go-car) to unpack the CAR file's contents: + +```bash +vaults retrieve -o - bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq | car extract +``` + ## Development ### Running diff --git a/cmd/vaults/commands.go b/cmd/vaults/commands.go index 12df9b6..e6a9358 100644 --- a/cmd/vaults/commands.go +++ b/cmd/vaults/commands.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path" "regexp" @@ -581,20 +582,32 @@ func newListEventsCommand() *cli.Command { } func newRetrieveCommand() *cli.Command { + var output string + return &cli.Command{ Name: "retrieve", Usage: "Retrieve an event by CID", - UsageText: "vaults retrieve ", - Description: "Retrieving an event will download the event's CAR file into the current directory.", + ArgsUsage: "", + Description: "Retrieving an event will download the event's CAR file into the current directory, a provided directory path, or to stdout.\n\nExample:\n\nvaults retrieve --output /path/to/dir bafy...", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "output", + Aliases: []string{"o"}, + Category: "OPTIONAL", + Usage: "Output directory path, or '-' for stdout", + DefaultText: "current directory", + Destination: &output, + }, + }, Action: func(cCtx *cli.Context) error { arg := cCtx.Args().Get(0) if arg == "" { - return errors.New("argument is empty") + return errors.New("must provide an event CID") } rootCid, err := cid.Parse(arg) if err != nil { - return errors.New("cid is invalid") + return errors.New("CID is invalid") } lassie, err := lassie.NewLassie(cCtx.Context) @@ -607,7 +620,35 @@ func newRetrieveCommand() *cli.Command { car.StoreIdentityCIDs(false), car.UseWholeCIDs(false), } - carWriter := deferred.NewDeferredCarWriterForPath(fmt.Sprintf("./%s.car", arg), []cid.Cid{rootCid}, carOpts...) + + var carWriter *deferred.DeferredCarWriter + var tmpFile *os.File + + if output == "-" { + // Create a temporary file only for writing to stdout case + tmpFile, err := os.CreateTemp("", fmt.Sprintf("%s.car", arg)) + if err != nil { + return fmt.Errorf("failed to create temporary file: %s", err) + } + defer os.Remove(tmpFile.Name()) + carWriter = deferred.NewDeferredCarWriterForPath(tmpFile.Name(), []cid.Cid{rootCid}, carOpts...) + } else { + // Write to the provided path or current directory + if output == "" { + output = "." // Default to current directory + } + // Ensure path is a valid directory + info, err := os.Stat(output) + if err != nil { + return fmt.Errorf("failed to access output directory: %s", err) + } + if !info.IsDir() { + return fmt.Errorf("output path is not a directory: %s", output) + } + carPath := path.Join(output, fmt.Sprintf("%s.car", arg)) + carWriter = deferred.NewDeferredCarWriterForPath(carPath, []cid.Cid{rootCid}, carOpts...) + } + defer func() { _ = carWriter.Close() }() @@ -627,6 +668,15 @@ func newRetrieveCommand() *cli.Command { return fmt.Errorf("failed to fetch: %s", err) } + // Write to stdout only if the output flag is set to '-' + if output == "-" && tmpFile != nil { + _, _ = tmpFile.Seek(0, io.SeekStart) + _, err = io.Copy(os.Stdout, tmpFile) + if err != nil { + return fmt.Errorf("failed to write to stdout: %s", err) + } + } + return nil }, } From 571935a592d4c5311a1a2c891da29421087416eb Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Mon, 18 Dec 2023 11:55:28 -0800 Subject: [PATCH 12/16] chore: linting fixes; capitalization, new lines, colon usage in doc strings --- cmd/vaults/commands.go | 135 ++++++++++++++++++++++++----------------- cmd/vaults/main.go | 9 ++- 2 files changed, 87 insertions(+), 57 deletions(-) diff --git a/cmd/vaults/commands.go b/cmd/vaults/commands.go index e6a9358..adc8c66 100644 --- a/cmd/vaults/commands.go +++ b/cmd/vaults/commands.go @@ -41,15 +41,17 @@ func newVaultCreateCommand() *cli.Command { var winSize, cache int64 return &cli.Command{ - Name: "create", - Usage: "Create a new vault", - ArgsUsage: "", - Description: "Create a vault for a given account's address as either database streaming or file uploading. Optionally, also set a cache duration for the data.\n\nExample:\n\nvaults create --account 0x1234abcd --cache 10 my.vault", + Name: "create", + Usage: "Create a new vault", + ArgsUsage: "", + Description: "Create a vault for a given account's address as either database streaming \n" + + "or file uploading. Optionally, also set a cache duration for the data.\n\nEXAMPLE:\n\n" + + "vaults create --account 0x1234abcd --cache 10 my.vault", Flags: []cli.Flag{ &cli.StringFlag{ Name: "account", Aliases: []string{"a"}, - Category: "REQUIRED", + Category: "REQUIRED:", Usage: "Ethereum wallet address", Destination: &address, Required: true, @@ -57,7 +59,7 @@ func newVaultCreateCommand() *cli.Command { &cli.StringFlag{ Name: "provider", Aliases: []string{"p"}, - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The provider's address and port (e.g., localhost:8080)", DefaultText: DefaultProviderHost, Destination: &provider, @@ -65,7 +67,7 @@ func newVaultCreateCommand() *cli.Command { }, &cli.Int64Flag{ Name: "cache", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "Time duration (in minutes) that the data will be available in the cache", DefaultText: "0", Destination: &cache, @@ -73,13 +75,13 @@ func newVaultCreateCommand() *cli.Command { }, &cli.StringFlag{ Name: "dburi", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "PostgreSQL connection string (e.g., postgresql://postgres:[PASSWORD]@[HOST]:[PORT]/postgres)", Destination: &dburi, }, &cli.Int64Flag{ Name: "window-size", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "Number of seconds for which WAL updates are buffered before being sent to the provider", DefaultText: fmt.Sprintf("%d", DefaultWindowSize), Destination: &winSize, @@ -162,15 +164,18 @@ func newStreamCommand() *cli.Command { var privateKey string return &cli.Command{ - Name: "stream", - Usage: "Starts a daemon process that streams Postgres changes to a vault", - ArgsUsage: "", - Description: "The daemon will continuously stream database changes (except deletions) to the vault, as long as the daemon is actively running.\n\nExample:\n\nvaults stream --vault my.vault --private-key 0x1234abcd", + Name: "stream", + Usage: "Starts a daemon process that streams Postgres changes to a vault", + ArgsUsage: "", + Description: "The daemon will continuously stream database changes (except deletions) \n" + + "to the vault, as long as the daemon is actively running.\n\n" + + "EXAMPLE:\n\nvaults stream --vault my.vault --private-key 0x1234abcd", + Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", Aliases: []string{"k"}, - Category: "REQUIRED", + Category: "REQUIRED:", Usage: "Ethereum wallet private key", Destination: &privateKey, Required: true, @@ -266,15 +271,17 @@ func newWriteCommand() *cli.Command { var timestamp string return &cli.Command{ - Name: "write", - Usage: "Write a Parquet file", - ArgsUsage: "", - Description: "A Parquet file can be pushed directly to the vault, as an alternative to continuous Postgres data streaming.\n\nExample:\n\nvaults write --vault my.vault --private-key 0x1234abcd /path/to/file.parquet", + Name: "write", + Usage: "Write a Parquet file", + ArgsUsage: "", + Description: "A Parquet file can be pushed directly to the vault, as an \n" + + "alternative to continuous Postgres data streaming.\n\n" + + "EXAMPLE:\n\nvaults write --vault my.vault --private-key 0x1234abcd /path/to/file.parquet", Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", Aliases: []string{"k"}, - Category: "REQUIRED", + Category: "REQUIRED:", Usage: "Ethereum wallet private key", Destination: &privateKey, Required: true, @@ -282,14 +289,14 @@ func newWriteCommand() *cli.Command { &cli.StringFlag{ Name: "vault", Aliases: []string{"v"}, - Category: "REQUIRED", + Category: "REQUIRED:", Usage: "Vault name", Destination: &vaultName, Required: true, }, &cli.StringFlag{ Name: "timestamp", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The time the file was created", DefaultText: "current epoch in UTC", Destination: ×tamp, @@ -364,14 +371,16 @@ func newListCommand() *cli.Command { var address, provider, format string return &cli.Command{ - Name: "list", - Usage: "List vaults of a given account", - Description: "Listing vaults will show all vaults that have been created by the provided account's address and logged as either line delimited text or a json array.\n\nExample:\n\nvaults list --account 0x1234abcd --format json", + Name: "list", + Usage: "List vaults of a given account", + Description: "Listing vaults will show all vaults that have been created by the provided \n" + + "account's address and logged as either line delimited text or a json array.\n\n" + + "EXAMPLE:\n\nvaults list --account 0x1234abcd --format json", Flags: []cli.Flag{ &cli.StringFlag{ Name: "account", Aliases: []string{"a"}, - Category: "REQUIRED", + Category: "REQUIRED:", Usage: "Ethereum wallet address", Destination: &address, Required: true, @@ -379,7 +388,7 @@ func newListCommand() *cli.Command { &cli.StringFlag{ Name: "provider", Aliases: []string{"p"}, - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The provider's address and port (e.g., localhost:8080)", DefaultText: DefaultProviderHost, Destination: &provider, @@ -387,7 +396,7 @@ func newListCommand() *cli.Command { }, &cli.StringFlag{ Name: "format", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The output format (text or json)", DefaultText: "text", Destination: &format, @@ -430,15 +439,20 @@ func newListEventsCommand() *cli.Command { var limit, offset, latest int return &cli.Command{ - Name: "events", - Usage: "List events of a given vault", - UsageText: "vaults events [command options]", - Description: "Vault events can be filtered by date ranges (unix, ISO 8601 date, or ISO 8601 date & time), returning the event metadata and corresponding CID.\n\nExample:\n\nvaults events --vault my.vault \\\n--limit 10 --offset 3 \\\n--after 2023-09-01 --before 2023-12-01 \\\n--format json", + Name: "events", + Usage: "List events of a given vault", + UsageText: "vaults events [command options]", + Description: "Vault events can be filtered by date ranges (unix, ISO 8601 date,\n" + + "or ISO 8601 date & time), returning the event metadata and \n" + + "corresponding CID.\n\n" + + "EXAMPLE:\n\nvaults events --vault my.vault \\\n" + + "--limit 10 --offset 3 \\\n--after 2023-09-01 --before 2023-12-01 \\\n" + + "--format json", Flags: []cli.Flag{ &cli.StringFlag{ Name: "vault", Aliases: []string{"v"}, - Category: "REQUIRED", + Category: "REQUIRED:", Usage: "Vault name", Destination: &vault, Required: true, @@ -446,7 +460,7 @@ func newListEventsCommand() *cli.Command { &cli.StringFlag{ Name: "provider", Aliases: []string{"p"}, - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The provider's address and port (e.g., localhost:8080)", DefaultText: DefaultProviderHost, Destination: &provider, @@ -454,7 +468,7 @@ func newListEventsCommand() *cli.Command { }, &cli.IntFlag{ Name: "limit", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The number of deals to fetch", DefaultText: "10", Destination: &limit, @@ -462,13 +476,13 @@ func newListEventsCommand() *cli.Command { }, &cli.IntFlag{ Name: "latest", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The latest N deals to fetch", Destination: &latest, }, &cli.IntFlag{ Name: "offset", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The epoch to start from", DefaultText: "0", Destination: &offset, @@ -476,28 +490,28 @@ func newListEventsCommand() *cli.Command { }, &cli.StringFlag{ Name: "before", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "Filter deals created before this timestamp", Destination: &before, Value: "", }, &cli.StringFlag{ Name: "after", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "Filter deals created after this timestamp", Destination: &after, Value: "", }, &cli.StringFlag{ Name: "at", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "Filter deals created at this timestamp", Destination: &at, Value: "", }, &cli.StringFlag{ Name: "format", - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "The output format (table or json)", DefaultText: "table", Destination: &format, @@ -585,15 +599,17 @@ func newRetrieveCommand() *cli.Command { var output string return &cli.Command{ - Name: "retrieve", - Usage: "Retrieve an event by CID", - ArgsUsage: "", - Description: "Retrieving an event will download the event's CAR file into the current directory, a provided directory path, or to stdout.\n\nExample:\n\nvaults retrieve --output /path/to/dir bafy...", + Name: "retrieve", + Usage: "Retrieve an event by CID", + ArgsUsage: "", + Description: "Retrieving an event will download the event's CAR file into the \n" + + "current directory, a provided directory path, or to stdout.\n\n" + + "EXAMPLE:\n\nvaults retrieve --output /path/to/dir bafy...", Flags: []cli.Flag{ &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, - Category: "OPTIONAL", + Category: "OPTIONAL:", Usage: "Output directory path, or '-' for stdout", DefaultText: "current directory", Destination: &output, @@ -630,7 +646,9 @@ func newRetrieveCommand() *cli.Command { if err != nil { return fmt.Errorf("failed to create temporary file: %s", err) } - defer os.Remove(tmpFile.Name()) + defer func() { + _ = os.Remove(tmpFile.Name()) + }() carWriter = deferred.NewDeferredCarWriterForPath(tmpFile.Name(), []cid.Cid{rootCid}, carOpts...) } else { // Write to the provided path or current directory @@ -689,10 +707,12 @@ func newWalletCommand() *cli.Command { UsageText: "vaults account [arguments...]", Subcommands: []*cli.Command{ { - Name: "create", - Usage: "Creates a new account", - UsageText: "vaults account create ", - Description: "Create an Ethereum-style wallet (secp256k1 key pair) at a provided file path.\n\nExample:\n\nvaults account create /path/to/file", + Name: "create", + Usage: "Creates a new account", + UsageText: "vaults account create ", + Description: "Create an Ethereum-style wallet (secp256k1 key pair) at a \n" + + "provided file path.\n\n" + + "EXAMPLE:\n\nvaults account create /path/to/file", Action: func(cCtx *cli.Context) error { filename := cCtx.Args().Get(0) if filename == "" { @@ -717,10 +737,12 @@ func newWalletCommand() *cli.Command { }, }, { - Name: "address", - Usage: "Print the public key for an account's private key", - UsageText: "vaults account address ", - Description: "The result of the `vaults account create` command will write a private key to a file, and this lets you retrieve that value for use in other commands.\n\nExample:\n\nvaults account address /path/to/file", + Name: "address", + Usage: "Print the public key for an account's private key", + UsageText: "vaults account address ", + Description: "The result of the `vaults account create` command will write a private key to a file, \n" + + "and this lets you retrieve the public key value for use in other commands.\n\n" + + "EXAMPLE:\n\nvaults account address /path/to/file", Action: func(cCtx *cli.Context) error { filename := cCtx.Args().Get(0) if filename == "" { @@ -746,7 +768,10 @@ func newWalletCommand() *cli.Command { func parseVaultName(name string) (ns string, rel string, err error) { match := pubNameRx.FindStringSubmatch(name) if len(match) != 3 { - return "", "", errors.New("vault name must be of the form `namespace.relation_name` using only letters, numbers, and underscores (_), where `namespace` and `relation` do not start with a number") // nolint + return "", "", errors.New( + "vault name must be of the form `namespace.relation_name` using only letters, numbers, " + + "and underscores (_), where `namespace` and `relation` do not start with a number", + ) // nolint } ns = match[1] rel = match[2] diff --git a/cmd/vaults/main.go b/cmd/vaults/main.go index d4dfd9b..1fb98db 100644 --- a/cmd/vaults/main.go +++ b/cmd/vaults/main.go @@ -9,7 +9,12 @@ import ( ) func init() { - cli.VersionFlag = &cli.BoolFlag{Name: "version", Aliases: []string{"V"}, Usage: "show version"} // Enforce uppercase + // Enforce uppercase version shorthand flag + cli.VersionFlag = &cli.BoolFlag{ + Name: "version", + Aliases: []string{"V"}, + Usage: "show version", + } cli.VersionPrinter = func(c *cli.Context) { fmt.Printf("%s\n", c.App.Version) } @@ -18,7 +23,7 @@ func init() { func main() { // migrate v1 config to v2 config migrateConfigV1ToV2() - var version = getVersion() + version := getVersion() cliApp := &cli.App{ Name: "vaults", From 91d733a511e2ded4e22a68806fe5881bf2e690e9 Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Tue, 19 Dec 2023 09:14:44 -0800 Subject: [PATCH 13/16] docs: fix incorrect vaults install link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 93608a9..6ad5fd9 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Textile Vaults is a secure and verifiable open data platform. The Vaults CLI is You can either install the CLI from the remote source: ```bash -go install github.com/tablelandnetwork/basin-cli/cmd/basin@latest +go install github.com/tablelandnetwork/basin-cli/cmd/vaults@latest ``` Or clone from source and run the Makefile `install` command: From a670d233219be0ddc5d48a469701e3625bd21f83 Mon Sep 17 00:00:00 2001 From: Dan Buchholz Date: Tue, 19 Dec 2023 11:51:23 -0800 Subject: [PATCH 14/16] chore: alter GH version workflow --- .github/workflows/update-version.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml index 209fd09..9bf00fb 100644 --- a/.github/workflows/update-version.yml +++ b/.github/workflows/update-version.yml @@ -1,10 +1,9 @@ name: Update version file on: - push: - branches: - - main - workflow_dispatch: + release: + types: + - created jobs: update_version: @@ -14,18 +13,13 @@ jobs: with: fetch-depth: 0 - - name: Get latest tag - id: get-tag - run: | - echo "LATEST_TAG=$(git describe --tags $(git rev-list --tags --max-count=1))" >> $GITHUB_OUTPUT - - name: Update version.json run: | echo "{ - \"version\": \"${{steps.get-tag.outputs.LATEST_TAG}}\" + \"version\": \"${{ github.event.release.tag_name }}\" }" > version.json - name: Commit changes uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: "chore: update version to ${{steps.get-tag.outputs.LATEST_TAG}}" + commit_message: "chore: update version to ${{ github.event.release.tag_name }}" From 67ece26d02a3fddef763a4bd48ab3206eff68ab2 Mon Sep 17 00:00:00 2001 From: Bruno Calza Date: Tue, 19 Dec 2023 18:29:03 -0300 Subject: [PATCH 15/16] changes the way version is passed Signed-off-by: Bruno Calza --- .github/workflows/update-version.yml | 25 ------------------------- cmd/vaults/main.go | 3 ++- cmd/vaults/utils.go | 23 ----------------------- version.json | 3 --- 4 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 .github/workflows/update-version.yml delete mode 100644 cmd/vaults/utils.go delete mode 100644 version.json diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml deleted file mode 100644 index 9bf00fb..0000000 --- a/.github/workflows/update-version.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Update version file - -on: - release: - types: - - created - -jobs: - update_version: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Update version.json - run: | - echo "{ - \"version\": \"${{ github.event.release.tag_name }}\" - }" > version.json - - - name: Commit changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "chore: update version to ${{ github.event.release.tag_name }}" diff --git a/cmd/vaults/main.go b/cmd/vaults/main.go index 1fb98db..e4dc078 100644 --- a/cmd/vaults/main.go +++ b/cmd/vaults/main.go @@ -20,10 +20,11 @@ func init() { } } +var version = "dev" + func main() { // migrate v1 config to v2 config migrateConfigV1ToV2() - version := getVersion() cliApp := &cli.App{ Name: "vaults", diff --git a/cmd/vaults/utils.go b/cmd/vaults/utils.go deleted file mode 100644 index b63c94e..0000000 --- a/cmd/vaults/utils.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "encoding/json" - "os" -) - -type version struct { - Version string `json:"version"` -} - -func getVersion() string { - var v version - data, err := os.ReadFile("version.json") - if err != nil { - return "unknown" - } - err = json.Unmarshal(data, &v) - if err != nil { - return "unknown" - } - return v.Version -} diff --git a/version.json b/version.json deleted file mode 100644 index 0c7dfa8..0000000 --- a/version.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "version": "v0.0.6" -} From 7576c692203266c26a44687616f1fb2dfa3fbf98 Mon Sep 17 00:00:00 2001 From: Bruno Calza Date: Tue, 19 Dec 2023 18:45:56 -0300 Subject: [PATCH 16/16] changes variable name Signed-off-by: Bruno Calza --- cmd/vaults/commands.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/vaults/commands.go b/cmd/vaults/commands.go index adc8c66..d2354fb 100644 --- a/cmd/vaults/commands.go +++ b/cmd/vaults/commands.go @@ -34,7 +34,7 @@ import ( "gopkg.in/yaml.v3" ) -var pubNameRx = regexp.MustCompile(`^([a-zA-Z_][a-zA-Z0-9_]*)[.]([a-zA-Z_][a-zA-Z0-9_]*$)`) +var vaultNameRx = regexp.MustCompile(`^([a-zA-Z_][a-zA-Z0-9_]*)[.]([a-zA-Z_][a-zA-Z0-9_]*$)`) func newVaultCreateCommand() *cli.Command { var address, dburi, provider string @@ -766,7 +766,7 @@ func newWalletCommand() *cli.Command { } func parseVaultName(name string) (ns string, rel string, err error) { - match := pubNameRx.FindStringSubmatch(name) + match := vaultNameRx.FindStringSubmatch(name) if len(match) != 3 { return "", "", errors.New( "vault name must be of the form `namespace.relation_name` using only letters, numbers, " +