diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml new file mode 100644 index 0000000..9bf00fb --- /dev/null +++ b/.github/workflows/update-version.yml @@ -0,0 +1,25 @@ +name: Update version file + +on: + release: + types: + - created + +jobs: + update_version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Update version.json + run: | + echo "{ + \"version\": \"${{ github.event.release.tag_name }}\" + }" > version.json + + - name: Commit changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: "chore: update version to ${{ github.event.release.tag_name }}" diff --git a/Makefile b/Makefile index f45d4ab..f3c09d6 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,11 @@ build: go build -o vaults cmd/vaults/* .PHONY: build +# Install +install: + go install ./cmd/vaults +.PHONY: install + # Test test: go test ./... -short -race -timeout 1m diff --git a/README.md b/README.md index e35aeab..6ad5fd9 100644 --- a/README.md +++ b/README.md @@ -3,43 +3,54 @@ [![License](https://img.shields.io/github/license/tablelandnetwork/basin-cli.svg)](./LICENSE) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg)](https://github.com/RichardLitt/standard-readme) -> Continuously publish data from your database to the Tableland network. +> Continuously publish data from your database or file uploads to the Tableland Vaults network. -# Table of Contents +## Table of Contents -- [Install](#install) -- [Postgres Setup](#postgres-setup) - - [Self-hosted](#self-hosted) - - [Amazon RDS](#amazon-rds) +- [Background](#background) +- [Usage](#usage) + - [Install](#install) + - [Postgres Setup](#postgres-setup) - [Supabase](#supabase) -- [Create a vault](#create-a-vault) -- [Start replicating a database](#start-replicating-a-database) -- [Write a Parquet file](#write-a-parquet-file) -- [Listing Vaults](#listing-vaults) -- [Listing Events](#listing-events) -- [Running](#running) -- [Run tests](#run-tests) -- [Retrieving](#retrieving) + - [Create a vault](#create-a-vault) + - [Start replicating a database](#start-replicating-a-database) + - [Write a Parquet file](#write-a-parquet-file) + - [Listing Vaults](#listing-vaults) + - [Listing Events](#listing-events) + - [Retrieving](#retrieving) +- [Development](#development) + - [Running](#running) + - [Run tests](#run-tests) +- [Contributing](#contributing) +- [License](#license) + +## Background + +Textile Vaults is a secure and verifiable open data platform. The Vaults CLI is a tool that allows you to continuously replicate a table or view from your database to the network (currently, only PostgreSQL is supported). Or, you can directly upload files to the vault (currently, parquet is only supported) -# Background +> 🚧 Vaults is currently not in a production-ready state. Any data that is pushed to the network may be subject to deletion. 🚧 -Textile Vaults is a secure and verifiable open data platform. The Vaults CLI is a tool that allows you to continuously replicate a table or view from your database to the network. Currently, only PostgreSQL is supported. +## Usage -> 🚧 Vaults is currently not in a production-ready state. Any data that is pushed to the network may be subject to deletion. 🚧 +### Install -# Usage +You can either install the CLI from the remote source: + +```bash +go install github.com/tablelandnetwork/basin-cli/cmd/vaults@latest +``` -## Install +Or clone from source and run the Makefile `install` command: ```bash git clone https://github.com/tablelandnetwork/basin-cli.git cd basin-cli -go install ./cmd/vaults +make install ``` -## Postgres Setup +### Postgres Setup -### Self-hosted +#### Self-hosted - Make sure you have access to a superuser role. For example, you can create a new role such as `CREATE ROLE vaults WITH PASSWORD NULL LOGIN SUPERUSER;`. - Check that your Postgres installation has the [wal2json](https://github.com/eulerto/wal2json) plugin installed. @@ -53,7 +64,7 @@ go install ./cmd/vaults - Restart the database in order for the new `wal_level` to take effect (be careful!). -### Amazon RDS +#### Amazon RDS - Make sure you have a user with the `rds_superuser` role, and use `psql` to connect to your database. @@ -88,14 +99,14 @@ go install ./cmd/vaults postgresql://postgres:[PASSWORD]@db.[PROJECT_ID].supabase.co:5432/postgres ``` -## Create a vault +### Create a vault _Vaults_ define the place you push data into. Vaults uses public key authentication, so you will need an Ethereum style (ECDSA, secp256k1) wallet to create a new vault. You can use an existing wallet or set up a new one with `vaults wallet create`. Your private key is only used locally for signing. ```bash -vaults wallet create [FILENAME] +vaults account create [FILENAME] ``` A new private key will be written to `FILENAME`. @@ -103,12 +114,12 @@ A new private key will be written to `FILENAME`. The name of a vault contains a `namespace` (e.g. `my_company`) and the name of an existing database relation (e.g. `my_table`), separated by a period (`.`). Use `vaults create` to create a new vault. See `vaults create --help` for more info. ```bash -vaults create --dburi [DBURI] --account [WALLET_ADDRESS] namespace.relation_name +vaults create --dburi [DBURI] --account [WALLET_ADDRESS] namespace.relation_name ``` 🚧 Vaults currently only replicates `INSERT` statements, which means that it only replicates append-only data (e.g., log-style data). Row updates and deletes will be ignored. 🚧 -## Start replicating a database +### Start replicating a database Use `vaults stream` to start a daemon that will continuously push changes to the underlying table/view to the network. See `vaults stream --help` for more info. @@ -116,7 +127,7 @@ Use `vaults stream` to start a daemon that will continuously push changes to the vaults stream --private-key [PRIVATE_KEY] namespace.relation_name ``` -## Write a Parquet file +### Write a Parquet file Before writing a Parquet file, you need to [Create a vault](#create-a-vault), if not already created. You can omit the `--dburi` flag, in this case. @@ -126,7 +137,7 @@ Then, use `vaults write` to write a Parquet file. vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] filepath ``` -You can attach a timestamp to that file write, e.g. +You can attach a timestamp to that file write, e.g. ```bash vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] --timestamp 1699984703 filepath @@ -140,7 +151,7 @@ vaults write --vault [namespace.relation_name] --private-key [PRIVATE_KEY] --tim If a timestamp is not provided, the CLI will assume the timestamp is the current client epoch in UTC. -## Listing Vaults +### Listing Vaults You can list the vaults from an account by running: @@ -148,7 +159,7 @@ You can list the vaults from an account by running: vaults list --account [ETH_ADDRESS] ``` -## Listing Events +### Listing Events You can list events of a given vault by running: @@ -162,18 +173,32 @@ Events command accept `--before`,`--after` , and `--at` flags to filter events b # examples vaults events --vault demotest.data --at 1699569502 vaults events --vault demotest.data --before 2023-11-09T19:38:23-03:00 -vaults events --vault demotest.data --after 2023-11-09 +vaults events --vault demotest.data --after 2023-11-09 ``` -## Retrieving +### Retrieving + +You can retrieve a file from a vault by running: ```bash vaults retrieve bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq ``` -# Development +You can also specify where to save the file: -## Running +```bash +vaults retrieve --output /path/to/dir bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq +``` + +Or stream the file to stdout the `-` value (note: the short form `-o` is for `--output`), and then pipe it to something like [`car extract`](https://github.com/ipld/go-car) to unpack the CAR file's contents: + +```bash +vaults retrieve -o - bafybeifr5njnrw67yyb2h2t7k6ukm3pml4fgphsxeurqcmgmeb7omc2vlq | car extract +``` + +## Development + +### Running You can make use of the scripts inside `scripts` to facilitate running the CLI locally without building. @@ -181,14 +206,14 @@ You can make use of the scripts inside `scripts` to facilitate running the CLI l # Starting the Provider Server PORT=8888 ./scripts/server.sh -# Create a wallet -./scripts/run.sh wallet create pk.out +# Create an account +./scripts/run.sh account create pk.out # Start replicating ./scripts/run.sh vaults stream --private-key [PRIVATE_KEY] namespace.relation_name ``` -## Run tests +### Run tests ```bash make test @@ -196,14 +221,13 @@ make test Note: One of the tests requires Docker Engine to be running. - -# Contributing +## Contributing PRs accepted. Small note: If editing the README, please conform to the [standard-readme](https://github.com/RichardLitt/standard-readme) specification. -# License +## License MIT AND Apache-2.0, © 2021-2023 Tableland Network Contributors diff --git a/cmd/vaults/commands.go b/cmd/vaults/commands.go index bc87d2a..adc8c66 100644 --- a/cmd/vaults/commands.go +++ b/cmd/vaults/commands.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path" "regexp" @@ -40,42 +41,56 @@ func newVaultCreateCommand() *cli.Command { var winSize, cache int64 return &cli.Command{ - Name: "create", - Usage: "create a new vault", + Name: "create", + Usage: "Create a new vault", + ArgsUsage: "", + Description: "Create a vault for a given account's address as either database streaming \n" + + "or file uploading. Optionally, also set a cache duration for the data.\n\nEXAMPLE:\n\n" + + "vaults create --account 0x1234abcd --cache 10 my.vault", Flags: []cli.Flag{ &cli.StringFlag{ Name: "account", + Aliases: []string{"a"}, + Category: "REQUIRED:", Usage: "Ethereum wallet address", Destination: &address, Required: true, }, - &cli.StringFlag{ - Name: "dburi", - Usage: "PostgreSQL connection string", - Destination: &dburi, - }, &cli.StringFlag{ Name: "provider", - Usage: "The provider's address and port (e.g. localhost:8080)", + Aliases: []string{"p"}, + Category: "OPTIONAL:", + Usage: "The provider's address and port (e.g., localhost:8080)", + DefaultText: DefaultProviderHost, Destination: &provider, Value: DefaultProviderHost, }, - &cli.Int64Flag{ - Name: "window-size", - Usage: "Number of seconds for which WAL updates are buffered before being sent to the provider", - Destination: &winSize, - Value: DefaultWindowSize, - }, &cli.Int64Flag{ Name: "cache", + Category: "OPTIONAL:", Usage: "Time duration (in minutes) that the data will be available in the cache", + DefaultText: "0", Destination: &cache, Value: 0, }, + &cli.StringFlag{ + Name: "dburi", + Category: "OPTIONAL:", + Usage: "PostgreSQL connection string (e.g., postgresql://postgres:[PASSWORD]@[HOST]:[PORT]/postgres)", + Destination: &dburi, + }, + &cli.Int64Flag{ + Name: "window-size", + Category: "OPTIONAL:", + Usage: "Number of seconds for which WAL updates are buffered before being sent to the provider", + DefaultText: fmt.Sprintf("%d", DefaultWindowSize), + Destination: &winSize, + Value: DefaultWindowSize, + }, }, Action: func(cCtx *cli.Context) error { if cCtx.NArg() != 1 { - return errors.New("one argument should be provided") + return errors.New("must provide a vault name") } pub := cCtx.Args().First() @@ -149,11 +164,18 @@ func newStreamCommand() *cli.Command { var privateKey string return &cli.Command{ - Name: "stream", - Usage: "starts a daemon process that streams Postgres changes to a vault", + Name: "stream", + Usage: "Starts a daemon process that streams Postgres changes to a vault", + ArgsUsage: "", + Description: "The daemon will continuously stream database changes (except deletions) \n" + + "to the vault, as long as the daemon is actively running.\n\n" + + "EXAMPLE:\n\nvaults stream --vault my.vault --private-key 0x1234abcd", + Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", + Aliases: []string{"k"}, + Category: "REQUIRED:", Usage: "Ethereum wallet private key", Destination: &privateKey, Required: true, @@ -161,7 +183,7 @@ func newStreamCommand() *cli.Command { }, Action: func(cCtx *cli.Context) error { if cCtx.NArg() != 1 { - return errors.New("one argument should be provided") + return errors.New("must provide a vault name") } vault := cCtx.Args().First() @@ -249,30 +271,40 @@ func newWriteCommand() *cli.Command { var timestamp string return &cli.Command{ - Name: "write", - Usage: "write a Parquet file", + Name: "write", + Usage: "Write a Parquet file", + ArgsUsage: "", + Description: "A Parquet file can be pushed directly to the vault, as an \n" + + "alternative to continuous Postgres data streaming.\n\n" + + "EXAMPLE:\n\nvaults write --vault my.vault --private-key 0x1234abcd /path/to/file.parquet", Flags: []cli.Flag{ &cli.StringFlag{ Name: "private-key", + Aliases: []string{"k"}, + Category: "REQUIRED:", Usage: "Ethereum wallet private key", Destination: &privateKey, Required: true, }, &cli.StringFlag{ Name: "vault", + Aliases: []string{"v"}, + Category: "REQUIRED:", Usage: "Vault name", Destination: &vaultName, Required: true, }, &cli.StringFlag{ Name: "timestamp", - Usage: "The time the file was created (default: current epoch in UTC)", + Category: "OPTIONAL:", + Usage: "The time the file was created", + DefaultText: "current epoch in UTC", Destination: ×tamp, }, }, Action: func(cCtx *cli.Context) error { if cCtx.NArg() != 1 { - return errors.New("one argument should be provided") + return errors.New("must provide a file path") } ns, rel, err := parseVaultName(vaultName) if err != nil { @@ -336,24 +368,40 @@ func newWriteCommand() *cli.Command { } func newListCommand() *cli.Command { - var address, provider string + var address, provider, format string return &cli.Command{ Name: "list", - Usage: "list vaults of a given account", + Usage: "List vaults of a given account", + Description: "Listing vaults will show all vaults that have been created by the provided \n" + + "account's address and logged as either line delimited text or a json array.\n\n" + + "EXAMPLE:\n\nvaults list --account 0x1234abcd --format json", Flags: []cli.Flag{ &cli.StringFlag{ Name: "account", + Aliases: []string{"a"}, + Category: "REQUIRED:", Usage: "Ethereum wallet address", Destination: &address, Required: true, }, &cli.StringFlag{ Name: "provider", - Usage: "The provider's address and port (e.g. localhost:8080)", + Aliases: []string{"p"}, + Category: "OPTIONAL:", + Usage: "The provider's address and port (e.g., localhost:8080)", + DefaultText: DefaultProviderHost, Destination: &provider, Value: DefaultProviderHost, }, + &cli.StringFlag{ + Name: "format", + Category: "OPTIONAL:", + Usage: "The output format (text or json)", + DefaultText: "text", + Destination: &format, + Value: "text", + }, }, Action: func(cCtx *cli.Context) error { account, err := app.NewAccount(address) @@ -367,8 +415,18 @@ func newListCommand() *cli.Command { return fmt.Errorf("failed to list vaults: %s", err) } - for _, vault := range vaults { - fmt.Printf("%s\n", vault) + if format == "text" { + for _, vault := range vaults { + fmt.Printf("%s\n", vault) + } + } else if format == "json" { + jsonData, err := json.Marshal(vaults) + if err != nil { + return fmt.Errorf("error serializing events to JSON") + } + fmt.Println(string(jsonData)) + } else { + return fmt.Errorf("invalid format: %s", format) } return nil @@ -381,59 +439,81 @@ func newListEventsCommand() *cli.Command { var limit, offset, latest int return &cli.Command{ - Name: "events", - Usage: "list events of a given vault", + Name: "events", + Usage: "List events of a given vault", + UsageText: "vaults events [command options]", + Description: "Vault events can be filtered by date ranges (unix, ISO 8601 date,\n" + + "or ISO 8601 date & time), returning the event metadata and \n" + + "corresponding CID.\n\n" + + "EXAMPLE:\n\nvaults events --vault my.vault \\\n" + + "--limit 10 --offset 3 \\\n--after 2023-09-01 --before 2023-12-01 \\\n" + + "--format json", Flags: []cli.Flag{ &cli.StringFlag{ Name: "vault", - Usage: "vault name", + Aliases: []string{"v"}, + Category: "REQUIRED:", + Usage: "Vault name", Destination: &vault, Required: true, }, &cli.StringFlag{ Name: "provider", - Usage: "The provider's address and port (e.g. localhost:8080)", + Aliases: []string{"p"}, + Category: "OPTIONAL:", + Usage: "The provider's address and port (e.g., localhost:8080)", + DefaultText: DefaultProviderHost, Destination: &provider, Value: DefaultProviderHost, }, &cli.IntFlag{ Name: "limit", + Category: "OPTIONAL:", Usage: "The number of deals to fetch", + DefaultText: "10", Destination: &limit, Value: 10, }, &cli.IntFlag{ Name: "latest", + Category: "OPTIONAL:", Usage: "The latest N deals to fetch", Destination: &latest, }, &cli.IntFlag{ Name: "offset", + Category: "OPTIONAL:", Usage: "The epoch to start from", + DefaultText: "0", Destination: &offset, Value: 0, }, &cli.StringFlag{ Name: "before", + Category: "OPTIONAL:", Usage: "Filter deals created before this timestamp", Destination: &before, Value: "", }, &cli.StringFlag{ Name: "after", + Category: "OPTIONAL:", Usage: "Filter deals created after this timestamp", Destination: &after, Value: "", }, &cli.StringFlag{ Name: "at", + Category: "OPTIONAL:", Usage: "Filter deals created at this timestamp", Destination: &at, Value: "", }, &cli.StringFlag{ Name: "format", + Category: "OPTIONAL:", Usage: "The output format (table or json)", + DefaultText: "table", Destination: &format, Value: "table", }, @@ -516,18 +596,34 @@ func newListEventsCommand() *cli.Command { } func newRetrieveCommand() *cli.Command { + var output string + return &cli.Command{ - Name: "retrieve", - Usage: "Retrieve an event by CID", + Name: "retrieve", + Usage: "Retrieve an event by CID", + ArgsUsage: "", + Description: "Retrieving an event will download the event's CAR file into the \n" + + "current directory, a provided directory path, or to stdout.\n\n" + + "EXAMPLE:\n\nvaults retrieve --output /path/to/dir bafy...", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "output", + Aliases: []string{"o"}, + Category: "OPTIONAL:", + Usage: "Output directory path, or '-' for stdout", + DefaultText: "current directory", + Destination: &output, + }, + }, Action: func(cCtx *cli.Context) error { arg := cCtx.Args().Get(0) if arg == "" { - return errors.New("argument is empty") + return errors.New("must provide an event CID") } rootCid, err := cid.Parse(arg) if err != nil { - return errors.New("cid is invalid") + return errors.New("CID is invalid") } lassie, err := lassie.NewLassie(cCtx.Context) @@ -540,7 +636,37 @@ func newRetrieveCommand() *cli.Command { car.StoreIdentityCIDs(false), car.UseWholeCIDs(false), } - carWriter := deferred.NewDeferredCarWriterForPath(fmt.Sprintf("./%s.car", arg), []cid.Cid{rootCid}, carOpts...) + + var carWriter *deferred.DeferredCarWriter + var tmpFile *os.File + + if output == "-" { + // Create a temporary file only for writing to stdout case + tmpFile, err := os.CreateTemp("", fmt.Sprintf("%s.car", arg)) + if err != nil { + return fmt.Errorf("failed to create temporary file: %s", err) + } + defer func() { + _ = os.Remove(tmpFile.Name()) + }() + carWriter = deferred.NewDeferredCarWriterForPath(tmpFile.Name(), []cid.Cid{rootCid}, carOpts...) + } else { + // Write to the provided path or current directory + if output == "" { + output = "." // Default to current directory + } + // Ensure path is a valid directory + info, err := os.Stat(output) + if err != nil { + return fmt.Errorf("failed to access output directory: %s", err) + } + if !info.IsDir() { + return fmt.Errorf("output path is not a directory: %s", output) + } + carPath := path.Join(output, fmt.Sprintf("%s.car", arg)) + carWriter = deferred.NewDeferredCarWriterForPath(carPath, []cid.Cid{rootCid}, carOpts...) + } + defer func() { _ = carWriter.Close() }() @@ -560,6 +686,15 @@ func newRetrieveCommand() *cli.Command { return fmt.Errorf("failed to fetch: %s", err) } + // Write to stdout only if the output flag is set to '-' + if output == "-" && tmpFile != nil { + _, _ = tmpFile.Seek(0, io.SeekStart) + _, err = io.Copy(os.Stdout, tmpFile) + if err != nil { + return fmt.Errorf("failed to write to stdout: %s", err) + } + } + return nil }, } @@ -567,12 +702,17 @@ func newRetrieveCommand() *cli.Command { func newWalletCommand() *cli.Command { return &cli.Command{ - Name: "wallet", - Usage: "wallet commands", + Name: "account", + Usage: "Account management for an Ethereum-style wallet", + UsageText: "vaults account [arguments...]", Subcommands: []*cli.Command{ { - Name: "create", - Usage: "creates a new wallet", + Name: "create", + Usage: "Creates a new account", + UsageText: "vaults account create ", + Description: "Create an Ethereum-style wallet (secp256k1 key pair) at a \n" + + "provided file path.\n\n" + + "EXAMPLE:\n\nvaults account create /path/to/file", Action: func(cCtx *cli.Context) error { filename := cCtx.Args().Get(0) if filename == "" { @@ -597,8 +737,12 @@ func newWalletCommand() *cli.Command { }, }, { - Name: "pubkey", - Usage: "print the public key for a private key", + Name: "address", + Usage: "Print the public key for an account's private key", + UsageText: "vaults account address ", + Description: "The result of the `vaults account create` command will write a private key to a file, \n" + + "and this lets you retrieve the public key value for use in other commands.\n\n" + + "EXAMPLE:\n\nvaults account address /path/to/file", Action: func(cCtx *cli.Context) error { filename := cCtx.Args().Get(0) if filename == "" { @@ -624,7 +768,10 @@ func newWalletCommand() *cli.Command { func parseVaultName(name string) (ns string, rel string, err error) { match := pubNameRx.FindStringSubmatch(name) if len(match) != 3 { - return "", "", errors.New("vault name must be of the form `namespace.relation_name` using only letters, numbers, and underscores (_), where `namespace` and `relation` do not start with a number") // nolint + return "", "", errors.New( + "vault name must be of the form `namespace.relation_name` using only letters, numbers, " + + "and underscores (_), where `namespace` and `relation` do not start with a number", + ) // nolint } ns = match[1] rel = match[2] diff --git a/cmd/vaults/main.go b/cmd/vaults/main.go index 300bd4e..1fb98db 100644 --- a/cmd/vaults/main.go +++ b/cmd/vaults/main.go @@ -1,19 +1,34 @@ package main import ( + "fmt" "os" "github.com/urfave/cli/v2" "golang.org/x/exp/slog" ) +func init() { + // Enforce uppercase version shorthand flag + cli.VersionFlag = &cli.BoolFlag{ + Name: "version", + Aliases: []string{"V"}, + Usage: "show version", + } + cli.VersionPrinter = func(c *cli.Context) { + fmt.Printf("%s\n", c.App.Version) + } +} + func main() { // migrate v1 config to v2 config migrateConfigV1ToV2() + version := getVersion() cliApp := &cli.App{ - Name: "vaults", - Usage: "Continuously publish data from your database to the Textile Vaults network.", + Name: "vaults", + Usage: "Continuously publish data from your database or file uploads to the Textile Vaults network.", + Version: version, Commands: []*cli.Command{ newVaultCreateCommand(), newStreamCommand(), diff --git a/cmd/vaults/utils.go b/cmd/vaults/utils.go new file mode 100644 index 0000000..b63c94e --- /dev/null +++ b/cmd/vaults/utils.go @@ -0,0 +1,23 @@ +package main + +import ( + "encoding/json" + "os" +) + +type version struct { + Version string `json:"version"` +} + +func getVersion() string { + var v version + data, err := os.ReadFile("version.json") + if err != nil { + return "unknown" + } + err = json.Unmarshal(data, &v) + if err != nil { + return "unknown" + } + return v.Version +} diff --git a/version.json b/version.json new file mode 100644 index 0000000..0c7dfa8 --- /dev/null +++ b/version.json @@ -0,0 +1,3 @@ +{ + "version": "v0.0.6" +}