From 14f9ef8a734a6b04e34110480090b3562df4071d Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Fri, 30 Oct 2020 17:16:09 -0600 Subject: [PATCH 1/9] feat: new api design Signed-off-by: Aaron Sutula --- src/admin/index.ts | 34 ++ src/admin/profiles.ts | 45 ++ src/admin/storage-jobs.ts | 125 +++++ src/admin/wallet.ts | 69 +++ src/asks/index.spec.ts | 19 - src/asks/index.ts | 49 -- src/asks/util.ts | 10 - src/data/index.ts | 233 +++++++++ src/data/types.ts | 24 + src/deals/index.ts | 78 +++ src/deals/types.ts | 34 ++ src/faults/index.spec.ts | 12 - src/faults/index.ts | 29 -- src/ffs/index.spec.ts | 766 ++++++++++++++-------------- src/ffs/index.ts | 669 ------------------------ src/ffs/types.ts | 76 --- src/health/index.spec.ts | 13 - src/health/index.ts | 26 - src/index.spec.ts | 50 -- src/index.ts | 118 ++--- src/integration.spec.ts | 601 ++++++++++++++++++++++ src/miners/index.spec.ts | 12 - src/miners/index.ts | 26 - src/net/index.spec.ts | 53 -- src/net/index.ts | 119 ----- src/{ffs => }/normalize.ts | 0 src/reputation/index.spec.ts | 12 - src/reputation/index.ts | 53 -- src/storage-config/index.spec.ts | 110 ++++ src/storage-config/index.ts | 111 ++++ src/storage-config/types.ts | 16 + src/{ffs => storage-config}/util.ts | 7 +- src/storage-jobs/index.ts | 157 ++++++ src/test-utils.ts | 98 ++++ src/types.ts | 7 +- src/util/grpc-helpers.ts | 15 +- src/wallet/index.spec.ts | 62 +-- src/wallet/index.ts | 138 +++-- 38 files changed, 2324 insertions(+), 1752 deletions(-) create mode 100644 src/admin/index.ts create mode 100644 src/admin/profiles.ts create mode 100644 src/admin/storage-jobs.ts create mode 100644 src/admin/wallet.ts delete mode 100644 src/asks/index.spec.ts delete mode 100644 src/asks/index.ts delete mode 100644 src/asks/util.ts create mode 100644 src/data/index.ts create mode 100644 src/data/types.ts create mode 100644 src/deals/index.ts create mode 100644 src/deals/types.ts delete mode 100644 src/faults/index.spec.ts delete mode 100644 src/faults/index.ts delete mode 100644 src/ffs/index.ts delete mode 100644 src/ffs/types.ts delete mode 100644 src/health/index.spec.ts delete mode 100644 src/health/index.ts delete mode 100644 src/index.spec.ts create mode 100644 src/integration.spec.ts delete mode 100644 src/miners/index.spec.ts delete mode 100644 src/miners/index.ts delete mode 100644 src/net/index.spec.ts delete mode 100644 src/net/index.ts rename src/{ffs => }/normalize.ts (100%) delete mode 100644 src/reputation/index.spec.ts delete mode 100644 src/reputation/index.ts create mode 100644 src/storage-config/index.spec.ts create mode 100644 src/storage-config/index.ts create mode 100644 src/storage-config/types.ts rename src/{ffs => storage-config}/util.ts (86%) create mode 100644 src/storage-jobs/index.ts create mode 100644 src/test-utils.ts diff --git a/src/admin/index.ts b/src/admin/index.ts new file mode 100644 index 00000000..704f2b17 --- /dev/null +++ b/src/admin/index.ts @@ -0,0 +1,34 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { Config } from "../types" +import { createProfiles, Profiles } from "./profiles" +import { createStorageJobs, StorageJobs } from "./storage-jobs" +import { createWallet, Wallet } from "./wallet" +export { Profiles, StorageJobs, Wallet } + +export interface Admin { + /** + * The admin Profiles API. + */ + profiles: Profiles + + /** + * The admin Wallet API. + */ + wallet: Wallet + + /** + * The admin Storage Jobs API. + */ + storageJobs: StorageJobs +} + +/** + * @ignore + */ +export const createAdmin = (config: Config, getMeta: () => grpc.Metadata): Admin => { + return { + profiles: createProfiles(config, getMeta), + wallet: createWallet(config, getMeta), + storageJobs: createStorageJobs(config, getMeta), + } +} diff --git a/src/admin/profiles.ts b/src/admin/profiles.ts new file mode 100644 index 00000000..cdeccd95 --- /dev/null +++ b/src/admin/profiles.ts @@ -0,0 +1,45 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + CreateStorageProfileRequest, + CreateStorageProfileResponse, + StorageProfilesRequest, + StorageProfilesResponse, +} from "@textile/grpc-powergate-client/dist/proto/admin/v1/powergate_admin_pb" +import { PowergateAdminServiceClient } from "@textile/grpc-powergate-client/dist/proto/admin/v1/powergate_admin_pb_service" +import { Config } from "../types" +import { promise } from "../util" + +export interface Profiles { + /** + * Create a new storage profile. + * @returns Information about the new storage profile. + */ + createStorageProfile: () => Promise + + /** + * List all storage profiles. + * @returns A list of all storage profiles. + */ + storageProfiles: () => Promise +} + +/** + * @ignore + */ +export const createProfiles = (config: Config, getMeta: () => grpc.Metadata): Profiles => { + const client = new PowergateAdminServiceClient(config.host, config) + return { + createStorageProfile: () => { + return promise( + (cb) => client.createStorageProfile(new CreateStorageProfileRequest(), getMeta(), cb), + (resp: CreateStorageProfileResponse) => resp.toObject(), + ) + }, + + storageProfiles: () => + promise( + (cb) => client.storageProfiles(new StorageProfilesRequest(), getMeta(), cb), + (resp: StorageProfilesResponse) => resp.toObject(), + ), + } +} diff --git a/src/admin/storage-jobs.ts b/src/admin/storage-jobs.ts new file mode 100644 index 00000000..5215b20b --- /dev/null +++ b/src/admin/storage-jobs.ts @@ -0,0 +1,125 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + ExecutingStorageJobsRequest, + ExecutingStorageJobsResponse, + LatestFinalStorageJobsRequest, + LatestFinalStorageJobsResponse, + LatestSuccessfulStorageJobsRequest, + LatestSuccessfulStorageJobsResponse, + QueuedStorageJobsRequest, + QueuedStorageJobsResponse, + StorageJobsSummaryRequest, + StorageJobsSummaryResponse, +} from "@textile/grpc-powergate-client/dist/proto/admin/v1/powergate_admin_pb" +import { PowergateAdminServiceClient } from "@textile/grpc-powergate-client/dist/proto/admin/v1/powergate_admin_pb_service" +import { Config } from "../types" +import { promise } from "../util" + +export interface StorageJobs { + /** + * List queued storgae jobs. + * @param profileId The storage profile id to query or an empty string for all storage profiles. + * @param cids An optional list of data cids to fileter the results with. + * @returns A list of queued storage jobs. + */ + queued: (profileId: string, ...cids: string[]) => Promise + + /** + * List executing storgae jobs. + * @param profileId The storage profile id to query or an empty string for all storage profiles. + * @param cids An optional list of data cids to fileter the results with. + * @returns A list of executing storage jobs. + */ + executing: ( + profileId: string, + ...cids: string[] + ) => Promise + + /** + * List the latest final storgae jobs. + * @param profileId The storage profile id to query or an empty string for all storage profiles. + * @param cids An optional list of data cids to fileter the results with. + * @returns A list of the latest final storage jobs. + */ + latestFinal: ( + profileId: string, + ...cids: string[] + ) => Promise + + /** + * List the latest successful storgae jobs. + * @param profileId The storage profile id to query or an empty string for all storage profiles. + * @param cids An optional list of data cids to fileter the results with. + * @returns A list of the latest successful storage jobs. + */ + latestSuccessful: ( + profileId: string, + ...cids: string[] + ) => Promise + + /** + * Get a summary of all jobs. + * @param profileId The storage profile id to query or an empty string for all storage profiles. + * @param cids An optional list of data cids to fileter the results with. + * @returns A summary of all jobs. + */ + summary: (profileId: string, ...cids: string[]) => Promise +} + +/** + * @ignore + */ +export const createStorageJobs = (config: Config, getMeta: () => grpc.Metadata): StorageJobs => { + const client = new PowergateAdminServiceClient(config.host, config) + return { + queued: (profileId: string, ...cids: string[]) => { + const req = new QueuedStorageJobsRequest() + req.setCidsList(cids) + req.setProfileId(profileId) + return promise( + (cb) => client.queuedStorageJobs(req, getMeta(), cb), + (resp: QueuedStorageJobsResponse) => resp.toObject(), + ) + }, + + executing: (profileId: string, ...cids: string[]) => { + const req = new ExecutingStorageJobsRequest() + req.setCidsList(cids) + req.setProfileId(profileId) + return promise( + (cb) => client.executingStorageJobs(req, getMeta(), cb), + (resp: ExecutingStorageJobsResponse) => resp.toObject(), + ) + }, + + latestFinal: (profileId: string, ...cids: string[]) => { + const req = new LatestFinalStorageJobsRequest() + req.setCidsList(cids) + req.setProfileId(profileId) + return promise( + (cb) => client.latestFinalStorageJobs(req, getMeta(), cb), + (resp: LatestFinalStorageJobsResponse) => resp.toObject(), + ) + }, + + latestSuccessful: (profileId: string, ...cids: string[]) => { + const req = new LatestSuccessfulStorageJobsRequest() + req.setCidsList(cids) + req.setProfileId(profileId) + return promise( + (cb) => client.latestSuccessfulStorageJobs(req, getMeta(), cb), + (resp: LatestSuccessfulStorageJobsResponse) => resp.toObject(), + ) + }, + + summary: (profileId: string, ...cids: string[]) => { + const req = new StorageJobsSummaryRequest() + req.setCidsList(cids) + req.setProfileId(profileId) + return promise( + (cb) => client.storageJobsSummary(req, getMeta(), cb), + (resp: StorageJobsSummaryResponse) => resp.toObject(), + ) + }, + } +} diff --git a/src/admin/wallet.ts b/src/admin/wallet.ts new file mode 100644 index 00000000..a908ef97 --- /dev/null +++ b/src/admin/wallet.ts @@ -0,0 +1,69 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + AddressesRequest, + AddressesResponse, + NewAddressRequest, + NewAddressResponse, + SendFilRequest, + SendFilResponse, +} from "@textile/grpc-powergate-client/dist/proto/admin/v1/powergate_admin_pb" +import { PowergateAdminServiceClient } from "@textile/grpc-powergate-client/dist/proto/admin/v1/powergate_admin_pb_service" +import { Config } from "../types" +import { promise } from "../util" + +export interface Wallet { + /** + * Create a new wallet address. + * @param type The type of address to create, bls or secp256k1. + * @returns The new address. + */ + newAddress: (type?: "bls" | "secp256k1") => Promise + + /** + * List all wallet addresses. + * @returns The list of wallet addresses. + */ + addresses: () => Promise + + /** + * Send Fil from one address to another. + * @param from The address to send from. + * @param to The address to send to. + * @param amount The amount of Fil to send. + */ + sendFil: (from: string, to: string, amount: number) => Promise +} + +/** + * @ignore + */ +export const createWallet = (config: Config, getMeta: () => grpc.Metadata): Wallet => { + const client = new PowergateAdminServiceClient(config.host, config) + return { + newAddress: (type: "bls" | "secp256k1" = "bls") => { + const req = new NewAddressRequest() + req.setType(type) + return promise( + (cb) => client.newAddress(req, getMeta(), cb), + (resp: NewAddressResponse) => resp.toObject(), + ) + }, + + addresses: () => + promise( + (cb) => client.addresses(new AddressesRequest(), getMeta(), cb), + (resp: AddressesResponse) => resp.toObject(), + ), + + sendFil: (from: string, to: string, amount: number) => { + const req = new SendFilRequest() + req.setFrom(from) + req.setTo(to) + req.setAmount(amount) + return promise( + (cb) => client.sendFil(req, getMeta(), cb), + (resp: SendFilResponse) => resp.toObject(), + ) + }, + } +} diff --git a/src/asks/index.spec.ts b/src/asks/index.spec.ts deleted file mode 100644 index b60481f1..00000000 --- a/src/asks/index.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Query } from "@textile/grpc-powergate-client/dist/index/ask/rpc/rpc_pb" -import { expect } from "chai" -import { createAsks } from "." -import { getTransport, host } from "../util" - -describe("asks", () => { - const c = createAsks({ host, transport: getTransport() }) - - it("should get", async () => { - const { index } = await c.get() - expect(index).not.undefined - }) - - it("should query", async () => { - const q = new Query().toObject() - const { asksList } = await c.query(q) - expect(asksList).not.undefined - }) -}) diff --git a/src/asks/index.ts b/src/asks/index.ts deleted file mode 100644 index fb69211b..00000000 --- a/src/asks/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - GetRequest, - GetResponse, - Query, - QueryRequest, - QueryResponse, -} from "@textile/grpc-powergate-client/dist/index/ask/rpc/rpc_pb" -import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/index/ask/rpc/rpc_pb_service" -import { Config } from "../types" -import { promise } from "../util" -import { queryObjectToMsg } from "./util" - -export interface Asks { - /** - * Gets the asks index. - * @returns The asks index. - */ - get: () => Promise - - /** - * Queries the asks index. - * @param query The query to run against the asks index. - * @returns The asks matching the provided query. - */ - query: (query: Query.AsObject) => Promise -} - -/** - * @ignore - */ -export const createAsks = (config: Config): Asks => { - const client = new RPCServiceClient(config.host, config) - return { - get: () => - promise( - (cb) => client.get(new GetRequest(), cb), - (resp: GetResponse) => resp.toObject(), - ), - - query: (query: Query.AsObject) => { - const req = new QueryRequest() - req.setQuery(queryObjectToMsg(query)) - return promise( - (cb) => client.query(req, cb), - (resp: QueryResponse) => resp.toObject(), - ) - }, - } -} diff --git a/src/asks/util.ts b/src/asks/util.ts deleted file mode 100644 index fada7195..00000000 --- a/src/asks/util.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Query } from "@textile/grpc-powergate-client/dist/index/ask/rpc/rpc_pb" - -export function queryObjectToMsg(query: Query.AsObject): Query { - const ret = new Query() - ret.setLimit(query.limit) - ret.setMaxPrice(query.maxPrice) - ret.setOffset(query.offset) - ret.setPieceSize(query.pieceSize) - return ret -} diff --git a/src/data/index.ts b/src/data/index.ts new file mode 100644 index 00000000..14cc5781 --- /dev/null +++ b/src/data/index.ts @@ -0,0 +1,233 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + CidInfoRequest, + CidInfoResponse, + GetRequest, + LogEntry, + ReplaceDataRequest, + ReplaceDataResponse, + StageRequest, + StageResponse, + WatchLogsRequest, +} from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" +import { + PowergateService, + PowergateServiceClient, +} from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb_service" +import fs from "fs" +import ipfsClient from "ipfs-http-client" +import block from "it-block" +import path from "path" +import { File, normaliseInput } from "../normalize" +import { Config } from "../types" +import { promise } from "../util" +import { GetFolderOptions, WatchLogsOptions } from "./types" + +export { GetFolderOptions, WatchLogsOptions } + +export interface Data { + /** + * A helper method to stage data in IPFS in preparation for storing using ffs.pushStorageConfig. + * This doesn't actually store data in FFS, you'll want to call pushStorageConfig for that. + * @param input The raw data to add. + * @returns The cid of the added data. + */ + stage: (input: Uint8Array) => Promise + + /** + * A helper method to stage a folder recursively in IPFS in preparation for storing using ffs.pushStorageConfig. + * This doesn't actually store data in FFS, you'll want to call pushStorageConfig for that. + * @param path The path to the folder to add. + * @returns The cid of the added folder. + */ + stageFolder: (path: string) => Promise + + /** + * Applies a StorageConfig for cid2 equal to that of cid1, and removes cid1. This operation + * is more efficient than manually removing and adding in two separate operations. + * @param cid1 The cid to replace. + * @param cid2 The new cid. + * @returns The job id of the job executing the storage configuration. + */ + replaceData: (cid1: string, cid2: string) => Promise + + /** + * Retrieve data stored in the current FFS instance. + * @param cid The cid of the data to retrieve. + * @returns The raw data. + */ + get: (cid: string) => Promise + + /** + * Retrieve a folder stored in the current FFS instance. + * @param cid The root cid of the folder to retrieve. + * @param outputPath The location to write the folder to + * @param opts Options controlling the behavior of retrieving the folder + */ + getFolder: (cid: string, output: string, opts?: GetFolderOptions) => Promise + + /** + * Listen for any updates for a stored cid. + * @param handler The callback to receive log updates. + * @param cid The cid to watch. + * @param opts Options that control the behavior of watching logs. + * @returns A function that can be used to cancel watching. + */ + watchLogs: ( + handler: (event: LogEntry.AsObject) => void, + cid: string, + opts?: WatchLogsOptions, + ) => () => void + + cidInfo: (...cids: string[]) => Promise +} + +/** + * @ignore + */ +export const createData = ( + config: Config, + getMeta: () => grpc.Metadata, + getHeaders: () => Record, +): Data => { + const client = new PowergateServiceClient(config.host, config) + const ipfs = ipfsClient(config.host) + return { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + stage: async (input: any) => { + // Only process the first input if there are more than one + const source: File | undefined = (await normaliseInput(input).next()).value + return new Promise(async (resolve, reject) => { + const client = grpc.client(PowergateService.Stage, config) + client.onMessage((message) => { + resolve(message.toObject() as StageResponse.AsObject) + }) + client.onEnd((code, msg) => { + if (code !== grpc.Code.OK) { + reject(`error code ${code} - ${msg}`) + } else { + reject("ended with no message") + } + }) + if (source?.content) { + client.start(getMeta()) + const process = await block({ size: 32000, noPad: true }) + for await (const chunk of process(source.content)) { + const buf = chunk.slice() + const req = new StageRequest() + req.setChunk(buf) + client.send(req) + } + client.finishSend() + } else { + reject(new Error("no content to stage")) + } + }) + }, + + stageFolder: async (path: string) => { + const src = ipfsClient.globSource(path, { recursive: true }) + const headers = getHeaders() + const res = await ipfs.add(src, { headers }) + return res.cid.string + }, + + get: (cid: string) => { + return new Promise((resolve, reject) => { + const append = (l: Uint8Array, r: Uint8Array) => { + const tmp = new Uint8Array(l.byteLength + r.byteLength) + tmp.set(l, 0) + tmp.set(r, l.byteLength) + return tmp + } + let final = new Uint8Array() + const req = new GetRequest() + req.setCid(cid) + const stream = client.get(req, getMeta()) + stream.on("data", (resp) => { + final = append(final, resp.getChunk_asU8()) + }) + stream.on("end", (status) => { + if (status?.code !== grpc.Code.OK) { + reject(`error code ${status?.code} - ${status?.details}`) + } else { + resolve(final) + } + }) + }) + }, + + getFolder: async (cid: string, output: string, opts: GetFolderOptions = {}) => { + const headers = getHeaders() + const options: Record = { headers } + if (opts.timeout) { + options["timeout"] = opts.timeout + } + for await (const file of ipfs.get(cid, options)) { + const noCidPath = file.path.replace(cid, "") + const fullFilePath = path.join(output, noCidPath) + if (file.content) { + await fs.promises.mkdir(path.join(output, path.dirname(file.path)), { recursive: true }) + const stream = fs.createWriteStream(fullFilePath) + for await (const chunk of file.content) { + const slice = chunk.slice() + await new Promise((resolve, reject) => { + stream.write(slice, (err) => { + if (err) { + reject(err) + } else { + resolve() + } + }) + }) + } + } else { + // this is a dir + await fs.promises.mkdir(fullFilePath, { recursive: true }) + } + } + }, + + watchLogs: ( + handler: (event: LogEntry.AsObject) => void, + cid: string, + opts: WatchLogsOptions = {}, + ) => { + const req = new WatchLogsRequest() + req.setCid(cid) + if (opts.includeHistory) { + req.setHistory(opts.includeHistory) + } + if (opts.jobId) { + req.setJobId(opts.jobId) + } + const stream = client.watchLogs(req, getMeta()) + stream.on("data", (res) => { + const logEntry = res.getLogEntry()?.toObject() + if (logEntry) { + handler(logEntry) + } + }) + return stream.cancel + }, + + replaceData: (cid1: string, cid2: string) => { + const req = new ReplaceDataRequest() + req.setCid1(cid1) + req.setCid2(cid2) + return promise( + (cb) => client.replaceData(req, getMeta(), cb), + (res: ReplaceDataResponse) => res.toObject(), + ) + }, + + cidInfo: (...cids: string[]) => { + const req = new CidInfoRequest() + req.setCidsList(cids) + return promise( + (cb) => client.cidInfo(req, getMeta(), cb), + (res: CidInfoResponse) => res.toObject(), + ) + }, + } +} diff --git a/src/data/types.ts b/src/data/types.ts new file mode 100644 index 00000000..a5f52e16 --- /dev/null +++ b/src/data/types.ts @@ -0,0 +1,24 @@ +/** + * Object that allows you to configure the call to getFolder + */ +export type GetFolderOptions = { + /** + * Timeout for fetching data + */ + timeout?: number +} + +/** + * Object control the behavior of watchLogs + */ +export type WatchLogsOptions = { + /** + * Control whether or not to include the history of log events + */ + includeHistory?: boolean + + /** + * Filter log events to only those associated with the provided job id + */ + jobId?: string +} diff --git a/src/deals/index.ts b/src/deals/index.ts new file mode 100644 index 00000000..c3b17df2 --- /dev/null +++ b/src/deals/index.ts @@ -0,0 +1,78 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + DealRecordsConfig, + RetrievalDealRecordsRequest, + RetrievalDealRecordsResponse, + StorageDealRecordsRequest, + StorageDealRecordsResponse, +} from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" +import { PowergateServiceClient } from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb_service" +import { Config } from "../types" +import { promise } from "../util" +import { DealRecordsOptions } from "./types" + +export { DealRecordsOptions } + +export interface Deals { + /** + * List storage deal records for the FFS instance according to the provided options. + * @param opts Options that control the behavior of listing records. + * @returns A list of storage deal records. + */ + storageDealRecords: (opts?: DealRecordsOptions) => Promise + + /** + * List retrieval deal records for the FFS instance according to the provided options. + * @param opts Options that control the behavior of listing records. + * @returns A list of retrieval deal records. + */ + retrievalDealRecords: ( + opts?: DealRecordsOptions, + ) => Promise +} + +/** + * @ignore + */ +export const createDeals = (config: Config, getMeta: () => grpc.Metadata): Deals => { + const client = new PowergateServiceClient(config.host, config) + return { + storageDealRecords: (opts: DealRecordsOptions = {}) => { + const req = new StorageDealRecordsRequest() + req.setConfig(listDealRecordsOptionsToConfig(opts)) + return promise( + (cb) => client.storageDealRecords(req, getMeta(), cb), + (res: StorageDealRecordsResponse) => res.toObject(), + ) + }, + + retrievalDealRecords: (opts: DealRecordsOptions = {}) => { + const req = new RetrievalDealRecordsRequest() + req.setConfig(listDealRecordsOptionsToConfig(opts)) + return promise( + (cb) => client.retrievalDealRecords(req, getMeta(), cb), + (res: RetrievalDealRecordsResponse) => res.toObject(), + ) + }, + } +} + +function listDealRecordsOptionsToConfig(opts: DealRecordsOptions) { + const conf = new DealRecordsConfig() + if (opts.ascending) { + conf.setAscending(opts.ascending) + } + if (opts.dataCids) { + conf.setDataCidsList(opts.dataCids) + } + if (opts.fromAddresses) { + conf.setFromAddrsList(opts.fromAddresses) + } + if (opts.includeFinal) { + conf.setIncludeFinal(opts.includeFinal) + } + if (opts.includePending) { + conf.setIncludePending(opts.includePending) + } + return conf +} diff --git a/src/deals/types.ts b/src/deals/types.ts new file mode 100644 index 00000000..141f9b3f --- /dev/null +++ b/src/deals/types.ts @@ -0,0 +1,34 @@ +/** + * Controls the behavior of listing deal records. + */ +export type DealRecordsOptions = { + /** + * Limits the results deals initiated from the provided wallet addresses. + */ + fromAddresses?: string[] + + /** + * Limits the results to deals for the provided data cids + */ + dataCids?: string[] + + /** + * Specifies whether or not to include pending deals in the results + * Default is false + * Ignored for listRetrievalDealRecords + */ + includePending?: boolean + + /** + * Specifies whether or not to include final deals in the results + * Default is false + * Ignored for listRetrievalDealRecords + */ + includeFinal?: boolean + + /** + * Specifies to sort the results in ascending order + * Default is descending order + * Records are sorted by timestamp */ + ascending?: boolean +} diff --git a/src/faults/index.spec.ts b/src/faults/index.spec.ts deleted file mode 100644 index af071923..00000000 --- a/src/faults/index.spec.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { expect } from "chai" -import { createFaults } from "." -import { getTransport, host } from "../util" - -describe("faults", () => { - const c = createFaults({ host, transport: getTransport() }) - - it("should get", async () => { - const { index } = await c.get() - expect(index).not.undefined - }) -}) diff --git a/src/faults/index.ts b/src/faults/index.ts deleted file mode 100644 index 6bb52337..00000000 --- a/src/faults/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { - GetRequest, - GetResponse, -} from "@textile/grpc-powergate-client/dist/index/faults/rpc/rpc_pb" -import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/index/faults/rpc/rpc_pb_service" -import { Config } from "../types" -import { promise } from "../util" - -export interface Faults { - /** - * Gets the faults index. - * @returns The faults index. - */ - get: () => Promise -} - -/** - * @ignore - */ -export const createFaults = (config: Config): Faults => { - const client = new RPCServiceClient(config.host, config) - return { - get: () => - promise( - (cb) => client.get(new GetRequest(), cb), - (resp: GetResponse) => resp.toObject(), - ), - } -} diff --git a/src/ffs/index.spec.ts b/src/ffs/index.spec.ts index 839a929d..56e68094 100644 --- a/src/ffs/index.spec.ts +++ b/src/ffs/index.spec.ts @@ -1,383 +1,383 @@ -import { - JobStatus, - JobStatusMap, - StorageConfig, -} from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" -import { expect } from "chai" -import fs from "fs" -import { createFFS } from "." -import { getTransport, host, useToken } from "../util" -import { PushStorageConfigOptions } from "./types" - -describe("ffs", function () { - this.timeout(180000) - - const { getMeta, getHeaders, setToken } = useToken("") - - const c = createFFS({ host, transport: getTransport() }, getMeta, getHeaders) - - it("should create an instance", async () => { - await expectNewInstance() - }) - - it("should list instances", async () => { - await expectNewInstance() - const res = await c.list() - expect(res.instancesList).length.greaterThan(0) - }) - - it("should get instance id", async () => { - const instanceInfo = await expectNewInstance() - const res = await c.id() - expect(res.id).eq(instanceInfo.id) - }) - - it("should get addrs", async () => { - await expectNewInstance() - await expectAddrs(1) - }) - - it("should get the default config", async () => { - await expectNewInstance() - await expectDefaultStorageConfig() - }) - - it("should create a new addr", async () => { - await expectNewInstance() - await expectAddrs(1) - await expectNewAddr() - await expectAddrs(2) - }) - - it("should set default config", async () => { - await expectNewInstance() - const defaultConfig = await expectDefaultStorageConfig() - await c.setDefaultStorageConfig(defaultConfig) - }) - - it("should get info", async () => { - await expectNewInstance() - const res = await c.info() - expect(res.info).not.undefined - }) - - it("should stage", async () => { - await expectNewInstance() - await expectStage("sample-data/samplefile") - }) - - it("should stage a folder", async () => { - await expectNewInstance() - const res = await c.stageFolder("./sample-data") - expect(res).length.greaterThan(0) - }) - - it("should push config", async () => { - await expectNewInstance() - const cid = await expectStage("sample-data/samplefile") - const config = await expectDefaultStorageConfig() - await expectPushStorageConfig(cid, { override: false, storageConfig: config }) - }) - - it("should get storage job", async () => { - await expectNewInstance() - const cid = await expectStage("sample-data/samplefile") - const jobId = await expectPushStorageConfig(cid) - const res = await c.getStorageJob(jobId) - expect(res.job?.cid).eq(cid) - }) - - it("should watch job", async () => { - await expectNewInstance() - const addrs = await expectAddrs(1) - await waitForBalance(addrs[0].addr, 0) - const cid = await expectStage("sample-data/samplefile") - const jobId = await expectPushStorageConfig(cid) - await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) - // Test that if we call watchJobs again after the job has finished - // that is calls us back with the latest status - await new Promise((r) => setTimeout(r, 10000)) - await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) - }) - - it("should watch logs", async () => { - await expectNewInstance() - const cid = await expectStage("sample-data/samplefile") - await expectPushStorageConfig(cid) - await new Promise((resolve, reject) => { - const cancel = c.watchLogs( - (logEvent) => { - if (logEvent.cid.length > 0) { - cancel() - resolve() - } else { - cancel() - reject("empty log cid") - } - }, - cid, - { includeHistory: true }, - ) - }) - }) - - it("should get storage deal records", async () => { - await expectNewInstance() - const addrs = await expectAddrs(1) - await waitForBalance(addrs[0].addr, 0) - - const [cid1, cid2, cid3] = await Promise.all([ - expectStage("sample-data/samplefile"), - expectStage("sample-data/samplefile2"), - expectStage("sample-data/samplefile3"), - ]) - - const jobId1 = await expectPushStorageConfig(cid1) - await new Promise((r) => setTimeout(r, 1000)) - const jobId2 = await expectPushStorageConfig(cid2) - await new Promise((r) => setTimeout(r, 1000)) - const jobId3 = await expectPushStorageConfig(cid3) - - await Promise.all([ - waitForJobStatus(jobId1, JobStatus.JOB_STATUS_EXECUTING), - waitForJobStatus(jobId2, JobStatus.JOB_STATUS_EXECUTING), - waitForJobStatus(jobId3, JobStatus.JOB_STATUS_EXECUTING), - ]) - - // wait for a second so some async work of actually starting a deal happens - await new Promise((r) => setTimeout(r, 1000)) - - const { recordsList: pendingRecords } = await c.listStorageDealRecords({ - includePending: true, - ascending: true, - fromAddresses: [addrs[0].addr], - dataCids: [cid1, cid2, cid3], - }) - expect(pendingRecords, "pending length").length(3) - expect(pendingRecords[0].time).lessThan(pendingRecords[1].time).lessThan(pendingRecords[2].time) - - const { recordsList: finalRecords } = await c.listStorageDealRecords({ includeFinal: true }) - expect(finalRecords, "final empty").empty - - await Promise.all([ - waitForJobStatus(jobId1, JobStatus.JOB_STATUS_SUCCESS), - waitForJobStatus(jobId2, JobStatus.JOB_STATUS_SUCCESS), - waitForJobStatus(jobId3, JobStatus.JOB_STATUS_SUCCESS), - ]) - - const { recordsList: pendingRecords2 } = await c.listStorageDealRecords({ - includePending: true, - }) - expect(pendingRecords2, "pending2 empty").empty - - const { recordsList: finalRecords2 } = await c.listStorageDealRecords({ - includeFinal: true, - ascending: true, - fromAddresses: [addrs[0].addr], - dataCids: [cid1, cid2, cid3], - }) - expect(finalRecords2, "final2 length").length(3) - expect(finalRecords2[0].time).lessThan(finalRecords2[1].time).lessThan(finalRecords2[2].time) - }) - - it("should get a retrieval deal record", async () => { - // ToDo: Figure out hot to make sure the data in the previous push isn't cached in hot - }) - - it("should get storage config", async () => { - await expectNewInstance() - const cid = await expectStage("sample-data/samplefile") - await expectPushStorageConfig(cid) - const res = await c.getStorageConfig(cid) - expect(res.config?.cold).not.undefined - expect(res.config?.hot).not.undefined - }) - - it("should show", async () => { - await expectNewInstance() - const addrs = await expectAddrs(1) - await waitForBalance(addrs[0].addr, 0) - const cid = await expectStage("sample-data/samplefile") - const jobId = await expectPushStorageConfig(cid) - await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) - const res = await c.show(cid) - expect(res.cidInfo).not.undefined - }) - - it("should show all", async () => { - await expectNewInstance() - const addrs = await expectAddrs(1) - await waitForBalance(addrs[0].addr, 0) - const cid = await expectStage("sample-data/samplefile") - const jobId = await expectPushStorageConfig(cid) - await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) - const res = await c.showAll() - expect(res.cidInfosList).not.empty - }) - - it("should replace", async () => { - await expectNewInstance() - const addrs = await expectAddrs(1) - await waitForBalance(addrs[0].addr, 0) - const cid = await expectStage("sample-data/samplefile") - const jobId = await expectPushStorageConfig(cid) - await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) - const cid2 = await expectStage("sample-data/samplefile2") - const res = await c.replace(cid, cid2) - expect(res.jobId).length.greaterThan(0) - }) - - it("should get", async () => { - await expectNewInstance() - const addrs = await expectAddrs(1) - await waitForBalance(addrs[0].addr, 0) - const cid = await expectStage("sample-data/samplefile") - const jobId = await expectPushStorageConfig(cid) - await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) - const bytes = await c.get(cid) - expect(bytes.byteLength).greaterThan(0) - }) - - it("should get a folder", async () => { - await expectNewInstance() - const res = await c.stageFolder("./sample-data") - expect(res).length.greaterThan(0) - await c.getFolder(res, "./output", { timeout: 10000 }) - }) - - it("should cancel a job", async () => { - await expectNewInstance() - const cid = await expectStage("sample-data/samplefile") - const jobId = await expectPushStorageConfig(cid) - await c.cancelJob(jobId) - }) - - it("should list payment channels", async () => { - // TODO - }) - - it("should create a payment channel", async () => { - // TODO - }) - - it("should redeem a payment channel", async () => { - // TODO - }) - - it("should remove", async () => { - await expectNewInstance() - const cid = await expectStage("sample-data/samplefile") - const conf: StorageConfig.AsObject = { - repairable: false, - cold: { - enabled: false, - }, - hot: { - allowUnfreeze: false, - enabled: false, - unfreezeMaxPrice: 0, - }, - } - const jobId = await expectPushStorageConfig(cid, { override: false, storageConfig: conf }) - await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) - await c.remove(cid) - }) - - it("should send fil", async () => { - await expectNewInstance() - const addrs = await expectAddrs(1) - await waitForBalance(addrs[0].addr, 0) - const addr = await expectNewAddr() - await c.sendFil(addrs[0].addr, addr, 10) - }) - - async function expectNewInstance() { - const res = await c.create() - expect(res.id).not.empty - expect(res.token).not.empty - setToken(res.token) - return res - } - - async function expectAddrs(length: number) { - const res = await c.addrs() - expect(res.addrsList).length(length) - return res.addrsList - } - - async function expectNewAddr() { - const res = await c.newAddr("my addr") - expect(res.addr).length.greaterThan(0) - return res.addr - } - - async function expectDefaultStorageConfig() { - const res = await c.defaultStorageConfig() - expect(res.defaultStorageConfig).not.undefined - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return res.defaultStorageConfig! - } - - async function expectStage(path: string) { - const buffer = fs.readFileSync(path) - const res = await c.stage(buffer) - expect(res.cid).length.greaterThan(0) - return res.cid - } - - async function expectPushStorageConfig(cid: string, opts?: PushStorageConfigOptions) { - const res = await c.pushStorageConfig(cid, opts) - expect(res.jobId).length.greaterThan(0) - return res.jobId - } - - function waitForJobStatus(jobId: string, status: JobStatusMap[keyof JobStatusMap]) { - return new Promise((resolve, reject) => { - try { - const cancel = c.watchJobs((job) => { - if (job.errCause.length > 0) { - reject(job.errCause) - } - if (job.status === JobStatus.JOB_STATUS_CANCELED) { - reject("job canceled") - } - if (job.status === JobStatus.JOB_STATUS_FAILED) { - reject("job failed") - } - if (job.status === status) { - cancel() - resolve() - } - }, jobId) - } catch (e) { - reject(e) - } - }) - } - - function waitForBalance(address: string, greaterThan: number) { - return new Promise(async (resolve, reject) => { - while (true) { - try { - const res = await c.info() - if (!res.info) { - reject("no balance info returned") - return - } - const info = res.info.balancesList.find((info) => info.addr?.addr === address) - if (!info) { - reject("address not in balances list") - return - } - if (info.balance > greaterThan) { - resolve(info.balance) - return - } - } catch (e) { - reject(e) - } - await new Promise((r) => setTimeout(r, 1000)) - } - }) - } -}) +// import { +// JobStatus, +// JobStatusMap, +// StorageConfig, +// } from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" +// import { expect } from "chai" +// import fs from "fs" +// import { createFFS } from "." +// import { getTransport, host, useTokens } from "../util" +// import { PushStorageConfigOptions } from "./types" + +// describe("ffs", function () { +// this.timeout(180000) + +// const { getMeta, getHeaders, setToken } = useTokens("") + +// const c = createFFS({ host, transport: getTransport() }, getMeta, getHeaders) + +// it("should create an instance", async () => { +// await expectNewInstance() +// }) + +// it("should list instances", async () => { +// await expectNewInstance() +// const res = await c.list() +// expect(res.instancesList).length.greaterThan(0) +// }) + +// it("should get instance id", async () => { +// const instanceInfo = await expectNewInstance() +// const res = await c.id() +// expect(res.id).eq(instanceInfo.id) +// }) + +// it("should get addrs", async () => { +// await expectNewInstance() +// await expectAddrs(1) +// }) + +// it("should get the default config", async () => { +// await expectNewInstance() +// await expectDefaultStorageConfig() +// }) + +// it("should create a new addr", async () => { +// await expectNewInstance() +// await expectAddrs(1) +// await expectNewAddr() +// await expectAddrs(2) +// }) + +// it("should set default config", async () => { +// await expectNewInstance() +// const defaultConfig = await expectDefaultStorageConfig() +// await c.setDefaultStorageConfig(defaultConfig) +// }) + +// it("should get info", async () => { +// await expectNewInstance() +// const res = await c.info() +// expect(res.info).not.undefined +// }) + +// it("should stage", async () => { +// await expectNewInstance() +// await expectStage("sample-data/samplefile") +// }) + +// it("should stage a folder", async () => { +// await expectNewInstance() +// const res = await c.stageFolder("./sample-data") +// expect(res).length.greaterThan(0) +// }) + +// it("should push config", async () => { +// await expectNewInstance() +// const cid = await expectStage("sample-data/samplefile") +// const config = await expectDefaultStorageConfig() +// await expectPushStorageConfig(cid, { override: false, storageConfig: config }) +// }) + +// it("should get storage job", async () => { +// await expectNewInstance() +// const cid = await expectStage("sample-data/samplefile") +// const jobId = await expectPushStorageConfig(cid) +// const res = await c.getStorageJob(jobId) +// expect(res.job?.cid).eq(cid) +// }) + +// it("should watch job", async () => { +// await expectNewInstance() +// const addrs = await expectAddrs(1) +// await waitForBalance(addrs[0].addr, 0) +// const cid = await expectStage("sample-data/samplefile") +// const jobId = await expectPushStorageConfig(cid) +// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) +// // Test that if we call watchJobs again after the job has finished +// // that is calls us back with the latest status +// await new Promise((r) => setTimeout(r, 10000)) +// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) +// }) + +// it("should watch logs", async () => { +// await expectNewInstance() +// const cid = await expectStage("sample-data/samplefile") +// await expectPushStorageConfig(cid) +// await new Promise((resolve, reject) => { +// const cancel = c.watchLogs( +// (logEvent) => { +// if (logEvent.cid.length > 0) { +// cancel() +// resolve() +// } else { +// cancel() +// reject("empty log cid") +// } +// }, +// cid, +// { includeHistory: true }, +// ) +// }) +// }) + +// it("should get storage deal records", async () => { +// await expectNewInstance() +// const addrs = await expectAddrs(1) +// await waitForBalance(addrs[0].addr, 0) + +// const [cid1, cid2, cid3] = await Promise.all([ +// expectStage("sample-data/samplefile"), +// expectStage("sample-data/samplefile2"), +// expectStage("sample-data/samplefile3"), +// ]) + +// const jobId1 = await expectPushStorageConfig(cid1) +// await new Promise((r) => setTimeout(r, 1000)) +// const jobId2 = await expectPushStorageConfig(cid2) +// await new Promise((r) => setTimeout(r, 1000)) +// const jobId3 = await expectPushStorageConfig(cid3) + +// await Promise.all([ +// waitForJobStatus(jobId1, JobStatus.JOB_STATUS_EXECUTING), +// waitForJobStatus(jobId2, JobStatus.JOB_STATUS_EXECUTING), +// waitForJobStatus(jobId3, JobStatus.JOB_STATUS_EXECUTING), +// ]) + +// // wait for a second so some async work of actually starting a deal happens +// await new Promise((r) => setTimeout(r, 1000)) + +// const { recordsList: pendingRecords } = await c.listStorageDealRecords({ +// includePending: true, +// ascending: true, +// fromAddresses: [addrs[0].addr], +// dataCids: [cid1, cid2, cid3], +// }) +// expect(pendingRecords, "pending length").length(3) +// expect(pendingRecords[0].time).lessThan(pendingRecords[1].time).lessThan(pendingRecords[2].time) + +// const { recordsList: finalRecords } = await c.listStorageDealRecords({ includeFinal: true }) +// expect(finalRecords, "final empty").empty + +// await Promise.all([ +// waitForJobStatus(jobId1, JobStatus.JOB_STATUS_SUCCESS), +// waitForJobStatus(jobId2, JobStatus.JOB_STATUS_SUCCESS), +// waitForJobStatus(jobId3, JobStatus.JOB_STATUS_SUCCESS), +// ]) + +// const { recordsList: pendingRecords2 } = await c.listStorageDealRecords({ +// includePending: true, +// }) +// expect(pendingRecords2, "pending2 empty").empty + +// const { recordsList: finalRecords2 } = await c.listStorageDealRecords({ +// includeFinal: true, +// ascending: true, +// fromAddresses: [addrs[0].addr], +// dataCids: [cid1, cid2, cid3], +// }) +// expect(finalRecords2, "final2 length").length(3) +// expect(finalRecords2[0].time).lessThan(finalRecords2[1].time).lessThan(finalRecords2[2].time) +// }) + +// it("should get a retrieval deal record", async () => { +// // ToDo: Figure out hot to make sure the data in the previous push isn't cached in hot +// }) + +// it("should get storage config", async () => { +// await expectNewInstance() +// const cid = await expectStage("sample-data/samplefile") +// await expectPushStorageConfig(cid) +// const res = await c.getStorageConfig(cid) +// expect(res.config?.cold).not.undefined +// expect(res.config?.hot).not.undefined +// }) + +// it("should show", async () => { +// await expectNewInstance() +// const addrs = await expectAddrs(1) +// await waitForBalance(addrs[0].addr, 0) +// const cid = await expectStage("sample-data/samplefile") +// const jobId = await expectPushStorageConfig(cid) +// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) +// const res = await c.show(cid) +// expect(res.cidInfo).not.undefined +// }) + +// it("should show all", async () => { +// await expectNewInstance() +// const addrs = await expectAddrs(1) +// await waitForBalance(addrs[0].addr, 0) +// const cid = await expectStage("sample-data/samplefile") +// const jobId = await expectPushStorageConfig(cid) +// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) +// const res = await c.showAll() +// expect(res.cidInfosList).not.empty +// }) + +// it("should replace", async () => { +// await expectNewInstance() +// const addrs = await expectAddrs(1) +// await waitForBalance(addrs[0].addr, 0) +// const cid = await expectStage("sample-data/samplefile") +// const jobId = await expectPushStorageConfig(cid) +// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) +// const cid2 = await expectStage("sample-data/samplefile2") +// const res = await c.replace(cid, cid2) +// expect(res.jobId).length.greaterThan(0) +// }) + +// it("should get", async () => { +// await expectNewInstance() +// const addrs = await expectAddrs(1) +// await waitForBalance(addrs[0].addr, 0) +// const cid = await expectStage("sample-data/samplefile") +// const jobId = await expectPushStorageConfig(cid) +// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) +// const bytes = await c.get(cid) +// expect(bytes.byteLength).greaterThan(0) +// }) + +// it("should get a folder", async () => { +// await expectNewInstance() +// const res = await c.stageFolder("./sample-data") +// expect(res).length.greaterThan(0) +// await c.getFolder(res, "./output", { timeout: 10000 }) +// }) + +// it("should cancel a job", async () => { +// await expectNewInstance() +// const cid = await expectStage("sample-data/samplefile") +// const jobId = await expectPushStorageConfig(cid) +// await c.cancelJob(jobId) +// }) + +// it("should list payment channels", async () => { +// // TODO +// }) + +// it("should create a payment channel", async () => { +// // TODO +// }) + +// it("should redeem a payment channel", async () => { +// // TODO +// }) + +// it("should remove", async () => { +// await expectNewInstance() +// const cid = await expectStage("sample-data/samplefile") +// const conf: StorageConfig.AsObject = { +// repairable: false, +// cold: { +// enabled: false, +// }, +// hot: { +// allowUnfreeze: false, +// enabled: false, +// unfreezeMaxPrice: 0, +// }, +// } +// const jobId = await expectPushStorageConfig(cid, { override: false, storageConfig: conf }) +// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) +// await c.remove(cid) +// }) + +// it("should send fil", async () => { +// await expectNewInstance() +// const addrs = await expectAddrs(1) +// await waitForBalance(addrs[0].addr, 0) +// const addr = await expectNewAddr() +// await c.sendFil(addrs[0].addr, addr, 10) +// }) + +// async function expectNewInstance() { +// const res = await c.create() +// expect(res.id).not.empty +// expect(res.token).not.empty +// setToken(res.token) +// return res +// } + +// async function expectAddrs(length: number) { +// const res = await c.addrs() +// expect(res.addrsList).length(length) +// return res.addrsList +// } + +// async function expectNewAddr() { +// const res = await c.newAddr("my addr") +// expect(res.addr).length.greaterThan(0) +// return res.addr +// } + +// async function expectDefaultStorageConfig() { +// const res = await c.defaultStorageConfig() +// expect(res.defaultStorageConfig).not.undefined +// // eslint-disable-next-line @typescript-eslint/no-non-null-assertion +// return res.defaultStorageConfig! +// } + +// async function expectStage(path: string) { +// const buffer = fs.readFileSync(path) +// const res = await c.stage(buffer) +// expect(res.cid).length.greaterThan(0) +// return res.cid +// } + +// async function expectPushStorageConfig(cid: string, opts?: PushStorageConfigOptions) { +// const res = await c.pushStorageConfig(cid, opts) +// expect(res.jobId).length.greaterThan(0) +// return res.jobId +// } + +// function waitForJobStatus(jobId: string, status: JobStatusMap[keyof JobStatusMap]) { +// return new Promise((resolve, reject) => { +// try { +// const cancel = c.watchJobs((job) => { +// if (job.errCause.length > 0) { +// reject(job.errCause) +// } +// if (job.status === JobStatus.JOB_STATUS_CANCELED) { +// reject("job canceled") +// } +// if (job.status === JobStatus.JOB_STATUS_FAILED) { +// reject("job failed") +// } +// if (job.status === status) { +// cancel() +// resolve() +// } +// }, jobId) +// } catch (e) { +// reject(e) +// } +// }) +// } + +// function waitForBalance(address: string, greaterThan: number) { +// return new Promise(async (resolve, reject) => { +// while (true) { +// try { +// const res = await c.info() +// if (!res.info) { +// reject("no balance info returned") +// return +// } +// const info = res.info.balancesList.find((info) => info.addr?.addr === address) +// if (!info) { +// reject("address not in balances list") +// return +// } +// if (info.balance > greaterThan) { +// resolve(info.balance) +// return +// } +// } catch (e) { +// reject(e) +// } +// await new Promise((r) => setTimeout(r, 1000)) +// } +// }) +// } +// }) diff --git a/src/ffs/index.ts b/src/ffs/index.ts deleted file mode 100644 index ef7ca0a8..00000000 --- a/src/ffs/index.ts +++ /dev/null @@ -1,669 +0,0 @@ -import { grpc } from "@improbable-eng/grpc-web" -import { - AddrsRequest, - AddrsResponse, - CancelJobRequest, - CreatePayChannelRequest, - CreatePayChannelResponse, - CreateRequest, - CreateResponse, - DefaultStorageConfigRequest, - DefaultStorageConfigResponse, - GetRequest, - GetStorageConfigRequest, - GetStorageConfigResponse, - GetStorageJobRequest, - GetStorageJobResponse, - IDRequest, - IDResponse, - InfoRequest, - InfoResponse, - Job, - ListAPIRequest, - ListAPIResponse, - ListDealRecordsConfig, - ListPayChannelsRequest, - ListPayChannelsResponse, - ListRetrievalDealRecordsRequest, - ListRetrievalDealRecordsResponse, - ListStorageDealRecordsRequest, - ListStorageDealRecordsResponse, - LogEntry, - NewAddrRequest, - NewAddrResponse, - PushStorageConfigRequest, - PushStorageConfigResponse, - RedeemPayChannelRequest, - RemoveRequest, - ReplaceRequest, - ReplaceResponse, - SendFilRequest, - SetDefaultStorageConfigRequest, - ShowAllRequest, - ShowAllResponse, - ShowRequest, - ShowResponse, - StageRequest, - StageResponse, - StorageConfig, - WatchJobsRequest, - WatchLogsRequest, -} from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" -import { - RPCService, - RPCServiceClient, -} from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb_service" -import fs from "fs" -import ipfsClient from "ipfs-http-client" -import block from "it-block" -import path from "path" -import { Config } from "../types" -import { promise } from "../util" -import { File, normaliseInput } from "./normalize" -import { - GetFolderOptions, - ListDealRecordsOptions, - PushStorageConfigOptions, - WatchLogsOptions, -} from "./types" -import { coldObjToMessage, hotObjToMessage } from "./util" - -export { GetFolderOptions, ListDealRecordsOptions, WatchLogsOptions, PushStorageConfigOptions } - -export interface FFS { - /** - * Creates a new FFS instance. - * @returns Information about the new FFS instance. - */ - create: () => Promise - - /** - * Lists all FFS instance IDs. - * @returns A list off all FFS instance IDs. - */ - list: () => Promise - - /** - * Get the FFS instance ID associated with the current auth token. - * @returns A Promise containing the FFS instance ID. - */ - id: () => Promise - - /** - * Get all wallet addresses associated with the current auth token. - * @returns A list of wallet addresses. - */ - addrs: () => Promise - - /** - * Get the default storage config associated with the current auth token. - * @returns The default storage config. - */ - defaultStorageConfig: () => Promise - - /** - * Create a new wallet address associates with the current auth token. - * @param name A human readable name for the address. - * @param type Address type, defaults to bls. - * @param makeDefault Specify if the new address should become the default address for this FFS instance, defaults to false. - * @returns Information about the newly created address. - */ - newAddr: ( - name: string, - type?: "bls" | "secp256k1" | undefined, - makeDefault?: boolean | undefined, - ) => Promise - - /** - * Get the desired storage config for the provided cid, this config may not yet be realized. - * @param cid The cid of the desired storage config. - * @returns The storage config for the provided cid. - */ - getStorageConfig: (cid: string) => Promise - - /** - * Set the default storage config for this FFS instance. - * @param config The new default storage config. - */ - setDefaultStorageConfig: (config: StorageConfig.AsObject) => Promise - - /** - * Get the current storage config for the provided cid, the reflects the actual storage state. - * @param cid The cid of the desired storage config. - * @returns The current storage config for the provided cid. - */ - show: (cid: string) => Promise - - /** - * Get general information about the current FFS instance. - * @returns Information about the FFS instance. - */ - info: () => Promise - - /** - * Get the current state of a storage job. - * @param jobId The job id to query. - * @returns The current state of the storage job. - */ - getStorageJob: (jobId: string) => Promise - - /** - * Listen for job updates for the provided job ids. - * @param handler The callback to receive job updates. - * @param jobs A list of job ids to watch. - * @returns A function that can be used to cancel watching. - */ - watchJobs: (handler: (event: Job.AsObject) => void, ...jobs: string[]) => () => void - - /** - * Cancel a job. - * @param jobId The id of the job to cancel. - */ - cancelJob: (jobId: string) => Promise - - /** - * Listen for any updates for a stored cid. - * @param handler The callback to receive log updates. - * @param cid The cid to watch. - * @param opts Options that control the behavior of watching logs. - * @returns A function that can be used to cancel watching. - */ - watchLogs: ( - handler: (event: LogEntry.AsObject) => void, - cid: string, - opts?: WatchLogsOptions, - ) => () => void - - /** - * Replace pushes a StorageConfig for cid2 equal to that of cid1, and removes cid1. This operation - * is more efficient than manually removing and adding in two separate operations. - * @param cid1 The cid to replace. - * @param cid2 The new cid. - * @returns The job id of the job executing the storage configuration. - */ - replace: (cid1: string, cid2: string) => Promise - - /** - * Push a storage config for the specified cid. - * @param cid The cid to store. - * @param opts Options controlling the behavior storage config execution. - * @returns An object containing the job id of the job executing the storage configuration. - */ - pushStorageConfig: ( - cid: string, - opts?: PushStorageConfigOptions, - ) => Promise - - /** - * Remove a cid from FFS storage. - * @param cid The cid to remove. - */ - remove: (cid: string) => Promise - - /** - * Retrieve data stored in the current FFS instance. - * @param cid The cid of the data to retrieve. - * @returns The raw data. - */ - get: (cid: string) => Promise - - /** - * Retrieve a folder stored in the current FFS instance. - * @param cid The root cid of the folder to retrieve. - * @param outputPath The location to write the folder to - * @param opts Options controlling the behavior of retrieving the folder - */ - getFolder: (cid: string, output: string, opts?: GetFolderOptions) => Promise - - /** - * Send FIL from an address associated with the current FFS instance to any other address. - * @param from The address to send FIL from. - * @param to The address to send FIL to. - * @param amount The amount of FIL to send. - */ - sendFil: (from: string, to: string, amount: number) => Promise - - /** - * A helper method to stage data in IPFS in preparation for storing using ffs.pushStorageConfig. - * This doesn't actually store data in FFS, you'll want to call pushStorageConfig for that. - * @param input The raw data to add. - * @returns The cid of the added data. - */ - stage: (input: Uint8Array) => Promise - - /** - * A helper method to stage a folder recursively in IPFS in preparation for storing using ffs.pushStorageConfig. - * This doesn't actually store data in FFS, you'll want to call pushStorageConfig for that. - * @param path The path to the folder to add. - * @returns The cid of the added folder. - */ - stageFolder: (path: string) => Promise - - /** - * List all payment channels for the current FFS instance. - * @returns A list of payment channel info. - */ - listPayChannels: () => Promise - - /** - * Create or get a payment channel. - * @param from The address to send FIL from. - * @param to The address to send FIL to. - * @param amt The amount to ensure exists in the payment channel. - * @returns Information about the payment channel. - */ - createPayChannel: ( - from: string, - to: string, - amt: number, - ) => Promise - - /** - * Redeem a payment channel. - * @param payChannelAddr The address of the payment channel to redeem. - */ - redeemPayChannel: (payChannelAddr: string) => Promise - - /** - * List storage deal records for the FFS instance according to the provided options. - * @param opts Options that control the behavior of listing records. - * @returns A list of storage deal records. - */ - listStorageDealRecords: ( - opts?: ListDealRecordsOptions, - ) => Promise - - /** - * List retrieval deal records for the FFS instance according to the provided options. - * @param opts Options that control the behavior of listing records. - * @returns A list of retrieval deal records. - */ - listRetrievalDealRecords: ( - opts?: ListDealRecordsOptions, - ) => Promise - - /** - * List cid infos for all data stored in the current FFS instance. - * @returns A list of cid info. - */ - showAll: () => Promise -} - -/** - * @ignore - */ -export const createFFS = ( - config: Config, - getMeta: () => grpc.Metadata, - getHeaders: () => Record, -): FFS => { - const client = new RPCServiceClient(config.host, config) - const ipfs = ipfsClient(config.host) - return { - create: () => - promise( - (cb) => client.create(new CreateRequest(), cb), - (res: CreateResponse) => res.toObject(), - ), - - list: () => - promise( - (cb) => client.listAPI(new ListAPIRequest(), cb), - (res: ListAPIResponse) => res.toObject(), - ), - - id: () => - promise( - (cb) => client.iD(new IDRequest(), getMeta(), cb), - (res: IDResponse) => res.toObject(), - ), - - addrs: () => - promise( - (cb) => client.addrs(new AddrsRequest(), getMeta(), cb), - (res: AddrsResponse) => res.toObject(), - ), - - defaultStorageConfig: () => - promise( - (cb) => client.defaultStorageConfig(new DefaultStorageConfigRequest(), getMeta(), cb), - (res: DefaultStorageConfigResponse) => res.toObject(), - ), - - newAddr: (name: string, type?: "bls" | "secp256k1", makeDefault?: boolean) => { - const req = new NewAddrRequest() - req.setName(name) - req.setAddressType(type || "bls") - req.setMakeDefault(makeDefault || false) - return promise( - (cb) => client.newAddr(req, getMeta(), cb), - (res: NewAddrResponse) => res.toObject(), - ) - }, - - getStorageConfig: (cid: string) => { - const req = new GetStorageConfigRequest() - req.setCid(cid) - return promise( - (cb) => client.getStorageConfig(req, getMeta(), cb), - (res: GetStorageConfigResponse) => res.toObject(), - ) - }, - - setDefaultStorageConfig: (config: StorageConfig.AsObject) => { - const c = new StorageConfig() - c.setRepairable(config.repairable) - if (config.hot) { - c.setHot(hotObjToMessage(config.hot)) - } - if (config.cold) { - c.setCold(coldObjToMessage(config.cold)) - } - const req = new SetDefaultStorageConfigRequest() - req.setConfig(c) - return promise( - (cb) => client.setDefaultStorageConfig(req, getMeta(), cb), - () => { - // nothing to return - }, - ) - }, - - show: (cid: string) => { - const req = new ShowRequest() - req.setCid(cid) - return promise( - (cb) => client.show(req, getMeta(), cb), - (res: ShowResponse) => res.toObject(), - ) - }, - - info: () => - promise( - (cb) => client.info(new InfoRequest(), getMeta(), cb), - (res: InfoResponse) => res.toObject(), - ), - - getStorageJob: (jobId: string) => { - const req = new GetStorageJobRequest() - req.setJid(jobId) - return promise( - (cb) => client.getStorageJob(req, getMeta(), cb), - (res: GetStorageJobResponse) => res.toObject(), - ) - }, - - watchJobs: (handler: (event: Job.AsObject) => void, ...jobs: string[]) => { - const req = new WatchJobsRequest() - req.setJidsList(jobs) - const stream = client.watchJobs(req, getMeta()) - stream.on("data", (res) => { - const job = res.getJob()?.toObject() - if (job) { - handler(job) - } - }) - return stream.cancel - }, - - cancelJob: (jobId: string) => { - const req = new CancelJobRequest() - req.setJid(jobId) - return promise( - (cb) => client.cancelJob(req, getMeta(), cb), - () => { - // nothing to return - }, - ) - }, - - watchLogs: ( - handler: (event: LogEntry.AsObject) => void, - cid: string, - opts: WatchLogsOptions = {}, - ) => { - const req = new WatchLogsRequest() - req.setCid(cid) - if (opts.includeHistory) { - req.setHistory(opts.includeHistory) - } - if (opts.jobId) { - req.setJid(opts.jobId) - } - const stream = client.watchLogs(req, getMeta()) - stream.on("data", (res) => { - const logEntry = res.getLogEntry()?.toObject() - if (logEntry) { - handler(logEntry) - } - }) - return stream.cancel - }, - - replace: (cid1: string, cid2: string) => { - const req = new ReplaceRequest() - req.setCid1(cid1) - req.setCid2(cid2) - return promise( - (cb) => client.replace(req, getMeta(), cb), - (res: ReplaceResponse) => res.toObject(), - ) - }, - - pushStorageConfig: (cid: string, opts: PushStorageConfigOptions = {}) => { - const req = new PushStorageConfigRequest() - req.setCid(cid) - if (opts.override) { - req.setOverrideConfig(opts.override) - req.setHasOverrideConfig(true) - } - if (opts.storageConfig) { - const c = new StorageConfig() - c.setRepairable(opts.storageConfig.repairable) - if (opts.storageConfig.hot) { - c.setHot(hotObjToMessage(opts.storageConfig.hot)) - } - if (opts.storageConfig.cold) { - c.setCold(coldObjToMessage(opts.storageConfig.cold)) - } - req.setConfig(c) - req.setHasConfig(true) - } - return promise( - (cb) => client.pushStorageConfig(req, getMeta(), cb), - (res: PushStorageConfigResponse) => res.toObject(), - ) - }, - - remove: (cid: string) => { - const req = new RemoveRequest() - req.setCid(cid) - return promise( - (cb) => client.remove(req, getMeta(), cb), - () => { - // nothing to return - }, - ) - }, - - get: (cid: string) => { - return new Promise((resolve, reject) => { - const append = (l: Uint8Array, r: Uint8Array) => { - const tmp = new Uint8Array(l.byteLength + r.byteLength) - tmp.set(l, 0) - tmp.set(r, l.byteLength) - return tmp - } - let final = new Uint8Array() - const req = new GetRequest() - req.setCid(cid) - const stream = client.get(req, getMeta()) - stream.on("data", (resp) => { - final = append(final, resp.getChunk_asU8()) - }) - stream.on("end", (status) => { - if (status?.code !== grpc.Code.OK) { - reject(`error code ${status?.code} - ${status?.details}`) - } else { - resolve(final) - } - }) - }) - }, - - getFolder: async (cid: string, output: string, opts: GetFolderOptions = {}) => { - const headers = getHeaders() - const options: Record = { headers } - if (opts.timeout) { - options["timeout"] = opts.timeout - } - for await (const file of ipfs.get(cid, options)) { - const noCidPath = file.path.replace(cid, "") - const fullFilePath = path.join(output, noCidPath) - if (file.content) { - await fs.promises.mkdir(path.join(output, path.dirname(file.path)), { recursive: true }) - const stream = fs.createWriteStream(fullFilePath) - for await (const chunk of file.content) { - const slice = chunk.slice() - await new Promise((resolve, reject) => { - stream.write(slice, (err) => { - if (err) { - reject(err) - } else { - resolve() - } - }) - }) - } - } else { - // this is a dir - await fs.promises.mkdir(fullFilePath, { recursive: true }) - } - } - }, - - sendFil: (from: string, to: string, amount: number) => { - const req = new SendFilRequest() - req.setFrom(from) - req.setTo(to) - req.setAmount(amount) - return promise( - (cb) => client.sendFil(req, getMeta(), cb), - () => { - // nothing to return - }, - ) - }, - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - stage: async (input: any) => { - // Only process the first input if there are more than one - const source: File | undefined = (await normaliseInput(input).next()).value - return new Promise(async (resolve, reject) => { - const client = grpc.client(RPCService.Stage, config) - client.onMessage((message) => { - resolve(message.toObject() as StageResponse.AsObject) - }) - client.onEnd((code, msg) => { - if (code !== grpc.Code.OK) { - reject(`error code ${code} - ${msg}`) - } else { - reject("ended with no message") - } - }) - if (source?.content) { - client.start(getMeta()) - const process = await block({ size: 32000, noPad: true }) - for await (const chunk of process(source.content)) { - const buf = chunk.slice() - const req = new StageRequest() - req.setChunk(buf as Buffer) - client.send(req) - } - client.finishSend() - } else { - reject(new Error("no content to stage")) - } - }) - }, - - stageFolder: async (path: string) => { - const src = ipfsClient.globSource(path, { recursive: true }) - const headers = getHeaders() - const res = await ipfs.add(src, { headers }) - return res.cid.string - }, - - listPayChannels: () => - promise( - (cb) => client.listPayChannels(new ListPayChannelsRequest(), getMeta(), cb), - (res: ListPayChannelsResponse) => res.toObject(), - ), - - createPayChannel: (from: string, to: string, amt: number) => { - const req = new CreatePayChannelRequest() - req.setFrom(from) - req.setTo(to) - req.setAmount(amt) - return promise( - (cb) => client.createPayChannel(req, getMeta(), cb), - (res: CreatePayChannelResponse) => res.toObject(), - ) - }, - - redeemPayChannel: (payChannelAddr: string) => { - const req = new RedeemPayChannelRequest() - req.setPayChannelAddr(payChannelAddr) - return promise( - (cb) => client.redeemPayChannel(req, getMeta(), cb), - () => { - // nothing to return - }, - ) - }, - - listStorageDealRecords: (opts: ListDealRecordsOptions = {}) => { - const req = new ListStorageDealRecordsRequest() - req.setConfig(listDealRecordsOptionsToConfig(opts)) - return promise( - (cb) => client.listStorageDealRecords(req, getMeta(), cb), - (res: ListStorageDealRecordsResponse) => res.toObject(), - ) - }, - - listRetrievalDealRecords: (opts: ListDealRecordsOptions = {}) => { - const req = new ListRetrievalDealRecordsRequest() - req.setConfig(listDealRecordsOptionsToConfig(opts)) - return promise( - (cb) => client.listRetrievalDealRecords(req, getMeta(), cb), - (res: ListRetrievalDealRecordsResponse) => res.toObject(), - ) - }, - - showAll: () => - promise( - (cb) => client.showAll(new ShowAllRequest(), getMeta(), cb), - (res: ShowAllResponse) => res.toObject(), - ), - } -} - -function listDealRecordsOptionsToConfig(opts: ListDealRecordsOptions) { - const conf = new ListDealRecordsConfig() - if (opts.ascending) { - conf.setAscending(opts.ascending) - } - if (opts.dataCids) { - conf.setDataCidsList(opts.dataCids) - } - if (opts.fromAddresses) { - conf.setFromAddrsList(opts.fromAddresses) - } - if (opts.includeFinal) { - conf.setIncludeFinal(opts.includeFinal) - } - if (opts.includePending) { - conf.setIncludePending(opts.includePending) - } - return conf -} diff --git a/src/ffs/types.ts b/src/ffs/types.ts deleted file mode 100644 index 6effe20c..00000000 --- a/src/ffs/types.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { StorageConfig } from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" - -/** - * Options to control the behavior of pushStorageConfig. - */ -export type PushStorageConfigOptions = { - /** - * Allows you to override an existing storage configuration - */ - override?: boolean - - /** - * Allows you to override the default storage config with a custom one - */ - storageConfig?: StorageConfig.AsObject -} - -/** - * Object that allows you to configure the call to getFolder - */ -export type GetFolderOptions = { - /** - * Timeout for fetching data - */ - timeout?: number -} - -/** - * Controls the behavior of listing deal records. - */ -export type ListDealRecordsOptions = { - /** - * Limits the results deals initiated from the provided wallet addresses. - */ - fromAddresses?: string[] - - /** - * Limits the results to deals for the provided data cids - */ - dataCids?: string[] - - /** - * Specifies whether or not to include pending deals in the results - * Default is false - * Ignored for listRetrievalDealRecords - */ - includePending?: boolean - - /** - * Specifies whether or not to include final deals in the results - * Default is false - * Ignored for listRetrievalDealRecords - */ - includeFinal?: boolean - - /** - * Specifies to sort the results in ascending order - * Default is descending order - * Records are sorted by timestamp */ - ascending?: boolean -} - -/** - * Object control the behavior of watchLogs - */ -export type WatchLogsOptions = { - /** - * Control whether or not to include the history of log events - */ - includeHistory?: boolean - - /** - * Filter log events to only those associated with the provided job id - */ - jobId?: string -} diff --git a/src/health/index.spec.ts b/src/health/index.spec.ts deleted file mode 100644 index b10725f4..00000000 --- a/src/health/index.spec.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Status } from "@textile/grpc-powergate-client/dist/health/rpc/rpc_pb" -import { expect } from "chai" -import { createHealth } from "." -import { getTransport, host } from "../util" - -describe("health", () => { - const c = createHealth({ host, transport: getTransport() }) - - it("should check health", async () => { - const status = await c.check() - expect(status.status).equal(Status.STATUS_OK) - }) -}) diff --git a/src/health/index.ts b/src/health/index.ts deleted file mode 100644 index b4ff7da6..00000000 --- a/src/health/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { CheckRequest, CheckResponse } from "@textile/grpc-powergate-client/dist/health/rpc/rpc_pb" -import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/health/rpc/rpc_pb_service" -import { Config } from "../types" -import { promise } from "../util" - -export interface Health { - /** - * Checks the Powergate node health. - * @returns Information about the health of the Powergate node. - */ - check: () => Promise -} - -/** - * @ignore - */ -export const createHealth = (config: Config): Health => { - const client = new RPCServiceClient(config.host, config) - return { - check: () => - promise( - (cb) => client.check(new CheckRequest(), cb), - (resp: CheckResponse) => resp.toObject(), - ), - } -} diff --git a/src/index.spec.ts b/src/index.spec.ts deleted file mode 100644 index 14d6175f..00000000 --- a/src/index.spec.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { expect } from "chai" -import cp from "child_process" -import wait from "wait-on" -import { createPow } from "." -import { host } from "./util" - -beforeEach(async function () { - this.timeout(120000) - cp.exec(`cd powergate-docker && BIGSECTORS=false make localnet`, (err) => { - if (err) { - throw err - } - }) - await wait({ - resources: ["http://0.0.0.0:6002"], - timeout: 120000, - validateStatus: function () { - return true // the call expectedly returns 404, so just allow that - }, - }) -}) - -afterEach(async function () { - this.timeout(120000) - await new Promise((resolve, reject) => { - cp.exec(`cd powergate-docker && make localnet-down`, (err, stdout) => { - if (err) { - reject(err) - } - resolve(stdout) - }) - }) -}) - -describe("client", () => { - const pow = createPow({ host }) - - it("should create a client", () => { - expect(pow.ffs).not.undefined - expect(pow.health).not.undefined - expect(pow.net).not.undefined - expect(pow.miners).not.undefined - expect(pow.host).equal(host) - }) - - it("should get build info", async () => { - const res = await pow.buildInfo() - expect(res.gitSummary).not.empty - }) -}) diff --git a/src/index.ts b/src/index.ts index 9623ee9e..f579d39b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,37 +1,24 @@ import { BuildInfoRequest, BuildInfoResponse, -} from "@textile/grpc-powergate-client/dist/buildinfo/rpc/rpc_pb" -import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/buildinfo/rpc/rpc_pb_service" -import { Asks, createAsks } from "./asks" -import { createFaults, Faults } from "./faults" -import { - createFFS, - FFS, - GetFolderOptions, - ListDealRecordsOptions, - PushStorageConfigOptions, - WatchLogsOptions, -} from "./ffs" -import { createHealth, Health } from "./health" -import { createMiners, Miners } from "./miners" -import { createNet, Net } from "./net" -import { createReputation, Reputation } from "./reputation" + StorageProfileIdentifierRequest, + StorageProfileIdentifierResponse, +} from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" +import { PowergateServiceClient } from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb_service" +import { Admin, createAdmin } from "./admin" +import { createData, Data, GetFolderOptions, WatchLogsOptions } from "./data" +import { createDeals, DealRecordsOptions, Deals } from "./deals" +import { ApplyOptions, createStorageConfig, StorageConfig } from "./storage-config" +import { createStorageJobs, StorageJobs } from "./storage-jobs" import { Config } from "./types" -import { getTransport, host, promise, useToken } from "./util" +import { getTransport, host, promise, useTokens } from "./util" import { createWallet, Wallet } from "./wallet" -export * as ffsTypes from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" -export * as healthTypes from "@textile/grpc-powergate-client/dist/health/rpc/rpc_pb" -export * as asksTypes from "@textile/grpc-powergate-client/dist/index/ask/rpc/rpc_pb" -export * as faultsTypes from "@textile/grpc-powergate-client/dist/index/faults/rpc/rpc_pb" -export * as minersTypes from "@textile/grpc-powergate-client/dist/index/miner/rpc/rpc_pb" -export * as netTypes from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb" -export * as reputationTypes from "@textile/grpc-powergate-client/dist/reputation/rpc/rpc_pb" -export * as walletTypes from "@textile/grpc-powergate-client/dist/wallet/rpc/rpc_pb" -export { GetFolderOptions, PushStorageConfigOptions, WatchLogsOptions, ListDealRecordsOptions } +export * as adminTypes from "@textile/grpc-powergate-client/dist/proto/admin/v1/powergate_admin_pb" +export * as powTypes from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" +export { GetFolderOptions, ApplyOptions, WatchLogsOptions, DealRecordsOptions } export { Config } -export { Asks, Faults, FFS, Health, Miners, Net, Reputation, Wallet } +export { Admin, Data, Deals, StorageConfig, StorageJobs, Wallet } const defaultConfig: Config = { host, @@ -40,61 +27,63 @@ const defaultConfig: Config = { export interface Pow { /** - * Set the active auth token + * Set the active storage profile auth token * @param t The token to set */ setToken: (t: string) => void /** - * Returns build information about the server - * @returns An object containing build information about the server + * Set the active admin auth token + * @param t The token to set */ - buildInfo: () => Promise + setAdminToken: (t: string) => void /** - * The host address the client is using + * Returns build information about the server + * @returns An object containing build information about the server */ - host: string + buildInfo: () => Promise /** - * The Asks API + * Get the storage profile ID. + * @returns A Promise containing the storage profile ID. */ - asks: Asks + storageProfileId: () => Promise /** - * The Faults API + * The host address the client is using */ - faults: Faults + host: string /** - * The FFS API + * The StorageConfig API */ - ffs: FFS + storageConfig: StorageConfig /** - * The Health API + * The Data API */ - health: Health + data: Data /** - * The Miners API + * The Wallet API */ - miners: Miners + wallet: Wallet /** - * The Net API + * The Deals API */ - net: Net + deals: Deals /** - * The Reputation API + * The StorageJobs API */ - reputation: Reputation + storageJobs: StorageJobs /** - * The Wallet API + * The Admin API */ - wallet: Wallet + admin: Admin } /** @@ -103,38 +92,43 @@ export interface Pow { * @returns A Powergate client API */ export const createPow = (config?: Partial): Pow => { - const c = { ...defaultConfig, ...removeEmpty(config) } + const c: Config = { ...defaultConfig, ...removeEmpty(config) } - const { getMeta, getHeaders, setToken } = useToken(c.authToken) + const { getMeta, getHeaders, setToken, setAdminToken } = useTokens(c.authToken, c.adminToken) - const buildInfoClient = new RPCServiceClient(c.host, c) + const client = new PowergateServiceClient(c.host, c) return { host: c.host, setToken, + setAdminToken, + buildInfo: () => promise( - (cb) => buildInfoClient.buildInfo(new BuildInfoRequest(), cb), + (cb) => client.buildInfo(new BuildInfoRequest(), cb), (resp: BuildInfoResponse) => resp.toObject(), ), - asks: createAsks(c), - - faults: createFaults(c), + storageProfileId: () => + promise( + (cb) => + client.storageProfileIdentifier(new StorageProfileIdentifierRequest(), getMeta(), cb), + (res: StorageProfileIdentifierResponse) => res.toObject(), + ), - ffs: createFFS(c, getMeta, getHeaders), + storageConfig: createStorageConfig(c, getMeta), - health: createHealth(c), + data: createData(c, getMeta, getHeaders), - miners: createMiners(c), + wallet: createWallet(c, getMeta), - net: createNet(c), + deals: createDeals(c, getMeta), - reputation: createReputation(c), + storageJobs: createStorageJobs(c, getMeta), - wallet: createWallet(c), + admin: createAdmin(c, getMeta), } } diff --git a/src/integration.spec.ts b/src/integration.spec.ts new file mode 100644 index 00000000..bb2fa8a5 --- /dev/null +++ b/src/integration.spec.ts @@ -0,0 +1,601 @@ +import { expect } from "chai" +import crypto from "crypto" +import fs from "fs" +import { ApplyOptions, createPow, Pow, powTypes } from "." +import { host } from "./util" + +// beforeEach(async function () { +// this.timeout(120000) +// cp.exec(`cd powergate-docker && BIGSECTORS=false make localnet`, (err) => { +// if (err) { +// throw err +// } +// }) +// await wait({ +// resources: ["http://0.0.0.0:6002"], +// timeout: 120000, +// validateStatus: function () { +// return true // the call expectedly returns 404, so just allow that +// }, +// }) +// }) + +// afterEach(async function () { +// this.timeout(120000) +// await new Promise((resolve, reject) => { +// cp.exec(`cd powergate-docker && make localnet-down`, (err, stdout) => { +// if (err) { +// reject(err) +// } +// resolve(stdout) +// }) +// }) +// }) + +describe("pow", () => { + it("should get build info", async () => { + const pow = newPow() + const res = await pow.buildInfo() + expect(res.gitSummary).not.empty + }) + + it("should check host", () => { + const pow = newPow() + expect(pow.host).equal(host) + }) + + describe("admin", () => { + describe("profiles", () => { + it("should create profile", async () => { + const pow = newPow() + await expectNewInstance(pow) + }) + + it("should list profiles", async () => { + const pow = newPow() + const res = await pow.admin.profiles.storageProfiles() + expect(res.authEntriesList).length.greaterThan(0) + }) + }) + + describe("profile storage jobs", async () => { + it("should get executing", async function () { + const pow = newPow() + const auth = await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + const res = await pow.admin.storageJobs.executing(auth.id, cid) + expect(res.storageJobsList).length(1) + expect(res.storageJobsList[0].id).equals(jobId) + expect(res.storageJobsList[0].cid).equals(cid) + }) + + it("should get latest final", async function () { + this.timeout(180000) + const pow = newPow() + const auth = await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.admin.storageJobs.latestFinal(auth.id, cid) + expect(res.storageJobsList).empty + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + res = await pow.admin.storageJobs.latestFinal(auth.id, cid) + expect(res.storageJobsList).length(1) + expect(res.storageJobsList[0].id).equals(jobId) + expect(res.storageJobsList[0].cid).equals(cid) + }) + + it("should get latest successful", async function () { + this.timeout(180000) + const pow = newPow() + const auth = await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.admin.storageJobs.latestSuccessful(auth.id, cid) + expect(res.storageJobsList).empty + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + res = await pow.admin.storageJobs.latestSuccessful(auth.id, cid) + expect(res.storageJobsList).length(1) + res = await pow.admin.storageJobs.latestSuccessful(auth.id, cid) + expect(res.storageJobsList).length(1) + expect(res.storageJobsList[0].id).equals(jobId) + expect(res.storageJobsList[0].cid).equals(cid) + }) + + it("should get queued", async function () { + const pow = newPow() + const auth = await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await expectApplyStorageConfig(pow, cid) + const res = await pow.admin.storageJobs.queued(auth.id, cid) + expect(res.storageJobsList).length.lessThan(2) + }) + + it("should get summary", async function () { + const pow = newPow() + const auth = await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.admin.storageJobs.summary(auth.id, cid) + expect(res.executingStorageJobsList).length(1) + res = await pow.admin.storageJobs.summary(auth.id, cid) + expect(res.executingStorageJobsList).length(1) + expect(res.executingStorageJobsList[0].cid).equals(cid) + expect(res.executingStorageJobsList[0].id).equals(jobId) + expect(res.jobCounts?.executing).equals(1) + expect(res.jobCounts?.latestFinal).equals(0) + expect(res.jobCounts?.latestSuccessful).equals(0) + expect(res.jobCounts?.queued).equals(0) + }) + }) + + describe("all storage jobs", () => { + it("should get summary", async function () { + const pow = newPow() + const res = await pow.admin.storageJobs.summary("") + expect(res.jobCounts?.latestFinal).greaterThan(0) + expect(res.jobCounts?.latestSuccessful).greaterThan(0) + expect(res.jobCounts?.queued).equals(0) + }) + }) + + describe("wallet", async () => { + it("should get addresses", async function () { + const pow = newPow() + const res = await pow.admin.wallet.addresses() + expect(res.addressesList).length.greaterThan(0) + }) + + it("should create an address", async function () { + const pow = newPow() + const res = await pow.admin.wallet.newAddress() + expect(res.address).not.empty + }) + + it("should send fil", async function () { + this.timeout(120000) + const pow = newPow() + const res0 = await pow.admin.wallet.newAddress() + const res1 = await pow.admin.wallet.newAddress() + await waitForBalance(pow, res0.address, 0) + const bal2 = await waitForBalance(pow, res1.address, 0) + await pow.admin.wallet.sendFil(res0.address, res1.address, 10) + // ToDo: see why fil is not sending + // const bal3 = await waitForBalance(pow, res1.address, bal2) + // expect(bal3).equals(bal2 + 10) + }) + }) + }) + + describe("data", () => { + it("should get cid info", async function () { + this.timeout(180000) + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.data.cidInfo(cid) + expect(res.cidInfosList).length(1) + res = await pow.data.cidInfo() + expect(res.cidInfosList).length(1) + expect(res.cidInfosList[0].cid).equals(cid) + expect(res.cidInfosList[0].currentStorageInfo).undefined + expect(res.cidInfosList[0].latestFinalStorageJob).undefined + expect(res.cidInfosList[0].latestSuccessfulStorageJob).undefined + expect(res.cidInfosList[0].latestPushedStorageConfig).not.undefined + expect(res.cidInfosList[0].queuedStorageJobsList).length.lessThan(2) + if (res.cidInfosList[0].executingStorageJob) { + expect(res.cidInfosList[0].executingStorageJob.cid).equals(cid) + } + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + res = await pow.data.cidInfo() + expect(res.cidInfosList).length(1) + expect(res.cidInfosList[0].cid).equals(cid) + expect(res.cidInfosList[0].currentStorageInfo?.cid).equals(cid) + expect(res.cidInfosList[0].latestFinalStorageJob).not.undefined + expect(res.cidInfosList[0].latestSuccessfulStorageJob).not.undefined + expect(res.cidInfosList[0].latestPushedStorageConfig).not.undefined + expect(res.cidInfosList[0].queuedStorageJobsList).length(0) + expect(res.cidInfosList[0].executingStorageJob).undefined + }) + + it("should get", async function () { + this.timeout(180000) + const pow = newPow() + await expectNewInstance(pow) + const addrs = await expectAddresses(pow, 1) + await waitForBalance(pow, addrs[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + const bytes = await pow.data.get(cid) + expect(bytes.byteLength).greaterThan(0) + }) + + it("should get a folder", async () => { + const pow = newPow() + await expectNewInstance(pow) + const res = await pow.data.stageFolder("./sample-data") + expect(res).length.greaterThan(0) + await pow.data.getFolder(res, "./output", { timeout: 10000 }) + }) + + it("should replace", async function () { + this.timeout(180000) + const pow = newPow() + await expectNewInstance(pow) + const addrs = await expectAddresses(pow, 1) + await waitForBalance(pow, addrs[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + const cid2 = await expectStageData(pow, crypto.randomBytes(1024)) + const res = await pow.data.replaceData(cid, cid2) + expect(res.jobId).length.greaterThan(0) + await watchJobUntil(pow, res.jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + }) + + it("should stage", async () => { + const pow = newPow() + await expectNewInstance(pow) + await expectStageData(pow, crypto.randomBytes(1024)) + }) + + it("should stage folder", async () => { + const pow = newPow() + await expectNewInstance(pow) + const res = await pow.data.stageFolder("sample-data") + expect(res).not.empty + }) + + it("should watch logs", async function () { + const pow = newPow() + await expectNewInstance(pow) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + const event = await new Promise((resolve) => { + pow.data.watchLogs((event) => resolve(event), cid, { includeHistory: true, jobId }) + }) + expect(event.cid).equals(cid) + expect(event.jobId).equals(jobId) + }) + }) + + describe("deals", () => { + it("should get storage deal records", async function () { + this.timeout(180000) + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + const pending = await pow.deals.storageDealRecords({ includePending: true }) + const final = await pow.deals.storageDealRecords({ includeFinal: true }) + expect(pending.recordsList).empty + expect(final.recordsList).length(1) + }) + + it("should get retreival deal records", () => { + // ToDo: Figre out how to force a retrieval to test this. + }) + }) + + describe("storage config", () => { + it("should get default", async () => { + const pow = newPow() + await expectNewInstance(pow) + await expectDefaultStorageConfig(pow) + }) + + it("should set default", async () => { + const pow = newPow() + await expectNewInstance(pow) + const conf = await expectDefaultStorageConfig(pow) + await pow.storageConfig.setDefault(conf) + }) + + it("should apply", async () => { + const pow = newPow() + await expectNewInstance(pow) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await expectApplyStorageConfig(pow, cid) + }) + + it("should remove", async () => { + const pow = newPow() + await expectNewInstance(pow) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const conf = await expectDefaultStorageConfig(pow) + conf.cold = { ...conf.cold, enabled: false } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + conf.hot = { ...conf.hot!, enabled: false } + await expectApplyStorageConfig(pow, cid, { storageConfig: conf }) + await pow.storageConfig.remove(cid) + }) + }) + + describe("storage jobs", () => { + it("should cancel", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + await pow.storageJobs.cancel(jobId) + }) + + it("should get executing", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.storageJobs.executing() + expect(res.storageJobsList).length(1) + res = await pow.storageJobs.executing(cid) + expect(res.storageJobsList).length(1) + expect(res.storageJobsList[0].id).equals(jobId) + expect(res.storageJobsList[0].cid).equals(cid) + }) + + it("should get latest final", async function () { + this.timeout(180000) + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.storageJobs.latestFinal() + expect(res.storageJobsList).empty + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + res = await pow.storageJobs.latestFinal() + expect(res.storageJobsList).length(1) + res = await pow.storageJobs.latestFinal(cid) + expect(res.storageJobsList).length(1) + expect(res.storageJobsList[0].id).equals(jobId) + expect(res.storageJobsList[0].cid).equals(cid) + }) + + it("should get latest successful", async function () { + this.timeout(180000) + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.storageJobs.latestSuccessful() + expect(res.storageJobsList).empty + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + res = await pow.storageJobs.latestSuccessful() + expect(res.storageJobsList).length(1) + res = await pow.storageJobs.latestSuccessful(cid) + expect(res.storageJobsList).length(1) + expect(res.storageJobsList[0].id).equals(jobId) + expect(res.storageJobsList[0].cid).equals(cid) + }) + + it("should get queued", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await expectApplyStorageConfig(pow, cid) + const res = await pow.storageJobs.queued(cid) + expect(res.storageJobsList).length.lessThan(2) + }) + + it("should get storage config for job", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + const res = await pow.storageJobs.storageConfigForJob(jobId) + expect(res.storageConfig).not.undefined + }) + + it("should get storage job", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + const res = await pow.storageJobs.storageJob(jobId) + expect(res.job?.id).equals(jobId) + }) + + it("should get summary", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.storageJobs.summary() + expect(res.executingStorageJobsList).length(1) + res = await pow.storageJobs.summary(cid) + expect(res.executingStorageJobsList).length(1) + expect(res.executingStorageJobsList[0].cid).equals(cid) + expect(res.executingStorageJobsList[0].id).equals(jobId) + expect(res.jobCounts?.executing).equals(1) + expect(res.jobCounts?.latestFinal).equals(0) + expect(res.jobCounts?.latestSuccessful).equals(0) + expect(res.jobCounts?.queued).equals(0) + }) + + it("should watch", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_EXECUTING) + }) + }) + + describe("wallet", () => { + it("should get addresses", async function () { + const pow = newPow() + await expectNewInstance(pow) + await expectAddresses(pow, 1) + }) + + it("should get balance", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address, 0) + }) + + it("should create new address", async function () { + const pow = newPow() + await expectNewInstance(pow) + const res = await pow.wallet.newAddress("new one") + expect(res.address).not.empty + }) + + it("should send fil", async function () { + this.timeout(120000) + const pow = newPow() + await expectNewInstance(pow) + await pow.wallet.newAddress("new one") + const addressees = await expectAddresses(pow, 2) + await waitForBalance(pow, addressees[0].address, 0) + await waitForBalance(pow, addressees[1].address, 0) + await pow.wallet.sendFil(addressees[0].address, addressees[1].address, 10) + // ToDo: Check that sendFil is working on the back end + // await waitForBalance(pow, addressees[1].address, bal + 9) + }) + + it("should sign message", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + const res = await pow.wallet.signMessage(addressees[0].address, crypto.randomBytes(1024)) + expect(res.signature).not.empty + }) + + it("should verify message signature", async function () { + const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + const message = crypto.randomBytes(1024) + const res0 = await pow.wallet.signMessage(addressees[0].address, message) + const res1 = await pow.wallet.verifyMessage(addressees[0].address, message, res0.signature) + expect(res1.ok).true + }) + }) +}) + +function newPow(): Pow { + return createPow({ host }) +} + +async function expectNewInstance(pow: Pow) { + const res = await pow.admin.profiles.createStorageProfile() + expect(res.authEntry?.id).not.empty + expect(res.authEntry?.token).not.empty + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + pow.setToken(res.authEntry!.token) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return res.authEntry! +} + +async function expectDefaultStorageConfig(pow: Pow) { + const res = await pow.storageConfig.default() + expect(res.defaultStorageConfig).not.undefined + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return res.defaultStorageConfig! +} + +async function expectStage(pow: Pow, path: string) { + const buffer = fs.readFileSync(path) + const res = await pow.data.stage(buffer) + expect(res.cid).length.greaterThan(0) + return res.cid +} + +async function expectStageData(pow: Pow, data: Buffer) { + const res = await pow.data.stage(data) + expect(res.cid).length.greaterThan(0) + return res.cid +} + +async function expectApplyStorageConfig(pow: Pow, cid: string, opts?: ApplyOptions) { + const res = await pow.storageConfig.apply(cid, opts) + expect(res.jobId).length.greaterThan(0) + return res.jobId +} + +function watchJobUntil( + pow: Pow, + jobId: string, + status: powTypes.JobStatusMap[keyof powTypes.JobStatusMap], +) { + return new Promise((resolve, reject) => { + try { + const cancel = pow.storageJobs.watch((job) => { + if (job.errorCause.length > 0) { + reject(job.errorCause) + } + if (job.status === powTypes.JobStatus.JOB_STATUS_CANCELED) { + reject("job canceled") + } + if (job.status === powTypes.JobStatus.JOB_STATUS_FAILED) { + reject("job failed") + } + if (job.status === status) { + cancel() + resolve() + } + }, jobId) + } catch (e) { + reject(e) + } + }) +} + +async function expectAddresses(pow: Pow, length: number) { + const res = await pow.wallet.addresses() + expect(res.addressesList).length(length) + return res.addressesList +} + +function waitForBalance(pow: Pow, address: string, greaterThan: number) { + return new Promise(async (resolve, reject) => { + while (true) { + try { + const res = await pow.wallet.balance(address) + if (res.balance > greaterThan) { + resolve(res.balance) + return + } + } catch (e) { + reject(e) + } + await new Promise((r) => setTimeout(r, 1000)) + } + }) +} diff --git a/src/miners/index.spec.ts b/src/miners/index.spec.ts deleted file mode 100644 index 3aaea820..00000000 --- a/src/miners/index.spec.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { expect } from "chai" -import { createMiners } from "." -import { getTransport, host } from "../util" - -describe("miners", () => { - const c = createMiners({ host, transport: getTransport() }) - - it("should get the index", async () => { - const res = await c.get() - expect(res.index).not.undefined - }) -}) diff --git a/src/miners/index.ts b/src/miners/index.ts deleted file mode 100644 index 6c3f41f1..00000000 --- a/src/miners/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { GetRequest, GetResponse } from "@textile/grpc-powergate-client/dist/index/miner/rpc/rpc_pb" -import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/index/miner/rpc/rpc_pb_service" -import { Config } from "../types" -import { promise } from "../util" - -export interface Miners { - /** - * Gets the miner index. - * @returns The miner index. - */ - get: () => Promise -} - -/** - * @ignore - */ -export const createMiners = (config: Config): Miners => { - const client = new RPCServiceClient(config.host, config) - return { - get: () => - promise( - (cb) => client.get(new GetRequest(), cb), - (resp: GetResponse) => resp.toObject(), - ), - } -} diff --git a/src/net/index.spec.ts b/src/net/index.spec.ts deleted file mode 100644 index 54abc7cf..00000000 --- a/src/net/index.spec.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Connectedness, PeersResponse } from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb" -import { assert, expect } from "chai" -import { createNet } from "." -import { getTransport, host } from "../util" - -describe("net", () => { - const c = createNet({ host, transport: getTransport() }) - - it("should query peers", async () => { - await expectPeers() - }) - - it("should get listen address", async () => { - const listenAddr = await c.listenAddr() - expect(listenAddr.addrInfo?.addrsList).length.greaterThan(0) - expect(listenAddr.addrInfo?.id).length.greaterThan(0) - }) - - it("should find a peer", async () => { - const peers = await expectPeers() - const peerId = expectPeerInfo(peers).id - const peer = await c.findPeer(peerId) - expect(peer.peerInfo).not.undefined - }) - - it("should get peer connectedness", async () => { - const peers = await expectPeers() - const peerId = expectPeerInfo(peers).id - const resp = await c.connectedness(peerId) - expect(resp.connectedness).equal(Connectedness.CONNECTEDNESS_CONNECTED) - }) - - it("should disconnect and reconnect to a peer", async () => { - const peers = await expectPeers() - const peerInfo = expectPeerInfo(peers) - await c.disconnectPeer(peerInfo.id) - await c.connectPeer(peerInfo) - }) - - async function expectPeers() { - const peers = await c.peers() - expect(peers.peersList).length.greaterThan(0) - return peers - } - - function expectPeerInfo(peersResp: PeersResponse.AsObject) { - const peerInfo = peersResp.peersList[0].addrInfo - if (!peerInfo) { - assert.fail("no peer info") - } - return peerInfo - } -}) diff --git a/src/net/index.ts b/src/net/index.ts deleted file mode 100644 index c5a1e1b5..00000000 --- a/src/net/index.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { - ConnectednessRequest, - ConnectednessResponse, - ConnectPeerRequest, - DisconnectPeerRequest, - FindPeerRequest, - FindPeerResponse, - ListenAddrRequest, - ListenAddrResponse, - PeerAddrInfo, - PeersRequest, - PeersResponse, -} from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb" -import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb_service" -import { Config } from "../types" -import { promise } from "../util" - -export interface Net { - /** - * Get the listen address of the filecoin node. - * @returns The listen address info. - */ - listenAddr: () => Promise - - /** - * List filecoin peers. - * @returns A list of filecoin peers. - */ - peers: () => Promise - - /** - * Find a peer by peer id. - * @param peerId The peer id to find info for. - * @returns The peer info. - */ - findPeer: (peerId: string) => Promise - - /** - * Connect to a peer. - * @param peerInfo The peer info specifying the peer to connect to. - */ - connectPeer: (peerInfo: PeerAddrInfo.AsObject) => Promise - - /** - * Get the current connectedness state to a peer. - * @param peerId The peer id. - * @returns Information about the connectedness to the peer. - */ - connectedness: (peerId: string) => Promise - - /** - * Disconnect from a peer. - * @param peerId The peer id to disconnect from. - */ - disconnectPeer: (peerId: string) => Promise -} - -/** - * @ignore - */ -export const createNet = (config: Config): Net => { - const client = new RPCServiceClient(config.host, config) - return { - listenAddr: () => - promise( - (cb) => client.listenAddr(new ListenAddrRequest(), cb), - (res: ListenAddrResponse) => res.toObject(), - ), - - peers: () => - promise( - (cb) => client.peers(new PeersRequest(), cb), - (res: PeersResponse) => res.toObject(), - ), - - findPeer: (peerId: string) => { - const req = new FindPeerRequest() - req.setPeerId(peerId) - return promise( - (cb) => client.findPeer(req, cb), - (res: FindPeerResponse) => res.toObject(), - ) - }, - - connectPeer: (peerInfo: PeerAddrInfo.AsObject) => { - const info = new PeerAddrInfo() - info.setId(peerInfo.id) - info.setAddrsList(peerInfo.addrsList) - const req = new ConnectPeerRequest() - req.setPeerInfo(info) - return promise( - (cb) => client.connectPeer(req, cb), - () => { - // nothing to return - }, - ) - }, - - connectedness: (peerId: string) => { - const req = new ConnectednessRequest() - req.setPeerId(peerId) - return promise( - (cb) => client.connectedness(req, cb), - (res: ConnectednessResponse) => res.toObject(), - ) - }, - - disconnectPeer: (peerId: string) => { - const req = new DisconnectPeerRequest() - req.setPeerId(peerId) - return promise( - (cb) => client.disconnectPeer(req, cb), - () => { - // nothing to return - }, - ) - }, - } -} diff --git a/src/ffs/normalize.ts b/src/normalize.ts similarity index 100% rename from src/ffs/normalize.ts rename to src/normalize.ts diff --git a/src/reputation/index.spec.ts b/src/reputation/index.spec.ts deleted file mode 100644 index 31976ad1..00000000 --- a/src/reputation/index.spec.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { expect } from "chai" -import { createReputation } from "." -import { getTransport, host } from "../util" - -describe("reputation", () => { - const c = createReputation({ host, transport: getTransport() }) - - it("should get top miners", async () => { - const { topMinersList } = await c.getTopMiners(10) - expect(topMinersList).not.undefined - }) -}) diff --git a/src/reputation/index.ts b/src/reputation/index.ts deleted file mode 100644 index 2836d218..00000000 --- a/src/reputation/index.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - AddSourceRequest, - GetTopMinersRequest, - GetTopMinersResponse, -} from "@textile/grpc-powergate-client/dist/reputation/rpc/rpc_pb" -import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/reputation/rpc/rpc_pb_service" -import { Config } from "../types" -import { promise } from "../util" - -export interface Reputation { - /** - * Adds a data source to the reputation index. - * @param id The id of the data source. - * @param multiaddress The multiaddress of the data source. - */ - addSource: (id: string, multiaddress: string) => Promise - - /** - * Gets the top ranked miners. - * @param limit Limits the number of results. - * @returns The list of miner scores. - */ - getTopMiners: (limit: number) => Promise -} - -/** - * @ignore - */ -export const createReputation = (config: Config): Reputation => { - const client = new RPCServiceClient(config.host, config) - return { - addSource: (id: string, multiaddress: string) => { - const req = new AddSourceRequest() - req.setId(id) - req.setMaddr(multiaddress) - return promise( - (cb) => client.addSource(req, cb), - () => { - // nothing to return - }, - ) - }, - - getTopMiners: (limit: number) => { - const req = new GetTopMinersRequest() - req.setLimit(limit) - return promise( - (cb) => client.getTopMiners(req, cb), - (resp: GetTopMinersResponse) => resp.toObject(), - ) - }, - } -} diff --git a/src/storage-config/index.spec.ts b/src/storage-config/index.spec.ts new file mode 100644 index 00000000..c5fc57d5 --- /dev/null +++ b/src/storage-config/index.spec.ts @@ -0,0 +1,110 @@ +// import { +// JobStatus, +// JobStatusMap, +// StorageConfig, +// } from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" +// import { expect } from "chai" +// import fs from "fs" +// import { createStorageConfig } from "." +// import { createAdmin } from "../admin" +// import { getTransport, host, useTokens } from "../util" +// import { ApplyOptions } from "./types" + +// describe("storage config", function () { +// this.timeout(180000) + +// const { getMeta, _, setToken } = useTokens("") + +// const c = createStorageConfig({ host, transport: getTransport() }, getMeta) +// const a = createAdmin({ host, transport: getTransport() }) + +// it("should get the default config", async () => { +// await expectNewInstance() +// await expectDefaultStorageConfig() +// }) + +// it("should set default config", async () => { +// await expectNewInstance() +// const defaultConfig = await expectDefaultStorageConfig() +// await c.setDefault(defaultConfig) +// }) + +// it("should apply config", async () => { +// await expectNewInstance() +// const cid = await expectStage("sample-data/samplefile") +// const config = await expectDefaultStorageConfig() +// await expectPushStorageConfig(cid, { override: false, storageConfig: config }) +// }) + +// it("should remove", async () => { +// await expectNewInstance() +// const cid = await expectStage("sample-data/samplefile") +// const conf: StorageConfig.AsObject = { +// repairable: false, +// cold: { +// enabled: false, +// }, +// hot: { +// allowUnfreeze: false, +// enabled: false, +// unfreezeMaxPrice: 0, +// }, +// } +// const jobId = await expectPushStorageConfig(cid, { override: false, storageConfig: conf }) +// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) +// await c.remove(cid) +// }) + +// async function expectNewInstance() { +// const res = await a.profiles.createStorageProfile() +// expect(res.authEntry?.id).not.empty +// expect(res.authEntry?.token).not.empty +// // eslint-disable-next-line @typescript-eslint/no-non-null-assertion +// setToken(res.authEntry!.token) +// return res +// } + +// async function expectDefaultStorageConfig() { +// const res = await c.default() +// expect(res.defaultStorageConfig).not.undefined +// // eslint-disable-next-line @typescript-eslint/no-non-null-assertion +// return res.defaultStorageConfig! +// } + +// async function expectStage(path: string) { +// const buffer = fs.readFileSync(path) +// const res = await c.stage(buffer) +// expect(res.cid).length.greaterThan(0) +// return res.cid +// } + +// async function expectPushStorageConfig(cid: string, opts?: ApplyOptions) { +// const res = await c.apply(cid, opts) +// expect(res.jobId).length.greaterThan(0) +// return res.jobId +// } + +// function waitForJobStatus(jobId: string, status: JobStatusMap[keyof JobStatusMap]) { +// return new Promise((resolve, reject) => { +// try { +// const cancel = c.watchJobs((job) => { +// if (job.errCause.length > 0) { +// reject(job.errCause) +// } +// if (job.status === JobStatus.JOB_STATUS_CANCELED) { +// reject("job canceled") +// } +// if (job.status === JobStatus.JOB_STATUS_FAILED) { +// reject("job failed") +// } +// if (job.status === status) { +// cancel() +// resolve() +// } +// }, jobId) +// } catch (e) { +// reject(e) +// } +// }) +// } +// }) diff --git a/src/storage-config/index.ts b/src/storage-config/index.ts new file mode 100644 index 00000000..c10578c0 --- /dev/null +++ b/src/storage-config/index.ts @@ -0,0 +1,111 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + ApplyStorageConfigRequest, + ApplyStorageConfigResponse, + DefaultStorageConfigRequest, + DefaultStorageConfigResponse, + RemoveRequest, + RemoveResponse, + SetDefaultStorageConfigRequest, + SetDefaultStorageConfigResponse, + StorageConfig as SConfig, +} from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" +import { PowergateServiceClient } from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb_service" +import { Config } from "../types" +import { promise } from "../util" +import { ApplyOptions } from "./types" +import { coldObjToMessage, hotObjToMessage } from "./util" + +export { ApplyOptions } +export interface StorageConfig { + /** + * Get the default storage config associated with the current storage profile. + * @returns The default storage config. + */ + default: () => Promise + + /** + * Set the default storage config for this storage profile. + * @param config The new default storage config. + */ + setDefault: (config: SConfig.AsObject) => Promise + + /** + * Apply a storage config or the default to the specified cid. + * @param cid The cid to store. + * @param opts Options controlling the behavior storage config execution. + * @returns An object containing the job id of the job executing the storage configuration. + */ + apply: (cid: string, opts?: ApplyOptions) => Promise + + /** + * Remove a cid from the storage profile storage. + * @param cid The cid to remove. + */ + remove: (cid: string) => Promise +} + +export const createStorageConfig = ( + config: Config, + getMeta: () => grpc.Metadata, +): StorageConfig => { + const client = new PowergateServiceClient(config.host, config) + return { + default: () => + promise( + (cb) => client.defaultStorageConfig(new DefaultStorageConfigRequest(), getMeta(), cb), + (res: DefaultStorageConfigResponse) => res.toObject(), + ), + + setDefault: (config: SConfig.AsObject) => { + const c = new SConfig() + c.setRepairable(config.repairable) + if (config.hot) { + c.setHot(hotObjToMessage(config.hot)) + } + if (config.cold) { + c.setCold(coldObjToMessage(config.cold)) + } + const req = new SetDefaultStorageConfigRequest() + req.setConfig(c) + return promise( + (cb) => client.setDefaultStorageConfig(req, getMeta(), cb), + (res: SetDefaultStorageConfigResponse) => res.toObject(), + ) + }, + + apply: (cid: string, opts?: ApplyOptions) => { + const req = new ApplyStorageConfigRequest() + req.setCid(cid) + if (opts?.override) { + req.setOverrideConfig(opts.override) + req.setHasOverrideConfig(true) + } + if (opts?.storageConfig) { + const c = new SConfig() + c.setRepairable(opts.storageConfig.repairable) + if (opts.storageConfig.hot) { + c.setHot(hotObjToMessage(opts.storageConfig.hot)) + } + if (opts.storageConfig.cold) { + c.setCold(coldObjToMessage(opts.storageConfig.cold)) + } + req.setConfig(c) + req.setHasConfig(true) + } + return promise( + (cb) => client.applyStorageConfig(req, getMeta(), cb), + (res: ApplyStorageConfigResponse) => res.toObject(), + ) + }, + + remove: (cid: string) => { + const req = new RemoveRequest() + req.setCid(cid) + return promise( + (cb) => client.remove(req, getMeta(), cb), + (res: RemoveResponse) => res.toObject(), + ) + }, + } +} diff --git a/src/storage-config/types.ts b/src/storage-config/types.ts new file mode 100644 index 00000000..3d2b71e9 --- /dev/null +++ b/src/storage-config/types.ts @@ -0,0 +1,16 @@ +import { StorageConfig } from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" + +/** + * Options to control the behavior of pushStorageConfig. + */ +export type ApplyOptions = { + /** + * Allows you to override an existing storage configuration + */ + override?: boolean + + /** + * Allows you to override the default storage config with a custom one + */ + storageConfig?: StorageConfig.AsObject +} diff --git a/src/ffs/util.ts b/src/storage-config/util.ts similarity index 86% rename from src/ffs/util.ts rename to src/storage-config/util.ts index 092ac622..8dc17db6 100644 --- a/src/ffs/util.ts +++ b/src/storage-config/util.ts @@ -4,18 +4,19 @@ import { FilRenew, HotConfig, IpfsConfig, -} from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" +} from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" + export function coldObjToMessage(obj: ColdConfig.AsObject): ColdConfig { const cold = new ColdConfig() cold.setEnabled(obj.enabled) if (obj.filecoin) { const fc = new FilConfig() - fc.setAddr(obj.filecoin.addr) + fc.setAddress(obj.filecoin.address) fc.setCountryCodesList(obj.filecoin.countryCodesList) fc.setDealMinDuration(obj.filecoin.dealMinDuration) fc.setExcludedMinersList(obj.filecoin.excludedMinersList) fc.setMaxPrice(obj.filecoin.maxPrice) - fc.setRepFactor(obj.filecoin.repFactor) + fc.setReplicationFactor(obj.filecoin.replicationFactor) fc.setTrustedMinersList(obj.filecoin.trustedMinersList) if (obj.filecoin.renew) { const renew = new FilRenew() diff --git a/src/storage-jobs/index.ts b/src/storage-jobs/index.ts new file mode 100644 index 00000000..40e84cd7 --- /dev/null +++ b/src/storage-jobs/index.ts @@ -0,0 +1,157 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + CancelStorageJobRequest, + CancelStorageJobResponse, + ExecutingStorageJobsRequest, + ExecutingStorageJobsResponse, + Job, + LatestFinalStorageJobsRequest, + LatestFinalStorageJobsResponse, + LatestSuccessfulStorageJobsRequest, + LatestSuccessfulStorageJobsResponse, + QueuedStorageJobsRequest, + QueuedStorageJobsResponse, + StorageConfigForJobRequest, + StorageConfigForJobResponse, + StorageJobRequest, + StorageJobResponse, + StorageJobsSummaryRequest, + StorageJobsSummaryResponse, + WatchStorageJobsRequest, +} from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" +import { PowergateServiceClient } from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb_service" +import { Config } from "../types" +import { promise } from "../util" + +export interface StorageJobs { + /** + * Get the current state of a storage job. + * @param jobId The job id to query. + * @returns The current state of the storage job. + */ + storageJob: (jobId: string) => Promise + + /** + * Get the desired storage config for the provided cid, this config may not yet be realized. + * @param cid The cid of the desired storage config. + * @returns The storage config for the provided cid. + */ + storageConfigForJob: (jobId: string) => Promise + + queued: (...cids: string[]) => Promise + + executing: (...cids: string[]) => Promise + + latestFinal: (...cids: string[]) => Promise + + latestSuccessful: (...cids: string[]) => Promise + + summary: (...cids: string[]) => Promise + + /** + * Listen for job updates for the provided job ids. + * @param handler The callback to receive job updates. + * @param jobs A list of job ids to watch. + * @returns A function that can be used to cancel watching. + */ + watch: (handler: (event: Job.AsObject) => void, ...jobs: string[]) => () => void + + /** + * Cancel a job. + * @param jobId The id of the job to cancel. + */ + cancel: (jobId: string) => Promise +} + +/** + * @ignore + */ +export const createStorageJobs = (config: Config, getMeta: () => grpc.Metadata): StorageJobs => { + const client = new PowergateServiceClient(config.host, config) + return { + storageJob: (jobId: string) => { + const req = new StorageJobRequest() + req.setJobId(jobId) + return promise( + (cb) => client.storageJob(req, getMeta(), cb), + (res: StorageJobResponse) => res.toObject(), + ) + }, + + storageConfigForJob: (jobId: string) => { + const req = new StorageConfigForJobRequest() + req.setJobId(jobId) + return promise( + (cb) => client.storageConfigForJob(req, getMeta(), cb), + (res: StorageConfigForJobResponse) => res.toObject(), + ) + }, + + queued: (...cids: string[]) => { + const req = new QueuedStorageJobsRequest() + req.setCidsList(cids) + return promise( + (cb) => client.queuedStorageJobs(req, getMeta(), cb), + (res: QueuedStorageJobsResponse) => res.toObject(), + ) + }, + + executing: (...cids: string[]) => { + const req = new ExecutingStorageJobsRequest() + req.setCidsList(cids) + return promise( + (cb) => client.executingStorageJobs(req, getMeta(), cb), + (res: ExecutingStorageJobsResponse) => res.toObject(), + ) + }, + + latestFinal: (...cids: string[]) => { + const req = new LatestFinalStorageJobsRequest() + req.setCidsList(cids) + return promise( + (cb) => client.latestFinalStorageJobs(req, getMeta(), cb), + (res: LatestFinalStorageJobsResponse) => res.toObject(), + ) + }, + + latestSuccessful: (...cids: string[]) => { + const req = new LatestSuccessfulStorageJobsRequest() + req.setCidsList(cids) + return promise( + (cb) => client.latestSuccessfulStorageJobs(req, getMeta(), cb), + (res: LatestSuccessfulStorageJobsResponse) => res.toObject(), + ) + }, + + summary: (...cids: string[]) => { + const req = new StorageJobsSummaryRequest() + req.setCidsList(cids) + return promise( + (cb) => client.storageJobsSummary(req, getMeta(), cb), + (res: StorageJobsSummaryResponse) => res.toObject(), + ) + }, + + watch: (handler: (event: Job.AsObject) => void, ...jobs: string[]) => { + const req = new WatchStorageJobsRequest() + req.setJobIdsList(jobs) + const stream = client.watchStorageJobs(req, getMeta()) + stream.on("data", (res) => { + const job = res.getJob()?.toObject() + if (job) { + handler(job) + } + }) + return stream.cancel + }, + + cancel: (jobId: string) => { + const req = new CancelStorageJobRequest() + req.setJobId(jobId) + return promise( + (cb) => client.cancelStorageJob(req, getMeta(), cb), + (res: CancelStorageJobResponse) => res.toObject(), + ) + }, + } +} diff --git a/src/test-utils.ts b/src/test-utils.ts new file mode 100644 index 00000000..b966352f --- /dev/null +++ b/src/test-utils.ts @@ -0,0 +1,98 @@ +import { CreateStorageProfileResponse } from "@textile/grpc-powergate-client/dist/proto/admin/v1/powergate_admin_pb" +import { expect } from "chai" +import { Profiles } from "./admin/profiles" + +export async function expectNewInstance( + p: Profiles, + setToken: (t: string) => void, +): Promise { + const res = await p.createStorageProfile() + expect(res.authEntry?.id).not.empty + expect(res.authEntry?.token).not.empty + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + setToken(res.authEntry!.token) + return res +} + +// async function expectAddrs(length: number) { +// const res = await c.addrs() +// expect(res.addrsList).length(length) +// return res.addrsList +// } + +// async function expectNewAddr() { +// const res = await c.newAddr("my addr") +// expect(res.addr).length.greaterThan(0) +// return res.addr +// } + +// async function expectDefaultStorageConfig() { +// const res = await c.defaultStorageConfig() +// expect(res.defaultStorageConfig).not.undefined +// // eslint-disable-next-line @typescript-eslint/no-non-null-assertion +// return res.defaultStorageConfig! +// } + +// async function expectStage(path: string) { +// const buffer = fs.readFileSync(path) +// const res = await c.stage(buffer) +// expect(res.cid).length.greaterThan(0) +// return res.cid +// } + +// async function expectPushStorageConfig(cid: string, opts?: PushStorageConfigOptions) { +// const res = await c.pushStorageConfig(cid, opts) +// expect(res.jobId).length.greaterThan(0) +// return res.jobId +// } + +// function waitForJobStatus(jobId: string, status: JobStatusMap[keyof JobStatusMap]) { +// return new Promise((resolve, reject) => { +// try { +// const cancel = c.watchJobs((job) => { +// if (job.errCause.length > 0) { +// reject(job.errCause) +// } +// if (job.status === JobStatus.JOB_STATUS_CANCELED) { +// reject("job canceled") +// } +// if (job.status === JobStatus.JOB_STATUS_FAILED) { +// reject("job failed") +// } +// if (job.status === status) { +// cancel() +// resolve() +// } +// }, jobId) +// } catch (e) { +// reject(e) +// } +// }) +// } + +// function waitForBalance(address: string, greaterThan: number) { +// return new Promise(async (resolve, reject) => { +// while (true) { +// try { +// const res = await c.info() +// if (!res.info) { +// reject("no balance info returned") +// return +// } +// const info = res.info.balancesList.find((info) => info.addr?.addr === address) +// if (!info) { +// reject("address not in balances list") +// return +// } +// if (info.balance > greaterThan) { +// resolve(info.balance) +// return +// } +// } catch (e) { +// reject(e) +// } +// await new Promise((r) => setTimeout(r, 1000)) +// } +// }) +// } +// }) diff --git a/src/types.ts b/src/types.ts index 4a0595d6..db757596 100644 --- a/src/types.ts +++ b/src/types.ts @@ -10,7 +10,12 @@ export interface Config extends grpc.RpcOptions { host: string /** - * A FFS auth token + * A storage profile auth token */ authToken?: string + + /** + * An admin auth token + */ + adminToken?: string } diff --git a/src/util/grpc-helpers.ts b/src/util/grpc-helpers.ts index 56e195cd..02650f5b 100644 --- a/src/util/grpc-helpers.ts +++ b/src/util/grpc-helpers.ts @@ -4,6 +4,7 @@ import { WebsocketTransport } from "@textile/grpc-transport" export const host = "http://0.0.0.0:6002" const tokenKey = "X-ffs-Token" +const adminTokenKey = "X-pow-admin-token" export function promise( handler: (callback: (error: V | null, resp: U | null) => void) => void, @@ -23,20 +24,26 @@ export function promise( }) } -export const useToken = ( +export const useTokens = ( initialToken?: string, + initialAdminToken?: string, ): Readonly<{ getMeta: () => grpc.Metadata getHeaders: () => Record setToken: (t: string) => void + setAdminToken: (t: string) => void }> => { let token = initialToken + let adminToken = initialAdminToken const getMeta = () => { const meta = new grpc.Metadata() if (token) { meta.set(tokenKey, token) } + if (adminToken) { + meta.set(adminTokenKey, adminToken) + } return meta } @@ -52,7 +59,11 @@ export const useToken = ( token = t } - return Object.freeze({ getMeta, getHeaders, setToken }) + const setAdminToken = (t: string) => { + adminToken = t + } + + return Object.freeze({ getMeta, getHeaders, setToken, setAdminToken }) } export const getTransport = (): grpc.TransportFactory => WebsocketTransport() diff --git a/src/wallet/index.spec.ts b/src/wallet/index.spec.ts index 5b579fa5..ed8c1932 100644 --- a/src/wallet/index.spec.ts +++ b/src/wallet/index.spec.ts @@ -1,38 +1,38 @@ -import { expect } from "chai" -import { createWallet } from "." -import { getTransport, host } from "../util" +// import { expect } from "chai" +// import { createWallet } from "." +// import { getTransport, host } from "../util" -describe("wallet", () => { - const c = createWallet({ host, transport: getTransport() }) +// describe("wallet", () => { +// const c = createWallet({ host, transport: getTransport() }) - it("should list addresses", async () => { - await expectAddresses(0) - }) +// it("should list addresses", async () => { +// await expectAddresses(0) +// }) - it("should create a new address", async () => { - await expectNewAddress() - }) +// it("should create a new address", async () => { +// await expectNewAddress() +// }) - it("should check balance", async () => { - const addrs = await expectAddresses(0) - await c.balance(addrs[0]) - }) +// it("should check balance", async () => { +// const addrs = await expectAddresses(0) +// await c.balance(addrs[0]) +// }) - it("should send fil", async () => { - const addrs = await expectAddresses(0) - const newAddr = await expectNewAddress() - await c.sendFil(addrs[0], newAddr, 10) - }) +// it("should send fil", async () => { +// const addrs = await expectAddresses(0) +// const newAddr = await expectNewAddress() +// await c.sendFil(addrs[0], newAddr, 10) +// }) - async function expectAddresses(lengthGreaterThan: number) { - const { addressesList } = await c.list() - expect(addressesList).length.greaterThan(lengthGreaterThan) - return addressesList - } +// async function expectAddresses(lengthGreaterThan: number) { +// const { addressesList } = await c.list() +// expect(addressesList).length.greaterThan(lengthGreaterThan) +// return addressesList +// } - async function expectNewAddress() { - const { address } = await c.newAddress() - expect(address).length.greaterThan(0) - return address - } -}) +// async function expectNewAddress() { +// const { address } = await c.newAddress() +// expect(address).length.greaterThan(0) +// return address +// } +// }) diff --git a/src/wallet/index.ts b/src/wallet/index.ts index f18ff20d..61df2032 100644 --- a/src/wallet/index.ts +++ b/src/wallet/index.ts @@ -1,86 +1,140 @@ +import { grpc } from "@improbable-eng/grpc-web" import { + AddressesRequest, + AddressesResponse, BalanceRequest, BalanceResponse, - ListRequest, - ListResponse, NewAddressRequest, NewAddressResponse, SendFilRequest, -} from "@textile/grpc-powergate-client/dist/wallet/rpc/rpc_pb" -import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/wallet/rpc/rpc_pb_service" + SendFilResponse, + SignMessageRequest, + SignMessageResponse, + VerifyMessageRequest, + VerifyMessageResponse, +} from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" +import { PowergateServiceClient } from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb_service" import { Config } from "../types" import { promise } from "../util" export interface Wallet { /** - * Create a new wallet address. - * @param type The type of address to create, bls or secp256k1. - * @returns The new address. + * Get the balance for a wallet address. + * @param address The address to get the balance for. + * @returns The address balance. */ - newAddress: (type?: "bls" | "secp256k1") => Promise + balance: (address: string) => Promise /** - * List all wallet addresses. - * @returns The list of wallet addresses. + * Create a new wallet address associates with the current storage profile. + * @param name A human readable name for the address. + * @param type Address type, defaults to bls. + * @param makeDefault Specify if the new address should become the default address for this FFS instance, defaults to false. + * @returns Information about the newly created address. */ - list: () => Promise + newAddress: ( + name: string, + type?: "bls" | "secp256k1" | undefined, + makeDefault?: boolean | undefined, + ) => Promise /** - * Get the balance for a wallet address. - * @param address The address to get the balance for. - * @returns The address balance. + * Get all wallet addresses associated with the current storage profile. + * @returns A list of wallet addresses. */ - balance: (address: string) => Promise + addresses: () => Promise /** - * Send Fil from one address to another. - * @param from The address to send from. - * @param to The address to send to. - * @param amount The amount of Fil to send. + * Send FIL from an address associated with the current storage profile to any other address. + * @param from The address to send FIL from. + * @param to The address to send FIL to. + * @param amount The amount of FIL to send. */ - sendFil: (from: string, to: string, amount: number) => Promise + sendFil: (from: string, to: string, amount: number) => Promise + + /** + * Sign a message with the specified address. + * @param address The address used to sign the message. + * @param message The message to sign. + * @returns The signature. + */ + signMessage: (address: string, message: Uint8Array) => Promise + + /** + * Verify a signed message. + * @param address The address that should have signed the message. + * @param message The message to verify. + * @param signatre The signature to verify. + * @returns Whether or not the signature is valid for the provided address and message. + */ + verifyMessage: ( + address: string, + message: Uint8Array, + signature: Uint8Array | string, + ) => Promise } /** * @ignore */ -export const createWallet = (config: Config): Wallet => { - const client = new RPCServiceClient(config.host, config) +export const createWallet = (config: Config, getMeta: () => grpc.Metadata): Wallet => { + const client = new PowergateServiceClient(config.host, config) return { - newAddress: (type: "bls" | "secp256k1" = "bls") => { - const req = new NewAddressRequest() - req.setType(type) + balance: (address: string) => { + const req = new BalanceRequest() + req.setAddress(address) return promise( - (cb) => client.newAddress(req, cb), - (resp: NewAddressResponse) => resp.toObject(), + (cb) => client.balance(req, getMeta(), cb), + (res: BalanceResponse) => res.toObject(), ) }, - list: () => - promise( - (cb) => client.list(new ListRequest(), cb), - (resp: ListResponse) => resp.toObject(), - ), - - balance: (address: string) => { - const req = new BalanceRequest() - req.setAddress(address) + newAddress: (name: string, type?: "bls" | "secp256k1", makeDefault?: boolean) => { + const req = new NewAddressRequest() + req.setName(name) + req.setAddressType(type || "bls") + req.setMakeDefault(makeDefault || false) return promise( - (cb) => client.balance(req, cb), - (resp: BalanceResponse) => resp.toObject(), + (cb) => client.newAddress(req, getMeta(), cb), + (res: NewAddressResponse) => res.toObject(), ) }, + addresses: () => + promise( + (cb) => client.addresses(new AddressesRequest(), getMeta(), cb), + (res: AddressesResponse) => res.toObject(), + ), + sendFil: (from: string, to: string, amount: number) => { const req = new SendFilRequest() req.setFrom(from) req.setTo(to) req.setAmount(amount) return promise( - (cb) => client.sendFil(req, cb), - () => { - // nothing to return - }, + (cb) => client.sendFil(req, getMeta(), cb), + (res: SendFilResponse) => res.toObject(), + ) + }, + + signMessage: (address: string, message: Uint8Array) => { + const req = new SignMessageRequest() + req.setAddress(address) + req.setMessage(message) + return promise( + (cb) => client.signMessage(req, getMeta(), cb), + (res: SignMessageResponse) => res.toObject(), + ) + }, + + verifyMessage: (address: string, message: Uint8Array, signature: Uint8Array | string) => { + const req = new VerifyMessageRequest() + req.setAddress(address) + req.setMessage(message) + req.setSignature(signature) + return promise( + (cb) => client.verifyMessage(req, getMeta(), cb), + (res: VerifyMessageResponse) => res.toObject(), ) }, } From 3e2d8205a0384a0dcf6cbd1e6bd88877134869ab Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Mon, 2 Nov 2020 11:16:38 -0600 Subject: [PATCH 2/9] fix: bigint use Signed-off-by: Aaron Sutula --- src/integration.spec.ts | 126 ++++++++++++++++++---------------------- src/wallet/index.ts | 6 +- 2 files changed, 61 insertions(+), 71 deletions(-) diff --git a/src/integration.spec.ts b/src/integration.spec.ts index bb2fa8a5..94184f66 100644 --- a/src/integration.spec.ts +++ b/src/integration.spec.ts @@ -1,6 +1,5 @@ import { expect } from "chai" import crypto from "crypto" -import fs from "fs" import { ApplyOptions, createPow, Pow, powTypes } from "." import { host } from "./util" @@ -63,8 +62,8 @@ describe("pow", () => { const pow = newPow() const auth = await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) const res = await pow.admin.storageJobs.executing(auth.id, cid) expect(res.storageJobsList).length(1) @@ -77,8 +76,8 @@ describe("pow", () => { const pow = newPow() const auth = await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.admin.storageJobs.latestFinal(auth.id, cid) expect(res.storageJobsList).empty @@ -94,8 +93,8 @@ describe("pow", () => { const pow = newPow() const auth = await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.admin.storageJobs.latestSuccessful(auth.id, cid) expect(res.storageJobsList).empty @@ -112,8 +111,8 @@ describe("pow", () => { const pow = newPow() const auth = await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) await expectApplyStorageConfig(pow, cid) const res = await pow.admin.storageJobs.queued(auth.id, cid) expect(res.storageJobsList).length.lessThan(2) @@ -123,8 +122,8 @@ describe("pow", () => { const pow = newPow() const auth = await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.admin.storageJobs.summary(auth.id, cid) expect(res.executingStorageJobsList).length(1) @@ -163,16 +162,14 @@ describe("pow", () => { }) it("should send fil", async function () { - this.timeout(120000) + this.timeout(20000) const pow = newPow() const res0 = await pow.admin.wallet.newAddress() const res1 = await pow.admin.wallet.newAddress() - await waitForBalance(pow, res0.address, 0) - const bal2 = await waitForBalance(pow, res1.address, 0) + await waitForBalance(pow, res0.address) + const bal = await waitForBalance(pow, res1.address) await pow.admin.wallet.sendFil(res0.address, res1.address, 10) - // ToDo: see why fil is not sending - // const bal3 = await waitForBalance(pow, res1.address, bal2) - // expect(bal3).equals(bal2 + 10) + await waitForBalance(pow, res1.address, bal) }) }) }) @@ -183,8 +180,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.data.cidInfo(cid) expect(res.cidInfosList).length(1) @@ -216,8 +213,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addrs = await expectAddresses(pow, 1) - await waitForBalance(pow, addrs[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addrs[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) const bytes = await pow.data.get(cid) @@ -237,11 +234,11 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addrs = await expectAddresses(pow, 1) - await waitForBalance(pow, addrs[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addrs[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) - const cid2 = await expectStageData(pow, crypto.randomBytes(1024)) + const cid2 = await expectStage(pow, crypto.randomBytes(1024)) const res = await pow.data.replaceData(cid, cid2) expect(res.jobId).length.greaterThan(0) await watchJobUntil(pow, res.jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) @@ -250,7 +247,7 @@ describe("pow", () => { it("should stage", async () => { const pow = newPow() await expectNewInstance(pow) - await expectStageData(pow, crypto.randomBytes(1024)) + await expectStage(pow, crypto.randomBytes(1024)) }) it("should stage folder", async () => { @@ -263,7 +260,7 @@ describe("pow", () => { it("should watch logs", async function () { const pow = newPow() await expectNewInstance(pow) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) const event = await new Promise((resolve) => { pow.data.watchLogs((event) => resolve(event), cid, { includeHistory: true, jobId }) @@ -279,8 +276,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) const pending = await pow.deals.storageDealRecords({ includePending: true }) @@ -311,14 +308,14 @@ describe("pow", () => { it("should apply", async () => { const pow = newPow() await expectNewInstance(pow) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const cid = await expectStage(pow, crypto.randomBytes(1024)) await expectApplyStorageConfig(pow, cid) }) it("should remove", async () => { const pow = newPow() await expectNewInstance(pow) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const conf = await expectDefaultStorageConfig(pow) conf.cold = { ...conf.cold, enabled: false } // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -333,8 +330,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) await pow.storageJobs.cancel(jobId) }) @@ -343,8 +340,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.storageJobs.executing() expect(res.storageJobsList).length(1) @@ -359,8 +356,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.storageJobs.latestFinal() expect(res.storageJobsList).empty @@ -378,8 +375,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.storageJobs.latestSuccessful() expect(res.storageJobsList).empty @@ -396,8 +393,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) await expectApplyStorageConfig(pow, cid) const res = await pow.storageJobs.queued(cid) expect(res.storageJobsList).length.lessThan(2) @@ -407,8 +404,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) const res = await pow.storageJobs.storageConfigForJob(jobId) expect(res.storageConfig).not.undefined @@ -418,8 +415,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) const res = await pow.storageJobs.storageJob(jobId) expect(res.job?.id).equals(jobId) @@ -429,8 +426,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.storageJobs.summary() expect(res.executingStorageJobsList).length(1) @@ -448,8 +445,8 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) - const cid = await expectStageData(pow, crypto.randomBytes(1024)) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_EXECUTING) }) @@ -466,7 +463,7 @@ describe("pow", () => { const pow = newPow() await expectNewInstance(pow) const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address, 0) + await waitForBalance(pow, addressees[0].address) }) it("should create new address", async function () { @@ -482,11 +479,10 @@ describe("pow", () => { await expectNewInstance(pow) await pow.wallet.newAddress("new one") const addressees = await expectAddresses(pow, 2) - await waitForBalance(pow, addressees[0].address, 0) - await waitForBalance(pow, addressees[1].address, 0) - await pow.wallet.sendFil(addressees[0].address, addressees[1].address, 10) - // ToDo: Check that sendFil is working on the back end - // await waitForBalance(pow, addressees[1].address, bal + 9) + await waitForBalance(pow, addressees[0].address) + const bal = await waitForBalance(pow, addressees[1].address) + await pow.wallet.sendFil(addressees[0].address, addressees[1].address, BigInt(10)) + await waitForBalance(pow, addressees[1].address, bal) }) it("should sign message", async function () { @@ -510,7 +506,7 @@ describe("pow", () => { }) function newPow(): Pow { - return createPow({ host }) + return createPow({ host, debug: true }) } async function expectNewInstance(pow: Pow) { @@ -530,14 +526,7 @@ async function expectDefaultStorageConfig(pow: Pow) { return res.defaultStorageConfig! } -async function expectStage(pow: Pow, path: string) { - const buffer = fs.readFileSync(path) - const res = await pow.data.stage(buffer) - expect(res.cid).length.greaterThan(0) - return res.cid -} - -async function expectStageData(pow: Pow, data: Buffer) { +async function expectStage(pow: Pow, data: Buffer) { const res = await pow.data.stage(data) expect(res.cid).length.greaterThan(0) return res.cid @@ -583,14 +572,15 @@ async function expectAddresses(pow: Pow, length: number) { return res.addressesList } -function waitForBalance(pow: Pow, address: string, greaterThan: number) { - return new Promise(async (resolve, reject) => { +function waitForBalance(pow: Pow, address: string, greaterThan?: bigint) { + return new Promise(async (resolve, reject) => { while (true) { try { const res = await pow.wallet.balance(address) - if (res.balance > greaterThan) { - resolve(res.balance) - return + const balace = BigInt(res.balance) + if (balace > (greaterThan || BigInt(0))) { + resolve(balace) + break } } catch (e) { reject(e) diff --git a/src/wallet/index.ts b/src/wallet/index.ts index 61df2032..9c6066a0 100644 --- a/src/wallet/index.ts +++ b/src/wallet/index.ts @@ -50,7 +50,7 @@ export interface Wallet { * @param to The address to send FIL to. * @param amount The amount of FIL to send. */ - sendFil: (from: string, to: string, amount: number) => Promise + sendFil: (from: string, to: string, amount: bigint) => Promise /** * Sign a message with the specified address. @@ -106,11 +106,11 @@ export const createWallet = (config: Config, getMeta: () => grpc.Metadata): Wall (res: AddressesResponse) => res.toObject(), ), - sendFil: (from: string, to: string, amount: number) => { + sendFil: (from: string, to: string, amount: bigint) => { const req = new SendFilRequest() req.setFrom(from) req.setTo(to) - req.setAmount(amount) + req.setAmount(amount.toString()) return promise( (cb) => client.sendFil(req, getMeta(), cb), (res: SendFilResponse) => res.toObject(), From a56dff3a7d79d04d9b9940d42f2821729366e1ab Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Mon, 2 Nov 2020 11:18:39 -0600 Subject: [PATCH 3/9] fix: deleting old test files Signed-off-by: Aaron Sutula --- src/ffs/index.spec.ts | 383 ------------------------------- src/storage-config/index.spec.ts | 110 --------- src/wallet/index.spec.ts | 38 --- 3 files changed, 531 deletions(-) delete mode 100644 src/ffs/index.spec.ts delete mode 100644 src/storage-config/index.spec.ts delete mode 100644 src/wallet/index.spec.ts diff --git a/src/ffs/index.spec.ts b/src/ffs/index.spec.ts deleted file mode 100644 index 56e68094..00000000 --- a/src/ffs/index.spec.ts +++ /dev/null @@ -1,383 +0,0 @@ -// import { -// JobStatus, -// JobStatusMap, -// StorageConfig, -// } from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" -// import { expect } from "chai" -// import fs from "fs" -// import { createFFS } from "." -// import { getTransport, host, useTokens } from "../util" -// import { PushStorageConfigOptions } from "./types" - -// describe("ffs", function () { -// this.timeout(180000) - -// const { getMeta, getHeaders, setToken } = useTokens("") - -// const c = createFFS({ host, transport: getTransport() }, getMeta, getHeaders) - -// it("should create an instance", async () => { -// await expectNewInstance() -// }) - -// it("should list instances", async () => { -// await expectNewInstance() -// const res = await c.list() -// expect(res.instancesList).length.greaterThan(0) -// }) - -// it("should get instance id", async () => { -// const instanceInfo = await expectNewInstance() -// const res = await c.id() -// expect(res.id).eq(instanceInfo.id) -// }) - -// it("should get addrs", async () => { -// await expectNewInstance() -// await expectAddrs(1) -// }) - -// it("should get the default config", async () => { -// await expectNewInstance() -// await expectDefaultStorageConfig() -// }) - -// it("should create a new addr", async () => { -// await expectNewInstance() -// await expectAddrs(1) -// await expectNewAddr() -// await expectAddrs(2) -// }) - -// it("should set default config", async () => { -// await expectNewInstance() -// const defaultConfig = await expectDefaultStorageConfig() -// await c.setDefaultStorageConfig(defaultConfig) -// }) - -// it("should get info", async () => { -// await expectNewInstance() -// const res = await c.info() -// expect(res.info).not.undefined -// }) - -// it("should stage", async () => { -// await expectNewInstance() -// await expectStage("sample-data/samplefile") -// }) - -// it("should stage a folder", async () => { -// await expectNewInstance() -// const res = await c.stageFolder("./sample-data") -// expect(res).length.greaterThan(0) -// }) - -// it("should push config", async () => { -// await expectNewInstance() -// const cid = await expectStage("sample-data/samplefile") -// const config = await expectDefaultStorageConfig() -// await expectPushStorageConfig(cid, { override: false, storageConfig: config }) -// }) - -// it("should get storage job", async () => { -// await expectNewInstance() -// const cid = await expectStage("sample-data/samplefile") -// const jobId = await expectPushStorageConfig(cid) -// const res = await c.getStorageJob(jobId) -// expect(res.job?.cid).eq(cid) -// }) - -// it("should watch job", async () => { -// await expectNewInstance() -// const addrs = await expectAddrs(1) -// await waitForBalance(addrs[0].addr, 0) -// const cid = await expectStage("sample-data/samplefile") -// const jobId = await expectPushStorageConfig(cid) -// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) -// // Test that if we call watchJobs again after the job has finished -// // that is calls us back with the latest status -// await new Promise((r) => setTimeout(r, 10000)) -// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) -// }) - -// it("should watch logs", async () => { -// await expectNewInstance() -// const cid = await expectStage("sample-data/samplefile") -// await expectPushStorageConfig(cid) -// await new Promise((resolve, reject) => { -// const cancel = c.watchLogs( -// (logEvent) => { -// if (logEvent.cid.length > 0) { -// cancel() -// resolve() -// } else { -// cancel() -// reject("empty log cid") -// } -// }, -// cid, -// { includeHistory: true }, -// ) -// }) -// }) - -// it("should get storage deal records", async () => { -// await expectNewInstance() -// const addrs = await expectAddrs(1) -// await waitForBalance(addrs[0].addr, 0) - -// const [cid1, cid2, cid3] = await Promise.all([ -// expectStage("sample-data/samplefile"), -// expectStage("sample-data/samplefile2"), -// expectStage("sample-data/samplefile3"), -// ]) - -// const jobId1 = await expectPushStorageConfig(cid1) -// await new Promise((r) => setTimeout(r, 1000)) -// const jobId2 = await expectPushStorageConfig(cid2) -// await new Promise((r) => setTimeout(r, 1000)) -// const jobId3 = await expectPushStorageConfig(cid3) - -// await Promise.all([ -// waitForJobStatus(jobId1, JobStatus.JOB_STATUS_EXECUTING), -// waitForJobStatus(jobId2, JobStatus.JOB_STATUS_EXECUTING), -// waitForJobStatus(jobId3, JobStatus.JOB_STATUS_EXECUTING), -// ]) - -// // wait for a second so some async work of actually starting a deal happens -// await new Promise((r) => setTimeout(r, 1000)) - -// const { recordsList: pendingRecords } = await c.listStorageDealRecords({ -// includePending: true, -// ascending: true, -// fromAddresses: [addrs[0].addr], -// dataCids: [cid1, cid2, cid3], -// }) -// expect(pendingRecords, "pending length").length(3) -// expect(pendingRecords[0].time).lessThan(pendingRecords[1].time).lessThan(pendingRecords[2].time) - -// const { recordsList: finalRecords } = await c.listStorageDealRecords({ includeFinal: true }) -// expect(finalRecords, "final empty").empty - -// await Promise.all([ -// waitForJobStatus(jobId1, JobStatus.JOB_STATUS_SUCCESS), -// waitForJobStatus(jobId2, JobStatus.JOB_STATUS_SUCCESS), -// waitForJobStatus(jobId3, JobStatus.JOB_STATUS_SUCCESS), -// ]) - -// const { recordsList: pendingRecords2 } = await c.listStorageDealRecords({ -// includePending: true, -// }) -// expect(pendingRecords2, "pending2 empty").empty - -// const { recordsList: finalRecords2 } = await c.listStorageDealRecords({ -// includeFinal: true, -// ascending: true, -// fromAddresses: [addrs[0].addr], -// dataCids: [cid1, cid2, cid3], -// }) -// expect(finalRecords2, "final2 length").length(3) -// expect(finalRecords2[0].time).lessThan(finalRecords2[1].time).lessThan(finalRecords2[2].time) -// }) - -// it("should get a retrieval deal record", async () => { -// // ToDo: Figure out hot to make sure the data in the previous push isn't cached in hot -// }) - -// it("should get storage config", async () => { -// await expectNewInstance() -// const cid = await expectStage("sample-data/samplefile") -// await expectPushStorageConfig(cid) -// const res = await c.getStorageConfig(cid) -// expect(res.config?.cold).not.undefined -// expect(res.config?.hot).not.undefined -// }) - -// it("should show", async () => { -// await expectNewInstance() -// const addrs = await expectAddrs(1) -// await waitForBalance(addrs[0].addr, 0) -// const cid = await expectStage("sample-data/samplefile") -// const jobId = await expectPushStorageConfig(cid) -// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) -// const res = await c.show(cid) -// expect(res.cidInfo).not.undefined -// }) - -// it("should show all", async () => { -// await expectNewInstance() -// const addrs = await expectAddrs(1) -// await waitForBalance(addrs[0].addr, 0) -// const cid = await expectStage("sample-data/samplefile") -// const jobId = await expectPushStorageConfig(cid) -// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) -// const res = await c.showAll() -// expect(res.cidInfosList).not.empty -// }) - -// it("should replace", async () => { -// await expectNewInstance() -// const addrs = await expectAddrs(1) -// await waitForBalance(addrs[0].addr, 0) -// const cid = await expectStage("sample-data/samplefile") -// const jobId = await expectPushStorageConfig(cid) -// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) -// const cid2 = await expectStage("sample-data/samplefile2") -// const res = await c.replace(cid, cid2) -// expect(res.jobId).length.greaterThan(0) -// }) - -// it("should get", async () => { -// await expectNewInstance() -// const addrs = await expectAddrs(1) -// await waitForBalance(addrs[0].addr, 0) -// const cid = await expectStage("sample-data/samplefile") -// const jobId = await expectPushStorageConfig(cid) -// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) -// const bytes = await c.get(cid) -// expect(bytes.byteLength).greaterThan(0) -// }) - -// it("should get a folder", async () => { -// await expectNewInstance() -// const res = await c.stageFolder("./sample-data") -// expect(res).length.greaterThan(0) -// await c.getFolder(res, "./output", { timeout: 10000 }) -// }) - -// it("should cancel a job", async () => { -// await expectNewInstance() -// const cid = await expectStage("sample-data/samplefile") -// const jobId = await expectPushStorageConfig(cid) -// await c.cancelJob(jobId) -// }) - -// it("should list payment channels", async () => { -// // TODO -// }) - -// it("should create a payment channel", async () => { -// // TODO -// }) - -// it("should redeem a payment channel", async () => { -// // TODO -// }) - -// it("should remove", async () => { -// await expectNewInstance() -// const cid = await expectStage("sample-data/samplefile") -// const conf: StorageConfig.AsObject = { -// repairable: false, -// cold: { -// enabled: false, -// }, -// hot: { -// allowUnfreeze: false, -// enabled: false, -// unfreezeMaxPrice: 0, -// }, -// } -// const jobId = await expectPushStorageConfig(cid, { override: false, storageConfig: conf }) -// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) -// await c.remove(cid) -// }) - -// it("should send fil", async () => { -// await expectNewInstance() -// const addrs = await expectAddrs(1) -// await waitForBalance(addrs[0].addr, 0) -// const addr = await expectNewAddr() -// await c.sendFil(addrs[0].addr, addr, 10) -// }) - -// async function expectNewInstance() { -// const res = await c.create() -// expect(res.id).not.empty -// expect(res.token).not.empty -// setToken(res.token) -// return res -// } - -// async function expectAddrs(length: number) { -// const res = await c.addrs() -// expect(res.addrsList).length(length) -// return res.addrsList -// } - -// async function expectNewAddr() { -// const res = await c.newAddr("my addr") -// expect(res.addr).length.greaterThan(0) -// return res.addr -// } - -// async function expectDefaultStorageConfig() { -// const res = await c.defaultStorageConfig() -// expect(res.defaultStorageConfig).not.undefined -// // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -// return res.defaultStorageConfig! -// } - -// async function expectStage(path: string) { -// const buffer = fs.readFileSync(path) -// const res = await c.stage(buffer) -// expect(res.cid).length.greaterThan(0) -// return res.cid -// } - -// async function expectPushStorageConfig(cid: string, opts?: PushStorageConfigOptions) { -// const res = await c.pushStorageConfig(cid, opts) -// expect(res.jobId).length.greaterThan(0) -// return res.jobId -// } - -// function waitForJobStatus(jobId: string, status: JobStatusMap[keyof JobStatusMap]) { -// return new Promise((resolve, reject) => { -// try { -// const cancel = c.watchJobs((job) => { -// if (job.errCause.length > 0) { -// reject(job.errCause) -// } -// if (job.status === JobStatus.JOB_STATUS_CANCELED) { -// reject("job canceled") -// } -// if (job.status === JobStatus.JOB_STATUS_FAILED) { -// reject("job failed") -// } -// if (job.status === status) { -// cancel() -// resolve() -// } -// }, jobId) -// } catch (e) { -// reject(e) -// } -// }) -// } - -// function waitForBalance(address: string, greaterThan: number) { -// return new Promise(async (resolve, reject) => { -// while (true) { -// try { -// const res = await c.info() -// if (!res.info) { -// reject("no balance info returned") -// return -// } -// const info = res.info.balancesList.find((info) => info.addr?.addr === address) -// if (!info) { -// reject("address not in balances list") -// return -// } -// if (info.balance > greaterThan) { -// resolve(info.balance) -// return -// } -// } catch (e) { -// reject(e) -// } -// await new Promise((r) => setTimeout(r, 1000)) -// } -// }) -// } -// }) diff --git a/src/storage-config/index.spec.ts b/src/storage-config/index.spec.ts deleted file mode 100644 index c5fc57d5..00000000 --- a/src/storage-config/index.spec.ts +++ /dev/null @@ -1,110 +0,0 @@ -// import { -// JobStatus, -// JobStatusMap, -// StorageConfig, -// } from "@textile/grpc-powergate-client/dist/proto/powergate/v1/powergate_pb" -// import { expect } from "chai" -// import fs from "fs" -// import { createStorageConfig } from "." -// import { createAdmin } from "../admin" -// import { getTransport, host, useTokens } from "../util" -// import { ApplyOptions } from "./types" - -// describe("storage config", function () { -// this.timeout(180000) - -// const { getMeta, _, setToken } = useTokens("") - -// const c = createStorageConfig({ host, transport: getTransport() }, getMeta) -// const a = createAdmin({ host, transport: getTransport() }) - -// it("should get the default config", async () => { -// await expectNewInstance() -// await expectDefaultStorageConfig() -// }) - -// it("should set default config", async () => { -// await expectNewInstance() -// const defaultConfig = await expectDefaultStorageConfig() -// await c.setDefault(defaultConfig) -// }) - -// it("should apply config", async () => { -// await expectNewInstance() -// const cid = await expectStage("sample-data/samplefile") -// const config = await expectDefaultStorageConfig() -// await expectPushStorageConfig(cid, { override: false, storageConfig: config }) -// }) - -// it("should remove", async () => { -// await expectNewInstance() -// const cid = await expectStage("sample-data/samplefile") -// const conf: StorageConfig.AsObject = { -// repairable: false, -// cold: { -// enabled: false, -// }, -// hot: { -// allowUnfreeze: false, -// enabled: false, -// unfreezeMaxPrice: 0, -// }, -// } -// const jobId = await expectPushStorageConfig(cid, { override: false, storageConfig: conf }) -// await waitForJobStatus(jobId, JobStatus.JOB_STATUS_SUCCESS) -// await c.remove(cid) -// }) - -// async function expectNewInstance() { -// const res = await a.profiles.createStorageProfile() -// expect(res.authEntry?.id).not.empty -// expect(res.authEntry?.token).not.empty -// // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -// setToken(res.authEntry!.token) -// return res -// } - -// async function expectDefaultStorageConfig() { -// const res = await c.default() -// expect(res.defaultStorageConfig).not.undefined -// // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -// return res.defaultStorageConfig! -// } - -// async function expectStage(path: string) { -// const buffer = fs.readFileSync(path) -// const res = await c.stage(buffer) -// expect(res.cid).length.greaterThan(0) -// return res.cid -// } - -// async function expectPushStorageConfig(cid: string, opts?: ApplyOptions) { -// const res = await c.apply(cid, opts) -// expect(res.jobId).length.greaterThan(0) -// return res.jobId -// } - -// function waitForJobStatus(jobId: string, status: JobStatusMap[keyof JobStatusMap]) { -// return new Promise((resolve, reject) => { -// try { -// const cancel = c.watchJobs((job) => { -// if (job.errCause.length > 0) { -// reject(job.errCause) -// } -// if (job.status === JobStatus.JOB_STATUS_CANCELED) { -// reject("job canceled") -// } -// if (job.status === JobStatus.JOB_STATUS_FAILED) { -// reject("job failed") -// } -// if (job.status === status) { -// cancel() -// resolve() -// } -// }, jobId) -// } catch (e) { -// reject(e) -// } -// }) -// } -// }) diff --git a/src/wallet/index.spec.ts b/src/wallet/index.spec.ts deleted file mode 100644 index ed8c1932..00000000 --- a/src/wallet/index.spec.ts +++ /dev/null @@ -1,38 +0,0 @@ -// import { expect } from "chai" -// import { createWallet } from "." -// import { getTransport, host } from "../util" - -// describe("wallet", () => { -// const c = createWallet({ host, transport: getTransport() }) - -// it("should list addresses", async () => { -// await expectAddresses(0) -// }) - -// it("should create a new address", async () => { -// await expectNewAddress() -// }) - -// it("should check balance", async () => { -// const addrs = await expectAddresses(0) -// await c.balance(addrs[0]) -// }) - -// it("should send fil", async () => { -// const addrs = await expectAddresses(0) -// const newAddr = await expectNewAddress() -// await c.sendFil(addrs[0], newAddr, 10) -// }) - -// async function expectAddresses(lengthGreaterThan: number) { -// const { addressesList } = await c.list() -// expect(addressesList).length.greaterThan(lengthGreaterThan) -// return addressesList -// } - -// async function expectNewAddress() { -// const { address } = await c.newAddress() -// expect(address).length.greaterThan(0) -// return address -// } -// }) From 731033b9c34b97f3f11ad3a8181f61343de5e557 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Mon, 2 Nov 2020 16:18:59 -0600 Subject: [PATCH 4/9] fix: couple more bigint updates Signed-off-by: Aaron Sutula --- src/admin/wallet.ts | 6 +++--- src/integration.spec.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/admin/wallet.ts b/src/admin/wallet.ts index a908ef97..31ff17a6 100644 --- a/src/admin/wallet.ts +++ b/src/admin/wallet.ts @@ -31,7 +31,7 @@ export interface Wallet { * @param to The address to send to. * @param amount The amount of Fil to send. */ - sendFil: (from: string, to: string, amount: number) => Promise + sendFil: (from: string, to: string, amount: bigint) => Promise } /** @@ -55,11 +55,11 @@ export const createWallet = (config: Config, getMeta: () => grpc.Metadata): Wall (resp: AddressesResponse) => resp.toObject(), ), - sendFil: (from: string, to: string, amount: number) => { + sendFil: (from: string, to: string, amount: bigint) => { const req = new SendFilRequest() req.setFrom(from) req.setTo(to) - req.setAmount(amount) + req.setAmount(amount.toString()) return promise( (cb) => client.sendFil(req, getMeta(), cb), (resp: SendFilResponse) => resp.toObject(), diff --git a/src/integration.spec.ts b/src/integration.spec.ts index 94184f66..d4292837 100644 --- a/src/integration.spec.ts +++ b/src/integration.spec.ts @@ -168,7 +168,7 @@ describe("pow", () => { const res1 = await pow.admin.wallet.newAddress() await waitForBalance(pow, res0.address) const bal = await waitForBalance(pow, res1.address) - await pow.admin.wallet.sendFil(res0.address, res1.address, 10) + await pow.admin.wallet.sendFil(res0.address, res1.address, BigInt(10)) await waitForBalance(pow, res1.address, bal) }) }) From 21f8689ee6bd09f4f584e8b79a3216d78393c97a Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Wed, 4 Nov 2020 09:41:06 -0600 Subject: [PATCH 5/9] fix: some comments and updated address type Signed-off-by: Aaron Sutula --- src/admin/wallet.ts | 2 +- src/storage-jobs/index.ts | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/admin/wallet.ts b/src/admin/wallet.ts index 31ff17a6..3d97c1db 100644 --- a/src/admin/wallet.ts +++ b/src/admin/wallet.ts @@ -42,7 +42,7 @@ export const createWallet = (config: Config, getMeta: () => grpc.Metadata): Wall return { newAddress: (type: "bls" | "secp256k1" = "bls") => { const req = new NewAddressRequest() - req.setType(type) + req.setAddressType(type) return promise( (cb) => client.newAddress(req, getMeta(), cb), (resp: NewAddressResponse) => resp.toObject(), diff --git a/src/storage-jobs/index.ts b/src/storage-jobs/index.ts index 40e84cd7..1888a708 100644 --- a/src/storage-jobs/index.ts +++ b/src/storage-jobs/index.ts @@ -38,14 +38,39 @@ export interface StorageJobs { */ storageConfigForJob: (jobId: string) => Promise + /** + * Get queued jobs in the storage profile for the specified cids or all cids. + * @param cids A list of cids to get jobs for, providing no cids means all cids. + * @returns An object containing a list of jobs. + */ queued: (...cids: string[]) => Promise + /** + * Get executing jobs in the storage profile for the specified cids or all cids. + * @param cids A list of cids to get jobs for, providing no cids means all cids. + * @returns An object containing a list of jobs. + */ executing: (...cids: string[]) => Promise + /** + * Get the latest final jobs in the storage profile for the specified cids or all cids. + * @param cids A list of cids to get jobs for, providing no cids means all cids. + * @returns An object containing a list of jobs. + */ latestFinal: (...cids: string[]) => Promise + /** + * Get latest successful jobs in the storage profile for the specified cids or all cids. + * @param cids A list of cids to get jobs for, providing no cids means all cids. + * @returns An object containing a list of jobs. + */ latestSuccessful: (...cids: string[]) => Promise + /** + * Get a summary of jobs in the storage profile for the specified cids or all cids. + * @param cids A list of cids to get a job summary for, providing no cids means all cids. + * @returns An object containing a summary of jobs. + */ summary: (...cids: string[]) => Promise /** From 660fa1afb8f7e4bffb744ef9b731d06928b619ce Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Wed, 4 Nov 2020 18:48:24 -0600 Subject: [PATCH 6/9] chore: update to release grpc lib Signed-off-by: Aaron Sutula --- package-lock.json | 14 ++++----- package.json | 2 +- src/integration.spec.ts | 67 ++++++++++++++++++++++++----------------- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9cf7feb7..c6d84255 100644 --- a/package-lock.json +++ b/package-lock.json @@ -562,12 +562,12 @@ } }, "@textile/grpc-powergate-client": { - "version": "0.6.8", - "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-0.6.8.tgz", - "integrity": "sha512-cBWyLMGRilQgFq9VZI9hN6sPO7xc+m+zoxIfab2AEUGFhfDa2Q39qnUKUDYkNc4BcT+BOj66dWpD2kYzq5D2/A==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-1.0.0.tgz", + "integrity": "sha512-TO/mlqdFTqFfemaa/pSd1Dwfw1KoqNW9Zz9JL7yo34C13aWbp9BTUY96dwRyEGROkdMmSl6RmAoa27tDgKjnvg==", "requires": { "@improbable-eng/grpc-web": "^0.13.0", - "@types/google-protobuf": "^3.7.3", + "@types/google-protobuf": "^3.7.4", "google-protobuf": "^3.13.0" } }, @@ -615,9 +615,9 @@ } }, "@types/google-protobuf": { - "version": "3.7.3", - "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.7.3.tgz", - "integrity": "sha512-FRwj40euE2bYkG+0X5w2nEA8yAzgJRcEa7RBd0Gsdkb9/tPM2pctVVAvnOUTbcXo2VmIHPo0Ae94Gl9vRHfKzg==" + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.7.4.tgz", + "integrity": "sha512-6PjMFKl13cgB4kRdYtvyjKl8VVa0PXS2IdVxHhQ8GEKbxBkyJtSbaIeK1eZGjDKN7dvUh4vkOvU9FMwYNv4GQQ==" }, "@types/json-schema": { "version": "7.0.6", diff --git a/package.json b/package.json index dfc2c23d..4c50ab5d 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ } }, "dependencies": { - "@textile/grpc-powergate-client": "0.6.8", + "@textile/grpc-powergate-client": "1.0.0", "@textile/grpc-transport": "0.0.3", "ipfs-http-client": "^47.0.1", "it-block": "^2.0.0" diff --git a/src/integration.spec.ts b/src/integration.spec.ts index d4292837..81d22db6 100644 --- a/src/integration.spec.ts +++ b/src/integration.spec.ts @@ -1,35 +1,37 @@ import { expect } from "chai" +import cp from "child_process" import crypto from "crypto" +import wait from "wait-on" import { ApplyOptions, createPow, Pow, powTypes } from "." import { host } from "./util" -// beforeEach(async function () { -// this.timeout(120000) -// cp.exec(`cd powergate-docker && BIGSECTORS=false make localnet`, (err) => { -// if (err) { -// throw err -// } -// }) -// await wait({ -// resources: ["http://0.0.0.0:6002"], -// timeout: 120000, -// validateStatus: function () { -// return true // the call expectedly returns 404, so just allow that -// }, -// }) -// }) - -// afterEach(async function () { -// this.timeout(120000) -// await new Promise((resolve, reject) => { -// cp.exec(`cd powergate-docker && make localnet-down`, (err, stdout) => { -// if (err) { -// reject(err) -// } -// resolve(stdout) -// }) -// }) -// }) +beforeEach(async function () { + this.timeout(120000) + cp.exec(`cd powergate-docker && BIGSECTORS=false make localnet`, (err) => { + if (err) { + throw err + } + }) + await wait({ + resources: ["http://0.0.0.0:6002"], + timeout: 120000, + validateStatus: function () { + return true // the call expectedly returns 404, so just allow that + }, + }) +}) + +afterEach(async function () { + this.timeout(120000) + await new Promise((resolve, reject) => { + cp.exec(`cd powergate-docker && make localnet-down`, (err, stdout) => { + if (err) { + reject(err) + } + resolve(stdout) + }) + }) +}) describe("pow", () => { it("should get build info", async () => { @@ -52,6 +54,7 @@ describe("pow", () => { it("should list profiles", async () => { const pow = newPow() + await expectNewInstance(pow) const res = await pow.admin.profiles.storageProfiles() expect(res.authEntriesList).length.greaterThan(0) }) @@ -140,11 +143,19 @@ describe("pow", () => { describe("all storage jobs", () => { it("should get summary", async function () { + this.timeout(180000) const pow = newPow() + await expectNewInstance(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) const res = await pow.admin.storageJobs.summary("") expect(res.jobCounts?.latestFinal).greaterThan(0) expect(res.jobCounts?.latestSuccessful).greaterThan(0) expect(res.jobCounts?.queued).equals(0) + expect(res.jobCounts?.executing).equals(0) }) }) @@ -506,7 +517,7 @@ describe("pow", () => { }) function newPow(): Pow { - return createPow({ host, debug: true }) + return createPow({ host }) } async function expectNewInstance(pow: Pow) { From 663fab4fa8482a99855aacd7ea9d03cfed25039b Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Wed, 4 Nov 2020 21:26:35 -0600 Subject: [PATCH 7/9] fix: update readme code samples Signed-off-by: Aaron Sutula --- README.md | 86 ++++++++++++++++++++++--------------------------------- 1 file changed, 35 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 9b0ded8f..795792b5 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ const host = "http://0.0.0.0:6002" // or whatever powergate instance you want const pow = createPow({ host }) ``` -Many APIs are immediately available and don't require authorization. +Most Powergate APIs require authorization in the form of a Storage Profile auth token. Storage Profiles are created using the `admin` API. Powergate's backend may be configured to secure the `admin` API with an auth token, and in that case, you'll neeed to set the admin auth token on the client as shown below. ```typescript import { createPow } from "@textile/powergate-client" @@ -58,31 +58,18 @@ const host = "http://0.0.0.0:6002" // or whatever powergate instance you want const pow = createPow({ host }) -async function exampleCode () { - const { status, messagesList } = await pow.health.check() - - const { peersList } = await pow.net.peers() -} -``` - -Other APIs require authorization. The main API you'll interact with is the Filecoin File System (FFS), and it requires authorization. First, create a new FFS instance. - -```typescript -import { createPow } from "@textile/powergate-client" - -const host = "http://0.0.0.0:6002" // or whatever powergate instance you want - -const pow = createPow({ host }) +// Set the admin auth token if required. +pow.setAdminToken("") async function exampleCode () { - const { token } = await pow.ffs.create() // save this token for later use! - return token + const { authEntry } = await pow.admin.profiles.createStorageProfile() // save this token for later use! + return authEntry?.token } ``` -Currently, the returned auth token is the only thing that gives you access to your FFS instance at a later time, so be sure to save it securely. +The returned auth token is the only thing that gives access to the corresponding Storage Profile at a later time, so be sure to save it securely. -Once you have an auth token, either by creating a new FFS instance or by reading one you previously saved, set the auth token you'd like the Powergate client to use. +A Storage Profile auth token can later be set for the Powergate client so that the client authenticates with the Storage Profile associated with the auth token. ```typescript import { createPow } from "@textile/powergate-client" @@ -91,66 +78,63 @@ const host = "http://0.0.0.0:6002" // or whatever powergate instance you want const pow = createPow({ host }) -const authToken = '' +const token = "" -pow.setToken(authToken) +pow.setToken(token) ``` -Now, the FFS API is available for you to use. +Now, all authenticated APIs are available for you to use. ```typescript -import { JobStatus } from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" import fs from "fs" -import { createPow } from "@textile/powergate-client" +import { createPow, powTypes } from "@textile/powergate-client" const host = "http://0.0.0.0:6002" // or whatever powergate instance you want const pow = createPow({ host }) async function exampleCode() { - // get wallet addresses associated with your FFS instance - const { addrsList } = await pow.ffs.addrs() + // get wallet addresses associated with your storage profile + const { addressesList } = await pow.wallet.addresses() - // create a new address associated with your ffs instance - const { addr } = await pow.ffs.newAddr("my new addr") + // create a new address associated with your storage profile + const { address } = await pow.wallet.newAddress("my new address") - // get general info about your ffs instance - const { info } = await pow.ffs.info() + // get build information about the powergate server + const res = await pow.buildInfo() - // cache data in IPFS in preparation to store it using FFS + // cache data in IPFS in preparation to store it const buffer = fs.readFileSync(`path/to/a/file`) - const { cid } = await pow.ffs.stage(buffer) + const { cid } = await pow.data.stage(buffer) - // store the data in FFS using the default storage configuration - const { jobId } = await pow.ffs.pushStorageConfig(cid) + // store the data using the default storage configuration + const { jobId } = await pow.storageConfig.apply(cid) - // watch the FFS job status to see the storage process progressing - const jobsCancel = pow.ffs.watchJobs((job) => { - if (job.status === JobStatus.JOB_STATUS_CANCELED) { + // watch the job status to see the storage process progressing + const jobsCancel = pow.storageJobs.watch((job) => { + if (job.status === powTypes.JobStatus.JOB_STATUS_CANCELED) { console.log("job canceled") - } else if (job.status === JobStatus.JOB_STATUS_FAILED) { + } else if (job.status === powTypes.JobStatus.JOB_STATUS_FAILED) { console.log("job failed") - } else if (job.status === JobStatus.JOB_STATUS_SUCCESS) { + } else if (job.status === powTypes.JobStatus.JOB_STATUS_SUCCESS) { console.log("job success!") } }, jobId) - // watch all FFS events for a cid - const logsCancel = pow.ffs.watchLogs((logEvent) => { + // watch all log events for a cid + const logsCancel = pow.data.watchLogs((logEvent) => { console.log(`received event for cid ${logEvent.cid}`) }, cid) - // get the current desired storage configuration for a cid (this configuration may not be realized yet) - const { config } = await pow.ffs.getStorageConfig(cid) - - // get the current actual storage configuration for a cid - const { cidInfo } = await pow.ffs.show(cid) + // get information about the latest applied storage configuration, + // current storage state, and all related Powegate storage jobs + const { cidInfosList } = await pow.data.cidInfo(cid) - // retrieve data from FFS by cid - const bytes = await pow.ffs.get(cid) + // retrieve data stored in the storage profile by cid + const bytes = await pow.data.get(cid) - // send FIL from an address managed by your FFS instance to any other address - await pow.ffs.sendFil(addrsList[0].addr, "", 1000) + // send FIL from an address managed by your storage profile to any other address + await pow.wallet.sendFil(addressesList[0].address, "", BigInt(1000)) } ``` From 7869e670a35d1ab856eadd69f11ff8e91656dce3 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Wed, 4 Nov 2020 21:30:08 -0600 Subject: [PATCH 8/9] fix: remove some ffs wording Signed-off-by: Aaron Sutula --- src/data/index.ts | 12 ++++++------ src/deals/index.ts | 4 ++-- src/wallet/index.ts | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/data/index.ts b/src/data/index.ts index 14cc5781..5df84c2c 100644 --- a/src/data/index.ts +++ b/src/data/index.ts @@ -27,16 +27,16 @@ export { GetFolderOptions, WatchLogsOptions } export interface Data { /** - * A helper method to stage data in IPFS in preparation for storing using ffs.pushStorageConfig. - * This doesn't actually store data in FFS, you'll want to call pushStorageConfig for that. + * A helper method to stage data in IPFS in preparation for storing using storageConfig.apply. + * This doesn't actually store data in Powergate, you'll want to call storageConfig.apply for that. * @param input The raw data to add. * @returns The cid of the added data. */ stage: (input: Uint8Array) => Promise /** - * A helper method to stage a folder recursively in IPFS in preparation for storing using ffs.pushStorageConfig. - * This doesn't actually store data in FFS, you'll want to call pushStorageConfig for that. + * A helper method to stage a folder recursively in IPFS in preparation for storing using storageConfig.apply. + * This doesn't actually store data in Powergate, you'll want to call storageConfig.apply for that. * @param path The path to the folder to add. * @returns The cid of the added folder. */ @@ -52,14 +52,14 @@ export interface Data { replaceData: (cid1: string, cid2: string) => Promise /** - * Retrieve data stored in the current FFS instance. + * Retrieve data stored in the current Storage Profile. * @param cid The cid of the data to retrieve. * @returns The raw data. */ get: (cid: string) => Promise /** - * Retrieve a folder stored in the current FFS instance. + * Retrieve a folder stored in the current Storage Profile. * @param cid The root cid of the folder to retrieve. * @param outputPath The location to write the folder to * @param opts Options controlling the behavior of retrieving the folder diff --git a/src/deals/index.ts b/src/deals/index.ts index c3b17df2..4d2ce556 100644 --- a/src/deals/index.ts +++ b/src/deals/index.ts @@ -15,14 +15,14 @@ export { DealRecordsOptions } export interface Deals { /** - * List storage deal records for the FFS instance according to the provided options. + * List storage deal records for the Storage Profile according to the provided options. * @param opts Options that control the behavior of listing records. * @returns A list of storage deal records. */ storageDealRecords: (opts?: DealRecordsOptions) => Promise /** - * List retrieval deal records for the FFS instance according to the provided options. + * List retrieval deal records for the Storage Profile according to the provided options. * @param opts Options that control the behavior of listing records. * @returns A list of retrieval deal records. */ diff --git a/src/wallet/index.ts b/src/wallet/index.ts index 9c6066a0..70fdcfaa 100644 --- a/src/wallet/index.ts +++ b/src/wallet/index.ts @@ -29,7 +29,7 @@ export interface Wallet { * Create a new wallet address associates with the current storage profile. * @param name A human readable name for the address. * @param type Address type, defaults to bls. - * @param makeDefault Specify if the new address should become the default address for this FFS instance, defaults to false. + * @param makeDefault Specify if the new address should become the default address for this Storage Profile, defaults to false. * @returns Information about the newly created address. */ newAddress: ( From 89f28316fe39df4918f15c9cd91b3a494a417010 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Wed, 4 Nov 2020 21:42:31 -0600 Subject: [PATCH 9/9] fix: profile id test Signed-off-by: Aaron Sutula --- src/integration.spec.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/integration.spec.ts b/src/integration.spec.ts index 81d22db6..5f8b3ec1 100644 --- a/src/integration.spec.ts +++ b/src/integration.spec.ts @@ -45,6 +45,13 @@ describe("pow", () => { expect(pow.host).equal(host) }) + it("should get profile id", async () => { + const pow = newPow() + await expectNewInstance(pow) + const res = await pow.storageProfileId() + expect(res.id).not.empty + }) + describe("admin", () => { describe("profiles", () => { it("should create profile", async () => {