From 924fc4bd94fec2ee6f2cd303c94c3268a7aec534 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Wed, 8 Jul 2020 22:22:46 -0500 Subject: [PATCH 01/12] feat: add deal records apis to ffs Signed-off-by: Aaron Sutula --- package-lock.json | 6 ++--- package.json | 4 +-- src/deals/options.ts | 64 ++++++++++++++++++++++++++++++++++++++++++++ src/ffs/index.ts | 37 ++++++++++++++++++++++++- src/ffs/options.ts | 20 ++++++-------- src/options.ts | 3 ++- src/types.ts | 3 ++- 7 files changed, 117 insertions(+), 20 deletions(-) create mode 100644 src/deals/options.ts diff --git a/package-lock.json b/package-lock.json index aef18207..5a762c50 100644 --- a/package-lock.json +++ b/package-lock.json @@ -464,9 +464,9 @@ "integrity": "sha512-+Kjz+Dktfz5LKTZA9ZW/Vlww6HF9KaKz4x2mVe1O8CJdOP2WfzC+KY8L6EWMqVLrV4MvdBuQdSgDmvSJz+OGuA==" }, "@textile/grpc-powergate-client": { - "version": "0.0.1-beta.13", - "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-0.0.1-beta.13.tgz", - "integrity": "sha512-ACYEOokCWkev1sxbZcTGFCPaUDpk95cOHnmKDXdCA2hS+qYdI0SJYVZXCmVkLQf2WQB8P6Losi+tc7Iw3A42dg==", + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-0.1.0.tgz", + "integrity": "sha512-Yo3nm0WE0D5p3zoMfjqaSiK++bAK6EG3jyKecQ5yrTe9cwHsyLyWzxBu0qTUv4aaChD6x7Q6lfN6yhpdPlBHZA==", "requires": { "@improbable-eng/grpc-web": "^0.12.0", "@types/google-protobuf": "^3.7.2", diff --git a/package.json b/package.json index d6d437ef..1e7c92c9 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,6 @@ }, "dependencies": { "@improbable-eng/grpc-web-node-http-transport": "^0.12.0", - "@textile/grpc-powergate-client": "0.0.1-beta.13" + "@textile/grpc-powergate-client": "0.1.0" } -} \ No newline at end of file +} diff --git a/src/deals/options.ts b/src/deals/options.ts new file mode 100644 index 00000000..918222a8 --- /dev/null +++ b/src/deals/options.ts @@ -0,0 +1,64 @@ +import { dealsTypes } from "../types" + +export type ListDealRecordsOption = (req: dealsTypes.ListDealRecordsConfig) => void + +/** + * Limits the results deals initiated from the provided wallet addresses + * @param addresses The list of addresses + * @returns The resulting option + */ +export const withFromAddresses = (...addresses: string[]): ListDealRecordsOption => { + return (req: dealsTypes.ListDealRecordsConfig) => { + req.setFromAddrsList(addresses) + } +} + +/** + * Limits the results to deals for the provided data cids + * @param cids The list of cids + * @returns The resulting option + */ +export const withDataCids = (...cids: string[]): ListDealRecordsOption => { + return (req: dealsTypes.ListDealRecordsConfig) => { + req.setDataCidsList(cids) + } +} + +/** + * Specifies whether or not to include pending deals in the results + * Default is false + * Ignored for listRetrievalDealRecords + * @param includePending Whether or not to include pending deal records + * @returns The resulting option + */ +export const withIncludePending = (includePending: boolean): ListDealRecordsOption => { + return (req: dealsTypes.ListDealRecordsConfig) => { + req.setIncludePending(includePending) + } +} + +/** + * Specifies whether or not to include final deals in the results + * Default is false + * Ignored for listRetrievalDealRecords + * @param includeFinal Whether or not to include final deal records + * @returns The resulting option + */ +export const withIncludeFinal = (includeFinal: boolean): ListDealRecordsOption => { + return (req: dealsTypes.ListDealRecordsConfig) => { + req.setIncludeFinal(includeFinal) + } +} + +/** + * Specifies to sort the results in ascending order + * Default is descending order + * Records are sorted by timestamp + * @param ascending Whether or not to sort the results in ascending order + * @returns The resulting option + */ +export const withAscending = (ascending: boolean): ListDealRecordsOption => { + return (req: dealsTypes.ListDealRecordsConfig) => { + req.setAscending(ascending) + } +} diff --git a/src/ffs/index.ts b/src/ffs/index.ts index d35efaae..417592e8 100644 --- a/src/ffs/index.ts +++ b/src/ffs/index.ts @@ -3,7 +3,8 @@ import { RPCService, RPCServiceClient, } from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb_service" -import { Config, ffsTypes } from "../types" +import { ListDealRecordsOption } from "../deals/options" +import { Config, dealsTypes, ffsTypes } from "../types" import { promise } from "../util" import { PushConfigOption, WatchLogsOption } from "./options" import { coldObjToMessage, hotObjToMessage } from "./util" @@ -395,6 +396,40 @@ export const createFFS = (config: Config, getMeta: () => grpc.Metadata) => { ) }, + /** + * List storage deal records for the FFS instance according to the provided options + * @returns A list of storage deal records + */ + listStorageDealRecords: (...opts: ListDealRecordsOption[]) => { + const conf = new dealsTypes.ListDealRecordsConfig() + opts.forEach((opt) => { + opt(conf) + }) + const req = new ffsTypes.ListStorageDealRecordsRequest() + req.setConfig(conf) + return promise( + (cb) => client.listStorageDealRecords(req, getMeta(), cb), + (res: ffsTypes.ListStorageDealRecordsResponse) => res.toObject().recordsList, + ) + }, + + /** + * List retrieval deal records for the FFS instance according to the provided options + * @returns A list of retrieval deal records + */ + listRetrievalDealRecords: (...opts: ListDealRecordsOption[]) => { + const conf = new dealsTypes.ListDealRecordsConfig() + opts.forEach((opt) => { + opt(conf) + }) + const req = new ffsTypes.ListRetrievalDealRecordsRequest() + req.setConfig(conf) + return promise( + (cb) => client.listRetrievalDealRecords(req, getMeta(), cb), + (res: ffsTypes.ListRetrievalDealRecordsResponse) => res.toObject().recordsList, + ) + }, + /** * List cid infos for all data stored in the current FFS instance * @returns A list of cid info diff --git a/src/ffs/options.ts b/src/ffs/options.ts index 36f71472..4b55b966 100644 --- a/src/ffs/options.ts +++ b/src/ffs/options.ts @@ -8,12 +8,11 @@ export type PushConfigOption = (req: ffsTypes.PushConfigRequest) => void * @param override Whether or not to override any existing storage configuration * @returns The resulting option */ -export const withOverrideConfig = (override: boolean) => { - const option: PushConfigOption = (req: ffsTypes.PushConfigRequest) => { +export const withOverrideConfig = (override: boolean): PushConfigOption => { + return (req: ffsTypes.PushConfigRequest) => { req.setHasOverrideConfig(true) req.setOverrideConfig(override) } - return option } /** @@ -21,8 +20,8 @@ export const withOverrideConfig = (override: boolean) => { * @param config The storage configuration to use * @returns The resulting option */ -export const withConfig = (config: ffsTypes.CidConfig.AsObject) => { - const option: PushConfigOption = (req: ffsTypes.PushConfigRequest) => { +export const withConfig = (config: ffsTypes.CidConfig.AsObject): PushConfigOption => { + return (req: ffsTypes.PushConfigRequest) => { const c = new ffsTypes.CidConfig() c.setCid(config.cid) c.setRepairable(config.repairable) @@ -35,7 +34,6 @@ export const withConfig = (config: ffsTypes.CidConfig.AsObject) => { req.setHasConfig(true) req.setConfig(c) } - return option } export type WatchLogsOption = (res: ffsTypes.WatchLogsRequest) => void @@ -45,11 +43,10 @@ export type WatchLogsOption = (res: ffsTypes.WatchLogsRequest) => void * @param includeHistory Whether or not to include the history of log events * @returns The resulting option */ -export const withHistory = (includeHistory: boolean) => { - const option: WatchLogsOption = (req: ffsTypes.WatchLogsRequest) => { +export const withHistory = (includeHistory: boolean): WatchLogsOption => { + return (req: ffsTypes.WatchLogsRequest) => { req.setHistory(includeHistory) } - return option } /** @@ -57,9 +54,8 @@ export const withHistory = (includeHistory: boolean) => { * @param jobId The job id to show events for * @returns The resulting option */ -export const withJobId = (jobId: string) => { - const option: WatchLogsOption = (req: ffsTypes.WatchLogsRequest) => { +export const withJobId = (jobId: string): WatchLogsOption => { + return (req: ffsTypes.WatchLogsRequest) => { req.setJid(jobId) } - return option } diff --git a/src/options.ts b/src/options.ts index 3ccff8c2..1021d1c4 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,3 +1,4 @@ +import * as dealsOptions from "./deals/options" import * as ffsOptions from "./ffs/options" -export { ffsOptions } +export { ffsOptions, dealsOptions } diff --git a/src/types.ts b/src/types.ts index 0a565994..ff729756 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,5 @@ import { grpc } from "@improbable-eng/grpc-web" +import * as dealsTypes from "@textile/grpc-powergate-client/dist/deals/rpc/rpc_pb" import * as ffsTypes from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" import * as healthTypes from "@textile/grpc-powergate-client/dist/health/rpc/rpc_pb" import * as minersTypes from "@textile/grpc-powergate-client/dist/index/miner/rpc/rpc_pb" @@ -12,4 +13,4 @@ export interface Config extends grpc.RpcOptions { authToken?: string } -export { ffsTypes, healthTypes, minersTypes, netTypes } +export { ffsTypes, healthTypes, minersTypes, netTypes, dealsTypes } From d6a35ec9074df08ea22547b3f2ca2cf486506951 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Thu, 9 Jul 2020 16:37:09 -0500 Subject: [PATCH 02/12] feat: test for storage record Signed-off-by: Aaron Sutula --- src/ffs/index.spec.ts | 58 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/src/ffs/index.spec.ts b/src/ffs/index.spec.ts index 9def99aa..b4ad5635 100644 --- a/src/ffs/index.spec.ts +++ b/src/ffs/index.spec.ts @@ -1,6 +1,7 @@ import { expect } from "chai" import fs from "fs" import { createFFS } from "." +import { withIncludeFinal } from "../deals/options" import { ffsTypes } from "../types" import { getTransport, host, useToken } from "../util" import { withConfig, withHistory, withOverrideConfig } from "./options" @@ -14,7 +15,6 @@ describe("ffs", () => { let initialAddrs: ffsTypes.AddrInfo.AsObject[] let defaultConfig: ffsTypes.DefaultConfig.AsObject let cid: string - let defaultCidConfig: ffsTypes.CidConfig.AsObject it("should create an instance", async function () { this.timeout(30000) @@ -76,16 +76,22 @@ describe("ffs", () => { it("should get default cid config", async () => { const res = await c.getDefaultCidConfig(cid) expect(res.config?.cid).equal(cid) - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - defaultCidConfig = res.config! }) let jobId: string it("should push config", async () => { - const res = await c.pushConfig(cid, withOverrideConfig(false), withConfig(defaultCidConfig)) - expect(res.jobId).length.greaterThan(0) - jobId = res.jobId + const res0 = await c.getDefaultCidConfig(cid) + expect(res0.config?.cid).equal(cid) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const config = res0.config! + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + config.hot!.enabled = false + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + config.hot!.allowUnfreeze = true + const res1 = await c.pushConfig(cid, withOverrideConfig(false), withConfig(config)) + expect(res1.jobId).length.greaterThan(0) + jobId = res1.jobId }) it("should watch job", function (done) { @@ -114,6 +120,46 @@ describe("ffs", () => { ) }) + it("should get a storage deal record", async () => { + const res = await c.listStorageDealRecords(withIncludeFinal(true)) + expect(res).length.greaterThan(0) + }) + + it("should push an unfreeze config change", async () => { + const res0 = await c.getDefaultCidConfig(cid) + expect(res0.config?.cid).equal(cid) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const config = res0.config! + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + config.hot!.enabled = true + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + config.hot!.allowUnfreeze = true + const res1 = await c.pushConfig(cid, withOverrideConfig(true), withConfig(config)) + expect(res1.jobId).length.greaterThan(0) + jobId = res1.jobId + }) + + it("should watch the config change job", function (done) { + this.timeout(180000) + const cancel = c.watchJobs((job) => { + expect(job.errCause).empty + expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_CANCELED) + expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_FAILED) + if (job.status === ffsTypes.JobStatus.JOB_STATUS_SUCCESS) { + cancel() + done() + } + }, jobId) + }) + + it("should get a retrieval deal record", async function () { + // ToDo: Figure out hot to make sure the data in the previous push isn't cached in hot + // this.timeout(30000) + // await new Promise((r) => setTimeout(r, 10000)) + // const res = await c.listRetrievalDealRecords() + // expect(res).length.greaterThan(0) + }) + it("should get cid config", async () => { const res = await c.getCidConfig(cid) expect(res.config?.cid).equal(cid) From d1edf9b072d1a0ece148a47b5e2aa53e4a3d6f2d Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Thu, 9 Jul 2020 17:47:57 -0500 Subject: [PATCH 03/12] feat: asks and faults apis Signed-off-by: Aaron Sutula --- src/asks/index.spec.ts | 19 +++++++++++++++++++ src/asks/index.ts | 39 +++++++++++++++++++++++++++++++++++++++ src/asks/util.ts | 10 ++++++++++ src/faults/index.spec.ts | 12 ++++++++++++ src/faults/index.ts | 24 ++++++++++++++++++++++++ src/ffs/util.ts | 4 ++-- src/index.ts | 21 +++++++++++++++++---- src/types.ts | 4 +++- 8 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 src/asks/index.spec.ts create mode 100644 src/asks/index.ts create mode 100644 src/asks/util.ts create mode 100644 src/faults/index.spec.ts create mode 100644 src/faults/index.ts diff --git a/src/asks/index.spec.ts b/src/asks/index.spec.ts new file mode 100644 index 00000000..de45bdff --- /dev/null +++ b/src/asks/index.spec.ts @@ -0,0 +1,19 @@ +import { expect } from "chai" +import { createAsks } from "." +import { asksTypes } from "../types" +import { getTransport, host } from "../util" + +describe("asks", () => { + const c = createAsks({ host, transport: getTransport() }) + + it("should get", async () => { + const index = await c.get() + expect(index).not.undefined + }) + + it("should query", async () => { + const q = new asksTypes.Query().toObject() + const res = await c.query(q) + expect(res).not.undefined + }) +}) diff --git a/src/asks/index.ts b/src/asks/index.ts new file mode 100644 index 00000000..58c58bd5 --- /dev/null +++ b/src/asks/index.ts @@ -0,0 +1,39 @@ +import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/index/ask/rpc/rpc_pb_service" +import { asksTypes, Config } from "../types" +import { promise } from "../util" +import { queryObjectToMsg } from "./util" + +/** + * Creates the Asks API client + * @param config A config object that changes the behavior of the client + * @returns The Asks API client + */ +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +export const createAsks = (config: Config) => { + const client = new RPCServiceClient(config.host, config) + return { + /** + * Gets the asks index + * @returns The asks index + */ + get: () => + promise( + (cb) => client.get(new asksTypes.GetRequest(), cb), + (resp: asksTypes.GetResponse) => resp.toObject().index, + ), + + /** + * Queries the asks index + * @param query The query to run against the asks index + * @returns The asks matching the provided query + */ + query: (query: asksTypes.Query.AsObject) => { + const req = new asksTypes.QueryRequest() + req.setQuery(queryObjectToMsg(query)) + return promise( + (cb) => client.query(req, cb), + (resp: asksTypes.QueryResponse) => resp.toObject().asksList, + ) + }, + } +} diff --git a/src/asks/util.ts b/src/asks/util.ts new file mode 100644 index 00000000..a3d441ad --- /dev/null +++ b/src/asks/util.ts @@ -0,0 +1,10 @@ +import { asksTypes } from "../types" + +export function queryObjectToMsg(query: asksTypes.Query.AsObject): asksTypes.Query { + const ret = new asksTypes.Query() + ret.setLimit(query.limit) + ret.setMaxPrice(query.maxPrice) + ret.setOffset(query.offset) + ret.setPieceSize(query.pieceSize) + return ret +} diff --git a/src/faults/index.spec.ts b/src/faults/index.spec.ts new file mode 100644 index 00000000..11f0e87d --- /dev/null +++ b/src/faults/index.spec.ts @@ -0,0 +1,12 @@ +import { expect } from "chai" +import { createFaults } from "." +import { getTransport, host } from "../util" + +describe("faults", () => { + const c = createFaults({ host, transport: getTransport() }) + + it("should get", async () => { + const index = await c.get() + expect(index).not.undefined + }) +}) diff --git a/src/faults/index.ts b/src/faults/index.ts new file mode 100644 index 00000000..0818df17 --- /dev/null +++ b/src/faults/index.ts @@ -0,0 +1,24 @@ +import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/index/faults/rpc/rpc_pb_service" +import { Config, faultsTypes } from "../types" +import { promise } from "../util" + +/** + * Creates the Faults API client + * @param config A config object that changes the behavior of the client + * @returns The Faults API client + */ +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +export const createFaults = (config: Config) => { + const client = new RPCServiceClient(config.host, config) + return { + /** + * Gets the faults index + * @returns The faults index + */ + get: () => + promise( + (cb) => client.get(new faultsTypes.GetRequest(), cb), + (resp: faultsTypes.GetResponse) => resp.toObject().index, + ), + } +} diff --git a/src/ffs/util.ts b/src/ffs/util.ts index 8d371b75..88a5aca2 100644 --- a/src/ffs/util.ts +++ b/src/ffs/util.ts @@ -1,6 +1,6 @@ import { ffsTypes } from "../types" -export function coldObjToMessage(obj: ffsTypes.ColdConfig.AsObject) { +export function coldObjToMessage(obj: ffsTypes.ColdConfig.AsObject): ffsTypes.ColdConfig { const cold = new ffsTypes.ColdConfig() cold.setEnabled(obj.enabled) if (obj.filecoin) { @@ -23,7 +23,7 @@ export function coldObjToMessage(obj: ffsTypes.ColdConfig.AsObject) { return cold } -export function hotObjToMessage(obj: ffsTypes.HotConfig.AsObject) { +export function hotObjToMessage(obj: ffsTypes.HotConfig.AsObject): ffsTypes.HotConfig { const hot = new ffsTypes.HotConfig() hot.setAllowUnfreeze(obj.allowUnfreeze) hot.setEnabled(obj.enabled) diff --git a/src/index.ts b/src/index.ts index 191ac8d2..6d6eb93d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,5 @@ +import { createAsks } from "./asks" +import { createFaults } from "./faults" import { createFFS } from "./ffs" import { createHealth } from "./health" import { createMiners } from "./miners" @@ -18,6 +20,7 @@ const defaultConfig: Config = { * @param config A config object that changes the behavior of the client * @returns A Powergate client API */ +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types export const createPow = (config?: Partial) => { const c = { ...defaultConfig, ...removeEmpty(config) } @@ -31,24 +34,34 @@ export const createPow = (config?: Partial) => { setToken, /** - * The Health API + * The Asks API */ - health: createHealth(c), + asks: createAsks(c), /** - * The Net API + * The Faults API */ - net: createNet(c), + faults: createFaults(c), /** * The FFS API */ ffs: createFFS(c, getMeta), + /** + * The Health API + */ + health: createHealth(c), + /** * The Miners API */ miners: createMiners(c), + + /** + * The Net API + */ + net: createNet(c), } } diff --git a/src/types.ts b/src/types.ts index ff729756..7f069f6f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,6 +2,8 @@ import { grpc } from "@improbable-eng/grpc-web" import * as dealsTypes from "@textile/grpc-powergate-client/dist/deals/rpc/rpc_pb" import * as ffsTypes from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" import * as healthTypes from "@textile/grpc-powergate-client/dist/health/rpc/rpc_pb" +import * as asksTypes from "@textile/grpc-powergate-client/dist/index/ask/rpc/rpc_pb" +import * as faultsTypes from "@textile/grpc-powergate-client/dist/index/faults/rpc/rpc_pb" import * as minersTypes from "@textile/grpc-powergate-client/dist/index/miner/rpc/rpc_pb" import * as netTypes from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb" @@ -13,4 +15,4 @@ export interface Config extends grpc.RpcOptions { authToken?: string } -export { ffsTypes, healthTypes, minersTypes, netTypes, dealsTypes } +export { ffsTypes, healthTypes, minersTypes, netTypes, dealsTypes, asksTypes, faultsTypes } From 63ec2379b91639d8eedfb3df6620243ee9a0d856 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Thu, 9 Jul 2020 18:23:56 -0500 Subject: [PATCH 04/12] feat: reputation api Signed-off-by: Aaron Sutula --- src/index.ts | 33 +++++++++++++++++++++++--- src/reputation/index.spec.ts | 12 ++++++++++ src/reputation/index.ts | 45 ++++++++++++++++++++++++++++++++++++ src/types.ts | 12 +++++++++- 4 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 src/reputation/index.spec.ts create mode 100644 src/reputation/index.ts diff --git a/src/index.ts b/src/index.ts index 6d6eb93d..7c094126 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,11 +4,33 @@ import { createFFS } from "./ffs" import { createHealth } from "./health" import { createMiners } from "./miners" import { createNet } from "./net" -import { ffsOptions } from "./options" -import { Config, ffsTypes, healthTypes, minersTypes, netTypes } from "./types" +import { dealsOptions, ffsOptions } from "./options" +import { createReputation } from "./reputation" +import { + asksTypes, + Config, + dealsTypes, + faultsTypes, + ffsTypes, + healthTypes, + minersTypes, + netTypes, + reputationTypes, +} from "./types" import { getTransport, host, useToken } from "./util" -export { Config, ffsTypes, healthTypes, minersTypes, netTypes, ffsOptions } +export { dealsOptions, ffsOptions } +export { + asksTypes, + Config, + dealsTypes, + faultsTypes, + ffsTypes, + healthTypes, + minersTypes, + netTypes, + reputationTypes, +} const defaultConfig: Config = { host, @@ -62,6 +84,11 @@ export const createPow = (config?: Partial) => { * The Net API */ net: createNet(c), + + /** + * The Reputation API + */ + reputation: createReputation(c), } } diff --git a/src/reputation/index.spec.ts b/src/reputation/index.spec.ts new file mode 100644 index 00000000..eee059e1 --- /dev/null +++ b/src/reputation/index.spec.ts @@ -0,0 +1,12 @@ +import { expect } from "chai" +import { createReputation } from "." +import { getTransport, host } from "../util" + +describe("reputation", () => { + const c = createReputation({ host, transport: getTransport() }) + + it("should get top miners", async () => { + const scores = await c.getTopMiners(10) + expect(scores).not.undefined + }) +}) diff --git a/src/reputation/index.ts b/src/reputation/index.ts new file mode 100644 index 00000000..470d5d94 --- /dev/null +++ b/src/reputation/index.ts @@ -0,0 +1,45 @@ +import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/reputation/rpc/rpc_pb_service" +import { Config, reputationTypes } from "../types" +import { promise } from "../util" + +/** + * Creates the Reputation API client + * @param config A config object that changes the behavior of the client + * @returns The Reputation API client + */ +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +export const createReputation = (config: Config) => { + const client = new RPCServiceClient(config.host, config) + return { + /** + * Adds a data source to the reputation index + * @param id The id of the data source + * @param multiaddress The multiaddress of the data source + */ + addSource: (id: string, multiaddress: string) => { + const req = new reputationTypes.AddSourceRequest() + req.setId(id) + req.setMaddr(multiaddress) + return promise( + (cb) => client.addSource(req, cb), + () => { + // nothing to return + }, + ) + }, + + /** + * Gets the top ranked miners + * @param limit Limits the number of results + * @returns The list of miner scores + */ + getTopMiners: (limit: number) => { + const req = new reputationTypes.GetTopMinersRequest() + req.setLimit(limit) + return promise( + (cb) => client.getTopMiners(req, cb), + (resp: reputationTypes.GetTopMinersResponse) => resp.toObject().topMinersList, + ) + }, + } +} diff --git a/src/types.ts b/src/types.ts index 7f069f6f..5402d62b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,6 +6,7 @@ import * as asksTypes from "@textile/grpc-powergate-client/dist/index/ask/rpc/rp import * as faultsTypes from "@textile/grpc-powergate-client/dist/index/faults/rpc/rpc_pb" import * as minersTypes from "@textile/grpc-powergate-client/dist/index/miner/rpc/rpc_pb" import * as netTypes from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb" +import * as reputationTypes from "@textile/grpc-powergate-client/dist/reputation/rpc/rpc_pb" /** * Object that allows you to configure the Powergate client @@ -15,4 +16,13 @@ export interface Config extends grpc.RpcOptions { authToken?: string } -export { ffsTypes, healthTypes, minersTypes, netTypes, dealsTypes, asksTypes, faultsTypes } +export { + ffsTypes, + healthTypes, + minersTypes, + netTypes, + dealsTypes, + asksTypes, + faultsTypes, + reputationTypes, +} From 7d4d448228bba322e9ca3e77858ee6fa53e3e48d Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Thu, 9 Jul 2020 19:33:10 -0500 Subject: [PATCH 05/12] feat: wallet api Signed-off-by: Aaron Sutula --- src/index.ts | 8 ++++++ src/types.ts | 2 ++ src/wallet/index.spec.ts | 25 +++++++++++++++++++ src/wallet/index.ts | 54 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 src/wallet/index.spec.ts create mode 100644 src/wallet/index.ts diff --git a/src/index.ts b/src/index.ts index 7c094126..04bcc956 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,8 +16,10 @@ import { minersTypes, netTypes, reputationTypes, + walletTypes, } from "./types" import { getTransport, host, useToken } from "./util" +import { createWallet } from "./wallet" export { dealsOptions, ffsOptions } export { @@ -30,6 +32,7 @@ export { minersTypes, netTypes, reputationTypes, + walletTypes, } const defaultConfig: Config = { @@ -89,6 +92,11 @@ export const createPow = (config?: Partial) => { * The Reputation API */ reputation: createReputation(c), + + /** + * The Wallet API + */ + wallet: createWallet(c), } } diff --git a/src/types.ts b/src/types.ts index 5402d62b..3df596a7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -7,6 +7,7 @@ import * as faultsTypes from "@textile/grpc-powergate-client/dist/index/faults/r import * as minersTypes from "@textile/grpc-powergate-client/dist/index/miner/rpc/rpc_pb" import * as netTypes from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb" import * as reputationTypes from "@textile/grpc-powergate-client/dist/reputation/rpc/rpc_pb" +import * as walletTypes from "@textile/grpc-powergate-client/dist/wallet/rpc/rpc_pb" /** * Object that allows you to configure the Powergate client @@ -25,4 +26,5 @@ export { asksTypes, faultsTypes, reputationTypes, + walletTypes, } diff --git a/src/wallet/index.spec.ts b/src/wallet/index.spec.ts new file mode 100644 index 00000000..a901b54d --- /dev/null +++ b/src/wallet/index.spec.ts @@ -0,0 +1,25 @@ +import { expect } from "chai" +import { createWallet } from "." +import { getTransport, host } from "../util" + +describe("wallet", () => { + const c = createWallet({ host, transport: getTransport() }) + + let address: string + + it("should list addresses", async () => { + const addresses = await c.list() + expect(addresses).length(1) + address = addresses[0] + }) + + it("should create a new address", async () => { + const address = await c.newAddress() + expect(address).length.greaterThan(0) + }) + + it("should check balance", async () => { + const bal = await c.balance(address) + expect(bal).greaterThan(0) + }) +}) diff --git a/src/wallet/index.ts b/src/wallet/index.ts new file mode 100644 index 00000000..1d7d2067 --- /dev/null +++ b/src/wallet/index.ts @@ -0,0 +1,54 @@ +import { RPCServiceClient } from "@textile/grpc-powergate-client/dist/wallet/rpc/rpc_pb_service" +import { Config, walletTypes } from "../types" +import { promise } from "../util" + +/** + * Creates the Wallet API client + * @param config A config object that changes the behavior of the client + * @returns The Wallet API client + */ +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +export const createWallet = (config: Config) => { + const client = new RPCServiceClient(config.host, config) + return { + /** + * Create a new wallet address + * @param type The type of address to create, bls or secp256k1 + * @returns The new address + */ + newAddress: (type: "bls" | "secp256k1" = "bls") => { + const req = new walletTypes.NewAddressRequest() + req.setType(type) + return promise( + (cb) => client.newAddress(req, cb), + (resp: walletTypes.NewAddressResponse) => resp.toObject().address, + ) + }, + + /** + * List all wallet addresses + * @returns The list of wallet addresses + */ + list: () => + promise( + (cb) => client.list(new walletTypes.ListRequest(), cb), + (resp: walletTypes.ListResponse) => resp.toObject().addressesList, + ), + + /** + * Get the balance for a wallet address + * @param address The address to get the balance for + * @returns The address balance + */ + balance: (address: string) => { + const req = new walletTypes.WalletBalanceRequest() + req.setAddress(address) + return promise( + (cb) => client.walletBalance(req, cb), + (resp: walletTypes.WalletBalanceResponse) => resp.toObject().balance, + ) + }, + + // TODO: SendFil after next grpc bindings update + } +} From 51f89d8ec0fc1cdd6594beb871cdd0f1fcc2c2b0 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Thu, 9 Jul 2020 19:46:01 -0500 Subject: [PATCH 06/12] test: fix test Signed-off-by: Aaron Sutula --- src/wallet/index.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/wallet/index.spec.ts b/src/wallet/index.spec.ts index a901b54d..77412151 100644 --- a/src/wallet/index.spec.ts +++ b/src/wallet/index.spec.ts @@ -9,7 +9,7 @@ describe("wallet", () => { it("should list addresses", async () => { const addresses = await c.list() - expect(addresses).length(1) + expect(addresses).length.greaterThan(0) address = addresses[0] }) @@ -19,7 +19,6 @@ describe("wallet", () => { }) it("should check balance", async () => { - const bal = await c.balance(address) - expect(bal).greaterThan(0) + await c.balance(address) }) }) From f7dac749044260bd88266b1ad900c8c0fc69b7ec Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Fri, 10 Jul 2020 16:13:28 -0500 Subject: [PATCH 07/12] feat: wip on deals api Signed-off-by: Aaron Sutula --- src/deals/index.spec.ts | 0 src/deals/index.ts | 152 ++++++++++++++++++++++++++++++++++++++++ src/deals/util.ts | 8 +++ src/ffs/index.ts | 2 + 4 files changed, 162 insertions(+) create mode 100644 src/deals/index.spec.ts create mode 100644 src/deals/index.ts create mode 100644 src/deals/util.ts diff --git a/src/deals/index.spec.ts b/src/deals/index.spec.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/deals/index.ts b/src/deals/index.ts new file mode 100644 index 00000000..752ccf3d --- /dev/null +++ b/src/deals/index.ts @@ -0,0 +1,152 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + RPCService, + RPCServiceClient, +} from "@textile/grpc-powergate-client/dist/deals/rpc/rpc_pb_service" +import { Config, dealsTypes } from "../types" +import { promise } from "../util" +import { ListDealRecordsOption } from "./options" +import { dealConfigObjToMessage } from "./util" + +/** + * Creates the Deals API client + * @param config A config object that changes the behavior of the client + * @returns The Deals API client + */ +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +export const createDeals = (config: Config) => { + const client = new RPCServiceClient(config.host, config) + return { + /** + * Stores data according to the provided deal parameters + * @returns Information about the storage deals + */ + store: ( + addr: string, + data: Uint8Array, + dealConfigs: dealsTypes.DealConfig.AsObject[], + minDuration: number, + ) => { + // TODO: figure out how to stream data in here, or at least stream to the server + return new Promise((resolve, reject) => { + const client = grpc.client(RPCService.Store, config) + client.onMessage((message) => { + resolve(message.toObject() as dealsTypes.StoreResponse.AsObject) + }) + client.onEnd((code, msg) => { + if (code !== grpc.Code.OK) { + reject(`error code ${code} - ${msg}`) + } else { + reject("ended with no message") + } + }) + client.start() + + const dealConfigMsgs = dealConfigs.map((val) => dealConfigObjToMessage(val)) + const params = new dealsTypes.StoreParams() + params.setAddress(addr) + params.setDealConfigsList(dealConfigMsgs) + params.setMinDuration(minDuration) + const req0 = new dealsTypes.StoreRequest() + req0.setStoreParams(params) + client.send(req0) + + const req1 = new dealsTypes.StoreRequest() + req1.setChunk(data) + client.send(req1) + + client.finishSend() + }) + }, + + /** + * Listen for deal proposal updates for the provided proposal ids + * @param handler The callback to receive proposal updates + * @param proposals A list of proposal ids to watch + * @returns A function that can be used to cancel watching + */ + watch: ( + handler: (event: dealsTypes.StorageDealInfo.AsObject) => void, + ...proposals: string[] + ) => { + const req = new dealsTypes.WatchRequest() + req.setProposalsList(proposals) + const stream = client.watch(req) + stream.on("data", (res) => { + const dealInfo = res.getDealInfo()?.toObject() + if (dealInfo) { + handler(dealInfo) + } + }) + return stream.cancel + }, + + /** + * Retrieve data stored in Filecoin + * @param address The wallet address to fund the retrieval + * @param cid The cid of the data to retrieve + * @returns The raw data + */ + retrieve: (address: string, cid: string) => { + return new Promise((resolve, reject) => { + const append = (l: Uint8Array, r: Uint8Array) => { + const tmp = new Uint8Array(l.byteLength + r.byteLength) + tmp.set(l, 0) + tmp.set(r, l.byteLength) + return tmp + } + let final = new Uint8Array() + const req = new dealsTypes.RetrieveRequest() + req.setAddress(address) + req.setCid(cid) + const stream = client.retrieve(req) + stream.on("data", (resp) => { + final = append(final, resp.getChunk_asU8()) + }) + stream.on("end", (status) => { + if (status?.code !== grpc.Code.OK) { + reject(`error code ${status?.code} - ${status?.details}`) + } else { + resolve(final) + } + }) + }) + }, + + /** + * List storage deal records according to the provided options + * @param opts Options that control the behavior of listing records + * @returns A list of storage deal records + */ + listStorageDealRecords: (...opts: ListDealRecordsOption[]) => { + const conf = new dealsTypes.ListDealRecordsConfig() + opts.forEach((opt) => { + opt(conf) + }) + const req = new dealsTypes.ListStorageDealRecordsRequest() + req.setConfig(conf) + return promise( + (cb) => client.listStorageDealRecords(req, cb), + (res: dealsTypes.ListStorageDealRecordsResponse) => res.toObject().recordsList, + ) + }, + + /** + * List retrieval deal records according to the provided options + * @param opts Options that control the behavior of listing records + * @returns A list of retrieval deal records + */ + listRetrievalDealRecords: (...opts: ListDealRecordsOption[]) => { + const conf = new dealsTypes.ListDealRecordsConfig() + opts.forEach((opt) => { + opt(conf) + }) + const req = new dealsTypes.ListRetrievalDealRecordsRequest() + req.setConfig(conf) + return promise( + (cb) => client.listRetrievalDealRecords(req, cb), + (res: dealsTypes.ListRetrievalDealRecordsResponse) => res.toObject().recordsList, + ) + }, + } +} diff --git a/src/deals/util.ts b/src/deals/util.ts new file mode 100644 index 00000000..e97679ce --- /dev/null +++ b/src/deals/util.ts @@ -0,0 +1,8 @@ +import { dealsTypes } from "../types" + +export function dealConfigObjToMessage(obj: dealsTypes.DealConfig.AsObject): dealsTypes.DealConfig { + const msg = new dealsTypes.DealConfig() + msg.setEpochPrice(obj.epochPrice) + msg.setMiner(obj.miner) + return msg +} diff --git a/src/ffs/index.ts b/src/ffs/index.ts index 417592e8..c425c08f 100644 --- a/src/ffs/index.ts +++ b/src/ffs/index.ts @@ -398,6 +398,7 @@ export const createFFS = (config: Config, getMeta: () => grpc.Metadata) => { /** * List storage deal records for the FFS instance according to the provided options + * @param opts Options that control the behavior of listing records * @returns A list of storage deal records */ listStorageDealRecords: (...opts: ListDealRecordsOption[]) => { @@ -415,6 +416,7 @@ export const createFFS = (config: Config, getMeta: () => grpc.Metadata) => { /** * List retrieval deal records for the FFS instance according to the provided options + * @param opts Options that control the behavior of listing records * @returns A list of retrieval deal records */ listRetrievalDealRecords: (...opts: ListDealRecordsOption[]) => { From 63925f9936c582691687730a0909161426b853da Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Mon, 13 Jul 2020 20:09:16 -0500 Subject: [PATCH 08/12] feat: update to powergate pre release, implement the rest Signed-off-by: Aaron Sutula --- package-lock.json | 6 +- package.json | 4 +- src/deals/index.ts | 127 ++++++++++++++++++++++++++++----------- src/index.spec.ts | 22 +++++++ src/wallet/index.spec.ts | 12 ++++ src/wallet/index.ts | 25 ++++++-- 6 files changed, 151 insertions(+), 45 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5a762c50..e5aeea56 100644 --- a/package-lock.json +++ b/package-lock.json @@ -464,9 +464,9 @@ "integrity": "sha512-+Kjz+Dktfz5LKTZA9ZW/Vlww6HF9KaKz4x2mVe1O8CJdOP2WfzC+KY8L6EWMqVLrV4MvdBuQdSgDmvSJz+OGuA==" }, "@textile/grpc-powergate-client": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-0.1.0.tgz", - "integrity": "sha512-Yo3nm0WE0D5p3zoMfjqaSiK++bAK6EG3jyKecQ5yrTe9cwHsyLyWzxBu0qTUv4aaChD6x7Q6lfN6yhpdPlBHZA==", + "version": "0.1.1-pre.2", + "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-0.1.1-pre.2.tgz", + "integrity": "sha512-E7aAbFIMHny/recjL9k8z8ZKEG377yyDl/Hy8ISAbuyEZzPNdYkBfYVJLG3llRDJZZS63kycxW8LzFr3NpPnkw==", "requires": { "@improbable-eng/grpc-web": "^0.12.0", "@types/google-protobuf": "^3.7.2", diff --git a/package.json b/package.json index 6ccdf73c..86506c92 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,6 @@ }, "dependencies": { "@improbable-eng/grpc-web-node-http-transport": "^0.12.0", - "@textile/grpc-powergate-client": "0.1.0" + "@textile/grpc-powergate-client": "0.1.1-pre.2" } -} +} \ No newline at end of file diff --git a/src/deals/index.ts b/src/deals/index.ts index 752ccf3d..0f419b76 100644 --- a/src/deals/index.ts +++ b/src/deals/index.ts @@ -18,20 +18,16 @@ export const createDeals = (config: Config) => { const client = new RPCServiceClient(config.host, config) return { /** - * Stores data according to the provided deal parameters - * @returns Information about the storage deals + * Import raw data into the Filecoin client to prepare for storage + * @param data The raw data to import + * @param isCAR Specifies if the data is already CAR encoded, default false */ - store: ( - addr: string, - data: Uint8Array, - dealConfigs: dealsTypes.DealConfig.AsObject[], - minDuration: number, - ) => { + import: (data: Uint8Array, isCAR: boolean = false) => { // TODO: figure out how to stream data in here, or at least stream to the server - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { const client = grpc.client(RPCService.Store, config) client.onMessage((message) => { - resolve(message.toObject() as dealsTypes.StoreResponse.AsObject) + resolve(message.toObject() as dealsTypes.ImportResponse.AsObject) }) client.onEnd((code, msg) => { if (code !== grpc.Code.OK) { @@ -42,16 +38,13 @@ export const createDeals = (config: Config) => { }) client.start() - const dealConfigMsgs = dealConfigs.map((val) => dealConfigObjToMessage(val)) - const params = new dealsTypes.StoreParams() - params.setAddress(addr) - params.setDealConfigsList(dealConfigMsgs) - params.setMinDuration(minDuration) - const req0 = new dealsTypes.StoreRequest() - req0.setStoreParams(params) + const params = new dealsTypes.ImportParams() + params.setIsCar(isCAR) + const req0 = new dealsTypes.ImportRequest() + req0.setImportParams(params) client.send(req0) - const req1 = new dealsTypes.StoreRequest() + const req1 = new dealsTypes.ImportRequest() req1.setChunk(data) client.send(req1) @@ -60,34 +53,59 @@ export const createDeals = (config: Config) => { }, /** - * Listen for deal proposal updates for the provided proposal ids - * @param handler The callback to receive proposal updates - * @param proposals A list of proposal ids to watch - * @returns A function that can be used to cancel watching + * Stores data according to the provided deal parameters + * @param walletAddress The wallet address to fund the deals + * @param dataCid The cid of the data to store + * @param pieceSize The size of the data to store + * @param dealConfigs The list of deals to execute + * @param minDuration The amount of time to store the data in epochs + * @returns A list of information about the storage deals */ - watch: ( - handler: (event: dealsTypes.StorageDealInfo.AsObject) => void, - ...proposals: string[] + store: ( + walletAddress: string, + dataCid: string, + pieceSize: number, + dealConfigs: dealsTypes.StorageDealConfig.AsObject[], + minDuration: number, ) => { - const req = new dealsTypes.WatchRequest() - req.setProposalsList(proposals) - const stream = client.watch(req) - stream.on("data", (res) => { - const dealInfo = res.getDealInfo()?.toObject() - if (dealInfo) { - handler(dealInfo) - } - }) - return stream.cancel + const req = new dealsTypes.StoreRequest() + req.setAddress(walletAddress) + req.setDataCid(dataCid) + req.setPieceSize(pieceSize) + req.setStorageDealConfigsList(dealConfigs.map((c) => dealConfigObjToMessage(c))) + req.setMinDuration(minDuration) + + return promise( + (cb) => client.store(req, cb), + (res: dealsTypes.StoreResponse) => res.toObject().resultsList, + ) + }, + + /** + * Fetches deal data to the underlying blockstore of the Filecoin client + * @param walletAddress The address to fund the retrieval + * @param setDataCid The cid of the data to fetch + */ + fetch: (walletAddress: string, dataCid: string) => { + const req = new dealsTypes.FetchRequest() + req.setAddress(walletAddress) + req.setCid(dataCid) + return promise( + (cb) => client.fetch(req, cb), + () => { + // nothing to return + }, + ) }, /** * Retrieve data stored in Filecoin * @param address The wallet address to fund the retrieval * @param cid The cid of the data to retrieve + * @param isCAR Indicates if the data is CAR encoded * @returns The raw data */ - retrieve: (address: string, cid: string) => { + retrieve: (address: string, cid: string, isCAR: boolean = false) => { return new Promise((resolve, reject) => { const append = (l: Uint8Array, r: Uint8Array) => { const tmp = new Uint8Array(l.byteLength + r.byteLength) @@ -99,6 +117,7 @@ export const createDeals = (config: Config) => { const req = new dealsTypes.RetrieveRequest() req.setAddress(address) req.setCid(cid) + req.setCarEncoding(isCAR) const stream = client.retrieve(req) stream.on("data", (resp) => { final = append(final, resp.getChunk_asU8()) @@ -113,6 +132,42 @@ export const createDeals = (config: Config) => { }) }, + /** + * Returns the current status of the deal and a flag indicating if the miner of the deal was slashed + * @param proposalCid The proposal cid of the deal to check + * @returns An object containing the status of the deal and a flag indicating if the miner of the deal was slashed + */ + getDealStatus: (proposalCid: string) => { + const req = new dealsTypes.GetDealStatusRequest() + req.setProposalCid(proposalCid) + return promise( + (cb) => client.getDealStatus(req, cb), + (res: dealsTypes.GetDealStatusResponse) => res.toObject(), + ) + }, + + /** + * Listen for deal proposal updates for the provided proposal ids + * @param handler The callback to receive proposal updates + * @param proposals A list of proposal ids to watch + * @returns A function that can be used to cancel watching + */ + watch: ( + handler: (event: dealsTypes.StorageDealInfo.AsObject) => void, + ...proposals: string[] + ) => { + const req = new dealsTypes.WatchRequest() + req.setProposalsList(proposals) + const stream = client.watch(req) + stream.on("data", (res) => { + const dealInfo = res.getDealInfo()?.toObject() + if (dealInfo) { + handler(dealInfo) + } + }) + return stream.cancel + }, + /** * List storage deal records according to the provided options * @param opts Options that control the behavior of listing records diff --git a/src/index.spec.ts b/src/index.spec.ts index 9565d4ba..fa634ac9 100644 --- a/src/index.spec.ts +++ b/src/index.spec.ts @@ -21,6 +21,28 @@ after(() => { cp.exec(`cd powergate-docker && make down`) }) +// beforeEach(async function () { +// this.timeout(130000) +// cp.exec(`cd powergate-docker && BIGSECTORS=false make localnet`, (err) => { +// if (err) { +// throw err +// } +// }) +// await wait({ +// resources: ["http://0.0.0.0:6002"], +// timeout: 120000, +// }) +// }) + +// afterEach(function (done) { +// cp.exec(`cd powergate-docker && make down`, (err) => { +// if (err) { +// throw err +// } +// done() +// }) +// }) + describe("client", () => { it("should create a client", () => { const pow = createPow({ host }) diff --git a/src/wallet/index.spec.ts b/src/wallet/index.spec.ts index 77412151..a934db3a 100644 --- a/src/wallet/index.spec.ts +++ b/src/wallet/index.spec.ts @@ -21,4 +21,16 @@ describe("wallet", () => { it("should check balance", async () => { await c.balance(address) }) + + // it("should send fil", async () => { + // const addresses = await c.list() + // expect(addresses).length(1) + // const bal = await c.balance(addresses[0]) + // expect(bal).greaterThan(0) + + // const address = await c.newAddress() + // expect(address).length.greaterThan(0) + + // await c.sendFil(addresses[0], address, 10) + // }) }) diff --git a/src/wallet/index.ts b/src/wallet/index.ts index 1d7d2067..235b68fe 100644 --- a/src/wallet/index.ts +++ b/src/wallet/index.ts @@ -41,14 +41,31 @@ export const createWallet = (config: Config) => { * @returns The address balance */ balance: (address: string) => { - const req = new walletTypes.WalletBalanceRequest() + const req = new walletTypes.BalanceRequest() req.setAddress(address) return promise( - (cb) => client.walletBalance(req, cb), - (resp: walletTypes.WalletBalanceResponse) => resp.toObject().balance, + (cb) => client.balance(req, cb), + (resp: walletTypes.BalanceResponse) => resp.toObject().balance, ) }, - // TODO: SendFil after next grpc bindings update + /** + * Send Fil from one address to another + * @param from The address to send from + * @param to The address to send to + * @param amount The amount of Fil to send + */ + sendFil: (from: string, to: string, amount: number) => { + const req = new walletTypes.SendFilRequest() + req.setFrom(from) + req.setTo(to) + req.setAmount(amount) + return promise( + (cb) => client.sendFil(req, cb), + () => { + // nothing to return + }, + ) + }, } } From 3afffdd77ad7f8e5e0c14121868b39d1c8790c03 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Tue, 14 Jul 2020 17:53:53 -0500 Subject: [PATCH 09/12] feat: isolated tests Signed-off-by: Aaron Sutula --- src/deals/index.spec.ts | 9 + src/deals/index.ts | 2 +- src/deals/util.ts | 6 +- src/ffs/index.spec.ts | 383 +++++++++++++++++++++++---------------- src/index.spec.ts | 38 ++-- src/net/index.spec.ts | 40 ++-- src/wallet/index.spec.ts | 36 ++-- 7 files changed, 292 insertions(+), 222 deletions(-) diff --git a/src/deals/index.spec.ts b/src/deals/index.spec.ts index e69de29b..d0e38243 100644 --- a/src/deals/index.spec.ts +++ b/src/deals/index.spec.ts @@ -0,0 +1,9 @@ +describe("deals", () => { + // const c = createDeals({ host, transport: getTransport() }) + // it("should import", async () => { + // const buffer = fs.readFileSync(`sample-data/samplefile`) + // const { dataCid, size } = await c.import(buffer, false) + // expect(dataCid).length.greaterThan(0) + // expect(size).greaterThan(0) + // }) +}) diff --git a/src/deals/index.ts b/src/deals/index.ts index 0f419b76..1824f0a5 100644 --- a/src/deals/index.ts +++ b/src/deals/index.ts @@ -25,7 +25,7 @@ export const createDeals = (config: Config) => { import: (data: Uint8Array, isCAR: boolean = false) => { // TODO: figure out how to stream data in here, or at least stream to the server return new Promise((resolve, reject) => { - const client = grpc.client(RPCService.Store, config) + const client = grpc.client(RPCService.Import, config) client.onMessage((message) => { resolve(message.toObject() as dealsTypes.ImportResponse.AsObject) }) diff --git a/src/deals/util.ts b/src/deals/util.ts index e97679ce..634fbc25 100644 --- a/src/deals/util.ts +++ b/src/deals/util.ts @@ -1,7 +1,9 @@ import { dealsTypes } from "../types" -export function dealConfigObjToMessage(obj: dealsTypes.DealConfig.AsObject): dealsTypes.DealConfig { - const msg = new dealsTypes.DealConfig() +export function dealConfigObjToMessage( + obj: dealsTypes.StorageDealConfig.AsObject, +): dealsTypes.StorageDealConfig { + const msg = new dealsTypes.StorageDealConfig() msg.setEpochPrice(obj.epochPrice) msg.setMiner(obj.miner) return msg diff --git a/src/ffs/index.spec.ts b/src/ffs/index.spec.ts index b4ad5635..09d9c1b2 100644 --- a/src/ffs/index.spec.ts +++ b/src/ffs/index.spec.ts @@ -4,218 +4,185 @@ import { createFFS } from "." import { withIncludeFinal } from "../deals/options" import { ffsTypes } from "../types" import { getTransport, host, useToken } from "../util" -import { withConfig, withHistory, withOverrideConfig } from "./options" +import { PushConfigOption, withConfig, withHistory, withOverrideConfig } from "./options" + +describe("ffs", function () { + this.timeout(180000) -describe("ffs", () => { const { getMeta, setToken } = useToken("") const c = createFFS({ host, transport: getTransport() }, getMeta) - let instanceId: string - let initialAddrs: ffsTypes.AddrInfo.AsObject[] - let defaultConfig: ffsTypes.DefaultConfig.AsObject - let cid: string - - it("should create an instance", async function () { - this.timeout(30000) - const res = await c.create() - expect(res.id).not.empty - expect(res.token).not.empty - instanceId = res.id - setToken(res.token) - // wait for 10 seconds so our wallet address gets funded - await new Promise((r) => setTimeout(r, 10000)) + it("should create an instance", async () => { + await expectNewInstance() }) it("should list instances", async () => { + await expectNewInstance() const res = await c.list() expect(res.instancesList).length.greaterThan(0) }) it("should get instance id", async () => { + const instanceInfo = await expectNewInstance() const res = await c.id() - expect(res.id).eq(instanceId) + expect(res.id).eq(instanceInfo.id) }) it("should get addrs", async () => { - const res = await c.addrs() - initialAddrs = res.addrsList - expect(initialAddrs).length.greaterThan(0) + await expectNewInstance() + await expectAddrs(1) }) it("should get the default config", async () => { - const res = await c.defaultConfig() - expect(res.defaultConfig).not.undefined - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - defaultConfig = res.defaultConfig! + await expectNewInstance() + await expectDefaultConfig() }) it("should create a new addr", async () => { - const res = await c.newAddr("my addr") - expect(res.addr).length.greaterThan(0) - const addrsRes = await c.addrs() - expect(addrsRes.addrsList).length(initialAddrs.length + 1) + await expectNewInstance() + await expectAddrs(1) + await expectNewAddr() + await expectAddrs(2) }) it("should set default config", async () => { + await expectNewInstance() + const defaultConfig = await expectDefaultConfig() await c.setDefaultConfig(defaultConfig) }) it("should get info", async () => { + await expectNewInstance() const res = await c.info() expect(res.info).not.undefined }) it("should add to hot", async () => { - const buffer = fs.readFileSync(`sample-data/samplefile`) - const res = await c.addToHot(buffer) - expect(res.cid).length.greaterThan(0) - cid = res.cid + await expectNewInstance() + await expectAddToHot("sample-data/samplefile") }) it("should get default cid config", async () => { - const res = await c.getDefaultCidConfig(cid) - expect(res.config?.cid).equal(cid) + await expectNewInstance() + const cid = await expectAddToHot("sample-data/samplefile") + await expectDefaultCidConfig(cid) }) - let jobId: string - it("should push config", async () => { - const res0 = await c.getDefaultCidConfig(cid) - expect(res0.config?.cid).equal(cid) - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const config = res0.config! - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - config.hot!.enabled = false - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - config.hot!.allowUnfreeze = true - const res1 = await c.pushConfig(cid, withOverrideConfig(false), withConfig(config)) - expect(res1.jobId).length.greaterThan(0) - jobId = res1.jobId - }) - - it("should watch job", function (done) { - this.timeout(180000) - const cancel = c.watchJobs((job) => { - expect(job.errCause).empty - expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_CANCELED) - expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_FAILED) - if (job.status === ffsTypes.JobStatus.JOB_STATUS_SUCCESS) { - cancel() - done() - } - }, jobId) - }) - - it("should watch logs", function (done) { - this.timeout(10000) - const cancel = c.watchLogs( - (logEvent) => { - expect(logEvent.cid).not.empty - cancel() - done() - }, - cid, - withHistory(true), - ) + await expectNewInstance() + const cid = await expectAddToHot("sample-data/samplefile") + const config = await expectDefaultCidConfig(cid) + await expectPushConfig(cid, false, config) + }) + + it("should watch job", async () => { + await expectNewInstance() + const addrs = await expectAddrs(1) + await waitForBalance(addrs[0].addr, 0) + const cid = await expectAddToHot("sample-data/samplefile") + const jobId = await expectPushConfig(cid) + await waitForJobStatus(jobId, ffsTypes.JobStatus.JOB_STATUS_SUCCESS) + }) + + it("should watch logs", async () => { + await expectNewInstance() + const cid = await expectAddToHot("sample-data/samplefile") + await expectPushConfig(cid) + await new Promise((resolve, reject) => { + const cancel = c.watchLogs( + (logEvent) => { + if (logEvent.cid.length > 0) { + cancel() + resolve() + } else { + cancel() + reject("empty log cid") + } + }, + cid, + withHistory(true), + ) + }) }) it("should get a storage deal record", async () => { + await expectNewInstance() + const addrs = await expectAddrs(1) + await waitForBalance(addrs[0].addr, 0) + const cid = await expectAddToHot("sample-data/samplefile") + const jobId = await expectPushConfig(cid) + await waitForJobStatus(jobId, ffsTypes.JobStatus.JOB_STATUS_SUCCESS) const res = await c.listStorageDealRecords(withIncludeFinal(true)) expect(res).length.greaterThan(0) }) - it("should push an unfreeze config change", async () => { - const res0 = await c.getDefaultCidConfig(cid) - expect(res0.config?.cid).equal(cid) - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const config = res0.config! - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - config.hot!.enabled = true - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - config.hot!.allowUnfreeze = true - const res1 = await c.pushConfig(cid, withOverrideConfig(true), withConfig(config)) - expect(res1.jobId).length.greaterThan(0) - jobId = res1.jobId - }) - - it("should watch the config change job", function (done) { - this.timeout(180000) - const cancel = c.watchJobs((job) => { - expect(job.errCause).empty - expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_CANCELED) - expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_FAILED) - if (job.status === ffsTypes.JobStatus.JOB_STATUS_SUCCESS) { - cancel() - done() - } - }, jobId) - }) - - it("should get a retrieval deal record", async function () { + it("should get a retrieval deal record", async () => { // ToDo: Figure out hot to make sure the data in the previous push isn't cached in hot - // this.timeout(30000) - // await new Promise((r) => setTimeout(r, 10000)) - // const res = await c.listRetrievalDealRecords() - // expect(res).length.greaterThan(0) }) it("should get cid config", async () => { + await expectNewInstance() + const cid = await expectAddToHot("sample-data/samplefile") + await expectPushConfig(cid) const res = await c.getCidConfig(cid) expect(res.config?.cid).equal(cid) }) it("should show", async () => { + await expectNewInstance() + const addrs = await expectAddrs(1) + await waitForBalance(addrs[0].addr, 0) + const cid = await expectAddToHot("sample-data/samplefile") + const jobId = await expectPushConfig(cid) + await waitForJobStatus(jobId, ffsTypes.JobStatus.JOB_STATUS_SUCCESS) const res = await c.show(cid) expect(res.cidInfo).not.undefined }) it("should show all", async () => { + await expectNewInstance() + const addrs = await expectAddrs(1) + await waitForBalance(addrs[0].addr, 0) + const cid = await expectAddToHot("sample-data/samplefile") + const jobId = await expectPushConfig(cid) + await waitForJobStatus(jobId, ffsTypes.JobStatus.JOB_STATUS_SUCCESS) const res = await c.showAll() expect(res).not.empty }) - let buffer: Buffer - it("should replace", async () => { - buffer = fs.readFileSync(`sample-data/samplefile2`) - const res0 = await c.addToHot(buffer) - expect(res0.cid).length.greaterThan(0) - const res1 = await c.replace(cid, res0.cid) - expect(res1.jobId).length.greaterThan(0) - cid = res0.cid - jobId = res1.jobId - }) - - it("should watch replace job", function (done) { - this.timeout(180000) - const cancel = c.watchJobs((job) => { - expect(job.errCause).empty - expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_CANCELED) - expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_FAILED) - if (job.status === ffsTypes.JobStatus.JOB_STATUS_SUCCESS) { - cancel() - done() - } - }, jobId) + await expectNewInstance() + const addrs = await expectAddrs(1) + await waitForBalance(addrs[0].addr, 0) + const cid = await expectAddToHot("sample-data/samplefile") + const jobId = await expectPushConfig(cid) + await waitForJobStatus(jobId, ffsTypes.JobStatus.JOB_STATUS_SUCCESS) + const cid2 = await expectAddToHot("sample-data/samplefile2") + const res = await c.replace(cid, cid2) + expect(res.jobId).length.greaterThan(0) }) it("should get", async () => { + await expectNewInstance() + const addrs = await expectAddrs(1) + await waitForBalance(addrs[0].addr, 0) + const cid = await expectAddToHot("sample-data/samplefile") + const jobId = await expectPushConfig(cid) + await waitForJobStatus(jobId, ffsTypes.JobStatus.JOB_STATUS_SUCCESS) const bytes = await c.get(cid) - expect(bytes.byteLength).equal(buffer.byteLength) + expect(bytes.byteLength).greaterThan(0) }) it("should cancel a job", async () => { - buffer = fs.readFileSync(`sample-data/samplefile3`) - const res0 = await c.addToHot(buffer) - expect(res0.cid).length.greaterThan(0) - const res1 = await c.pushConfig(res0.cid) - expect(res1.jobId).length.greaterThan(0) - await c.cancelJob(res1.jobId) + await expectNewInstance() + const cid = await expectAddToHot("sample-data/samplefile") + const jobId = await expectPushConfig(cid) + await c.cancelJob(jobId) }) it("should list payment channels", async () => { - await c.listPayChannels() + // TODO }) it("should create a payment channel", async () => { @@ -226,8 +193,10 @@ describe("ffs", () => { // TODO }) - it("should push disable storage job", async () => { - const newConf: ffsTypes.CidConfig.AsObject = { + it("should remove", async () => { + await expectNewInstance() + const cid = await expectAddToHot("sample-data/samplefile") + const conf: ffsTypes.CidConfig.AsObject = { cid, repairable: false, cold: { @@ -238,35 +207,131 @@ describe("ffs", () => { enabled: false, }, } - const res0 = await c.pushConfig(cid, withOverrideConfig(true), withConfig(newConf)) - expect(res0).not.undefined - jobId = res0.jobId - }) - - it("should watch disable storage job", function (done) { - this.timeout(180000) - const cancel = c.watchJobs((job) => { - expect(job.errCause).empty - expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_CANCELED) - expect(job.status).not.equal(ffsTypes.JobStatus.JOB_STATUS_FAILED) - if (job.status === ffsTypes.JobStatus.JOB_STATUS_SUCCESS) { - cancel() - done() - } - }, jobId) - }) - - it("should remove", async () => { + const jobId = await expectPushConfig(cid, false, conf) + await waitForJobStatus(jobId, ffsTypes.JobStatus.JOB_STATUS_SUCCESS) await c.remove(cid) }) it("should send fil", async () => { - const addrs = await c.addrs() - expect(addrs.addrsList).lengthOf(2) - await c.sendFil(addrs.addrsList[0].addr, addrs.addrsList[1].addr, 10) + await expectNewInstance() + const addrs = await expectAddrs(1) + await waitForBalance(addrs[0].addr, 0) + const addr = await expectNewAddr() + await c.sendFil(addrs[0].addr, addr, 10) }) it("should close", async () => { + await expectNewInstance() await c.close() }) + + async function expectNewInstance() { + const res = await c.create() + expect(res.id).not.empty + expect(res.token).not.empty + setToken(res.token) + return res + } + + async function expectAddrs(length: number) { + const res = await c.addrs() + expect(res.addrsList).length(length) + return res.addrsList + } + + async function expectNewAddr() { + const res = await c.newAddr("my addr") + expect(res.addr).length.greaterThan(0) + return res.addr + } + + async function expectDefaultConfig() { + const res = await c.defaultConfig() + expect(res.defaultConfig).not.undefined + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return res.defaultConfig! + } + + async function expectAddToHot(path: string) { + const buffer = fs.readFileSync(path) + const res = await c.addToHot(buffer) + expect(res.cid).length.greaterThan(0) + return res.cid + } + + async function expectDefaultCidConfig(cid: string) { + const res = await c.getDefaultCidConfig(cid) + expect(res.config).not.undefined + expect(res.config?.cid).equal(cid) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return res.config! + } + + async function expectPushConfig( + cid: string, + override: boolean = false, + config?: ffsTypes.CidConfig.AsObject, + ) { + const opts: PushConfigOption[] = [] + opts.push(withOverrideConfig(override)) + if (config) { + opts.push(withConfig(config)) + } + const res = await c.pushConfig(cid, ...opts) + expect(res.jobId).length.greaterThan(0) + return res.jobId + } + + function waitForJobStatus( + jobId: string, + status: ffsTypes.JobStatusMap[keyof ffsTypes.JobStatusMap], + ) { + return new Promise((resolve, reject) => { + try { + const cancel = c.watchJobs((job) => { + if (job.errCause.length > 0) { + reject(job.errCause) + } + if (job.status === ffsTypes.JobStatus.JOB_STATUS_CANCELED) { + reject("job canceled") + } + if (job.status === ffsTypes.JobStatus.JOB_STATUS_FAILED) { + reject("job failed") + } + if (job.status === status) { + cancel() + resolve() + } + }, jobId) + } catch (e) { + reject(e) + } + }) + } + + function waitForBalance(address: string, greaterThan: number) { + return new Promise(async (resolve, reject) => { + while (true) { + try { + const res = await c.info() + if (!res.info) { + reject("no balance info returned") + return + } + const info = res.info.balancesList.find((info) => info.addr?.addr === address) + if (!info) { + reject("address not in balances list") + return + } + if (info.balance > greaterThan) { + resolve(info.balance) + return + } + } catch (e) { + reject(e) + } + await new Promise((r) => setTimeout(r, 1000)) + } + }) + } }) diff --git a/src/index.spec.ts b/src/index.spec.ts index fa634ac9..0931b39b 100644 --- a/src/index.spec.ts +++ b/src/index.spec.ts @@ -4,8 +4,8 @@ import wait from "wait-on" import { createPow } from "." import { host } from "./util" -before(async function () { - this.timeout(130000) +beforeEach(async function () { + this.timeout(10000) cp.exec(`cd powergate-docker && BIGSECTORS=false make localnet`, (err) => { if (err) { throw err @@ -17,32 +17,18 @@ before(async function () { }) }) -after(() => { - cp.exec(`cd powergate-docker && make down`) +afterEach(async function () { + this.timeout(10000) + await new Promise((resolve, reject) => { + cp.exec(`cd powergate-docker && make down`, (err, stdout) => { + if (err) { + reject(err) + } + resolve(stdout) + }) + }) }) -// beforeEach(async function () { -// this.timeout(130000) -// cp.exec(`cd powergate-docker && BIGSECTORS=false make localnet`, (err) => { -// if (err) { -// throw err -// } -// }) -// await wait({ -// resources: ["http://0.0.0.0:6002"], -// timeout: 120000, -// }) -// }) - -// afterEach(function (done) { -// cp.exec(`cd powergate-docker && make down`, (err) => { -// if (err) { -// throw err -// } -// done() -// }) -// }) - describe("client", () => { it("should create a client", () => { const pow = createPow({ host }) diff --git a/src/net/index.spec.ts b/src/net/index.spec.ts index 2eb62279..5837cc70 100644 --- a/src/net/index.spec.ts +++ b/src/net/index.spec.ts @@ -1,16 +1,14 @@ -import { Connectedness, PeersResponse } from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb" +import { Connectedness } from "@textile/grpc-powergate-client/dist/net/rpc/rpc_pb" import { assert, expect } from "chai" import { createNet } from "." +import { netTypes } from "../types" import { getTransport, host } from "../util" describe("net", () => { const c = createNet({ host, transport: getTransport() }) - let peers: PeersResponse.AsObject - it("should query peers", async () => { - peers = await c.peers() - expect(peers.peersList).length.greaterThan(0) + await expectPeers() }) it("should get listen address", async () => { @@ -20,29 +18,37 @@ describe("net", () => { }) it("should find a peer", async () => { - const peerId = peers.peersList[0].addrInfo?.id - if (!peerId) { - assert.fail("no peer id") - } + const peers = await expectPeers() + const peerId = expectPeerInfo(peers).id const peer = await c.findPeer(peerId) expect(peer.peerInfo).not.undefined }) it("should get peer connectedness", async () => { - const peerId = peers.peersList[0].addrInfo?.id - if (!peerId) { - assert.fail("no peer id") - } + const peers = await expectPeers() + const peerId = expectPeerInfo(peers).id const resp = await c.connectedness(peerId) expect(resp.connectedness).equal(Connectedness.CONNECTEDNESS_CONNECTED) }) it("should disconnect and reconnect to a peer", async () => { - const peerInfo = peers.peersList[0].addrInfo - if (!peerInfo) { - assert.fail("no peer info") - } + const peers = await expectPeers() + const peerInfo = expectPeerInfo(peers) await c.disconnectPeer(peerInfo.id) await c.connectPeer(peerInfo) }) + + async function expectPeers() { + const peers = await c.peers() + expect(peers.peersList).length.greaterThan(0) + return peers + } + + function expectPeerInfo(peersResp: netTypes.PeersResponse.AsObject) { + const peerInfo = peersResp.peersList[0].addrInfo + if (!peerInfo) { + assert.fail("no peer info") + } + return peerInfo + } }) diff --git a/src/wallet/index.spec.ts b/src/wallet/index.spec.ts index a934db3a..12e97b46 100644 --- a/src/wallet/index.spec.ts +++ b/src/wallet/index.spec.ts @@ -5,32 +5,34 @@ import { getTransport, host } from "../util" describe("wallet", () => { const c = createWallet({ host, transport: getTransport() }) - let address: string - it("should list addresses", async () => { - const addresses = await c.list() - expect(addresses).length.greaterThan(0) - address = addresses[0] + await expectAddresses(0) }) it("should create a new address", async () => { - const address = await c.newAddress() - expect(address).length.greaterThan(0) + await expectNewAddress() }) it("should check balance", async () => { - await c.balance(address) + const addrs = await expectAddresses(0) + await c.balance(addrs[0]) }) - // it("should send fil", async () => { - // const addresses = await c.list() - // expect(addresses).length(1) - // const bal = await c.balance(addresses[0]) - // expect(bal).greaterThan(0) + it("should send fil", async () => { + const addrs = await expectAddresses(0) + const newAddr = await expectNewAddress() + await c.sendFil(addrs[0], newAddr, 10) + }) - // const address = await c.newAddress() - // expect(address).length.greaterThan(0) + async function expectAddresses(lengthGreaterThan: number) { + const addresses = await c.list() + expect(addresses).length.greaterThan(lengthGreaterThan) + return addresses + } - // await c.sendFil(addresses[0], address, 10) - // }) + async function expectNewAddress() { + const address = await c.newAddress() + expect(address).length.greaterThan(0) + return address + } }) From c9e4f8aae95d0603ffb729810b31e1f2ffe5c4c6 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Tue, 14 Jul 2020 18:28:47 -0500 Subject: [PATCH 10/12] fix: longer timeout on starting powergate Signed-off-by: Aaron Sutula --- src/index.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.spec.ts b/src/index.spec.ts index 0931b39b..5a6efa80 100644 --- a/src/index.spec.ts +++ b/src/index.spec.ts @@ -5,7 +5,7 @@ import { createPow } from "." import { host } from "./util" beforeEach(async function () { - this.timeout(10000) + this.timeout(120000) cp.exec(`cd powergate-docker && BIGSECTORS=false make localnet`, (err) => { if (err) { throw err From bbd6ede8a36956c615d9eac1a8a04f23d436a4da Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Wed, 15 Jul 2020 14:27:32 -0500 Subject: [PATCH 11/12] chore: update to pg release, remove deals Signed-off-by: Aaron Sutula --- package-lock.json | 14 +-- package.json | 3 +- src/deals/index.spec.ts | 9 -- src/deals/index.ts | 207 ---------------------------------------- src/deals/options.ts | 64 ------------- src/deals/util.ts | 10 -- src/ffs/index.spec.ts | 9 +- src/ffs/index.ts | 9 +- src/ffs/options.ts | 63 ++++++++++++ src/index.ts | 6 +- src/options.ts | 3 +- src/types.ts | 2 - 12 files changed, 85 insertions(+), 314 deletions(-) delete mode 100644 src/deals/index.spec.ts delete mode 100644 src/deals/index.ts delete mode 100644 src/deals/options.ts delete mode 100644 src/deals/util.ts diff --git a/package-lock.json b/package-lock.json index e5aeea56..5dcf40e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -451,9 +451,9 @@ } }, "@improbable-eng/grpc-web": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@improbable-eng/grpc-web/-/grpc-web-0.12.0.tgz", - "integrity": "sha512-uJjgMPngreRTYPBuo6gswMj1gK39Wbqre/RgE0XnSDXJRg6ST7ZhuS53dFE6Vc2CX4jxgl+cO+0B3op8LA4Q0Q==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@improbable-eng/grpc-web/-/grpc-web-0.13.0.tgz", + "integrity": "sha512-vaxxT+Qwb7GPqDQrBV4vAAfH0HywgOLw6xGIKXd9Q8hcV63CQhmS3p4+pZ9/wVvt4Ph3ZDK9fdC983b9aGMUFg==", "requires": { "browser-headers": "^0.4.0" } @@ -464,11 +464,11 @@ "integrity": "sha512-+Kjz+Dktfz5LKTZA9ZW/Vlww6HF9KaKz4x2mVe1O8CJdOP2WfzC+KY8L6EWMqVLrV4MvdBuQdSgDmvSJz+OGuA==" }, "@textile/grpc-powergate-client": { - "version": "0.1.1-pre.2", - "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-0.1.1-pre.2.tgz", - "integrity": "sha512-E7aAbFIMHny/recjL9k8z8ZKEG377yyDl/Hy8ISAbuyEZzPNdYkBfYVJLG3llRDJZZS63kycxW8LzFr3NpPnkw==", + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-0.1.1.tgz", + "integrity": "sha512-PJ1VIKALEvYxGHi70y1tIBOFrpK7e4XvAGOfDnBpiVXqzjlTiNijDxzgJIWjNXLugkp4e1QZVrbJ7LG0+EQfBg==", "requires": { - "@improbable-eng/grpc-web": "^0.12.0", + "@improbable-eng/grpc-web": "^0.13.0", "@types/google-protobuf": "^3.7.2", "google-protobuf": "^3.12.2" } diff --git a/package.json b/package.json index 86506c92..8d9d086b 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,6 @@ "types": "dist/index", "scripts": { "prepublishOnly": "npm run build", - "prepare": "npm run compile", "prebuild": "npm run clean", "build": "npm run compile", "compile": "tsc -b tsconfig.json", @@ -66,6 +65,6 @@ }, "dependencies": { "@improbable-eng/grpc-web-node-http-transport": "^0.12.0", - "@textile/grpc-powergate-client": "0.1.1-pre.2" + "@textile/grpc-powergate-client": "0.1.1" } } \ No newline at end of file diff --git a/src/deals/index.spec.ts b/src/deals/index.spec.ts deleted file mode 100644 index d0e38243..00000000 --- a/src/deals/index.spec.ts +++ /dev/null @@ -1,9 +0,0 @@ -describe("deals", () => { - // const c = createDeals({ host, transport: getTransport() }) - // it("should import", async () => { - // const buffer = fs.readFileSync(`sample-data/samplefile`) - // const { dataCid, size } = await c.import(buffer, false) - // expect(dataCid).length.greaterThan(0) - // expect(size).greaterThan(0) - // }) -}) diff --git a/src/deals/index.ts b/src/deals/index.ts deleted file mode 100644 index 1824f0a5..00000000 --- a/src/deals/index.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { grpc } from "@improbable-eng/grpc-web" -import { - RPCService, - RPCServiceClient, -} from "@textile/grpc-powergate-client/dist/deals/rpc/rpc_pb_service" -import { Config, dealsTypes } from "../types" -import { promise } from "../util" -import { ListDealRecordsOption } from "./options" -import { dealConfigObjToMessage } from "./util" - -/** - * Creates the Deals API client - * @param config A config object that changes the behavior of the client - * @returns The Deals API client - */ -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types -export const createDeals = (config: Config) => { - const client = new RPCServiceClient(config.host, config) - return { - /** - * Import raw data into the Filecoin client to prepare for storage - * @param data The raw data to import - * @param isCAR Specifies if the data is already CAR encoded, default false - */ - import: (data: Uint8Array, isCAR: boolean = false) => { - // TODO: figure out how to stream data in here, or at least stream to the server - return new Promise((resolve, reject) => { - const client = grpc.client(RPCService.Import, config) - client.onMessage((message) => { - resolve(message.toObject() as dealsTypes.ImportResponse.AsObject) - }) - client.onEnd((code, msg) => { - if (code !== grpc.Code.OK) { - reject(`error code ${code} - ${msg}`) - } else { - reject("ended with no message") - } - }) - client.start() - - const params = new dealsTypes.ImportParams() - params.setIsCar(isCAR) - const req0 = new dealsTypes.ImportRequest() - req0.setImportParams(params) - client.send(req0) - - const req1 = new dealsTypes.ImportRequest() - req1.setChunk(data) - client.send(req1) - - client.finishSend() - }) - }, - - /** - * Stores data according to the provided deal parameters - * @param walletAddress The wallet address to fund the deals - * @param dataCid The cid of the data to store - * @param pieceSize The size of the data to store - * @param dealConfigs The list of deals to execute - * @param minDuration The amount of time to store the data in epochs - * @returns A list of information about the storage deals - */ - store: ( - walletAddress: string, - dataCid: string, - pieceSize: number, - dealConfigs: dealsTypes.StorageDealConfig.AsObject[], - minDuration: number, - ) => { - const req = new dealsTypes.StoreRequest() - req.setAddress(walletAddress) - req.setDataCid(dataCid) - req.setPieceSize(pieceSize) - req.setStorageDealConfigsList(dealConfigs.map((c) => dealConfigObjToMessage(c))) - req.setMinDuration(minDuration) - - return promise( - (cb) => client.store(req, cb), - (res: dealsTypes.StoreResponse) => res.toObject().resultsList, - ) - }, - - /** - * Fetches deal data to the underlying blockstore of the Filecoin client - * @param walletAddress The address to fund the retrieval - * @param setDataCid The cid of the data to fetch - */ - fetch: (walletAddress: string, dataCid: string) => { - const req = new dealsTypes.FetchRequest() - req.setAddress(walletAddress) - req.setCid(dataCid) - return promise( - (cb) => client.fetch(req, cb), - () => { - // nothing to return - }, - ) - }, - - /** - * Retrieve data stored in Filecoin - * @param address The wallet address to fund the retrieval - * @param cid The cid of the data to retrieve - * @param isCAR Indicates if the data is CAR encoded - * @returns The raw data - */ - retrieve: (address: string, cid: string, isCAR: boolean = false) => { - return new Promise((resolve, reject) => { - const append = (l: Uint8Array, r: Uint8Array) => { - const tmp = new Uint8Array(l.byteLength + r.byteLength) - tmp.set(l, 0) - tmp.set(r, l.byteLength) - return tmp - } - let final = new Uint8Array() - const req = new dealsTypes.RetrieveRequest() - req.setAddress(address) - req.setCid(cid) - req.setCarEncoding(isCAR) - const stream = client.retrieve(req) - stream.on("data", (resp) => { - final = append(final, resp.getChunk_asU8()) - }) - stream.on("end", (status) => { - if (status?.code !== grpc.Code.OK) { - reject(`error code ${status?.code} - ${status?.details}`) - } else { - resolve(final) - } - }) - }) - }, - - /** - * Returns the current status of the deal and a flag indicating if the miner of the deal was slashed - * @param proposalCid The proposal cid of the deal to check - * @returns An object containing the status of the deal and a flag indicating if the miner of the deal was slashed - */ - getDealStatus: (proposalCid: string) => { - const req = new dealsTypes.GetDealStatusRequest() - req.setProposalCid(proposalCid) - return promise( - (cb) => client.getDealStatus(req, cb), - (res: dealsTypes.GetDealStatusResponse) => res.toObject(), - ) - }, - - /** - * Listen for deal proposal updates for the provided proposal ids - * @param handler The callback to receive proposal updates - * @param proposals A list of proposal ids to watch - * @returns A function that can be used to cancel watching - */ - watch: ( - handler: (event: dealsTypes.StorageDealInfo.AsObject) => void, - ...proposals: string[] - ) => { - const req = new dealsTypes.WatchRequest() - req.setProposalsList(proposals) - const stream = client.watch(req) - stream.on("data", (res) => { - const dealInfo = res.getDealInfo()?.toObject() - if (dealInfo) { - handler(dealInfo) - } - }) - return stream.cancel - }, - - /** - * List storage deal records according to the provided options - * @param opts Options that control the behavior of listing records - * @returns A list of storage deal records - */ - listStorageDealRecords: (...opts: ListDealRecordsOption[]) => { - const conf = new dealsTypes.ListDealRecordsConfig() - opts.forEach((opt) => { - opt(conf) - }) - const req = new dealsTypes.ListStorageDealRecordsRequest() - req.setConfig(conf) - return promise( - (cb) => client.listStorageDealRecords(req, cb), - (res: dealsTypes.ListStorageDealRecordsResponse) => res.toObject().recordsList, - ) - }, - - /** - * List retrieval deal records according to the provided options - * @param opts Options that control the behavior of listing records - * @returns A list of retrieval deal records - */ - listRetrievalDealRecords: (...opts: ListDealRecordsOption[]) => { - const conf = new dealsTypes.ListDealRecordsConfig() - opts.forEach((opt) => { - opt(conf) - }) - const req = new dealsTypes.ListRetrievalDealRecordsRequest() - req.setConfig(conf) - return promise( - (cb) => client.listRetrievalDealRecords(req, cb), - (res: dealsTypes.ListRetrievalDealRecordsResponse) => res.toObject().recordsList, - ) - }, - } -} diff --git a/src/deals/options.ts b/src/deals/options.ts deleted file mode 100644 index 918222a8..00000000 --- a/src/deals/options.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { dealsTypes } from "../types" - -export type ListDealRecordsOption = (req: dealsTypes.ListDealRecordsConfig) => void - -/** - * Limits the results deals initiated from the provided wallet addresses - * @param addresses The list of addresses - * @returns The resulting option - */ -export const withFromAddresses = (...addresses: string[]): ListDealRecordsOption => { - return (req: dealsTypes.ListDealRecordsConfig) => { - req.setFromAddrsList(addresses) - } -} - -/** - * Limits the results to deals for the provided data cids - * @param cids The list of cids - * @returns The resulting option - */ -export const withDataCids = (...cids: string[]): ListDealRecordsOption => { - return (req: dealsTypes.ListDealRecordsConfig) => { - req.setDataCidsList(cids) - } -} - -/** - * Specifies whether or not to include pending deals in the results - * Default is false - * Ignored for listRetrievalDealRecords - * @param includePending Whether or not to include pending deal records - * @returns The resulting option - */ -export const withIncludePending = (includePending: boolean): ListDealRecordsOption => { - return (req: dealsTypes.ListDealRecordsConfig) => { - req.setIncludePending(includePending) - } -} - -/** - * Specifies whether or not to include final deals in the results - * Default is false - * Ignored for listRetrievalDealRecords - * @param includeFinal Whether or not to include final deal records - * @returns The resulting option - */ -export const withIncludeFinal = (includeFinal: boolean): ListDealRecordsOption => { - return (req: dealsTypes.ListDealRecordsConfig) => { - req.setIncludeFinal(includeFinal) - } -} - -/** - * Specifies to sort the results in ascending order - * Default is descending order - * Records are sorted by timestamp - * @param ascending Whether or not to sort the results in ascending order - * @returns The resulting option - */ -export const withAscending = (ascending: boolean): ListDealRecordsOption => { - return (req: dealsTypes.ListDealRecordsConfig) => { - req.setAscending(ascending) - } -} diff --git a/src/deals/util.ts b/src/deals/util.ts deleted file mode 100644 index 634fbc25..00000000 --- a/src/deals/util.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { dealsTypes } from "../types" - -export function dealConfigObjToMessage( - obj: dealsTypes.StorageDealConfig.AsObject, -): dealsTypes.StorageDealConfig { - const msg = new dealsTypes.StorageDealConfig() - msg.setEpochPrice(obj.epochPrice) - msg.setMiner(obj.miner) - return msg -} diff --git a/src/ffs/index.spec.ts b/src/ffs/index.spec.ts index 09d9c1b2..d09359eb 100644 --- a/src/ffs/index.spec.ts +++ b/src/ffs/index.spec.ts @@ -1,10 +1,15 @@ import { expect } from "chai" import fs from "fs" import { createFFS } from "." -import { withIncludeFinal } from "../deals/options" import { ffsTypes } from "../types" import { getTransport, host, useToken } from "../util" -import { PushConfigOption, withConfig, withHistory, withOverrideConfig } from "./options" +import { + PushConfigOption, + withConfig, + withHistory, + withIncludeFinal, + withOverrideConfig, +} from "./options" describe("ffs", function () { this.timeout(180000) diff --git a/src/ffs/index.ts b/src/ffs/index.ts index c425c08f..918da4f9 100644 --- a/src/ffs/index.ts +++ b/src/ffs/index.ts @@ -3,10 +3,9 @@ import { RPCService, RPCServiceClient, } from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb_service" -import { ListDealRecordsOption } from "../deals/options" -import { Config, dealsTypes, ffsTypes } from "../types" +import { Config, ffsTypes } from "../types" import { promise } from "../util" -import { PushConfigOption, WatchLogsOption } from "./options" +import { ListDealRecordsOption, PushConfigOption, WatchLogsOption } from "./options" import { coldObjToMessage, hotObjToMessage } from "./util" /** @@ -402,7 +401,7 @@ export const createFFS = (config: Config, getMeta: () => grpc.Metadata) => { * @returns A list of storage deal records */ listStorageDealRecords: (...opts: ListDealRecordsOption[]) => { - const conf = new dealsTypes.ListDealRecordsConfig() + const conf = new ffsTypes.ListDealRecordsConfig() opts.forEach((opt) => { opt(conf) }) @@ -420,7 +419,7 @@ export const createFFS = (config: Config, getMeta: () => grpc.Metadata) => { * @returns A list of retrieval deal records */ listRetrievalDealRecords: (...opts: ListDealRecordsOption[]) => { - const conf = new dealsTypes.ListDealRecordsConfig() + const conf = new ffsTypes.ListDealRecordsConfig() opts.forEach((opt) => { opt(conf) }) diff --git a/src/ffs/options.ts b/src/ffs/options.ts index 4b55b966..c5b00b7c 100644 --- a/src/ffs/options.ts +++ b/src/ffs/options.ts @@ -59,3 +59,66 @@ export const withJobId = (jobId: string): WatchLogsOption => { req.setJid(jobId) } } + +export type ListDealRecordsOption = (req: ffsTypes.ListDealRecordsConfig) => void + +/** + * Limits the results deals initiated from the provided wallet addresses + * @param addresses The list of addresses + * @returns The resulting option + */ +export const withFromAddresses = (...addresses: string[]): ListDealRecordsOption => { + return (req: ffsTypes.ListDealRecordsConfig) => { + req.setFromAddrsList(addresses) + } +} + +/** + * Limits the results to deals for the provided data cids + * @param cids The list of cids + * @returns The resulting option + */ +export const withDataCids = (...cids: string[]): ListDealRecordsOption => { + return (req: ffsTypes.ListDealRecordsConfig) => { + req.setDataCidsList(cids) + } +} + +/** + * Specifies whether or not to include pending deals in the results + * Default is false + * Ignored for listRetrievalDealRecords + * @param includePending Whether or not to include pending deal records + * @returns The resulting option + */ +export const withIncludePending = (includePending: boolean): ListDealRecordsOption => { + return (req: ffsTypes.ListDealRecordsConfig) => { + req.setIncludePending(includePending) + } +} + +/** + * Specifies whether or not to include final deals in the results + * Default is false + * Ignored for listRetrievalDealRecords + * @param includeFinal Whether or not to include final deal records + * @returns The resulting option + */ +export const withIncludeFinal = (includeFinal: boolean): ListDealRecordsOption => { + return (req: ffsTypes.ListDealRecordsConfig) => { + req.setIncludeFinal(includeFinal) + } +} + +/** + * Specifies to sort the results in ascending order + * Default is descending order + * Records are sorted by timestamp + * @param ascending Whether or not to sort the results in ascending order + * @returns The resulting option + */ +export const withAscending = (ascending: boolean): ListDealRecordsOption => { + return (req: ffsTypes.ListDealRecordsConfig) => { + req.setAscending(ascending) + } +} diff --git a/src/index.ts b/src/index.ts index 04bcc956..2bf7795a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,12 +4,11 @@ import { createFFS } from "./ffs" import { createHealth } from "./health" import { createMiners } from "./miners" import { createNet } from "./net" -import { dealsOptions, ffsOptions } from "./options" +import { ffsOptions } from "./options" import { createReputation } from "./reputation" import { asksTypes, Config, - dealsTypes, faultsTypes, ffsTypes, healthTypes, @@ -21,11 +20,10 @@ import { import { getTransport, host, useToken } from "./util" import { createWallet } from "./wallet" -export { dealsOptions, ffsOptions } +export { ffsOptions } export { asksTypes, Config, - dealsTypes, faultsTypes, ffsTypes, healthTypes, diff --git a/src/options.ts b/src/options.ts index 1021d1c4..3ccff8c2 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,4 +1,3 @@ -import * as dealsOptions from "./deals/options" import * as ffsOptions from "./ffs/options" -export { ffsOptions, dealsOptions } +export { ffsOptions } diff --git a/src/types.ts b/src/types.ts index 3df596a7..81598aee 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,4 @@ import { grpc } from "@improbable-eng/grpc-web" -import * as dealsTypes from "@textile/grpc-powergate-client/dist/deals/rpc/rpc_pb" import * as ffsTypes from "@textile/grpc-powergate-client/dist/ffs/rpc/rpc_pb" import * as healthTypes from "@textile/grpc-powergate-client/dist/health/rpc/rpc_pb" import * as asksTypes from "@textile/grpc-powergate-client/dist/index/ask/rpc/rpc_pb" @@ -22,7 +21,6 @@ export { healthTypes, minersTypes, netTypes, - dealsTypes, asksTypes, faultsTypes, reputationTypes, From 0a6e6435a7896319503b2db3104402eaada1d945 Mon Sep 17 00:00:00 2001 From: Aaron Sutula Date: Wed, 15 Jul 2020 14:41:41 -0500 Subject: [PATCH 12/12] fix: increase timeout Signed-off-by: Aaron Sutula --- src/index.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.spec.ts b/src/index.spec.ts index 5a6efa80..9178f592 100644 --- a/src/index.spec.ts +++ b/src/index.spec.ts @@ -18,7 +18,7 @@ beforeEach(async function () { }) afterEach(async function () { - this.timeout(10000) + this.timeout(120000) await new Promise((resolve, reject) => { cp.exec(`cd powergate-docker && make down`, (err, stdout) => { if (err) {