From 82a1583501db7d0d897f7438a4483a65b79a3d77 Mon Sep 17 00:00:00 2001 From: Tai Nguyen Date: Tue, 5 Dec 2023 13:54:57 +0700 Subject: [PATCH 1/2] fix concurrency issue --- package-lock.json | 155 +++++++++++++++++++++++++++++++++++++++++ package.json | 2 + src/database/redis.ts | 18 +++++ src/routes/metadata.ts | 104 ++++++++++++++++----------- 4 files changed, 239 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8bd5180..92b0019 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "elliptic": "^6.5.4", "express": "^4.18.2", "helmet": "^6.1.5", + "ioredis": "^5.3.2", "js-sha3": "^0.8.0", "json-stable-stringify": "^1.0.2", "knex": "^2.4.2", @@ -33,6 +34,7 @@ "multihashing-async": "^2.1.4", "mysql": "^2.18.1", "redis": "^4.6.6", + "redlock": "^5.0.0-beta.2", "socket.io": "^4.6.1" }, "devDependencies": { @@ -208,6 +210,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.7.tgz", @@ -2207,6 +2214,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -3885,6 +3900,29 @@ "node": ">= 0.10" } }, + "node_modules/ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ip-regex": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", @@ -4395,6 +4433,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -4862,6 +4910,11 @@ "node": ">= 0.6" } }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==" + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -5425,6 +5478,36 @@ "@redis/time-series": "1.0.4" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/redlock": { + "version": "5.0.0-beta.2", + "resolved": "https://registry.npmjs.org/redlock/-/redlock-5.0.0-beta.2.tgz", + "integrity": "sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==", + "dependencies": { + "node-abort-controller": "^3.0.1" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/regenerator-runtime": { "version": "0.13.11", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", @@ -5805,6 +5888,11 @@ "node": ">= 0.6" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -6658,6 +6746,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@jridgewell/resolve-uri": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.7.tgz", @@ -8143,6 +8236,11 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", "dev": true }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -9388,6 +9486,22 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==" }, + "ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "ip-regex": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", @@ -9736,6 +9850,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -10106,6 +10230,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, + "node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==" + }, "node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -10497,6 +10626,27 @@ "@redis/time-series": "1.0.4" } }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, + "redlock": { + "version": "5.0.0-beta.2", + "resolved": "https://registry.npmjs.org/redlock/-/redlock-5.0.0-beta.2.tgz", + "integrity": "sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==", + "requires": { + "node-abort-controller": "^3.0.1" + } + }, "regenerator-runtime": { "version": "0.13.11", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", @@ -10781,6 +10931,11 @@ "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", diff --git a/package.json b/package.json index f4611ff..a97d747 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "elliptic": "^6.5.4", "express": "^4.18.2", "helmet": "^6.1.5", + "ioredis": "^5.3.2", "js-sha3": "^0.8.0", "json-stable-stringify": "^1.0.2", "knex": "^2.4.2", @@ -46,6 +47,7 @@ "multihashing-async": "^2.1.4", "mysql": "^2.18.1", "redis": "^4.6.6", + "redlock": "^5.0.0-beta.2", "socket.io": "^4.6.1" }, "devDependencies": { diff --git a/src/database/redis.ts b/src/database/redis.ts index a7c8ab7..1e38db1 100644 --- a/src/database/redis.ts +++ b/src/database/redis.ts @@ -1,5 +1,7 @@ +import Client from "ioredis"; import log from "loglevel"; import { createClient } from "redis"; +import Redlock, { ResourceLockedError } from "redlock"; const { REDIS_PORT, REDIS_HOSTNAME } = process.env; const client = createClient({ socket: { host: REDIS_HOSTNAME, port: Number(REDIS_PORT) } }); @@ -12,4 +14,20 @@ client.on("ready", () => { log.info("Connected to redis"); }); +export const redlock = new Redlock([new Client({ host: REDIS_HOSTNAME, port: Number(REDIS_PORT) })], { + driftFactor: 0.01, // multiplied by lock ttl to determine drift time + retryCount: 10, + retryDelay: 200, // time in ms + retryJitter: 200, // time in ms + automaticExtensionThreshold: 500, // time in ms +}); + +redlock.on("error", (error) => { + // Ignore cases where a resource is explicitly marked as locked on a client. + if (error instanceof ResourceLockedError) { + return; + } + log.error(error); +}); + export default client; diff --git a/src/routes/metadata.ts b/src/routes/metadata.ts index 6cdf510..767aa1d 100644 --- a/src/routes/metadata.ts +++ b/src/routes/metadata.ts @@ -8,7 +8,7 @@ import multer from "multer"; import { getHashAndWriteAsync } from "../database/ipfs"; import { knexRead, knexWrite } from "../database/knex"; -import redis from "../database/redis"; +import redis, { redlock } from "../database/redis"; import { serializeStreamBody, validateDataTimeStamp, @@ -365,62 +365,86 @@ router.post( let pubNonce: string | { x: string; y: string }; let ipfs: string[]; - try { - nonce = await redis.get(key); - } catch (error) { - log.warn("redis get failed", error); - } - - if (!nonce) { - const newRetrievedNonce = await knexRead(tableName).where({ key }).orderBy("created_at", "desc").orderBy("id", "desc").first(); - nonce = newRetrievedNonce?.value || undefined; - } - - if (nonce === "" || (!nonce && data !== "getOrSetNonce")) return res.json({ typeOfUser: "v1" }); // This is a v1 user who didn't have a nonce before we rolled out v2, if he sets his nonce in the future, this value will be ignored + const getNonce = async (): Promise => { + let nonceVal: string; + try { + nonceVal = await redis.get(key); + } catch (error) { + log.warn("redis get failed", error); + } + if (!nonceVal) { + const newRetrievedNonce = await knexRead(tableName).where({ key }).orderBy("created_at", "desc").orderBy("id", "desc").first(); + nonceVal = newRetrievedNonce?.value || undefined; + } + return nonceVal; + }; - if (nonce) { + const getPubNonce = async (): Promise => { + let pubNonceVal: string; try { - pubNonce = await redis.get(keyForPubNonce); + pubNonceVal = await redis.get(keyForPubNonce); } catch (error) { log.warn("redis get failed", error); } - if (!pubNonce) { + if (!pubNonceVal) { const retrievedPubNonce = await knexRead(tableName) .where({ key: keyForPubNonce }) .orderBy("created_at", "desc") .orderBy("id", "desc") .first(); - pubNonce = retrievedPubNonce?.value; + pubNonceVal = retrievedPubNonce?.value; } - if (!pubNonce) throw new Error("pub nonce value is null"); - pubNonce = JSON.parse(pubNonce as string); + if (!pubNonceVal) throw new Error("pub nonce value is null"); + return JSON.parse(pubNonceVal as string); + }; + + nonce = await getNonce(); + + if (nonce === "" || (!nonce && data !== "getOrSetNonce")) return res.json({ typeOfUser: "v1" }); // This is a v1 user who didn't have a nonce before we rolled out v2, if he sets his nonce in the future, this value will be ignored + + if (nonce) { + pubNonce = await getPubNonce(); } // its a new v2 user, lets set his nonce if (!nonce) { - nonce = generatePrivate().toString("hex"); - - const unformattedPubNonce = elliptic.keyFromPrivate(nonce).getPublic(); - pubNonce = { - x: unformattedPubNonce.getX().toString("hex"), - y: unformattedPubNonce.getY().toString("hex"), - }; - - // We just created new nonce and pub nonce above, write to db - const pubNonceStr = JSON.stringify(pubNonce); - await insertDataInBatchForTable(tableName, [ - [ - { key, value: nonce }, - { key: keyForPubNonce, value: pubNonceStr }, - ], - ]); - [ipfs] = await Promise.all([ - getHashAndWriteAsync({ [tableName]: [{ key, value: pubNonceStr }] }), - redis.setEx(key, REDIS_TIMEOUT, nonce).catch((error) => log.warn("redis set failed", error)), - redis.setEx(keyForPubNonce, REDIS_TIMEOUT, pubNonceStr).catch((error) => log.warn("redis set failed", error)), - ]); + const lockKey = `metadata-lock-${key}`; + const lock = await redlock.acquire([lockKey], 5000); + try { + // check if someone else has set it + nonce = await getNonce(); + if (nonce) { + pubNonce = await getPubNonce(); + } else { + // create new nonce + nonce = generatePrivate().toString("hex"); + + const unformattedPubNonce = elliptic.keyFromPrivate(nonce).getPublic(); + pubNonce = { + x: unformattedPubNonce.getX().toString("hex"), + y: unformattedPubNonce.getY().toString("hex"), + }; + + // We just created new nonce and pub nonce above, write to db + const pubNonceStr = JSON.stringify(pubNonce); + await insertDataInBatchForTable(tableName, [ + [ + { key, value: nonce }, + { key: keyForPubNonce, value: pubNonceStr }, + ], + ]); + [ipfs] = await Promise.all([ + getHashAndWriteAsync({ [tableName]: [{ key, value: pubNonceStr }] }), + redis.setEx(key, REDIS_TIMEOUT, nonce).catch((error) => log.warn("redis set failed", error)), + redis.setEx(keyForPubNonce, REDIS_TIMEOUT, pubNonceStr).catch((error) => log.warn("redis set failed", error)), + ]); + } + } finally { + // Release the lock. + await lock.release(); + } } const returnResponse = { From 76501de064e3280787edd78410753bb7aa02bfeb Mon Sep 17 00:00:00 2001 From: Tai Nguyen Date: Wed, 6 Dec 2023 12:56:46 +0700 Subject: [PATCH 2/2] read nonce from master after getting lock --- src/routes/metadata.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/routes/metadata.ts b/src/routes/metadata.ts index 767aa1d..a979463 100644 --- a/src/routes/metadata.ts +++ b/src/routes/metadata.ts @@ -365,7 +365,7 @@ router.post( let pubNonce: string | { x: string; y: string }; let ipfs: string[]; - const getNonce = async (): Promise => { + const getNonce = async (strongConsistency = false): Promise => { let nonceVal: string; try { nonceVal = await redis.get(key); @@ -373,13 +373,14 @@ router.post( log.warn("redis get failed", error); } if (!nonceVal) { - const newRetrievedNonce = await knexRead(tableName).where({ key }).orderBy("created_at", "desc").orderBy("id", "desc").first(); + const knexClient = strongConsistency ? knexWrite : knexRead; + const newRetrievedNonce = await knexClient(tableName).where({ key }).orderBy("created_at", "desc").orderBy("id", "desc").first(); nonceVal = newRetrievedNonce?.value || undefined; } return nonceVal; }; - const getPubNonce = async (): Promise => { + const getPubNonce = async (strongConsistency = false): Promise => { let pubNonceVal: string; try { pubNonceVal = await redis.get(keyForPubNonce); @@ -388,7 +389,8 @@ router.post( } if (!pubNonceVal) { - const retrievedPubNonce = await knexRead(tableName) + const knexClient = strongConsistency ? knexWrite : knexRead; + const retrievedPubNonce = await knexClient(tableName) .where({ key: keyForPubNonce }) .orderBy("created_at", "desc") .orderBy("id", "desc") @@ -414,9 +416,9 @@ router.post( const lock = await redlock.acquire([lockKey], 5000); try { // check if someone else has set it - nonce = await getNonce(); + nonce = await getNonce(true); if (nonce) { - pubNonce = await getPubNonce(); + pubNonce = await getPubNonce(true); } else { // create new nonce nonce = generatePrivate().toString("hex");