Sign transactions early and dump them to spreadsheet (#79)

* Replace `jest` with `vitest` for testing

* Create test for getting route from squidrouter

* Use vitest dependencies

* Remove all references to jest

* Add usdce and allow to pass fromToken to getRoute

* Add refactoring

* Add spreadsheet packages

* Add env variables for google credentials

* Implement spreadsheet storage

* Upgrade vite version

* Increase vite test timeout

* Refactor tests

* Add small changes

* Improve handling for missing credentials in unit tests

* Refactor code

* Amend merge

* Update api-solang package

* Create GlobalSpreadsheet

* Add phase 'prepareTransactions'

* Split nabla extrinsic creation and submission

* Bump api-solang package

* Fix spreadsheet creation

* Split functions for transactions into creation and submission

* Add phase to prepare transactions

* Fix tests

* Fix some lint errors

* Move storage logic to backend

* Remove mutex

* Increase timebounds of Stellar transactions

* Move nabla code before swap again

* Remove unused vars and config

* Turn off mockSep24

package.json

@@ -26,7 +26,7 @@
     "@mui/icons-material": "^5.14.19",
     "@mui/material": "^5.14.20",
     "@pendulum-chain/api": "^0.3.1",
-    "@pendulum-chain/api-solang": "^0.4.0",
+    "@pendulum-chain/api-solang": "^0.6.0",
     "@polkadot/api": "^9.9.1",
     "@polkadot/api-base": "^9.9.1",
     "@polkadot/api-contract": "^9.9.1",

signer-service/package.json

@@ -18,8 +18,7 @@
     "lint": "eslint ./src/ --ignore-path .gitignore --ignore-pattern internals/scripts",
     "lint:fix": "yarn lint --fix",
     "lint:watch": "yarn lint --watch",
-    "test": "echo \"No tests defined yet\" && exit 0",
-    "test:cov": "echo \"No tests defined yet\" && exit 0",
+    "test": "vitest",
     "validate": "yarn lint && yarn test",
     "postpublish": "git push --tags",
     "docs": "apidoc -i src -o docs",
@@ -38,6 +37,8 @@
     "express": "^4.15.2",
     "express-rate-limit": "^6.7.0",
     "express-validation": "^1.0.2",
+    "google-auth-library": "^9.11.0",
+    "google-spreadsheet": "^4.1.2",
     "helmet": "^4.6.0",
     "http-status": "^1.0.1",
     "joi": "^10.4.1",
@@ -74,6 +75,7 @@
     "prettier": "^2.8.7",
     "sinon": "^7.5.0",
     "sinon-chai": "^3.0.0",
-    "supertest": "^6.1.3"
+    "supertest": "^6.1.3",
+    "vitest": "^2.0.5"
   }
 }

signer-service/src/api/controllers/storage.controller.js

@@ -0,0 +1,40 @@
+require('dotenv').config();
+
+const { spreadsheet } = require('../../config/vars');
+const { initGoogleSpreadsheet, getOrCreateSheet, appendData } = require('../services/spreadsheet.service');
+
+// These are the headers for the Google Spreadsheet
+exports.SHEET_HEADER_VALUES = [
+  'timestamp',
+  'polygonAddress',
+  'stellarEphemeralPublicKey',
+  'pendulumEphemeralPublicKey',
+  'nablaApprovalTx',
+  'nablaSwapTx',
+  'spacewalkRedeemTx',
+  'stellarOfframpTx',
+  'stellarCleanupTx',
+];
+
+exports.storeData = async (req, res, next) => {
+  try {
+    // We expect the data to be an object that matches our schema
+    const data = req.body;
+
+    // Try dumping transactions to spreadsheet
+    const sheet = await initGoogleSpreadsheet(spreadsheet.sheetId, spreadsheet.googleCredentials).then((doc) => {
+      return getOrCreateSheet(doc, this.SHEET_HEADER_VALUES);
+    });
+
+    if (sheet) {
+      console.log('Appending data to sheet');
+      await appendData(sheet, data);
+      return res.status(200).json({ message: 'Data stored successfully' });
+    }
+
+    return res.status(500).json({ error: 'Failed to store data. Sheet unavailable.', details: error.message });
+  } catch (error) {
+    console.error('Error in storeData:', error);
+    return res.status(500).json({ error: 'Failed to store data', details: error.message });
+  }
+};

signer-service/src/api/middlewares/validators.js

@@ -1,3 +1,5 @@
+const { SHEET_HEADER_VALUES } = require('../controllers/storage.controller');
+
 const validateCreationInput = (req, res, next) => {
   const { accountId, maxTime, assetCode } = req.body;
   if (!accountId || !maxTime) {
@@ -30,4 +32,17 @@ const validateChangeOpInput = (req, res, next) => {
   next();
 };
 
-module.exports = { validateChangeOpInput, validateCreationInput };
+const validateStorageInput = (req, res, next) => {
+  const data = req.body;
+  // Check if the data contains values for all the headers
+  if (!SHEET_HEADER_VALUES.every((header) => data[header])) {
+    const missingItems = SHEET_HEADER_VALUES.filter((header) => !data[header]);
+    let errorMessage = 'Data does not match schema. Missing items: ' + missingItems.join(', ');
+    console.error(errorMessage);
+    return res.status(400).json({ error: errorMessage });
+  }
+
+  next();
+};
+
+module.exports = { validateChangeOpInput, validateCreationInput, validateStorageInput };

signer-service/src/api/routes/v1/index.js

@@ -1,6 +1,6 @@
 const express = require('express');
-const httpStatus = require('http-status');
 const stellarRoutes = require('./stellar.route');
+const storageRoutes = require('./storage.route');
 
 const router = express.Router({ mergeParams: true });
 const { sendStatusWithPk } = require('../../controllers/stellar.controller');
@@ -21,4 +21,9 @@ router.get('/status', sendStatusWithPk);
  */
 router.use('/stellar', stellarRoutes);
 
+/**
+ * POST v1/storage
+ */
+router.use('/storage', storageRoutes);
+
 module.exports = router;

signer-service/src/api/routes/v1/storage.route.js

@@ -0,0 +1,9 @@
+const express = require('express');
+const controller = require('../../controllers/storage.controller');
+const { validateStorageInput } = require('../../middlewares/validators');
+
+const router = express.Router({ mergeParams: true });
+
+router.route('/create').post(validateStorageInput, controller.storeData);
+
+module.exports = router;

signer-service/src/api/services/spreadsheet.service.js

@@ -0,0 +1,59 @@
+const { GoogleSpreadsheet } = require('google-spreadsheet');
+const { JWT } = require('google-auth-library');
+
+const SCOPES = ['https://www.googleapis.com/auth/spreadsheets'];
+
+// googleCredentials: { email: string, key: string },
+exports.initGoogleSpreadsheet = async (sheetId, googleCredentials) => {
+  // Initialize auth - see https://theoephraim.github.io/node-google-spreadsheet/#/guides/authentication
+  if (!googleCredentials.email || !googleCredentials.key) {
+    throw new Error('Missing some google credentials');
+  }
+
+  const serviceAccountAuth = new JWT({
+    // env var values here are copied from service account credentials generated by google
+    // see "Authentication" section in docs for more info
+    email: googleCredentials.email,
+    key: googleCredentials.key,
+    scopes: SCOPES,
+  });
+
+  const doc = new GoogleSpreadsheet(sheetId, serviceAccountAuth);
+  try {
+    await doc.loadInfo();
+  } catch (error) {
+    console.error(`Error loading Google Spreadsheet ${sheetId}:`, error);
+    throw error;
+  }
+
+  return doc;
+};
+
+// doc: GoogleSpreadsheet, headerValues: string[]
+exports.getOrCreateSheet = async (doc, headerValues) => {
+  let sheet = doc.sheetsByIndex[0];
+  try {
+    await sheet.loadHeaderRow();
+    const sheetHeaders = sheet.headerValues;
+
+    // Compare the header values to the expected header values
+    if (
+      sheetHeaders.length !== headerValues.length &&
+      sheetHeaders.every((value, index) => value === headerValues[index])
+    ) {
+      // Create a new sheet if the headers don't match
+      console.log('Creating new sheet');
+      sheet = await doc.addSheet({ headerValues });
+    }
+  } catch (error) {
+    // Assume the error is due to the sheet not having any rows
+    await sheet.setHeaderRow(headerValues);
+  }
+
+  return sheet;
+};
+
+// sheet: GoogleSpreadsheetWorksheet, data: Record<string, string>
+exports.appendData = async (sheet, data) => {
+  await sheet.addRow(data);
+};

signer-service/src/config/vars.js

@@ -14,4 +14,11 @@ module.exports = {
   rateLimitWindowMinutes: process.env.RATE_LIMIT_WINDOW_MINUTES || 15,
   rateLimitNumberOfProxies: process.env.RATE_LIMIT_NUMBER_OF_PROXIES || 1,
   logs: process.env.NODE_ENV === 'production' ? 'combined' : 'dev',
+  spreadsheet: {
+    googleCredentials: {
+      email: process.env.GOOGLE_SERVICE_ACCOUNT_EMAIL,
+      key: process.env.GOOGLE_PRIVATE_KEY?.split(String.raw`\n`).join('\n'),
+    },
+    sheetId: process.env.GOOGLE_SPREADSHEET_ID,
+  },
 };