Add oracle service for remote shutdown (#531)

CONFIGURATION.md

@@ -44,6 +44,11 @@ ORACLE_ALWAYS_RELAY_SIGNATURES | If set to `true`, the oracle will always relay
 ORACLE_RPC_REQUEST_TIMEOUT | Timeout in milliseconds for a single RPC request. Default value is `ORACLE_*_RPC_POLLING_INTERVAL * 2`. | integer
 ORACLE_HOME_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Home sender service. Defaults to 20 minutes. | integer  
 ORACLE_FOREIGN_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Foreign sender service. Defaults to 20 minutes. | integer  
+ORACLE_SHUTDOWN_SERVICE_URL | Optional external URL to some other service/monitor/configuration manager that controls the remote shutdown process. GET request should return `application/json` message with the following schema: `{ shutdown: true/false }`. | URL
+ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL | Optional interval in milliseconds used to request the side RPC node or external shutdown service. Default is 120000. | integer
+ORACLE_SIDE_RPC_URL | Optional HTTPS URL(s) for communication with the external shutdown service or side RPC nodes, used for shutdown manager activities. Several URLs can be specified, delimited by spaces. If the connection to one of these nodes is lost the next URL is used for connection. | URL(s)
+ORACLE_SHUTDOWN_CONTRACT_ADDRESS | Optional contract address in the side chain accessible through `ORACLE_SIDE_RPC_URL`, where the method passed in `ORACLE_SHUTDOWN_CONTRACT_METHOD` is implemented. | `address`
+ORACLE_SHUTDOWN_CONTRACT_METHOD | Method signature to be used in the side chain to identify the current shutdown status. Method should return boolean. Default value is `isShutdown()`. | `function signature`
 
 
 ## UI configuration

deployment-e2e/molecule/multiple/tests/test_multiple.py

@@ -24,6 +24,7 @@ def test_services(host, service):
     ("oracle_bridge_affirmation_1"),
     ("oracle_bridge_senderhome_1"),
     ("oracle_bridge_senderforeign_1"),
+    ("oracle_bridge_shutdown_1"),
     ("ui_ui_1"),
     ("monitor_monitor_1")
 ])

deployment-e2e/molecule/oracle/tests/test_oracle.py

@@ -14,6 +14,7 @@
     ("oracle_bridge_affirmation_1"),
     ("oracle_bridge_senderhome_1"),
     ("oracle_bridge_senderforeign_1"),
+    ("oracle_bridge_shutdown_1"),
 ])
 def test_docker_containers(host, name):
     container = host.docker(name)

e2e-commons/up.sh

@@ -42,6 +42,7 @@ startValidator () {
     fi
     docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:home
     docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:foreign
+    docker-compose $1 run $2 $3 -d oracle yarn manager:shutdown
 }
 
 startAMBValidator () {
@@ -52,6 +53,7 @@ startAMBValidator () {
     docker-compose $1 run $2 $3 -d oracle-amb yarn watcher:affirmation-request
     docker-compose $1 run $2 $3 -d oracle-amb yarn sender:home
     docker-compose $1 run $2 $3 -d oracle-amb yarn sender:foreign
+    docker-compose $1 run $2 $3 -d oracle-amb yarn manager:shutdown
 }
 
 while [ "$1" != "" ]; do
@@ -120,13 +122,7 @@ while [ "$1" != "" ]; do
   fi
 
   if [ "$1" == "alm-e2e" ]; then
-    docker-compose up -d redis rabbit
-
-    docker-compose run -d oracle-amb yarn watcher:signature-request
-    docker-compose run -d oracle-amb yarn watcher:collected-signatures
-    docker-compose run -d oracle-amb yarn watcher:affirmation-request
-    docker-compose run -d oracle-amb yarn sender:home
-    docker-compose run -d oracle-amb yarn sender:foreign
+    startAMBValidator "" "" "" "redis" "rabbit"
 
     oracle2name="-p validator2"
     oracle2Values="-e ORACLE_VALIDATOR_ADDRESS=0xdCC784657C78054aa61FbcFFd2605F32374816A4 -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=5a5c3645d0f04e9eb4f27f94ed4c244a225587405b8838e7456f7781ce3a9513"

oracle/config/base.config.js

@@ -69,7 +69,8 @@ const bridgeConfig = {
   foreignBridgeAbi: foreignAbi,
   eventFilter: {},
   validatorAddress: ORACLE_VALIDATOR_ADDRESS || privateKeyToAddress(ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY),
-  maxProcessingTime
+  maxProcessingTime,
+  shutdownKey: 'oracle-shutdown'
 }
 
 const homeConfig = {

oracle/config/shutdown-manager.config.js

@@ -0,0 +1,20 @@
+const baseConfig = require('./base.config')
+
+const {
+  ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL,
+  ORACLE_SHUTDOWN_SERVICE_URL,
+  ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
+  ORACLE_SHUTDOWN_CONTRACT_METHOD
+} = process.env
+
+module.exports = {
+  ...baseConfig.bridgeConfig,
+  id: 'shutdown-manager',
+  name: 'shutdown-manager',
+  pollingInterval: ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL || 120000,
+  checksBeforeResume: 3,
+  checksBeforeStop: 1,
+  shutdownServiceURL: ORACLE_SHUTDOWN_SERVICE_URL,
+  shutdownContractAddress: ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
+  shutdownMethod: (ORACLE_SHUTDOWN_CONTRACT_METHOD || 'isShutdown()').trim()
+}

oracle/docker-compose-erc-native.yml

@@ -77,6 +77,12 @@ services:
     networks:
       - net_db_bridge_request
       - net_rabbit_bridge_request
+  bridge_shutdown:
+    extends:
+      file: docker-compose.yml
+      service: bridge_shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -91,6 +97,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/docker-compose-transfer.yml

@@ -61,6 +61,12 @@ services:
     networks:
       - net_db_bridge_request
       - net_rabbit_bridge_request
+  bridge_shutdown:
+    extends:
+      file: docker-compose.yml
+      service: bridge_shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -75,6 +81,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/docker-compose.yml

@@ -21,21 +21,22 @@ services:
     command: [redis-server, --appendonly, 'yes']
     hostname: redis
     image: redis:4
-    networks:    
+    networks:
       - net_db_bridge_request
       - net_db_bridge_collected
       - net_db_bridge_affirmation
       - net_db_bridge_senderhome
       - net_db_bridge_senderforeign
+      - net_db_bridge_shutdown
     restart: unless-stopped
     volumes: ['~/bridge_data/redis:/data']
   bridge_request:
     cpus: 0.1
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn watcher:signature-request
@@ -47,8 +48,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS=${ORACLE_VALIDATOR_ADDRESS}
     restart: unless-stopped
     entrypoint: yarn watcher:collected-signatures
@@ -60,8 +61,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS=${ORACLE_VALIDATOR_ADDRESS}
     restart: unless-stopped
     entrypoint: yarn watcher:affirmation-request
@@ -73,8 +74,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn sender:home
@@ -86,14 +87,25 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn sender:foreign
     networks:
       - net_db_bridge_senderforeign
       - net_rabbit_bridge_senderforeign
+  bridge_shutdown:
+    cpus: 0.1
+    mem_limit: 500m
+    image: poanetwork/tokenbridge-oracle:latest
+    env_file: ./.env
+    environment:
+      - NODE_ENV=production
+    restart: unless-stopped
+    entrypoint: yarn manager:shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -106,6 +118,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/package.json

@@ -13,6 +13,7 @@
     "sender:home": "./scripts/start-worker.sh sender home-sender",
     "sender:foreign": "./scripts/start-worker.sh sender foreign-sender",
     "confirm:transfer": "./scripts/start-worker.sh confirmRelay transfer-watcher",
+    "manager:shutdown": "./scripts/start-worker.sh shutdownManager shutdown-manager",
     "dev": "concurrently -n 'watcher:signature-request,watcher:collected-signatures,watcher:affirmation-request,watcher:transfer, sender:home,sender:foreign' -c 'red,green,yellow,blue,magenta,cyan' 'yarn watcher:signature-request' 'yarn watcher:collected-signatures' 'yarn watcher:affirmation-request' 'yarn watcher:transfer' 'yarn sender:home' 'yarn sender:foreign'",
     "test": "NODE_ENV=test mocha",
     "test:watch": "NODE_ENV=test mocha --watch --reporter=min",

oracle/src/sender.js

@@ -4,6 +4,7 @@ const { connectSenderToQueue } = require('./services/amqpClient')
 const { redis } = require('./services/redisClient')
 const GasPrice = require('./services/gasPrice')
 const logger = require('./services/logger')
+const { getShutdownFlag } = require('./services/shutdownState')
 const { sendTx } = require('./tx/sendTx')
 const { getNonce, getChainId } = require('./tx/web3')
 const {
@@ -102,6 +103,14 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
       return
     }
 
+    const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
+    if (await getShutdownFlag(logger, config.shutdownKey, true)) {
+      if (!wasShutdown) {
+        logger.info('Oracle sender was suspended via the remote shutdown process')
+      }
+      return
+    }
+
     const txArray = JSON.parse(msg.content)
     logger.debug(`Msg received with ${txArray.length} Tx to send`)
     const gasPrice = GasPrice.getPrice().toString(10)

oracle/src/services/logger.js

@@ -3,6 +3,7 @@ const path = require('path')
 const {
   web3Home,
   web3Foreign,
+  web3Side,
   web3HomeFallback,
   web3ForeignFallback,
   web3HomeRedundant,
@@ -30,4 +31,8 @@ web3ForeignFallback.currentProvider.setLogger(logger)
 web3HomeRedundant.currentProvider.setLogger(logger)
 web3ForeignRedundant.currentProvider.setLogger(logger)
 
+if (web3Side) {
+  web3Side.currentProvider.setLogger(logger)
+}
+
 module.exports = logger

oracle/src/services/shutdownState.js

@@ -0,0 +1,23 @@
+const { redis } = require('./redisClient')
+
+let isShutdown = false
+async function getShutdownFlag(logger, shutdownKey, force = false) {
+  if (force) {
+    logger.debug('Reading current shutdown state from the DB')
+    isShutdown = (await redis.get(shutdownKey)) === 'true'
+    logger.debug({ isShutdown }, 'Read shutdown state from the DB')
+  }
+  return isShutdown
+}
+
+async function setShutdownFlag(logger, shutdownKey, value) {
+  logger.info({ isShutdown: value }, 'Updating current shutdown state in the DB')
+  isShutdown = value
+  await redis.set(shutdownKey, value)
+  logger.debug('Updated state in the DB')
+}
+
+module.exports = {
+  getShutdownFlag,
+  setShutdownFlag
+}

oracle/src/services/web3.js

@@ -6,6 +6,7 @@ const { RETRY_CONFIG } = require('../utils/constants')
 const {
   COMMON_HOME_RPC_URL,
   COMMON_FOREIGN_RPC_URL,
+  ORACLE_SIDE_RPC_URL,
   ORACLE_RPC_REQUEST_TIMEOUT,
   ORACLE_HOME_RPC_POLLING_INTERVAL,
   ORACLE_FOREIGN_RPC_POLLING_INTERVAL
@@ -41,6 +42,18 @@ const web3Home = new Web3(homeProvider)
 const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions)
 const web3Foreign = new Web3(foreignProvider)
 
+let web3Side = null
+if (ORACLE_SIDE_RPC_URL) {
+  const sideUrls = ORACLE_SIDE_RPC_URL.split(' ').filter(url => url.length > 0)
+  const sideOptions = {
+    requestTimeout: configuredTimeout || 2000,
+    retry: RETRY_CONFIG
+  }
+
+  const sideProvider = new HttpListProvider(sideUrls, sideOptions)
+  web3Side = new Web3(sideProvider)
+}
+
 // secondary fallback providers are intended to be used in places where
 // it is more likely that RPC calls to the local non-archive nodes can fail
 // e.g. for checking status of the old transaction via eth_getTransactionByHash
@@ -70,6 +83,7 @@ if (foreignUrls.length > 1) {
 module.exports = {
   web3Home,
   web3Foreign,
+  web3Side,
   web3HomeRedundant,
   web3ForeignRedundant,
   web3HomeFallback,