Merge pull request #2007 from LiskHQ/2005-optimize-ws-client-handling

Optimize WS client handling

docker-compose.yml

@@ -153,8 +153,9 @@ services:
       - ENABLE_TESTING_MODE=${ENABLE_TESTING_MODE}
       - ENABLE_BLOCK_CACHING=${ENABLE_BLOCK_CACHING}
       - EXPIRY_IN_HOURS=${EXPIRY_IN_HOURS}
-      - CLIENT_ALIVE_ASSUMPTION_TIME=${CLIENT_ALIVE_ASSUMPTION_TIME}
-      - HEARTBEAT_ACK_MAX_WAIT_TIME=${HEARTBEAT_ACK_MAX_WAIT_TIME}
+      - CLIENT_POOL_SIZE=${CLIENT_POOL_SIZE}
+      - WS_SERVER_PING_INTERVAL=${WS_SERVER_PING_INTERVAL}
+      - WS_SERVER_PING_INTERVAL_BUFFER=${WS_SERVER_PING_INTERVAL_BUFFER}
       - ENDPOINT_INVOKE_MAX_RETRIES=${ENDPOINT_INVOKE_MAX_RETRIES}
       - ENDPOINT_INVOKE_RETRY_DELAY=${ENDPOINT_INVOKE_RETRY_DELAY}
       - CONNECTOR_EXIT_DELAY_IN_HOURS=${CONNECTOR_EXIT_DELAY_IN_HOURS}

docker/example.env

@@ -36,10 +36,11 @@
 # ==================================== #
 
 # GENESIS_BLOCK_URL='https://downloads.lisk.com/lisk/mainnet/genesis_block.json.tar.gz'
-# CLIENT_ALIVE_ASSUMPTION_TIME=5000
-# HEARTBEAT_ACK_MAX_WAIT_TIME=1000
+# CLIENT_POOL_SIZE=10
+# WS_SERVER_PING_INTERVAL=3000
+# WS_SERVER_PING_INTERVAL_BUFFER=1000
 # ENDPOINT_INVOKE_MAX_RETRIES=3
-# ENDPOINT_INVOKE_RETRY_DELAY=10
+# ENDPOINT_INVOKE_RETRY_DELAY=50
 # CONNECTOR_EXIT_DELAY_IN_HOURS=0
 
 # Moleculer jobs configuration 

docs/antora/modules/ROOT/pages/configuration/index.adoc

@@ -282,13 +282,13 @@ By default, it is set to `0`.
 | `JOB_INTERVAL_UPDATE_READINESS_STATUS`
 | number
 | Job run interval to update the readiness status.
-By default, it is set to 0.
+By default, it is set to `0`.
 | 0
 
 | `JOB_SCHEDULE_UPDATE_READINESS_STATUS`
 | string
 | Job run cron schedule to update the readiness status.
-By default, it is set to run every minute.
+By default, it is set to run `every` minute.
 | * * * * *
 
 |===
@@ -413,15 +413,23 @@ To disable it, set it to `false`.
 By default, it is set to `12` hours.
 | 12
 
-| `CLIENT_ALIVE_ASSUMPTION_TIME`
+| `CLIENT_POOL_SIZE`
+| number
+| Number of active API clients to be maintained in the pool.
+Only applicable when using the IPC or WS clients to connect with the Lisk node.
+By default, it is set to `10`.
+| 10
+
+| `WS_SERVER_PING_INTERVAL`
 | number
-| Interval (in milliseconds) for which the WS API Client is assumed to be alive since the last ping/pong success check.
-By default, it is set to `5000`.
-| 5000
+| Interval (in milliseconds) at which the WS server checks for liveliness of all the connected clients.
+This should not be modified unless explicitly recommended by the development team.
+By default, it is set to `3000`.
+| 3000
 
-| `HEARTBEAT_ACK_MAX_WAIT_TIME`
+| `WS_SERVER_PING_INTERVAL_BUFFER`
 | number
-| Maximum time (in milliseconds) within which the checkIsClientAlive's algorithm expects a corresponding `pong` for the `ping` sent to the WS server.
+| A conservative assumption of the latency (in milliseconds) for WS server pings to arrive at the client.
 By default, it is set to `1000`.
 | 1000
 
@@ -434,8 +442,8 @@ By default, it is set to `3`.
 | `ENDPOINT_INVOKE_RETRY_DELAY`
 | number
 | Delay (in milliseconds) between each endpoint invocation request retry.
-By default, it is set to `10`.
-| 10
+By default, it is set to `50`.
+| 50
 
 | `CONNECTOR_EXIT_DELAY_IN_HOURS`
 | number
@@ -448,13 +456,13 @@ By default, it is set to `0`.
 | `JOB_INTERVAL_CACHE_CLEANUP`
 | number
 | Job run interval to clean up block cache.
-By default, it is set to 0.
+By default, it is set to `0`.
 | 0
 
 | `JOB_SCHEDULE_CACHE_CLEANUP`
 | string
 | Job run cron schedule to clean up block cache.
-By default, it is set to run every 12 hours.
+By default, it is set to run every `12` hours.
 |0 */12 * * *
 
 | `JOB_INTERVAL_REFRESH_PEERS`
@@ -543,7 +551,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_INDEX_MISSING_BLOCKS`
 | string
 | Job run cron schedule to index missing blocks.
-By default, it is set to run every 5 minutes.
+By default, it is set to run every `5` minutes.
 | */5 * * * *
 |===
 
@@ -649,7 +657,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS`
 | string
 | Job run cron schedule to delete serialized events.
-By default, it is set to run every 5 minutes.
+By default, it is set to run every `5` minutes.
 | */5 * * * *
 
 | `JOB_INTERVAL_REFRESH_VALIDATORS`
@@ -661,7 +669,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_REFRESH_VALIDATORS`
 | string
 | Job run cron schedule to refresh validators cache.
-By default, it is set to run every 5 minutes.
+By default, it is set to run every `5` minutes.
 | */5 * * * *
 
 | `JOB_INTERVAL_VALIDATE_VALIDATORS_RANK`
@@ -697,7 +705,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_REFRESH_BLOCKCHAIN_APPS_STATS`
 | string
 | Job run cron schedule to refresh blockchain application statistics.
-By default, it is set to run every 15 minutes.
+By default, it is set to run every `15` minutes.
 | */15 * * * *
 
 | `JOB_INTERVAL_REFRESH_ACCOUNT_KNOWLEDGE`
@@ -709,7 +717,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_REFRESH_ACCOUNT_KNOWLEDGE`
 | string
 | Job run cron schedule to refresh account knowledge.
-By default, it is set to run every 15 minutes.
+By default, it is set to run every `15` minutes.
 | */15 * * * *
 
 | `JOB_INTERVAL_DELETE_FINALIZED_CCU_METADATA`
@@ -721,7 +729,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_DELETE_FINALIZED_CCU_METADATA`
 | string
 | Job run cron schedule to delete finalized CCU metadata.
-By default, it is set to run once a day at 02:00 am.
+By default, it is set to run once a day at `02:00 am`.
 | 0 2 * * *
 
 | `JOB_INTERVAL_TRIGGER_ACCOUNT_UPDATES`
@@ -733,7 +741,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_TRIGGER_ACCOUNT_UPDATES`
 | string
 | Job run cron schedule to trigger account updates.
-By default, it is set to run every 15 minutes.
+By default, it is set to run every `15` minutes.
 | */15 * * * *
 
 | `ESTIMATES_BUFFER_BYTES_LENGTH`
@@ -789,7 +797,7 @@ To accelerate the indexing process, the blockchain-indexer microservice also sup
 
 |`DURABILITY_VERIFY_FREQUENCY`
 |number
-|Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 1.
+|Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to `1`.
 |1
 
 |`INDEX_SNAPSHOT_URL`
@@ -908,7 +916,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_DELETE_NON_METADATA_FILES`
 | string
 | Job run cron schedule to delete non-metadata files.
-By default, it is set to run every day at midnight.
+By default, it is set to run every day at `midnight`.
 | 0 0 * * *
 
 | `JOB_INTERVAL_UPDATE_METADATA`
@@ -1111,7 +1119,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_REFRESH_TRANSACTION_STATS`
 | string
 | Job run cron schedule to refresh transaction statistics.
-By default, it is set to run every 30 minutes.
+By default, it is set to run every `30` minutes.
 | */30 * * * *
 
 | `JOB_INTERVAL_VERIFY_TRANSACTION_STATS`
@@ -1123,7 +1131,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_VERIFY_TRANSACTION_STATS`
 | string
 |Job run cron schedule to verify if the transaction statistics have been built correctly.
-By default, it is set to run every 3rd hour after the first `15` minutes.
+By default, it is set to run every `3rd hour` after the first `15` minutes.
 | 15 */3 * * *
 
 |===
@@ -1212,7 +1220,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_REFRESH_PRICES_BINANCE`
 | string
 | Job run cron schedule to refresh prices from Binance.
-By default, it is set to run every minute.
+By default, it is set to run `every` minute.
 |* * * * *
 
 | `JOB_INTERVAL_REFRESH_PRICES_BITTREX`
@@ -1224,7 +1232,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_REFRESH_PRICES_BITTREX`
 | string
 | Job run cron schedule to refresh prices from Bittrex.
-By default, it is set to run every minute.
+By default, it is set to run `every` minute.
 |* * * * *
 
 | `JOB_INTERVAL_REFRESH_PRICES_EXCHANGERATESAPI`
@@ -1236,7 +1244,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_REFRESH_PRICES_EXCHANGERATESAPI`
 | string
 | Job run cron schedule to refresh prices from exchangeratesapi.
-By default, it is set to run every minute.
+By default, it is set to run `every` minute.
 |* * * * *
 
 | `JOB_INTERVAL_REFRESH_PRICES_KRAKEN`
@@ -1248,7 +1256,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_REFRESH_PRICES_KRAKEN`
 | string
 | Job run cron schedule to refresh prices from Kraken.
-By default, it is set to run every minute.
+By default, it is set to run `every` minute.
 |* * * * *
 
 | `JOB_INTERVAL_UPDATE_PRICES`
@@ -1399,7 +1407,7 @@ By default, it is set to `0`.
 | `JOB_SCHEDULE_CACHE_PURGE`
 | string
 | Job run cron schedule to clean up the cache.
-By default, it is set to run daily at 04:45 am.
+By default, it is set to run daily at `04:45 am`.
 | 45 4 * * *
 
 |===

ecosystem.config.js

@@ -129,10 +129,11 @@ module.exports = {
 				// SERVICE_LOG_FILE: false,
 				// DOCKER_HOST: 'local',
 				// GENESIS_BLOCK_URL: 'https://downloads.lisk.com/lisk/mainnet/genesis_block.json.tar.gz',
-				// CLIENT_ALIVE_ASSUMPTION_TIME: 5 * 1000,
-				// HEARTBEAT_ACK_MAX_WAIT_TIME: 1000,
+				// CLIENT_POOL_SIZE: 10,
+				// WS_SERVER_PING_INTERVAL=3000,
+				// WS_SERVER_PING_INTERVAL_BUFFER=1000,
 				// ENDPOINT_INVOKE_MAX_RETRIES: 3,
-				// ENDPOINT_INVOKE_RETRY_DELAY: 10,
+				// ENDPOINT_INVOKE_RETRY_DELAY: 50,
 				// CONNECTOR_EXIT_DELAY_IN_HOURS: 0,
 				// JOB_INTERVAL_CACHE_CLEANUP: 0,
 				// JOB_SCHEDULE_CACHE_CLEANUP: '0 */12 * * *',

jenkins/docker-compose.nightly.yml

@@ -153,7 +153,9 @@ services:
       - ENABLE_TESTING_MODE=${ENABLE_TESTING_MODE}
       - ENABLE_BLOCK_CACHING=${ENABLE_BLOCK_CACHING}
       - EXPIRY_IN_HOURS=${EXPIRY_IN_HOURS}
-      - CLIENT_ALIVE_ASSUMPTION_TIME=${CLIENT_ALIVE_ASSUMPTION_TIME}
+      - CLIENT_POOL_SIZE=${CLIENT_POOL_SIZE}
+      - WS_SERVER_PING_INTERVAL=${WS_SERVER_PING_INTERVAL}
+      - WS_SERVER_PING_INTERVAL_BUFFER=${WS_SERVER_PING_INTERVAL_BUFFER}
       - ENDPOINT_INVOKE_MAX_RETRIES=${ENDPOINT_INVOKE_MAX_RETRIES}
       - ENDPOINT_INVOKE_RETRY_DELAY=${ENDPOINT_INVOKE_RETRY_DELAY}
       - CONNECTOR_EXIT_DELAY_IN_HOURS=${CONNECTOR_EXIT_DELAY_IN_HOURS}

services/blockchain-app-registry/README.md

@@ -30,9 +30,9 @@ A list of the most commonly used environment variables is presented below:
 - `SERVICE_APP_REGISTRY_MYSQL`: Connection string of the MySQL instance that the microservice connects to.
 - `ENABLE_REBUILD_INDEX_AT_INIT`: Boolean flag to truncate the index and rebuild at application init.
 - `DEFAULT_APPS`: Default blockchain applications. By default, it is set to `lisk_mainchain`.
-- `JOB_INTERVAL_DELETE_NON_METADATA_FILES`: Job run interval to delete non-metadata files. By default, it is set to 0.
+- `JOB_INTERVAL_DELETE_NON_METADATA_FILES`: Job run interval to delete non-metadata files. By default, it is set to `0`.
 - `JOB_SCHEDULE_DELETE_NON_METADATA_FILES`: Job run cron schedule to delete non-metadata files. By default, it is set to run every day at midnight (`0 0 * * *`).
-- `JOB_INTERVAL_UPDATE_METADATA`: Job run interval to update off-chain metadata. By default, it is set to 0.
+- `JOB_INTERVAL_UPDATE_METADATA`: Job run interval to update off-chain metadata. By default, it is set to `0`.
 - `JOB_SCHEDULE_UPDATE_METADATA`: Job run cron schedule to update off-chain metadata. By default, it is set to run every 10 minutes (`*/10 * * * *`).
 
 > **Note**: `interval` takes priority over `schedule` and must be greater than 0 to be valid for all the moleculer job configurations.

services/blockchain-connector/README.md

@@ -35,16 +35,17 @@ A list of the most commonly used environment variables is presented below:
 - `GENESIS_BLOCK_URL`: URL of the Lisk SDK-based application' genesis block. Only to be used when the genesis block is large enough to be transmitted over API calls within the timeout.
 - `GEOIP_JSON`: URL of GeoIP server.
 - `ENABLE_BLOCK_CACHING`: Boolean flag to enable the block caching. Enabled by default. To disable, set it to `false`.
-- `EXPIRY_IN_HOURS`: Expiry time (in hours) for block cache. By default, it is set to 12.
-- `CLIENT_ALIVE_ASSUMPTION_TIME`: Interval (in milliseconds) for which the WS API Client is assumed to be alive since the last ping/pong success check. By default, it is set to 5000.
-- `HEARTBEAT_ACK_MAX_WAIT_TIME`: Maximum time (in milliseconds) within which the checkIsClientAlive's algorithm expects a corresponding `pong` for the `ping` sent to the WS server. By default, it is set to 1000.
-- `ENDPOINT_INVOKE_MAX_RETRIES`: Maximum number of endpoint invocation request retries to the node. By default, it is set to 3.
-- `ENDPOINT_INVOKE_RETRY_DELAY`: Delay (in milliseconds) between each endpoint invocation request retry. By default, it is set to 10.
+- `EXPIRY_IN_HOURS`: Expiry time (in hours) for block cache. By default, it is set to `12`.
+- `CLIENT_POOL_SIZE`: Number of active API clients to be maintained in the pool. Only applicable when using the IPC or WS clients to connect with the Lisk node. By default, it is set to `10`.
+- `WS_SERVER_PING_INTERVAL`: Interval (in milliseconds) at which the WS server checks for liveliness of all the connected clients. This should not be modified unless explicitly recommended by the development team. By default, it is set to `3000`.
+- `WS_SERVER_PING_INTERVAL_BUFFER`: A conservative assumption of the latency (in milliseconds) for WS server pings to arrive at the client. By default, it is set to `1000`.
+- `ENDPOINT_INVOKE_MAX_RETRIES`: Maximum number of endpoint invocation request retries to the node. By default, it is set to `3`.
+- `ENDPOINT_INVOKE_RETRY_DELAY`: Delay (in milliseconds) between each endpoint invocation request retry. By default, it is set to `50`.
 - `CONNECTOR_EXIT_DELAY_IN_HOURS`: Delay (in hours) after which the blockchain-connector microservice exits. The service should restart automatically if deployed using Docker or PM2. To be removed eventually. To enable it, set it higher than `0`. By default, it is set to `0`.
-- `JOB_INTERVAL_CACHE_CLEANUP`: Job run interval to cleanup block cache. By default, it is set to 0.
+- `JOB_INTERVAL_CACHE_CLEANUP`: Job run interval to cleanup block cache. By default, it is set to `0`.
 - `JOB_SCHEDULE_CACHE_CLEANUP`: Job run cron schedule to cleanup block cache. By default, it is set to run every 12 hours (`0 */12 * * *`).
-- `JOB_INTERVAL_REFRESH_PEERS`: Job run interval to refresh the peers list. By default, it is set to run every 60 seconds.
-- `JOB_SCHEDULE_REFRESH_PEERS`: Job run cron schedule to refresh the peers list. By default, it is set to ''.
+- `JOB_INTERVAL_REFRESH_PEERS`: Job run interval to refresh the peers list. By default, it is set to run every `60` seconds.
+- `JOB_SCHEDULE_REFRESH_PEERS`: Job run cron schedule to refresh the peers list. By default, it is set to `''`.
 
 > **Note**: `interval` takes priority over `schedule` and must be greater than 0 to be valid for all the moleculer job configurations.
 

services/blockchain-connector/config.js

@@ -116,12 +116,12 @@ config.job = {
 };
 
 config.apiClient = {
-	heartbeatAckMaxWaitTime: Number(process.env.HEARTBEAT_ACK_MAX_WAIT_TIME) || 1000, // in millisecs
-	aliveAssumptionTime: Number(process.env.CLIENT_ALIVE_ASSUMPTION_TIME) || 5 * 1000, // in millisecs
-	connectionLimit: 5,
+	poolSize: Number(process.env.CLIENT_POOL_SIZE) || 10,
+	wsServerPingInterval: Number(process.env.WS_SERVER_PING_INTERVAL) || 3 * 1000, // in millisecs
+	pingIntervalBuffer: Number(process.env.WS_SERVER_PING_INTERVAL_BUFFER) || 1000, // in millisecs
 	request: {
 		maxRetries: Number(process.env.ENDPOINT_INVOKE_MAX_RETRIES) || 3,
-		retryDelay: Number(process.env.ENDPOINT_INVOKE_RETRY_DELAY) || 10, // in millisecs
+		retryDelay: Number(process.env.ENDPOINT_INVOKE_RETRY_DELAY) || 50, // in millisecs
 	},
 };