fix: NODE 16,logs fixed, proper pg escaping, proper env loading

snode/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20
+FROM node:16.20.2
 WORKDIR /app
 COPY . .
 RUN yarn install

snode/src/api/routes/storageRoutes.ts

@@ -26,7 +26,6 @@ function logRequest(req: Request) {
 }
 
 // todo ValidatorContractState
-// todo remove logic from router
 export function storageRoutes(app: Router) {
   app.use(bodyParser.json());
 
@@ -85,6 +84,8 @@ export function storageRoutes(app: Router) {
     }
   );
 
+
+
   // todo move to StorageNode
   // todo not tested with new sharing (we don't use it anymore)
   route.post(
@@ -200,3 +201,4 @@ export function storageRoutes(app: Router) {
     }
   );
 };
+// todo remove logic from router

snode/src/appInit.ts

@@ -1,3 +1,6 @@
+import {EnvLoader} from "./utilz/envLoader";
+EnvLoader.loadEnvOrFail();
+
 import 'reflect-metadata'; // We need this in order to use @Decorators
 import express from 'express';
 import chalk from 'chalk';
@@ -6,12 +9,12 @@ import StorageNode from "./services/messaging/storageNode";
 import {ValidatorContractState} from "./services/messaging-common/validatorContractState";
 import {MySqlUtil} from "./utilz/mySqlUtil";
 
+
 async function startServer(logLevel = null) {
   if (logLevel) {
     const changeLogLevel = (await require('./config/index')).changeLogLevel;
     changeLogLevel(logLevel);
   }
-
   // Continue Loading normally
   const config = (await require('./config/index')).default;
   logLevel = logLevel || config.logs.level;

snode/src/config/index.ts

@@ -1,19 +1,7 @@
-import dotenv from 'dotenv';
 // import {logLevel} from '../app'
 // Set the NODE_ENV to 'development' by default
 process.env.NODE_ENV = process.env.NODE_ENV || 'development';
 
-// loads all .env variables into process.env.* variables
-// Optional support for CONFIG_DIR variable
-console.log(`config dir is ${process.env.CONFIG_DIR}`);
-let options = {};
-if(process.env.CONFIG_DIR) {
-  options = {path: `${process.env.CONFIG_DIR}/.env`};
-}
-const envFound = dotenv.config(options);
-if (envFound.error) {
-  throw new Error("⚠️  Couldn't find .env file  ⚠️")
-}
 
 export const changeLogLevel = (level: string) => {
   if (level) {

snode/src/helpers/dbHelper.ts

@@ -23,8 +23,12 @@ var mysqlPool = mysql.createPool({
 })
 MySqlUtil.init(mysqlPool);
 
-
+// postgres
+// todo fix variable substitution, see #putValueInTable()
 // todo move everything into PgUtil including connection management
+// todo use PgUtil
+// todo use placeholders (?)
+
 let logger = WinstonUtil.newLog('pg');
 let options = {
     query: function (e) {
@@ -38,8 +42,6 @@ const pg: pgPromise.IMain<{}, IClient> = pgPromise(options);
 export const pgPool = pg(`postgres://${EnvLoader.getPropertyOrFail('PG_USER')}:${EnvLoader.getPropertyOrFail('PG_PASS')}@${EnvLoader.getPropertyOrFail('PG_HOST')}:5432/${EnvLoader.getPropertyOrFail('PG_NAME')}`);
 PgUtil.init(pgPool);
 
-// todo use PgUtil
-// todo use placeholders (?)
 export default class DbHelper {
 
     public static async createStorageTablesIfNeeded() {
@@ -261,12 +263,19 @@ END $$ LANGUAGE plpgsql;
                                  storageTable: string, ts: string, skey: string, body: string) {
         log.debug(`putValueInTable() namespace=${ns}, namespaceShardId=${shardId}
         ,storageTable=${storageTable}, skey=${skey}, jsonValue=${body}`);
-        const sql = `INSERT INTO ${storageTable} 
-                    (namespace, namespace_shard_id, namespace_id, ts, skey, dataschema, payload)
-                    values ('${ns}',  '${shardId}', '${nsIndex}', to_timestamp(${ts}),'${skey}', 'v1', '${body}')
-                    ON CONFLICT (namespace,namespace_shard_id,namespace_id,skey) DO UPDATE SET payload = '${body}'`
-        log.debug(sql);
-        return pgPool.none(sql).then(data => {
+        const sql = `INSERT INTO ${storageTable} (namespace, namespace_shard_id, namespace_id, ts, skey, dataschema, payload)
+                     values (\${ns}, \${shardId}, \${nsIndex}, to_timestamp(\${ts}), \${skey}, 'v1', \${body})
+                     ON CONFLICT (namespace, namespace_shard_id, namespace_id, skey) DO UPDATE SET payload = \${body}`;
+        const params = {
+            ns,
+            shardId,
+            nsIndex,
+            ts,
+            skey,
+            body
+        }
+        console.log(sql, params);
+        return pgPool.none(sql, params).then(data => {
             log.debug(data);
             return Promise.resolve();
         }).catch(err => {

snode/src/loaders/logger.ts

@@ -1,23 +1,7 @@
-import Transport from 'winston-transport';
 import winston from 'winston';
-
 import config from '../config';
-const moment = require('moment'); // time library
-
-class DynamicLoggerTransport extends Transport {
-  private dynamicLogging: object = null;
-  private formatLogInfo: Function = null;
-
-  constructor(opts, formatLogInfo) {
-    super(opts);
-    this.formatLogInfo = formatLogInfo;
-  }
-
-  public setDynamicLoggerObject(object) {
-    this.dynamicLogging = object;
-  }
+import {WinstonUtil} from "../utilz/winstonUtil";
 
-}
 
 const customLevels = {
   levels: {
@@ -41,73 +25,20 @@ const customLevels = {
   }
 };
 
-var options = {
-  file: {
-    level: 'verbose',
-    filename: `${__dirname}/../../logs/app.log`,
-    handleExceptions: true,
-    json: true,
-    maxsize: 5242880, // 5MB
-    maxFiles: 5,
-    colorize: true
-  }
-};
-
-const parser = (param: any): string => {
-  if (!param) {
-    return '';
-  }
-  if (typeof param === 'string') {
-    return param;
-  }
-
-  return Object.keys(param).length ? JSON.stringify(param, undefined, 2) : '';
-};
-
-const formatLogInfo = info => {
-  const { timestamp, level, message, meta } = info;
-
-  const ts = moment(timestamp)
-      .local()
-      .format('HH:MM:ss');
-  const metaMsg = meta ? `: ${parser(meta)}` : '';
-
-  return `${ts} ${level}    ${parser(message)} ${metaMsg}`;
-};
-
-const formatter = winston.format.combine(
-    winston.format.errors({ stack: true }),
-    winston.format.splat(),
-    winston.format.printf(info => {
-      return formatLogInfo(info);
-    }),
-    winston.format.colorize({
-      all: true
-    })
-);
-
 let transports = [];
-
 transports.push(
-    // Console should always be at 0 and dynamic log should always be at 2
-    // remember and not change it as it's manually baked in hijackLogger
-    new winston.transports.Console({
-      format: formatter
-    }),
-    new winston.transports.File(options.file)
+  // Console should always be at 0 and dynamic log should always be at 2
+  // remember and not change it as it's manually baked in hijackLogger
+  WinstonUtil.consoleTransport,
+  WinstonUtil.debugFileTransport,
+  WinstonUtil.errorFileTransport,
 );
-
+// WE SIMPLY REDIRECT ALL TO winstonUtil formatter x winstonUtil transports
+// this instance is being used across the whole codebase
 const LoggerInstance = winston.createLogger({
   level: config.logs.level,
   levels: customLevels.levels,
-  format: winston.format.combine(
-      winston.format.timestamp({
-        format: 'YYYY-MM-DD HH:mm:ss',
-      }),
-      winston.format.errors({ stack: true }),
-      winston.format.splat(),
-      winston.format.json()
-  ),
+  format: WinstonUtil.createFormat2WhichRendersClassName(),
   transports
 });
 

snode/src/services/messaging/BlockStorage.ts

@@ -78,7 +78,7 @@ export class BlockStorage {
       null, calculatedHash);
     if (hashFromDb != null) {
       this.log.info('received block with hash %s, ' +
-        'already exists in the storage at index %d, ignoring',
+        'already exists in the storage at index %s, ignoring',
         calculatedHash, hashFromDb);
       return false;
     }

snode/src/utilz/envLoader.ts

@@ -2,10 +2,17 @@ import dotenv from 'dotenv'
 import StrUtil from './strUtil'
 
 export class EnvLoader {
+
   public static loadEnvOrFail() {
-    const envFound = dotenv.config()
+    // loads all .env variables into process.env.* variables
+    // Optional support for CONFIG_DIR variable
+    console.log(`config dir is ${process.env.CONFIG_DIR}`);
+    let options = {};
+    if (process.env.CONFIG_DIR) {
+      options = {path: `${process.env.CONFIG_DIR}/.env`};
+    }
+    const envFound = dotenv.config(options);
     if (envFound.error) {
-      // This error should crash whole process
       throw new Error("⚠️  Couldn't find .env file  ⚠️")
     }
   }

snode/src/utilz/pgUtil.ts

@@ -6,7 +6,10 @@ import StrUtil from "./strUtil";
 import pg from "pg-promise/typescript/pg-subset";
 import {IDatabase} from "pg-promise";
 
+// PG PROMISE https://github.com/vitaly-t/pg-promise
 
+// NOTE: here are how named params are working in pg-async
+// https://github.com/vitaly-t/pg-promise/wiki/Learn-by-Example#named-parameters
 export class PgUtil {
   private static log: Logger = WinstonUtil.newLog('pg')
   static logSql = false