From ec4016d021fffebf356620562a09e45357660b4b Mon Sep 17 00:00:00 2001 From: Igx22 Date: Thu, 11 Jul 2024 18:02:18 +0700 Subject: [PATCH] fix: NODE 16,logs fixed, proper pg escaping, proper env loading --- dstorage-common/src/util/envLoader.ts | 21 ----- snode/Dockerfile | 2 +- snode/src/api/routes/storageRoutes.ts | 4 +- snode/src/appInit.ts | 5 +- snode/src/config/index.ts | 12 --- snode/src/helpers/dbHelper.ts | 27 ++++-- snode/src/loaders/logger.ts | 87 ++----------------- snode/src/services/messaging/BlockStorage.ts | 2 +- snode/src/utilz/envLoader.ts | 11 ++- snode/src/utilz/pgUtil.ts | 3 + snode/src/utilz/winstonUtil.ts | 89 +++++++++----------- 11 files changed, 90 insertions(+), 173 deletions(-) delete mode 100644 dstorage-common/src/util/envLoader.ts diff --git a/dstorage-common/src/util/envLoader.ts b/dstorage-common/src/util/envLoader.ts deleted file mode 100644 index 0857314..0000000 --- a/dstorage-common/src/util/envLoader.ts +++ /dev/null @@ -1,21 +0,0 @@ -import dotenv from 'dotenv'; -import StrUtil from "./strUtil"; - -export default class EnvLoader { - - public static loadEnvOrFail() { - const envFound = dotenv.config(); - if (envFound.error) { - // This error should crash whole process - throw new Error("⚠️ Couldn't find .env file ⚠️"); - } - } - - public static getPropertyOrFail(propName:string):string { - let val = process.env[propName]; - if(StrUtil.isEmpty(val)) { - throw new Error(`process.env.${propName} is empty`); - } - return val; - } -} \ No newline at end of file diff --git a/snode/Dockerfile b/snode/Dockerfile index 7c4b46f..4fc57d9 100644 --- a/snode/Dockerfile +++ b/snode/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 +FROM node:16.20.2 WORKDIR /app COPY . . RUN yarn install diff --git a/snode/src/api/routes/storageRoutes.ts b/snode/src/api/routes/storageRoutes.ts index 9d9a8b5..5d31dae 100755 --- a/snode/src/api/routes/storageRoutes.ts +++ b/snode/src/api/routes/storageRoutes.ts @@ -26,7 +26,6 @@ function logRequest(req: Request) { } // todo ValidatorContractState -// todo remove logic from router export function storageRoutes(app: Router) { app.use(bodyParser.json()); @@ -85,6 +84,8 @@ export function storageRoutes(app: Router) { } ); + + // todo move to StorageNode // todo not tested with new sharing (we don't use it anymore) route.post( @@ -200,3 +201,4 @@ export function storageRoutes(app: Router) { } ); }; +// todo remove logic from router diff --git a/snode/src/appInit.ts b/snode/src/appInit.ts index 55131bb..09a5a34 100755 --- a/snode/src/appInit.ts +++ b/snode/src/appInit.ts @@ -1,3 +1,6 @@ +import {EnvLoader} from "./utilz/envLoader"; +EnvLoader.loadEnvOrFail(); + import 'reflect-metadata'; // We need this in order to use @Decorators import express from 'express'; import chalk from 'chalk'; @@ -6,12 +9,12 @@ import StorageNode from "./services/messaging/storageNode"; import {ValidatorContractState} from "./services/messaging-common/validatorContractState"; import {MySqlUtil} from "./utilz/mySqlUtil"; + async function startServer(logLevel = null) { if (logLevel) { const changeLogLevel = (await require('./config/index')).changeLogLevel; changeLogLevel(logLevel); } - // Continue Loading normally const config = (await require('./config/index')).default; logLevel = logLevel || config.logs.level; diff --git a/snode/src/config/index.ts b/snode/src/config/index.ts index e298276..8e249af 100755 --- a/snode/src/config/index.ts +++ b/snode/src/config/index.ts @@ -1,19 +1,7 @@ -import dotenv from 'dotenv'; // import {logLevel} from '../app' // Set the NODE_ENV to 'development' by default process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -// loads all .env variables into process.env.* variables -// Optional support for CONFIG_DIR variable -console.log(`config dir is ${process.env.CONFIG_DIR}`); -let options = {}; -if(process.env.CONFIG_DIR) { - options = {path: `${process.env.CONFIG_DIR}/.env`}; -} -const envFound = dotenv.config(options); -if (envFound.error) { - throw new Error("⚠️ Couldn't find .env file ⚠️") -} export const changeLogLevel = (level: string) => { if (level) { diff --git a/snode/src/helpers/dbHelper.ts b/snode/src/helpers/dbHelper.ts index 162fa5e..4621112 100755 --- a/snode/src/helpers/dbHelper.ts +++ b/snode/src/helpers/dbHelper.ts @@ -23,8 +23,12 @@ var mysqlPool = mysql.createPool({ }) MySqlUtil.init(mysqlPool); - +// postgres +// todo fix variable substitution, see #putValueInTable() // todo move everything into PgUtil including connection management +// todo use PgUtil +// todo use placeholders (?) + let logger = WinstonUtil.newLog('pg'); let options = { query: function (e) { @@ -38,8 +42,6 @@ const pg: pgPromise.IMain<{}, IClient> = pgPromise(options); export const pgPool = pg(`postgres://${EnvLoader.getPropertyOrFail('PG_USER')}:${EnvLoader.getPropertyOrFail('PG_PASS')}@${EnvLoader.getPropertyOrFail('PG_HOST')}:5432/${EnvLoader.getPropertyOrFail('PG_NAME')}`); PgUtil.init(pgPool); -// todo use PgUtil -// todo use placeholders (?) export default class DbHelper { public static async createStorageTablesIfNeeded() { @@ -261,12 +263,19 @@ END $$ LANGUAGE plpgsql; storageTable: string, ts: string, skey: string, body: string) { log.debug(`putValueInTable() namespace=${ns}, namespaceShardId=${shardId} ,storageTable=${storageTable}, skey=${skey}, jsonValue=${body}`); - const sql = `INSERT INTO ${storageTable} - (namespace, namespace_shard_id, namespace_id, ts, skey, dataschema, payload) - values ('${ns}', '${shardId}', '${nsIndex}', to_timestamp(${ts}),'${skey}', 'v1', '${body}') - ON CONFLICT (namespace,namespace_shard_id,namespace_id,skey) DO UPDATE SET payload = '${body}'` - log.debug(sql); - return pgPool.none(sql).then(data => { + const sql = `INSERT INTO ${storageTable} (namespace, namespace_shard_id, namespace_id, ts, skey, dataschema, payload) + values (\${ns}, \${shardId}, \${nsIndex}, to_timestamp(\${ts}), \${skey}, 'v1', \${body}) + ON CONFLICT (namespace, namespace_shard_id, namespace_id, skey) DO UPDATE SET payload = \${body}`; + const params = { + ns, + shardId, + nsIndex, + ts, + skey, + body + } + console.log(sql, params); + return pgPool.none(sql, params).then(data => { log.debug(data); return Promise.resolve(); }).catch(err => { diff --git a/snode/src/loaders/logger.ts b/snode/src/loaders/logger.ts index 4bf4fda..752b321 100755 --- a/snode/src/loaders/logger.ts +++ b/snode/src/loaders/logger.ts @@ -1,23 +1,7 @@ -import Transport from 'winston-transport'; import winston from 'winston'; - import config from '../config'; -const moment = require('moment'); // time library - -class DynamicLoggerTransport extends Transport { - private dynamicLogging: object = null; - private formatLogInfo: Function = null; - - constructor(opts, formatLogInfo) { - super(opts); - this.formatLogInfo = formatLogInfo; - } - - public setDynamicLoggerObject(object) { - this.dynamicLogging = object; - } +import {WinstonUtil} from "../utilz/winstonUtil"; -} const customLevels = { levels: { @@ -41,73 +25,20 @@ const customLevels = { } }; -var options = { - file: { - level: 'verbose', - filename: `${__dirname}/../../logs/app.log`, - handleExceptions: true, - json: true, - maxsize: 5242880, // 5MB - maxFiles: 5, - colorize: true - } -}; - -const parser = (param: any): string => { - if (!param) { - return ''; - } - if (typeof param === 'string') { - return param; - } - - return Object.keys(param).length ? JSON.stringify(param, undefined, 2) : ''; -}; - -const formatLogInfo = info => { - const { timestamp, level, message, meta } = info; - - const ts = moment(timestamp) - .local() - .format('HH:MM:ss'); - const metaMsg = meta ? `: ${parser(meta)}` : ''; - - return `${ts} ${level} ${parser(message)} ${metaMsg}`; -}; - -const formatter = winston.format.combine( - winston.format.errors({ stack: true }), - winston.format.splat(), - winston.format.printf(info => { - return formatLogInfo(info); - }), - winston.format.colorize({ - all: true - }) -); - let transports = []; - transports.push( - // Console should always be at 0 and dynamic log should always be at 2 - // remember and not change it as it's manually baked in hijackLogger - new winston.transports.Console({ - format: formatter - }), - new winston.transports.File(options.file) + // Console should always be at 0 and dynamic log should always be at 2 + // remember and not change it as it's manually baked in hijackLogger + WinstonUtil.consoleTransport, + WinstonUtil.debugFileTransport, + WinstonUtil.errorFileTransport, ); - +// WE SIMPLY REDIRECT ALL TO winstonUtil formatter x winstonUtil transports +// this instance is being used across the whole codebase const LoggerInstance = winston.createLogger({ level: config.logs.level, levels: customLevels.levels, - format: winston.format.combine( - winston.format.timestamp({ - format: 'YYYY-MM-DD HH:mm:ss', - }), - winston.format.errors({ stack: true }), - winston.format.splat(), - winston.format.json() - ), + format: WinstonUtil.createFormat2WhichRendersClassName(), transports }); diff --git a/snode/src/services/messaging/BlockStorage.ts b/snode/src/services/messaging/BlockStorage.ts index 5df72bf..4a36685 100644 --- a/snode/src/services/messaging/BlockStorage.ts +++ b/snode/src/services/messaging/BlockStorage.ts @@ -78,7 +78,7 @@ export class BlockStorage { null, calculatedHash); if (hashFromDb != null) { this.log.info('received block with hash %s, ' + - 'already exists in the storage at index %d, ignoring', + 'already exists in the storage at index %s, ignoring', calculatedHash, hashFromDb); return false; } diff --git a/snode/src/utilz/envLoader.ts b/snode/src/utilz/envLoader.ts index eafdd9b..939fa82 100644 --- a/snode/src/utilz/envLoader.ts +++ b/snode/src/utilz/envLoader.ts @@ -2,10 +2,17 @@ import dotenv from 'dotenv' import StrUtil from './strUtil' export class EnvLoader { + public static loadEnvOrFail() { - const envFound = dotenv.config() + // loads all .env variables into process.env.* variables + // Optional support for CONFIG_DIR variable + console.log(`config dir is ${process.env.CONFIG_DIR}`); + let options = {}; + if (process.env.CONFIG_DIR) { + options = {path: `${process.env.CONFIG_DIR}/.env`}; + } + const envFound = dotenv.config(options); if (envFound.error) { - // This error should crash whole process throw new Error("⚠️ Couldn't find .env file ⚠️") } } diff --git a/snode/src/utilz/pgUtil.ts b/snode/src/utilz/pgUtil.ts index 03ea7f2..eeab171 100644 --- a/snode/src/utilz/pgUtil.ts +++ b/snode/src/utilz/pgUtil.ts @@ -6,7 +6,10 @@ import StrUtil from "./strUtil"; import pg from "pg-promise/typescript/pg-subset"; import {IDatabase} from "pg-promise"; +// PG PROMISE https://github.com/vitaly-t/pg-promise +// NOTE: here are how named params are working in pg-async +// https://github.com/vitaly-t/pg-promise/wiki/Learn-by-Example#named-parameters export class PgUtil { private static log: Logger = WinstonUtil.newLog('pg') static logSql = false diff --git a/snode/src/utilz/winstonUtil.ts b/snode/src/utilz/winstonUtil.ts index 5fcbeba..3850205 100644 --- a/snode/src/utilz/winstonUtil.ts +++ b/snode/src/utilz/winstonUtil.ts @@ -1,7 +1,6 @@ import {Format, TransformableInfo} from 'logform' import {DateTime} from 'ts-luxon' import winston from 'winston' -import {consoleTransport, jsonLogTransport} from '../loaders/logger' import StrUtil from './strUtil' import {EnvLoader} from './envLoader' @@ -52,10 +51,43 @@ format2 with class name: I 230811 174624 [MyClass] Got alias List (SendMessage) */ + + export class WinstonUtil { private static readonly CLASS_NAME_LENGTH = 23 + private static readonly LOG_DIR = EnvLoader.getPropertyOrFail('LOG_DIR'); + private static readonly LOG_LEVEL = EnvLoader.getPropertyOrFail('LOG_LEVEL'); + private static loggerMap: Map = new Map(); + + // all console writes drop here + public static consoleTransport = new winston.transports.Console({ + format: WinstonUtil.createFormat2WhichRendersClassName() + }); + + // add debug writes drop here + + public static debugFileTransport = new winston.transports.File({ + level: 'debug', + filename: `${WinstonUtil.LOG_DIR}/debug.log`, + handleExceptions: true, + maxsize: 5242880, + maxFiles: 5, + colorize: false, + tailable: true, + format: WinstonUtil.createFormat2WhichRendersClassName() + }) - static loggerMap: Map = new Map() + // all exceptions drop here + public static errorFileTransport = new winston.transports.File({ + level: 'error', + filename: `${WinstonUtil.LOG_DIR}/error.log`, + handleExceptions: true, + maxsize: 5242880, + maxFiles: 5, + colorize: false, + tailable: true, + format: WinstonUtil.createFormat2WhichRendersClassName() + }) /* { "message": "Checking Node Version", "level": "info", "timestamp": "230809 180338", className?: "myClass"} @@ -103,30 +135,6 @@ export class WinstonUtil { ) } - public static debugFileTransport = new winston.transports.File({ - level: 'debug', - filename: `${EnvLoader.getPropertyOrFail('LOG_DIR')}/debug.log`, - handleExceptions: true, - json: false, - maxsize: 5242880, - maxFiles: 5, - colorize: false, - tailable: true, - format: WinstonUtil.createFormat2WhichRendersClassName() - }) - - public static errorFileTransport = new winston.transports.File({ - level: 'error', - filename: `${EnvLoader.getPropertyOrFail('LOG_DIR')}/error.log`, - handleExceptions: true, - json: false, - maxsize: 5242880, - maxFiles: 5, - colorize: false, - tailable: true, - format: WinstonUtil.createFormat2WhichRendersClassName() - }) - public static newLog(classNameOrClass: string | { name: string }): winston.Logger { let loggerName = null if (typeof classNameOrClass === 'string') { @@ -140,28 +148,15 @@ export class WinstonUtil { if (loggerObj != null) { return loggerObj } - const logLevel = EnvLoader.getPropertyOrFail('LOG_LEVEL'); - const transports = []; - if (consoleTransport != null) { - transports.push(consoleTransport); - } else { - const console = new winston.transports.Console({ - format: WinstonUtil.createFormat2WhichRendersClassName() - }); - transports.push(console); - } - if (jsonLogTransport != null) { - transports.push(jsonLogTransport); - } - /*transports.push([ - this.debugFileTransport, // formats a class name from the log object - this.errorFileTransport // formats a class name from the log object - ]);*/ loggerObj = winston.createLogger({ - level: logLevel, - format: this.createFormat1WhichSetsClassName(loggerName), // puts a class name into the log object + formats in loaders/logger format - transports: transports - }) + level: WinstonUtil.LOG_LEVEL, + format: this.createFormat1WhichSetsClassName(loggerName), //winston.format.json(), + transports: [ + WinstonUtil.consoleTransport, + WinstonUtil.debugFileTransport, + WinstonUtil.errorFileTransport + ] + }); WinstonUtil.loggerMap.set(loggerName, loggerObj) return loggerObj }