Merge pull request #288 from w3f/git-config

Git config

README.md

@@ -50,6 +50,12 @@ If an expected destination address is specified in the config file, it is implic
 
 A sample config file is provided [here](config/main.sample.yaml)
 
+### Validators from Git
+
+The list of your addresses can be dynamically retrieved (app startup/restart) from a Git file. Check the [GitConfigLoader](src/gitConfigLoader) implementation.  
+
+- [GitLab API](https://docs.gitlab.com/ee/api/repository_files.html)
+
 ### Prometheus
 
 A Prometheus instance can be attached to this application thanks to the endpoint exposed at [/metrics](https://github.com/w3f/polkadot-watcher-validator/blob/master/src/prometheus.ts#L114).  

charts/polkadot-watcher/Chart.yaml

@@ -1,5 +1,5 @@
 description: Polkadot Watcher
 name: polkadot-watcher
-version: v4.2.0
-appVersion: v4.2.0
+version: v4.3.0
+appVersion: v4.3.0
 apiVersion: v2

charts/polkadot-watcher/templates/_helpers.tpl

@@ -0,0 +1,4 @@
+{{/* Returns the app name */}}
+{{- define "app.name" -}}
+{{- default .Release.Name .Values.nameOverride -}}
+{{- end }}

charts/polkadot-watcher/templates/alertrules.yaml

@@ -11,10 +11,10 @@ kind: PrometheusRule
 metadata:
   labels:
 {{ toYaml .Values.prometheusRules.labels | indent 4 }}
-  name: {{ .Release.Name }}
+  name: {{ include "app.name" . }}
 spec:
   groups:
-  - name: {{ .Release.Name }}.rules
+  - name: {{ include "app.name" . }}-{{ .Values.config.environment }}.rules
     rules:
     - alert: ValidatorOutOfActiveSet
       annotations:

charts/polkadot-watcher/templates/configmap.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ .Release.Name }}
+  name: {{ include "app.name" . }}
 data:
   main.yaml: |-
 {{ toYaml .Values.config | indent 4 }}

charts/polkadot-watcher/templates/deployment.yaml

@@ -1,26 +1,26 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ .Release.Name }}
+  name: {{ include "app.name" . }}
   labels:
-    app: {{ .Release.Name }}
+    app: {{ include "app.name" . }}
 spec:
   replicas: 1
   revisionHistoryLimit: 3
   strategy:
     type: RollingUpdate
   selector:
     matchLabels:
-      app: {{ .Release.Name }}
+      app: {{ include "app.name" . }}
   template:
     metadata:
       labels:
-        app: {{ .Release.Name }}
+        app: {{ include "app.name" . }}
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
     spec:
       containers:
-      - name: {{ .Release.Name }}
+      - name: {{ include "app.name" . }}
         image: {{ .Values.image.repo }}:{{ .Values.image.tag | default .Chart.AppVersion }}
         imagePullPolicy: {{ .Values.image.pullPolicy | default "Always" }}
         args:
@@ -45,4 +45,4 @@ spec:
       volumes:
       - name: config
         configMap:
-          name: {{ .Release.Name }}
+          name: {{ include "app.name" . }}

charts/polkadot-watcher/templates/service.yaml

@@ -1,12 +1,12 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ .Release.Name }}
+  name: {{ include "app.name" . }}
   labels:
-    app: {{ .Release.Name }}
+    app: {{ include "app.name" . }}
 spec:
   ports:
   - name: metrics
     port: {{ .Values.config.port }}
   selector:
-    app: {{ .Release.Name }}
+    app: {{ include "app.name" . }}

charts/polkadot-watcher/templates/servicemonitor.yaml

@@ -2,13 +2,13 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  name: {{ .Release.Name }}
+  name: {{ include "app.name" . }}
   labels:
 {{ toYaml .Values.serviceMonitor.labels | indent 4 }}
 spec:
   selector:
     matchLabels:
-      app: {{ .Release.Name }}
+      app: {{ include "app.name" . }}
   endpoints:
   - port: metrics
 {{ end }}

charts/polkadot-watcher/values.yaml

@@ -7,7 +7,15 @@ config:
   port: 3000
   logLevel: info
   environment: production #it is used also to filter the prometheusrules, i.e. to avoid conflicts between production and staging
-  validators: []
+  validators: [] #optional
+  validatorFromGit: #optional
+    enabled: false
+    platform: gitLab
+    private: 
+      enabled: true
+      apiToken: xxx
+    network: kusama
+    url: http://your.gitlab.domain/api/v4/projects/number/repository/files/accounts.yaml/raw?ref=main"
 
 serviceMonitor:
   enabled: true #to be enabled for each instance

config/main.sample.yaml

@@ -2,7 +2,15 @@ endpoint: "wss://kusama-rpc.polkadot.io/"
 port: 3000
 logLevel: info
 environment: production
-validators:
+validatorFromGit: #optional
+  enabled: false
+  platform: gitLab
+  private: 
+    enabled: true
+    apiToken: xxx
+  network: kusama
+  url: http://your.gitlab.domain/api/v4/projects/number/repository/files/accounts.yaml/raw?ref=main"
+validators: #optional
 - name: test
   address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5
   expected: #optional

package.json

@@ -1,6 +1,6 @@
 {
   "name": "polkadot-watcher",
-  "version": "4.2.0",
+  "version": "4.3.0",
   "description": "Monitor events on Polkadot networks",
   "repository": "git@github.com:w3f/polkadot-watcher.git",
   "author": "W3F Infrastructure Team <devops@web3.foundation>",
@@ -26,6 +26,7 @@
     "commander": "^4.0.0",
     "express": "^4.18.1",
     "got": "^11.8.5",
+    "node-fetch": "^2.6.6",
     "prom-client": "^14.0.1",
     "ws": "^6.1.2"
   },

src/actions/start.ts

@@ -7,6 +7,7 @@ import { Prometheus } from '../prometheus';
 import { InputConfig } from '../types';
 import { Client } from '../client';
 import { environment } from '../constants';
+import { GitConfigLoaderFactory } from '../gitConfigLoader/gitConfigLoaderFactory';
 
 const _addTestEndpoint = (server: express.Application, subscriber: Subscriber): void =>{
 
@@ -17,9 +18,24 @@ const _addTestEndpoint = (server: express.Application, subscriber: Subscriber):
       })
 }
 
-export async function startAction(cmd): Promise<void> {
-    const cfg = new Config<InputConfig>().parse(cmd.config);
+const _loadConfig = async (config: any): Promise<InputConfig> =>{
+    const cfg = new Config<InputConfig>().parse(config);
+    const gitList = await new GitConfigLoaderFactory(cfg).makeGitConfigLoader().downloadAndLoad();
+
+    const seen = new Set();
+    if(!cfg.validators) cfg.validators = []
+    const filteredArr = [...cfg.validators,...gitList].filter(el=>{ //priority given to locals over downloaded ones
+        const isDuplicate = seen.has(el.name);
+        seen.add(el.name)
+        return !isDuplicate
+    })
+    cfg.validators = filteredArr
+    return cfg
+}
 
+export async function startAction(cmd): Promise<void> {
+    const cfg = await _loadConfig(cmd.config)
+
     const server = express();
     server.get('/healthcheck',
         async (req: express.Request, res: express.Response): Promise<void> => {

src/gitConfigLoader/disabled.ts

@@ -0,0 +1,8 @@
+import { GitConfigLoader } from "./gitConfigLoaderInterface";
+import { Subscribable } from "../types";
+
+export class Disabled implements GitConfigLoader {
+  async downloadAndLoad(): Promise<Array<Subscribable>> {    
+    return []
+  }
+}

src/gitConfigLoader/gitConfigLoaderFactory.ts

@@ -0,0 +1,24 @@
+import { InputConfig } from "../types"
+import { Disabled } from "./disabled"
+import { GitConfigLoader } from "./gitConfigLoaderInterface"
+import { GitLabPrivate } from "./gitLabPrivate"
+
+export class GitConfigLoaderFactory {
+  constructor(private readonly cfg: InputConfig){}
+  makeGitConfigLoader = (): GitConfigLoader => {
+
+    const gitConfig = this.cfg.validatorsFromGit
+
+    if(!gitConfig?.enabled)
+      return new Disabled()
+
+    switch (gitConfig.platform.toLowerCase()) {  
+      case "gitlab":
+        if(gitConfig.private.enabled) return new GitLabPrivate(gitConfig.url,gitConfig.private.apiToken,gitConfig.network) //implemented just GitLab private for now
+        else new Disabled()
+        break;
+      default:
+        return new Disabled()
+    }  
+  }
+}

src/gitConfigLoader/gitConfigLoaderInterface.ts

@@ -0,0 +1,5 @@
+import { Subscribable } from "../types";
+
+export interface GitConfigLoader {
+  downloadAndLoad(): Promise<Array<Subscribable>>;
+}

src/gitConfigLoader/gitLabPrivate.ts

@@ -0,0 +1,39 @@
+import { GitConfigLoader } from "./gitConfigLoaderInterface";
+import fetch from 'node-fetch';
+import fs from 'fs';
+import { Config } from '@w3f/config';
+import { InputConfigFromGit, Subscribable } from "../types";
+
+export class GitLabPrivate implements GitConfigLoader {
+
+  constructor(
+    protected readonly url: string, 
+    protected readonly apiToken: string, 
+    protected readonly network: string
+    ) { }
+
+  async downloadAndLoad(): Promise<Array<Subscribable>> {
+    const response = await fetch(this.url, {
+    headers: {
+        'PRIVATE-TOKEN': this.apiToken
+    }
+    });
+    const data = await response.text();
+    if(!response.ok) throw new Error("git config download probelm: " + data)
+
+    fs.writeFileSync("./tmp.yaml",data)
+    const cfg = new Config<InputConfigFromGit>().parse("./tmp.yaml");
+    fs.rmSync("./tmp.yaml")
+
+    switch (this.network.toLowerCase()) {
+      case "kusama":
+        return cfg.Kusama
+
+      case "polkadot":
+        return cfg.Polkadot
+
+      default:
+        throw new Error("unexpected configuration")
+    }
+  }
+}

src/types.ts

@@ -16,7 +16,22 @@ export interface InputConfig {
     environment: string;
     port: number;
     endpoint: string;
-    validators: Array<Subscribable>;
+    validators?: Array<Subscribable>;
+    validatorsFromGit?: {
+        enabled: boolean;
+        platform: string;
+        private: {
+            enabled: boolean;
+            apiToken: string;
+        };
+        network: string;
+        url: string;
+    };
+}
+
+export interface InputConfigFromGit {
+    Kusama: Array<Subscribable>;
+    Polkadot: Array<Subscribable>;
 }
 
 export interface PromClient {