Node Visualizer (Refactor)

.github/workflows/test.yml

@@ -14,21 +14,21 @@ jobs:
         run: |
           GO111MODULES=off; go mod vendor
           curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.61.0
+          make install-shellcheck
+      - name: Check code formatting, linting, ...
+        run: make check
       - name: Setup SSH Key Build and run e2e
         run : |
           make e2e-build
           make e2e-run
-          make install-shellcheck
         shell: bash
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
           GIT_TOKEN: ${{secrets.GIT_TOKEN}}
           GIT_USER: ${{secrets.GIT_USER}}
       - name: Test e2e
-        run: |
-          make check
-          make e2e-test
+        run: make e2e-test
       - name: Clean e2e
         run: |
           make e2e-stop

Makefile

@@ -45,7 +45,6 @@ export REGISTRY=${DOCKER_REGISTRY}
 ## : ## _ [Prepare code]
 
 dep: ## Sorts dependencies
-	@command -v yarn >/dev/null 2>&1 && yarn --cwd ./pkg/node-visualizer/web install || true
 #	GO111MODULE=on GOPRIVATE=github.com/skycoin/* go get -v github.com/skycoin/skywire@master
 #	GO111MODULE=on GOPRIVATE=github.com/skycoin/* go mod vendor -v
 
@@ -58,11 +57,6 @@ format: dep ## Formats the code. Must have goimports and goimports-reviser insta
 build: dep ## Build binaries
 	${OPTS} go build ${BUILD_OPTS} -o ./bin/skywire-services ./cmd/skywire-services
 
-	# yarn --cwd ./pkg/node-visualizer/web build
-	# rm -rf ./pkg/node-visualizer/api/build/static
-	# mv ./pkg/node-visualizer/web/build/* ./pkg/node-visualizer/api/build
-	# ${OPTS} go build ${BUILD_OPTS} -o ./bin/node-visualizer ./cmd/node-visualizer
-
 build-deploy: ## Build for deployment Docker images
 	${DOCKER_OPTS} go build ${BUILD_OPTS_DEPLOY} -mod=vendor -o ./release/skywire-services ./cmd/skywire-services
 
@@ -174,7 +168,7 @@ check-help: ## Cursory check of the help menus
 ## : ## _ [E2E tests suite]
 
 e2e-build: set-forwarding ## E2E. Build dockers and containers for e2e-tests
-	./docker/docker_build.sh e2e ${BUILD_OPTS_DEPLOY}
+	./docker/docker_build.sh e2e ${BUILD_OPTS_DEPLOY} $(BUILD_ARCH)
 
 e2e-run: ## E2E. Start e2e environment
 	bash -c "DOCKER_TAG=e2e docker compose up -d"
@@ -231,7 +225,7 @@ reset-forwarding:
 ## : ## _ [Interactive integration tests]
 
 integration-env-build: set-forwarding #build
-	./docker/docker_build.sh integration ${BUILD_OPTS_DEPLOY}
+	./docker/docker_build.sh integration ${BUILD_OPTS_DEPLOY} $(BUILD_ARCH)
 	bash -c "DOCKER_TAG=integration docker compose up -d"
 
 integration-env-start: set-forwarding #start

README.md

@@ -63,7 +63,6 @@ Check the [docs](docs/Deployments.md) for more documentation on the deployments.
 
 - [Address Resolver](cmd/address-resolver/README.md)
 - [Config Bootstrapper](cmd/config-bootstrapper/README.md)
-- [Liveness Checker](cmd/liveness-checker/README.md)
 - [Network Monitor](cmd/network-monitor/README.md)
 - [Public Visor Monitor](cmd/public-visor-monitor/README.md)
 - [Route Finder](cmd/route-finder/README.md)

cmd/dmsg-monitor/README.md

@@ -0,0 +1,16 @@
+# DMSG Monitor
+
+## API endpoints
+
+### GET `/health`
+Gets the health info of the service. e.g.
+```
+{
+    "build_info": {
+        "version": "v1.0.1-267-ge1617c5b",
+        "commit": "e1617c5b0121182cfd2b610dc518e4753e56440e",
+        "date": "2022-10-25T11:01:52Z"
+    },
+    "started_at": "2022-10-25T11:10:45.152629597Z"
+}
+```

cmd/dmsg-monitor/commands/root.go

@@ -0,0 +1,123 @@
+// Package commands cmd/dmsg-monitor/commands/root.go
+package commands
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/buildinfo"
+	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/cipher"
+	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/cmdutil"
+	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/logging"
+	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/tcpproxy"
+	"github.com/spf13/cobra"
+
+	"github.com/skycoin/skywire-services/pkg/dmsg-monitor/api"
+)
+
+var (
+	confPath            string
+	dmsgURL             string
+	utURL               string
+	addr                string
+	tag                 string
+	logLvl              string
+	sleepDeregistration time.Duration
+	batchSize           int
+)
+
+func init() {
+	RootCmd.Flags().StringVarP(&addr, "addr", "a", ":9080", "address to bind to.\033[0m")
+	RootCmd.Flags().DurationVarP(&sleepDeregistration, "sleep-deregistration", "s", 60, "Sleep time for derigstration process in minutes\033[0m")
+	RootCmd.Flags().IntVarP(&batchSize, "batchsize", "b", 20, "Batch size of deregistration\033[0m")
+	RootCmd.Flags().StringVarP(&confPath, "config", "c", "dmsg-monitor.json", "config file location.\033[0m")
+	RootCmd.Flags().StringVarP(&dmsgURL, "dmsg-url", "d", "", "url to dmsg data.\033[0m")
+	RootCmd.Flags().StringVarP(&utURL, "ut-url", "u", "", "url to uptime tracker visor data.\033[0m")
+	RootCmd.Flags().StringVar(&tag, "tag", "dmsg_monitor", "logging tag\033[0m")
+	RootCmd.Flags().StringVarP(&logLvl, "loglvl", "l", "info", "set log level one of: info, error, warn, debug, trace, panic")
+}
+
+// RootCmd contains the root command
+var RootCmd = &cobra.Command{
+	Use: func() string {
+		return strings.Split(filepath.Base(strings.ReplaceAll(strings.ReplaceAll(fmt.Sprintf("%v", os.Args), "[", ""), "]", "")), " ")[0]
+	}(),
+	Short: "DMSG monitor of DMSG discovery entries.",
+	Long: `
+	┌┬┐┌┬┐┌─┐┌─┐   ┌┬┐┌─┐┌┐┌┬┌┬┐┌─┐┬─┐
+	 │││││└─┐│ ┬───││││ │││││ │ │ │├┬┘
+	─┴┘┴ ┴└─┘└─┘   ┴ ┴└─┘┘└┘┴ ┴ └─┘┴└─
+`,
+	SilenceErrors:         true,
+	SilenceUsage:          true,
+	DisableSuggestions:    true,
+	DisableFlagsInUseLine: true,
+	Version:               buildinfo.Version(),
+	Run: func(_ *cobra.Command, _ []string) {
+		if _, err := buildinfo.Get().WriteTo(os.Stdout); err != nil {
+			log.Printf("Failed to output build info: %v", err)
+		}
+
+		mLogger := logging.NewMasterLogger()
+		lvl, err := logging.LevelFromString(logLvl)
+		if err != nil {
+			mLogger.Fatal("Invalid log level")
+		}
+		logging.SetLevel(lvl)
+
+		conf := api.InitConfig(confPath, mLogger)
+
+		if dmsgURL == "" {
+			dmsgURL = conf.Dmsg.Discovery
+		}
+		if utURL == "" {
+			utURL = conf.UptimeTracker.Addr + "/uptimes"
+		}
+
+		var srvURLs api.ServicesURLs
+		srvURLs.DMSG = dmsgURL
+		srvURLs.UT = utURL
+
+		logger := mLogger.PackageLogger(tag)
+
+		logger.WithField("addr", addr).Info("Serving DMSG-Monitor API...")
+
+		monitorSign, _ := cipher.SignPayload([]byte(conf.PK.Hex()), conf.SK) //nolint
+
+		var monitorConfig api.DMSGMonitorConfig
+		monitorConfig.PK = conf.PK
+		monitorConfig.Sign = monitorSign
+		monitorConfig.BatchSize = batchSize
+
+		dmsgMonitorAPI := api.New(logger, srvURLs, monitorConfig)
+
+		ctx, cancel := cmdutil.SignalContext(context.Background(), logger)
+		defer cancel()
+
+		go dmsgMonitorAPI.InitDeregistrationLoop(ctx, conf, sleepDeregistration)
+
+		go func() {
+			if err := tcpproxy.ListenAndServe(addr, dmsgMonitorAPI); err != nil {
+				logger.Errorf("serve: %v", err)
+				cancel()
+			}
+		}()
+
+		<-ctx.Done()
+		if err := dmsgMonitorAPI.Visor.Close(); err != nil {
+			logger.WithError(err).Error("Visor closed with error.")
+		}
+	},
+}
+
+// Execute executes root CLI command.
+func Execute() {
+	if err := RootCmd.Execute(); err != nil {
+		log.Fatal("Failed to execute command: ", err)
+	}
+}

cmd/dmsg-monitor/dmsg-monitor.go

@@ -0,0 +1,43 @@
+// Package main cmd/dmsg-monitor/dmsg-monitor.go
+package main
+
+import (
+	cc "github.com/ivanpirog/coloredcobra"
+	"github.com/spf13/cobra"
+
+	"github.com/skycoin/skywire-services/cmd/dmsg-monitor/commands"
+)
+
+func init() {
+	var helpflag bool
+	commands.RootCmd.SetUsageTemplate(help)
+	commands.RootCmd.PersistentFlags().BoolVarP(&helpflag, "help", "h", false, "help for dmsgpty-cli")
+	commands.RootCmd.SetHelpCommand(&cobra.Command{Hidden: true})
+	commands.RootCmd.PersistentFlags().MarkHidden("help") //nolint
+}
+
+func main() {
+	cc.Init(&cc.Config{
+		RootCmd:         commands.RootCmd,
+		Headings:        cc.HiBlue + cc.Bold,
+		Commands:        cc.HiBlue + cc.Bold,
+		CmdShortDescr:   cc.HiBlue,
+		Example:         cc.HiBlue + cc.Italic,
+		ExecName:        cc.HiBlue + cc.Bold,
+		Flags:           cc.HiBlue + cc.Bold,
+		FlagsDescr:      cc.HiBlue,
+		NoExtraNewlines: true,
+		NoBottomNewline: true,
+	})
+	commands.Execute()
+}
+
+const help = "Usage:\r\n" +
+	"  {{.UseLine}}{{if .HasAvailableSubCommands}}{{end}} {{if gt (len .Aliases) 0}}\r\n\r\n" +
+	"{{.NameAndAliases}}{{end}}{{if .HasAvailableSubCommands}}\r\n\r\n" +
+	"Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand)}}\r\n  " +
+	"{{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}\r\n\r\n" +
+	"Flags:\r\n" +
+	"{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}\r\n\r\n" +
+	"Global Flags:\r\n" +
+	"{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}\r\n\r\n"

cmd/node-visualizer/commands/root.go

@@ -14,27 +14,25 @@ import (
 	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/buildinfo"
 	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/cmdutil"
 	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/logging"
-	"github.com/skycoin/skywire/pkg/skywire-utilities/pkg/metricsutil"
 	"github.com/spf13/cobra"
 
-	"github.com/skycoin/skywire-services/internal/tpdiscmetrics"
 	"github.com/skycoin/skywire-services/pkg/node-visualizer/api"
 )
 
 var (
-	addr        string
-	metricsAddr string
-	logEnabled  bool
-	tag         string
-	testing     bool
+	addr       string
+	logEnabled bool
+	tag        string
+	utURL      string
+	tpdURL     string
 )
 
 func init() {
-	RootCmd.Flags().StringVarP(&addr, "addr", "a", ":9081", "address to bind to\033[0m")
-	RootCmd.Flags().StringVarP(&metricsAddr, "metrics", "m", "", "address to bind metrics API to\033[0m")
+	RootCmd.Flags().StringVarP(&addr, "addr", "a", ":9089", "address to bind to\033[0m")
 	RootCmd.Flags().BoolVarP(&logEnabled, "log", "l", true, "enable request logging\033[0m")
 	RootCmd.Flags().StringVar(&tag, "tag", "node-visualizer", "logging tag\033[0m")
-	RootCmd.Flags().BoolVarP(&testing, "testing", "t", false, "enable testing to start without redis\033[0m")
+	RootCmd.Flags().StringVar(&utURL, "ut-url", "https://ut.skywire.skycoin.com/uptimes?status=on", "url of uptime tracker endpoint\033[0m")
+	RootCmd.Flags().StringVar(&tpdURL, "tpd-url", "https://tpd.skywire.skycoin.com/all-transports", "url of transport discovery endpoint\033[0m")
 }
 
 // RootCmd contains the root command
@@ -60,22 +58,14 @@ var RootCmd = &cobra.Command{
 		const loggerTag = "node_visualizer"
 		logger := logging.MustGetLogger(loggerTag)
 
-		metricsutil.ServeHTTPMetrics(logger, metricsAddr)
-
-		var m tpdiscmetrics.Metrics
-		if metricsAddr == "" {
-			m = tpdiscmetrics.NewEmpty()
-		} else {
-			m = tpdiscmetrics.NewVictoriaMetrics()
-		}
-
-		enableMetrics := metricsAddr != ""
-		nvAPI := api.New(logger, enableMetrics, m)
+		nvAPI := api.New(logger)
 
 		logger.Infof("Listening on %s", addr)
 		ctx, cancel := cmdutil.SignalContext(context.Background(), logger)
 		defer cancel()
-		go nvAPI.RunBackgroundTasks(ctx, logger)
+
+		go nvAPI.BackgroundTask(utURL, tpdURL)
+
 		go func() {
 			srv := &http.Server{
 				Addr:              addr,

cmd/skywire-services/commands/root.go

@@ -14,6 +14,7 @@ import (
 	ar "github.com/skycoin/skywire-services/cmd/address-resolver/commands"
 	confbs "github.com/skycoin/skywire-services/cmd/config-bootstrapper/commands"
 	kg "github.com/skycoin/skywire-services/cmd/keys-gen/commands"
+	nv "github.com/skycoin/skywire-services/cmd/node-visualizer/commands"
 	rf "github.com/skycoin/skywire-services/cmd/route-finder/commands"
 	se "github.com/skycoin/skywire-services/cmd/sw-env/commands"
 	tpd "github.com/skycoin/skywire-services/cmd/transport-discovery/commands"
@@ -31,6 +32,7 @@ func init() {
 		kg.RootCmd,
 		se.RootCmd,
 		ut.RootCmd,
+		nv.RootCmd,
 	)
 	tpd.RootCmd.Use = "tpd"
 	tps.RootCmd.Use = "tps"
@@ -40,6 +42,7 @@ func init() {
 	kg.RootCmd.Use = "kg"
 	se.RootCmd.Use = "se"
 	ut.RootCmd.Use = "ut"
+	nv.RootCmd.Use = "nv"
 }
 
 // RootCmd contains all subcommands

docker/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.7"
-
 networks:
   srv:
     driver: "bridge"
@@ -348,27 +346,24 @@ services:
     stdin_open: true # docker run -i
     tty: true        # docker run -t
 
-  # node-visualizer:
-  #   image: "${REGISTRY}/node-visualizer:${DOCKER_TAG}"
-  #   hostname: node-visualizer
-  #   container_name: "node-visualizer"
-  #   networks:
-  #     srv:
-  #       ipv4_address: 175.0.0.16
-  #     intra:
-  #       ipv4_address: 174.0.0.16
-  #   ports:
-  #     - "9089:9089"
-  #   entrypoint: "/release/node-visualizer"
-  #   depends_on:
-  #     - transport-discovery
-  #     - uptime-tracker
-  #     - visor-a
-  #     - visor-b
-  #     - visor-c
-  #   environment:
-  #     - UT_URL=http://uptime-tracker:9096/visor-ips
-  #     - TPD_URL=http://transport-discovery:9091/all-transports
+  node-visualizer:
+    image: "${REGISTRY}/node-visualizer:${DOCKER_TAG}"
+    hostname: node-visualizer
+    container_name: "node-visualizer"
+    networks:
+      srv:
+        ipv4_address: 175.0.0.17
+      intra:
+        ipv4_address: 174.0.0.17
+    ports:
+      - "9089:9089"
+    entrypoint: "/release/node-visualizer --ut-url http://uptime-tracker:9096 --tpd-url  http://transport-discovery:9091"
+    depends_on:
+      - transport-discovery
+      - uptime-tracker
+      - visor-a
+      - visor-b
+      - visor-c
 
   # skywire visors, which by being on a different network are to be isolated from the first one