Merge pull request #256 from flow-hydraulics/latenssi/dockerized-e2e-…

…tests

Dockerized e2e test suite

.dockerignore

@@ -22,14 +22,15 @@ tmp/
 *.pid
 
 # Docker specific
-flow/
-api/
 .github/
+api/
 examples/
 docker/
+docker-compose.yml
+docker-compose.test-suite.yml
+Makefile
 
 # Not required for build
-flow/
 api-test-scripts/
 *.md
 *.svg

Makefile

@@ -10,18 +10,21 @@ ifeq (, $(shell which go))
 $(error "No go in PATH")
 endif
 
+
+dev = docker-compose -f docker-compose.dev.yml -p flow-wallet-api-dev
+test = docker-compose -f docker-compose.test-suite.yml -p flow-wallet-api-test
+
 .PHONY: dev
 dev:
-	@docker-compose up -d db pgadmin emulator redis
-	@docker-compose logs -f
+	@$(dev) up --remove-orphans -d db pgadmin emulator redis
 
 .PHONY: stop
 stop:
-	@docker-compose stop
+	@$(dev) stop
 
 .PHONY: down
 down:
-	@docker-compose down
+	@$(dev) down --remove-orphans
 
 .PHONY: reset
 reset: down dev
@@ -58,3 +61,16 @@ emulator.pid:
 .PHONY: lint
 lint:
 	@golangci-lint run
+
+.PHONY: run-test-suite
+run-test-suite:
+	@$(test) build flow api
+	@$(test) up --remove-orphans -d db redis flow
+	@echo "\nRunning tests, hang on...\n" \
+	; $(test) run --rm api go test ./... -p 1 \
+	; echo "\nRunning linter, hang on...\n" \
+	; $(test) run --rm lint golangci-lint run
+
+.PHONY: stop-test-suite
+stop-test-suite:
+	@$(test) down --remove-orphans

README.md

@@ -257,7 +257,6 @@ If you want to use an AWS KMS symmetric encryption key for encrypting the stored
 | `EncryptionKeyType` | `FLOW_WALLET_ENCRYPTION_KEY_TYPE` | Encryption key type    | `local` | `aws_kms`                                                                       |
 | `EncryptionKey`     | `FLOW_WALLET_ENCRYPTION_KEY`      | KMS encryption key ARN | -       | `arn:aws:kms:eu-central-1:012345678910:key/00000000-aaaa-bbbb-cccc-12345678910` |
 
-
 ### Idempotency middleware
 
 Idempotency middleware ensures that `POST` requests are idempotent. When the middleware is enabled an `Idempotency-Key` HTTP header is required for `POST` requests. The header value should be a unique identifier for the request (UUID or similar is recommended). Trying to send a request with a duplicate idempotency key will result in a `409 Conflict` HTTP response.
@@ -271,6 +270,7 @@ To configure the middleware set the following configuration settings;
 | `IdempotencyMiddlewareRedisURL`     | `FLOW_WALLET_IDEMPOTENCY_MIDDLEWARE_REDIS_URL`     | Redis URL for idempotency key middleware storage | -       | `redis://walletapi:wallet-api-redis@localhost:6379/` |
 
 NOTE:
+
 - The `local` option for `IdempotencyMiddlewareDatabaseType` does not support multiple instances.
 - The provided `docker-compose.yml` provides a basic Redis instance for local development purposes, with basic configuration files in the [`redis-config`](redis-config) directory.
 - There is currently no automatic cleanup of old idempotency keys when using the `shared` (sql) database. Redis is recommended for production use.
@@ -305,3 +305,13 @@ The Flow Wallet API is developed and maintained by [Equilibrium](https://equilib
 with support from the Flow core contributors.
 
 <a href="https://equilibrium.co/"><img src="equilibrium.svg" alt="Equilibrium" width="200"/></a>
+
+## Testing
+
+You can run a fully dockerized test suite if you have _Docker_ and _docker-compose_ installed.
+
+    # Run the test suite
+    make run-test-suite
+
+    # If you don't want to leave background services running (database, redis, flow-emulator)
+    make stop-test-suite

chain_events/listener.go

@@ -224,8 +224,6 @@ func (l *ListenerImpl) Stop() {
 	if l.ticker != nil {
 		l.ticker.Stop()
 	}
-
-	l.ticker = nil
 }
 
 func (l *ListenerImpl) systemHalted() (bool, error) {

docker-compose.yml → docker-compose.dev.yml

@@ -2,7 +2,6 @@ version: "3.9"
 services:
   db:
     image: postgres:13-alpine
-    restart: unless-stopped
     ports:
       - "5432:5432"
     environment:
@@ -12,30 +11,42 @@ services:
 
   redis:
     image: redis:6.2-alpine
-    restart: unless-stopped
     command: redis-server /usr/local/etc/redis/redis.conf
     ports:
       - "6379:6379"
     volumes:
-      - redis_data:/data
       - ./redis-config/redis.conf:/usr/local/etc/redis/redis.conf
       - ./redis-config/users.acl:/usr/local/etc/redis/users.acl
 
   pgadmin:
     image: dpage/pgadmin4
-    restart: unless-stopped
     environment:
       PGADMIN_DEFAULT_EMAIL: admin@example.com
       PGADMIN_DEFAULT_PASSWORD: admin
     ports:
       - "5050:80"
 
-  wallet:
+  emulator:
+    image: gcr.io/flow-container-registry/emulator:0.27.3
+    command: emulator -v --persist
+    ports:
+      - "3569:3569"
+    volumes:
+      - emulator_persist:/flowdb
+    env_file:
+      - ./.env
+    environment:
+      - FLOW_SERVICEPRIVATEKEY=${FLOW_WALLET_ADMIN_PRIVATE_KEY}
+      - FLOW_SERVICEKEYSIGALGO=ECDSA_P256
+      - FLOW_SERVICEKEYHASHALGO=SHA3_256
+      - FLOW_DBPATH=/flowdb
+
+  api:
     build:
       context: .
       dockerfile: ./docker/wallet/Dockerfile
+      target: dependencies
       network: host # docker build sometimes has problems fetching from alpine's CDN
-    restart: unless-stopped
     ports:
       - "3000:3000"
     env_file:
@@ -45,27 +56,17 @@ services:
       FLOW_WALLET_DATABASE_TYPE: psql
       FLOW_WALLET_ACCESS_API_HOST: emulator:3569
       FLOW_WALLET_CHAIN_ID: flow-emulator
+    working_dir: /flow-wallet-api
+    volumes:
+      - .:/flow-wallet-api:ro
+      - go-modules:/go/pkg/mod
+      - go-cache:/root/.cache/go-build
     depends_on:
       - db
       - emulator
       - redis
 
-  emulator:
-    image: gcr.io/flow-container-registry/emulator:0.27.3
-    restart: unless-stopped
-    command: emulator -v --persist
-    ports:
-      - "3569:3569"
-    volumes:
-      - emulator_persist:/flowdb
-    env_file:
-      - ./.env
-    environment:
-      - FLOW_SERVICEPRIVATEKEY=${FLOW_WALLET_ADMIN_PRIVATE_KEY}
-      - FLOW_SERVICEKEYSIGALGO=ECDSA_P256
-      - FLOW_SERVICEKEYHASHALGO=SHA3_256
-      - FLOW_DBPATH=/flowdb
-
 volumes:
-  redis_data:
   emulator_persist:
+  go-modules:
+  go-cache:

docker-compose.test-suite.yml

@@ -0,0 +1,77 @@
+version: "3.9"
+services:
+  db:
+    image: postgres:13-alpine
+    environment:
+      POSTGRES_DB: wallet_test
+      POSTGRES_USER: wallet_test
+      POSTGRES_PASSWORD: wallet_test
+
+  redis:
+    image: redis:6.2-alpine
+    command: redis-server /usr/local/etc/redis/redis.conf
+    volumes:
+      - ./redis-config/redis.conf:/usr/local/etc/redis/redis.conf
+      - ./redis-config/users.acl:/usr/local/etc/redis/users.acl
+
+  flow:
+    build:
+      context: ./docker/flow-cli
+      args:
+        FLOW_CLI_VERSION: v0.28.3
+    entrypoint: ["/bin/sh","-c"]
+    command:
+       - |
+        flow emulator -b 100ms &
+        sleep 1
+        flow project deploy --network=emulator --update=true
+        tail -F anything
+    environment:
+      FLOW_SERVICEPRIVATEKEY: 91a22fbd87392b019fbe332c32695c14cf2ba5b6521476a8540228bdf1987068
+      FLOW_SERVICEKEYSIGALGO: ECDSA_P256
+      FLOW_SERVICEKEYHASHALGO: SHA3_256
+    working_dir: /flow
+    volumes:
+      - "./flow:/flow:ro"
+
+  api:
+    build:
+      context: .
+      dockerfile: ./docker/wallet/Dockerfile
+      target: dependencies
+      network: host # docker build sometimes has problems fetching from alpine's CDN
+    environment:
+      FLOW_WALLET_ACCESS_API_HOST: flow:3569
+      FLOW_WALLET_ADMIN_ADDRESS: "0xf8d6e0586b0a20c7"
+      FLOW_WALLET_ADMIN_PRIVATE_KEY: 91a22fbd87392b019fbe332c32695c14cf2ba5b6521476a8540228bdf1987068
+      FLOW_WALLET_ADMIN_PROPOSAL_KEY_COUNT: 100
+      FLOW_WALLET_CHAIN_ID: flow-emulator
+      FLOW_WALLET_ENABLED_TOKENS: FUSD:0xf8d6e0586b0a20c7:fusd,FlowToken:0x0ae53cb6e3f42a79:flowToken
+      FLOW_WALLET_ENCRYPTION_KEY: faae4ed1c30f4e4555ee3a71f1044a8e
+      FLOW_WALLET_ENCRYPTION_KEY_TYPE: local
+      FLOW_WALLET_DATABASE_DSN: postgresql://wallet_test:wallet_test@db:5432/wallet_test
+      FLOW_WALLET_DATABASE_TYPE: psql
+      FLOW_WALLET_DEFAULT_KEY_TYPE: local
+      FLOW_WALLET_IDEMPOTENCY_MIDDLEWARE_DATABASE_TYPE: redis
+      FLOW_WALLET_IDEMPOTENCY_MIDDLEWARE_REDIS_URL: redis://walletapi:wallet-api-redis@localhost:6379/
+    working_dir: /flow-wallet-api
+    volumes:
+      - .:/flow-wallet-api:ro
+      - go-modules:/go/pkg/mod
+      - go-cache:/root/.cache/go-build
+    depends_on:
+      - db
+      - redis
+      - flow
+
+  lint:
+    image: golangci/golangci-lint
+    working_dir: /flow-wallet-api
+    volumes:
+      - .:/flow-wallet-api:ro
+      - go-modules:/go/pkg/mod
+      - go-cache:/root/.cache/go-build
+
+volumes:
+  go-modules:
+  go-cache:

docker/flow-cli/Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:20.04
+
+ARG FLOW_CLI_VERSION
+
+RUN apt-get update && apt-get install -y \
+  ca-certificates \
+  curl \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN sh -c "$(curl -fsSL https://storage.googleapis.com/flow-cli/install.sh)" 0 $FLOW_CLI_VERSION \
+  && mv /root/.local/bin/flow /usr/local/bin
+
+ENTRYPOINT ["/usr/local/bin/flow"]

docker/wallet/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:alpine AS builder
+FROM golang:1.17-alpine AS dependencies
 
 RUN apk update && apk add --no-cache \
   ca-certificates \
@@ -14,10 +14,11 @@ ENV GO111MODULE=on \
 
 WORKDIR /build
 
-COPY go.mod .
-COPY go.sum .
+COPY go.mod go.sum ./
 RUN go mod download
 
+FROM dependencies AS builder
+
 COPY . .
 
 RUN ./build.sh
@@ -26,7 +27,7 @@ WORKDIR /dist
 
 RUN cp /build/main .
 
-FROM scratch
+FROM scratch as dist
 
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /dist/main /

tests/test/config.go

@@ -35,12 +35,13 @@ func LoadConfig(t *testing.T, cfgFile ...string) *configs.Config {
 		t.Fatal(err)
 	}
 
-	cfg.DatabaseDSN = path.Join(t.TempDir(), "test.db")
-	cfg.DatabaseType = "sqlite"
-	cfg.ChainID = flow.Emulator
+	// Check if using default
+	if cfg.DatabaseDSN == "wallet.db" {
+		cfg.DatabaseDSN = path.Join(t.TempDir(), "test.db")
+		cfg.DatabaseType = "sqlite"
+	}
 
-	cfg.WorkerQueueCapacity = 100
-	cfg.WorkerCount = 1
+	cfg.ChainID = flow.Emulator
 
 	return cfg
 }