Bug fix updating ingestion test

.github/workflows/tests.yml

@@ -24,17 +24,9 @@ jobs:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       BINDIR: /home/runner/.local/bin/
       SLEEP: ${{ secrets.SLEEP }}
-      CEGA_MQ_CONNECTION: ${{ secrets.CEGA_MQ_CONNECTION }}
+      CEGA_AUTH_URL: ${{ secrets.CEGA_AUTH_URL }}
       CEGA_USERNAME: ${{ secrets.CEGA_USERNAME }}
       CEGA_PASSWORD: ${{ secrets.CEGA_PASSWORD }}
-      BROKER_HOST: ${{ secrets.BROKER_HOST }}
-      BROKER_PORT: ${{ secrets.BROKER_PORT }}
-      BROKER_USERNAME: ${{ secrets.BROKER_USERNAME }}
-      BROKER_PASSWORD: ${{ secrets.BROKER_PASSWORD }}
-      BROKER_VHOST: ${{ secrets.BROKER_VHOST }}
-      BROKER_VALIDATE: ${{ secrets.BROKER_VALIDATE }}
-      EXCHANGE: ${{ secrets.EXCHANGE }}
-      CEGA_AUTH_URL: ${{ secrets.CEGA_AUTH_URL }}
       EGA_BOX_USERNAME: ${{ secrets.EGA_BOX_USERNAME }}
       EGA_BOX_PASSWORD: ${{ secrets.EGA_BOX_PASSWORD }}
 

Makefile

@@ -1,7 +1,42 @@
 SHELL := /bin/bash -O expand_aliases
+# sudo docker macro to run docker commands as sudo in Ubuntu systems
+DOCKER := sudo docker
 
 FILES := localhost+5.pem localhost+5-key.pem localhost+5-client.pem localhost+5-client-key.pem rootCA.pem rootCA.p12 localhost+5.p12 localhost+5-client.p12 localhost+5-client-key.der rootCA-key.pem docker-stack.yml jwt.pub.pem jwt.priv.pem ega.pub.pem ega.sec.pass ega.sec.pem server.pem server-key.pem server.p12 client.pem client-key.pem client-key.der client.p12 init-mappings-db.sh
 
+# Previously manually entered secrets not available in the repo
+# Now with default values in mock containers instead
+####################################################
+
+# proxy service CEGA NSS (to new mock CEGA auth service)
+export CEGA_AUTH_URL=http://cega-auth:8443/lega/v1/legas/users/
+export CEGA_USERNAME=dummy
+export CEGA_PASSWORD=dummy
+
+# Test user in CEGA NSS mock service
+# (confusing it's the same as ID and SECRET above)....
+export EGA_BOX_USERNAME=dummy
+export EGA_BOX_PASSWORD=dummy
+
+
+# Prefilled configs from the repo
+#################################
+
+#Interceptor CEGA details (using "mock" rabbitmq image)
+export CEGA_MQ_CONNECTION=amqps://test:test@cegamq:5671/lega?cacertfile=/etc/ega/ssl/CA.cert
+
+# Proxy service CEGA details (using "mock" rabbitmq image)
+export BROKER_HOST=cegamq
+export BROKER_PORT=5671
+export BROKER_USERNAME=test
+export BROKER_PASSWORD=test
+export BROKER_VHOST=lega
+
+export BROKER_VALIDATE=false
+export EXCHANGE=localega.v1
+
+
+
 export CAROOT := $(shell mkcert -CAROOT)
 export ROOT_CERT_PASSWORD=r00t_cert_passw0rd
 export TSD_ROOT_CERT_PASSWORD=r00t_cert_passw0rd
@@ -35,11 +70,11 @@ export POSTGRES_CONNECTION=postgres://postgres:p0stgres_passw0rd@postgres:5432/p
 bootstrap: init $(FILES)
 	@chmod 644 $(FILES)
 	@mkdir -p /tmp/tsd /tmp/vault /tmp/db
-	@sudo chown 65534:65534 /tmp/vault
+	@sudo chown 65534:65534 /tmp/vault /tmp/tsd
 	@sudo chmod 777 /tmp/tsd /tmp/vault /tmp/db
 
 init:
-	@-docker swarm init
+	@-$(DOCKER) swarm init
 
 mkcert:
 	@mkcert -install
@@ -65,19 +100,19 @@ localhost+5-client-key.der: localhost+5-client-key.pem
 
 jwt.priv.pem:
 	@openssl genpkey -algorithm RSA -out jwt.priv.pem -pkeyopt rsa_keygen_bits:4096
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 jwt.pub.pem: jwt.priv.pem
 	@openssl rsa -pubout -in jwt.priv.pem -out jwt.pub.pem
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 ega.sec.pass:
 	@printf $(KEY_PASSWORD) > ega.sec.pass
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 ega.sec.pem:
 	@crypt4gh generate -n ega -p $(KEY_PASSWORD)
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 ega.pub.pem: ega.sec.pem
 
@@ -86,44 +121,44 @@ docker-stack.yml:
 
 rootCA.pem: mkcert
 	@cp "$(CAROOT)/rootCA.pem" rootCA.pem
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 rootCA-key.pem: mkcert
 	@cp "$(CAROOT)/rootCA-key.pem" rootCA-key.pem
 	@chmod 600 rootCA-key.pem
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 rootCA.p12: rootCA.pem rootCA-key.pem
 	@openssl pkcs12 -export -out rootCA.p12 -in rootCA.pem -inkey rootCA-key.pem -passout pass:${ROOT_CERT_PASSWORD}
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 server.pem: localhost+5.pem
 	@cp localhost+5.pem server.pem
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 server-key.pem: localhost+5-key.pem
 	@cp localhost+5-key.pem server-key.pem
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 server.p12: localhost+5.p12
 	@cp localhost+5.p12 server.p12
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 client.pem: localhost+5-client.pem
 	@cp localhost+5-client.pem client.pem
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 client-key.pem: localhost+5-client-key.pem
 	@cp localhost+5-client-key.pem client-key.pem
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 client-key.der: localhost+5-client-key.der
 	@cp localhost+5-client-key.der client-key.der
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 client.p12: localhost+5-client.p12
 	@cp localhost+5-client.p12 client.p12
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 define mappings
 #!/bin/bash
@@ -144,22 +179,22 @@ export mappings
 
 init-mappings-db.sh:
 	@echo "$$mappings" > init-mappings-db.sh
-	@docker secret create $@ $@
+	@$(DOCKER) secret create $@ $@
 
 deploy: init
-	@docker stack deploy LEGA -c docker-stack.yml
+	@$(DOCKER) stack deploy LEGA -c docker-stack.yml
 
 ls:
-	@docker service list
+	@$(DOCKER) service list
 
 rm:
-	@docker stack rm LEGA
+	@$(DOCKER) stack rm LEGA
 	@sleep 10
 
 clean:
-	@rm -rf $(FILES)
-	@rm -rf /tmp/tsd /tmp/vault /tmp/db
-	@docker secret rm $(FILES)
+	@sudo rm -rf $(FILES)
+	@sudo rm -rf /tmp/tsd /tmp/vault /tmp/db
+	@$(DOCKER) secret rm $(FILES)
 
 test:
-	@mvn --no-transfer-progress test
+	@mvn -B test | grep -v 'Download.* http'

README.md

@@ -1,7 +1,15 @@
-# LocalEGA-deploy-swarm
+# LocalEGA-deploy-swarm-fork
 ![Integration tests](https://github.com/neicnordic/LocalEGA-deploy-swarm/workflows/Integration%20tests/badge.svg)
 
-Docker Swarm deployment of LocalEGA. Please, refer to the project documentation for getting more comprehensive information: https://neic-sda.readthedocs.io/
+Docker Swarm deployment of LocalEGA. This is a fork meant for internal testing purposes.
+Please, refer to the project documentation for getting more comprehensive information: https://neic-sda.readthedocs.io/
+
+## Purpose
+
+The `docker-template.yml` file can be used in several ways:
+1. Deploy all the NeIC nordic microservices in use by FEGA Norway locally as containers together with mock services for TSD and CEGA functionality needed.
+2. One can trim it down to a Public stack or a Private stack depending on what needs to be deployed.
+3. GitHub actions can test nightly if the master branch is building and testing ok (without connection to live CEGA services. For that, please visit the upstream neicnordic/localega-deploy-swarm repo)
 
 ## Development
 
@@ -11,11 +19,35 @@ Docker Swarm deployment of LocalEGA. Please, refer to the project documentation
 - `crypt4gh` (https://github.com/elixir-oslo/crypt4gh)
 - `j2cli` (https://github.com/kolypto/j2cli)
 
+**IMPORTANT**
+
+CEGA-related env-vars _is no more needed to be set_  manually in `Makefile` before running the makefile. They decker-template.yml file already contains default values expected by the micro-services to work. The variables are:
+
+```
+export CEGA_USERNAME= 
+export CEGA_PASSWORD=
+export BROKER_HOST= 
+export BROKER_PORT=
+export BROKER_USERNAME=
+export BROKER_PASSWORD=
+export CEGA_MQ_CONNECTION= 
+export BROKER_VALIDATE= 
+export BROKER_VHOST= 
+export EXCHANGE=
+```
+all these variables are required by proxy and interceptor micro services.
+
 ### How-to
 
-`make bootstrap deploy` (CEGA-related env-vars should be set manually, e.g. `CEGA_CONNECTION`)
+Run:
+```bash
+> make bootstrap deploy
+```
 
-Cleaning up: `make rm purge`.
+Clean:
+```bash
+> make rm clean
+```
 
 ## Production
 

cega-users/cega-mock.py

@@ -0,0 +1,103 @@
+import sys
+import os
+import logging
+import asyncio
+import json
+from base64 import b64decode
+from aiohttp import web
+
+#logging.basicConfig(format='[%(asctime)s][%(levelname)-8s] (L:%(lineno)s) %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+logging.basicConfig(format='[%(levelname)-8s] (L:%(lineno)s) %(message)s')
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
+filepath = None
+instances = {}
+store = None
+usernames = {}
+uids = {}
+
+def fetch_user_info(identifier, query):
+    id_type = query.get('idType', None)
+    if not id_type:
+        raise web.HTTPBadRequest(text='Missing or wrong idType')
+    LOG.info(f'Requesting User {identifier} [type {id_type}]')
+    if id_type == 'username':
+        pos = usernames.get(identifier, None)
+        return store[pos] if pos is not None else None
+    if id_type == 'uid':
+        try:
+            pos = uids.get(int(identifier), None)
+            return store[pos] if pos is not None else None
+        except Exception:
+            return None
+    raise web.HTTPBadRequest(text='Missing or wrong idType')
+
+async def user(request):
+    # Authenticate
+    auth_header = request.headers.get('AUTHORIZATION')
+    if not auth_header:
+        raise web.HTTPUnauthorized(text=f'Protected access\n')
+    _, token = auth_header.split(None, 1)  # Skipping the Basic keyword
+    LOG.debug(f'Token is {token}')
+    instance, passwd = b64decode(token).decode().split(':', 1)
+    LOG.debug(f'I am instance {instance} and the password is {passwd}')
+    info = instances.get(instance)
+    if info is None or info != passwd:
+        raise web.HTTPUnauthorized(text=f'Protected access\n')
+
+    # Reload users list
+    load_users()
+
+    # Find user
+    user_info = fetch_user_info(request.match_info['identifier'], request.rel_url.query)
+    if user_info is None:
+        raise web.HTTPBadRequest(text=f'No info for that user\n')
+    return web.json_response({ 'header': { "apiVersion": "v1",
+                                           "code": "200",
+                                           "service": "users",
+                                           "developerMessage": "",
+                                           "userMessage": "OK",
+                                           "errorCode": "1",
+                                           "docLink": "https://ega-archive.org",
+                                           "errorStack": "" },
+                               'response': { "numTotalResults": 1,
+                                             "resultType": "eu.crg.ega.microservice.dto.lega.v1.users.LocalEgaUser",
+                                             "result": [ user_info ]}
+    })
+
+def main():
+    print("Main is being run")
+    if len(sys.argv) < 3:
+        print('Usage: {sys.argv[0] <hostaddr> <port> <filepath>}', file=sys.stderr)
+        sys.exit(2)
+
+    host = sys.argv[1]
+    port = sys.argv[2]
+
+    global filepath
+    filepath = sys.argv[3]
+
+    server = web.Application()
+    load_users()
+
+    # Registering the routes
+    server.router.add_get('/lega/v1/legas/users/{identifier}', user, name='user')
+
+    # aaaand... cue music
+    web.run_app(server, host=host, port=port, shutdown_timeout=0)
+
+
+def load_users():
+    # Initialization
+    global filepath, instances, store, usernames, uids
+    instances[os.environ[f'CEGA_USERS_USER']] = os.environ[f'CEGA_USERS_PASSWORD']
+    with open(filepath, 'rt') as f:
+        store = json.load(f)
+    for i, d in enumerate(store):
+        usernames[d['username']] = i
+        uids[d['uid']] = i
+
+
+if __name__ == '__main__':
+    main()

cega-users/users.json

@@ -0,0 +1,7 @@
+[{"username": "dummy",
+  "uid": 1,
+  "passwordHash": "$2b$12$1gyKIjBc9/cT0MYkXX24xe1LjEUjNwgL4rEk8fDoO.vDQZzWkqrn.",
+  "gecos": "dummy user",
+  "sshPublicKey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDiEcu2czBfbQh6+A3DplO2DRHG4LmdUKLRPX1vFOpC30TZusio0cFcgi8TEQv9TlMwu2ujF/wn/0D2VwXiDGk/Rbeq9jgTLpVbDWg/3ZGxGSqjPV3fzl0NzmaSgF0+IZQaSr6OjGVpTmpk5G4d4qqFg4Shjjm+AwlgmruThHNS9KmdJ7Vru+rQ8LwcjuSWtBdf6JM3bjlw1swQt6776p+wTK51YSKdtFEE5yVZjVwxlcPre7sRiem0XwSCFsu9sAfUTbHNTfwQ8lVXbvgRGu9SoW0wwb0Qele1WXZ8YFF10KLxGKpb1u0NsXSIZrJZhk0nxKb5tGBSnKXquoAvLZfVEKc+AXw1sDSaKvZaDw0/GORoAVSt3LDKYydAlzpMw6am4fgcEzm0vwCieWvSxd9uLY9IxV4sx0n0ZcuG55Le2TaQMnSm5XQ8zHBFYOb9ux8h6TY6JO+HmSjoHXkGhILKg8Y7zpq0PWy7HUvWzQVVfShAMN/N2gZ3a2T7amg17/S0wtgvjxxpNtnLc0HnHjr/mwtBjrN8C4n+IYI13rqZzPPU1wZu5qiacmHmeR15XAktEFKrpuvViJcylksjwyl6aY9psm+dwocON/yA3pdJGA8mPvrnDpPkGpzqvTqqIMxQkgel46jF2B7+lLzq6wQOsb7Ct66CKKppM6kpVVHRWQ==",
+  "enabled": null
+}]

cega.conf

@@ -0,0 +1,16 @@
+listeners.ssl.default = 5671
+ssl_options.cacertfile = /etc/rabbitmq/ssl/ca.pem
+ssl_options.certfile = /etc/rabbitmq/ssl/mq.pem
+ssl_options.keyfile = /etc/rabbitmq/ssl/mq-key.pem
+ssl_options.verify = verify_none
+ssl_options.fail_if_no_peer_cert = false
+ssl_options.versions.1 = tlsv1.2
+management.load_definitions = /etc/rabbitmq/conf/cega.json
+management.listener.port = 15671
+management.listener.ssl = true
+management.listener.ssl_opts.cacertfile = /etc/rabbitmq/ssl/ca.pem
+management.listener.ssl_opts.certfile = /etc/rabbitmq/ssl/mq.pem
+management.listener.ssl_opts.keyfile = /etc/rabbitmq/ssl/mq-key.pem
+default_vhost = lega
+disk_free_limit.absolute = 1GB
+log.default.level = debug

cega.json

@@ -0,0 +1,23 @@
+{"rabbit_version":"3.8",
+     "users":[{"name":"test",
+            "password_hash":"C5ufXbYlww6ZBcEqDUB04YdUptO81s+ozI3Ll5GCHTnv8NAm","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"administrator"}],   "vhosts":[{"name":"lega"}],
+            "permissions":[{"user":"test", "vhost":"lega", "configure":".*", "write":".*", "read":".*"}],
+
+            "parameters":[], "global_parameters":[{"name":"cluster_name", "value":"rabbit@localhost"}],
+     "policies":[],
+            "queues":[{"name":"v1.files.inbox", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}},
+            {"name":"v1.stableIDs", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}},
+            {"name":"v1.files",           "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}},
+            {"name":"v1.files.completed",       "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}},
+            {"name":"v1.files.verified", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}},
+            {"name":"v1.files.error",          "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}}],
+            "exchanges":[{"name":"localega.v1", "vhost":"lega", "type":"topic", "durable":true, "auto_delete":false, "internal":false, "arguments":{}}],
+            "bindings":[
+              {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.stableIDs","routing_key":"stableIDs"},
+              {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files","routing_key":"files"},
+              {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.inbox","routing_key":"files.inbox"},
+              {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.error","routing_key":"files.error"},
+              {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.verified","routing_key":"files.verified"},
+              {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.completed","routing_key":"files.completed"}]
+
+}

cega.plugins

@@ -0,0 +1 @@
+[rabbitmq_federation,rabbitmq_federation_management,rabbitmq_management,rabbitmq_shovel,rabbitmq_shovel_management].