diff --git a/README.md b/README.md index 8af18da..333d78c 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ queries against Wikipedia and sister projects databases. ## Setting up a local dev environment ## +# docker-compose Quarry uses [Docker](https://docs.docker.com/engine/install/) to set up a local environment. You can set it up by: @@ -18,19 +19,43 @@ will imediatelly be taken into account. A worker node is also created to execute your queries in the background (uses the same image). Finally, redis and two database instances are also started. -One database is your quarry database the other is a wikireplica-like database -named `mywiki`. This (or `mywiki_p`) is the correct thing to enter in the -database field on all local test queries. - -In your local environment, you can query Quarry internal db itself. Use then -"quarry" as database name. - To stop, run `docker-compose stop` or hit CTRL-C on the terminal your docker-compose is running in. After that, to start with code changes, you'll want to `docker-compose down` to clean up. Also, this creates a docker volume where sqlite versions of query results are found. That will not be cleaned up unless you run `docker-compose down -v` + +# minikube +It is possible to run a quarry system inside [minikube](https://minikube.sigs.k8s.io/docs/)! +At this time, you need to set it up with a cluster version before 1.22, most likely. + +First build the containers: +``` +eval $(minikube docker-env) +docker build . -t quarry:01 +cd docker-replica/ +docker build . -t mywiki:01 +``` + +You will need to install minikube (tested on minikube 1.23) and [helm](https://helm.sh) and kubectl on your system. When you are confident those are working, start minikube with: + - `minikube start --kubernetes-version=v1.23.15` + - `minikube addons enable ingress` + - `kubectl create namespace quarry` + - `helm -n quarry install quarry helm-quarry -f helm-quarry/dev-env.yaml` + +The rest of the setup instructions will display on screen as long as the install is successful. + +# local databases +Both local setups will create two databases. + +One database is your quarry database the other is a wikireplica-like database +named `mywiki`. This (or `mywiki_p`) is the correct thing to enter in the +database field on all local test queries. + +The other database is the Quarry internal db. In your local environment, you can query Quarry internal db itself. Use then +"quarry" as database name. + ### Updating existing containers ### If you had already run a dev environment (that is, ran `docker-compose up`) you might want to update diff --git a/docker-replica/Dockerfile b/docker-replica/Dockerfile new file mode 100644 index 0000000..a257eb9 --- /dev/null +++ b/docker-replica/Dockerfile @@ -0,0 +1,3 @@ +FROM mariadb:10.4-focal + +COPY replica.sql /docker-entrypoint-initdb.d/replica.sql diff --git a/helm-quarry/.helmignore b/helm-quarry/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/helm-quarry/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm-quarry/Chart.yaml b/helm-quarry/Chart.yaml new file mode 100644 index 0000000..eb6c78f --- /dev/null +++ b/helm-quarry/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +description: A Helm chart for quarry +name: helm-quarry +version: "1.0" diff --git a/helm-quarry/dev-env.yaml b/helm-quarry/dev-env.yaml new file mode 100644 index 0000000..f8e2fa5 --- /dev/null +++ b/helm-quarry/dev-env.yaml @@ -0,0 +1,2 @@ +localdev: + enabled: true diff --git a/helm-quarry/dev_config.yaml b/helm-quarry/dev_config.yaml new file mode 100644 index 0000000..2b02faf --- /dev/null +++ b/helm-quarry/dev_config.yaml @@ -0,0 +1,31 @@ +# The OAUTH tokens are callbacks to a quarry.local domain +# it is ok that they are public. +OAUTH_CONSUMER_TOKEN: "deeb8dbabbf157cb0ed120b89971d800" +OAUTH_SECRET_TOKEN: "38fdce038bbf848453a856aa329f8a06c456c791" +SECRET_KEY: 'aaarggghhpleaserescuemeiamstuckinarandomnumbergeneratorfactory' +DEBUG: True +DB_HOST: 'db' +DB_NAME: 'quarry' +DB_USER: 'quarry' +DB_PASSWORD: 'quarry' +broker_url: 'redis://redis' +result_backend: 'redis://redis' +worker_concurrency: 24 # Since all tasks are IO bound +task_acks_late: True # Tasks are idempotent! +task_track_started: True +worker_prefetch_multiplier: 1 # Tasks can run for a long time +# Just query the quarry database itself. +REPLICA_DOMAIN: '' +REPLICA_HOST: 'mywiki' +REPLICA_DB: 'mywiki_p' +REPLICA_USER: 'repl' +REPLICA_PASSWORD: 'repl' +REPLICA_PORT: 3306 +OUTPUT_PATH_TEMPLATE: '/results/%s/%s/%s.sqlite' +REDIS_HOST: 'redis' +REDIS_PORT: 6379 +REDIS_DB: 0 +QUERY_TIME_LIMIT: 60 # 1 minute +QUERY_RESULTS_PER_PAGE: 50 +KILLER_LOG_PATH: 'killer.log' +MAINTENANCE_MSG: 'This is your local development environment.' diff --git a/helm-quarry/prod-env.yaml b/helm-quarry/prod-env.yaml new file mode 100644 index 0000000..9ec3ebb --- /dev/null +++ b/helm-quarry/prod-env.yaml @@ -0,0 +1,2 @@ +localdev: + enabled: false diff --git a/helm-quarry/prod_config.yaml b/helm-quarry/prod_config.yaml new file mode 100644 index 0000000..1b2c0ba --- /dev/null +++ b/helm-quarry/prod_config.yaml @@ -0,0 +1,33 @@ +## Replace these with prod values and encrypt + +# The OAUTH tokens are callbacks to a quarry.local domain +# it is ok that they are public. +OAUTH_CONSUMER_TOKEN: "deeb8dbabbf157cb0ed120b89971d800" +OAUTH_SECRET_TOKEN: "38fdce038bbf848453a856aa329f8a06c456c791" +SECRET_KEY: 'aaarggghhpleaserescuemeiamstuckinarandomnumbergeneratorfactory' +DEBUG: True +DB_HOST: 'db' +DB_NAME: 'quarry' +DB_USER: 'quarry' +DB_PASSWORD: 'quarry' +broker_url: 'redis://redis' +result_backend: 'redis://redis' +worker_concurrency: 24 # Since all tasks are IO bound +task_acks_late: True # Tasks are idempotent! +task_track_started: True +worker_prefetch_multiplier: 1 # Tasks can run for a long time +# Just query the quarry database itself. +REPLICA_DOMAIN: '' +REPLICA_HOST: 'mywiki' +REPLICA_DB: 'mywiki_p' +REPLICA_USER: 'repl' +REPLICA_PASSWORD: 'repl' +REPLICA_PORT: 3306 +OUTPUT_PATH_TEMPLATE: '/results/%s/%s/%s.sqlite' +REDIS_HOST: 'redis' +REDIS_PORT: 6379 +REDIS_DB: 0 +QUERY_TIME_LIMIT: 60 # 1 minute +QUERY_RESULTS_PER_PAGE: 50 +KILLER_LOG_PATH: 'killer.log' +MAINTENANCE_MSG: 'This is your local development environment.' diff --git a/helm-quarry/templates/NOTES.txt b/helm-quarry/templates/NOTES.txt new file mode 100644 index 0000000..f995aea --- /dev/null +++ b/helm-quarry/templates/NOTES.txt @@ -0,0 +1,14 @@ +Thank you for installing {{ .Chart.Name }}! + +An essential part of setting up your dev environment is adding a hosts file entry. +Get the IP of minikube with: +minikube ip +Add this line to your hosts file: + quarry.local + +You can override any values you need in a file called dev-values.yaml. + +If you create one, run: +helm -n {{ .Release.Namespace }} upgrade {{ .Release.Name }} helm-quarry/ -f dev-values.yaml +Happy hacking. + diff --git a/helm-quarry/templates/config_yaml.yaml b/helm-quarry/templates/config_yaml.yaml new file mode 100644 index 0000000..1432bb8 --- /dev/null +++ b/helm-quarry/templates/config_yaml.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: config +data: +{{ if .Values.localdev.enabled }} + config.yaml: {{ .Files.Get "dev_config.yaml" | quote }} +{{ else }} + config.yaml: {{ .Files.Get "prod_config.yaml" | quote }} +{{ end }} diff --git a/helm-quarry/templates/db_deployment.yaml b/helm-quarry/templates/db_deployment.yaml new file mode 100644 index 0000000..7fbae0a --- /dev/null +++ b/helm-quarry/templates/db_deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: db + labels: + app: db +spec: + replicas: 1 + selector: + matchLabels: + app: db + template: + metadata: + labels: + app: db + spec: + containers: + - name: db + image: mariadb:10.1.48-bionic + imagePullPolicy: Always + env: + - name: MYSQL_USER + value: "quarry" + - name: MYSQL_PASSWORD + value: "quarry" + - name: MYSQL_DATABASE + value: "quarry" + - name: MYSQL_RANDOM_ROOT_PASSWORD + value: "1" + volumeMounts: + - name: schema + mountPath: /docker-entrypoint-initdb.d/ + volumes: + - name: schema + configMap: + name: db-schema diff --git a/helm-quarry/templates/db_schema_cm.yaml b/helm-quarry/templates/db_schema_cm.yaml new file mode 100644 index 0000000..24555a6 --- /dev/null +++ b/helm-quarry/templates/db_schema_cm.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: db-schema +data: + schema.sql: | + CREATE DATABASE IF NOT EXISTS quarry CHARACTER SET utf8; + USE quarry; + CREATE TABLE IF NOT EXISTS user( + id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + username VARCHAR(255) BINARY NOT NULL UNIQUE, + wiki_uid INT UNSIGNED NOT NULL UNIQUE + ); + CREATE UNIQUE INDEX IF NOT EXISTS user_username_index ON user( username); + CREATE UNIQUE INDEX IF NOT EXISTS user_wiki_uid ON user(wiki_uid); + + CREATE TABLE IF NOT EXISTS user_group( + id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + user_id INT UNSIGNED NOT NULL, + group_name VARCHAR(255) BINARY NOT NULL + ); + CREATE INDEX IF NOT EXISTS user_group_user_group_index ON user_group(user_id, group_name); + CREATE INDEX IF NOT EXISTS user_group_user_id_index ON user_group(user_id); + CREATE INDEX IF NOT EXISTS user_group_group_name_index ON user_group(group_name); + + CREATE TABLE IF NOT EXISTS query( + id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + user_id INT UNSIGNED NOT NULL, + title VARCHAR(1024) BINARY, + latest_rev_id INT UNSIGNED, + last_touched TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + published SMALLINT DEFAULT 0 NOT NULL, + description TEXT BINARY, + parent_id INT UNSIGNED + ); + CREATE INDEX IF NOT EXISTS query_parent_id_index ON query(parent_id); + + CREATE TABLE IF NOT EXISTS query_revision( + id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + text TEXT BINARY NOT NULL, + query_database VARCHAR(1024) BINARY, + query_id INT UNSIGNED NOT NULL, + latest_run_id INT UNSIGNED, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS query_rev_query_id_index ON query_revision(query_id); + CREATE INDEX IF NOT EXISTS query_rev_query_database_index ON query_revision(query_database); + + CREATE TABLE IF NOT EXISTS query_run( + id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + query_rev_id INT UNSIGNED NOT NULL, + status TINYINT UNSIGNED NOT NULL DEFAULT 0, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + task_id VARCHAR(36) BINARY, + extra_info TEXT BINARY + ); + CREATE INDEX IF NOT EXISTS query_run_status_index ON query_run(status); + + CREATE TABLE IF NOT EXISTS star( + id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + user_id INT UNSIGNED NOT NULL, + query_id INT UNSIGNED NOT NULL, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS star_user_id_index ON star(user_id); + CREATE INDEX IF NOT EXISTS star_query_id_index ON star(query_id); + CREATE UNIQUE INDEX IF NOT EXISTS star_user_query_index ON star(user_id, query_id); + diff --git a/helm-quarry/templates/db_service.yaml b/helm-quarry/templates/db_service.yaml new file mode 100644 index 0000000..f6201ed --- /dev/null +++ b/helm-quarry/templates/db_service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: db +spec: + selector: + app: db + ports: + - name: db + port: 3306 + targetPort: 3306 diff --git a/helm-quarry/templates/mywiki_deployment.yaml b/helm-quarry/templates/mywiki_deployment.yaml new file mode 100644 index 0000000..493e048 --- /dev/null +++ b/helm-quarry/templates/mywiki_deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mywiki + labels: + app: mywiki +spec: + replicas: 1 + selector: + matchLabels: + app: mywiki + template: + metadata: + labels: + app: mywiki + spec: + containers: + - name: mywiki + image: mywiki:01 # replace with quay.io + imagePullPolicy: Never + env: + - name: MYSQL_USER + value: "repl" + - name: MYSQL_PASSWORD + value: "repl" + - name: MYSQL_DATABASE + value: "repl" + - name: MYSQL_RANDOM_ROOT_PASSWORD + value: "1" diff --git a/helm-quarry/templates/mywiki_service.yaml b/helm-quarry/templates/mywiki_service.yaml new file mode 100644 index 0000000..838c670 --- /dev/null +++ b/helm-quarry/templates/mywiki_service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: mywiki +spec: + selector: + app: mywiki + ports: + - name: mywiki + port: 3306 + targetPort: 3306 diff --git a/helm-quarry/templates/redis_deployment.yaml b/helm-quarry/templates/redis_deployment.yaml new file mode 100644 index 0000000..63776cd --- /dev/null +++ b/helm-quarry/templates/redis_deployment.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + labels: + app: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:alpine + imagePullPolicy: Always diff --git a/helm-quarry/templates/redis_service.yaml b/helm-quarry/templates/redis_service.yaml new file mode 100644 index 0000000..d0c27be --- /dev/null +++ b/helm-quarry/templates/redis_service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + selector: + app: redis + ports: + - name: redis + port: 6379 + targetPort: 6379 diff --git a/helm-quarry/templates/results_pv.yaml b/helm-quarry/templates/results_pv.yaml new file mode 100644 index 0000000..3d6370b --- /dev/null +++ b/helm-quarry/templates/results_pv.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: results +spec: + storageClassName: manual + capacity: + storage: 1Gi + accessModes: + - ReadWriteMany + hostPath: + path: "/results" diff --git a/helm-quarry/templates/results_pvc.yaml b/helm-quarry/templates/results_pvc.yaml new file mode 100644 index 0000000..e400885 --- /dev/null +++ b/helm-quarry/templates/results_pvc.yaml @@ -0,0 +1,9 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: results +spec: + accessModes: [ReadWriteMany] + resources: + requests: + storage: 1Gi diff --git a/helm-quarry/templates/web_deployment.yaml b/helm-quarry/templates/web_deployment.yaml new file mode 100644 index 0000000..d5e34fc --- /dev/null +++ b/helm-quarry/templates/web_deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + labels: + app: web +spec: + replicas: 1 + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + containers: + - name: web + image: {{ .Values.web.repository }}:{{ .Values.web.tag }} +{{ if .Values.localdev.enabled }} + imagePullPolicy: Never +{{ else }} + imagePullPolicy: Always +{{ end }} + command: ["python"] + args: ["quarry.wsgi"] + volumeMounts: + - mountPath: "/results" + name: results + - name: config + mountPath: /config/ + volumes: + - name: results + persistentVolumeClaim: + claimName: results + - name: config + configMap: + name: config diff --git a/helm-quarry/templates/web_ingress.yaml b/helm-quarry/templates/web_ingress.yaml new file mode 100644 index 0000000..1978656 --- /dev/null +++ b/helm-quarry/templates/web_ingress.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + labels: + app: web + name: web + namespace: quarry +spec: + ingressClassName: nginx + rules: + - host: quarry.local + http: + paths: + - backend: + service: + name: web + port: + number: 5000 + path: / + pathType: Prefix diff --git a/helm-quarry/templates/web_service.yaml b/helm-quarry/templates/web_service.yaml new file mode 100644 index 0000000..cd83aad --- /dev/null +++ b/helm-quarry/templates/web_service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: web + name: web + namespace: quarry +spec: + ports: + - name: web + port: 5000 + nodePort: 32700 + protocol: TCP + targetPort: 5000 + #targetPort: http + selector: + app: web + type: NodePort diff --git a/helm-quarry/templates/worker_deployment.yaml b/helm-quarry/templates/worker_deployment.yaml new file mode 100644 index 0000000..3413639 --- /dev/null +++ b/helm-quarry/templates/worker_deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker + labels: + app: worker +spec: + replicas: 1 + selector: + matchLabels: + app: worker + template: + metadata: + labels: + app: worker + spec: + containers: + - name: worker + image: {{ .Values.worker.repository }}:{{ .Values.worker.tag }} +{{ if .Values.localdev.enabled }} + imagePullPolicy: Never +{{ else }} + imagePullPolicy: Always +{{ end }} + command: ["celery"] + args: ["--app", "quarry.web.worker", "worker"] + volumeMounts: + - mountPath: "/results" + name: results + - name: config + mountPath: /config/ + volumes: + - name: results + persistentVolumeClaim: + claimName: results + - name: config + configMap: + name: config diff --git a/helm-quarry/values.yaml b/helm-quarry/values.yaml new file mode 100644 index 0000000..5137ac9 --- /dev/null +++ b/helm-quarry/values.yaml @@ -0,0 +1,9 @@ +web: + repository: 'quarry' # replace with quay.io + pullPolicy: Never + tag: '01' + +worker: + repository: 'quarry' # replace with quay.io + pullPolicy: Never + tag: '01' diff --git a/quarry/default_config.yaml b/quarry/default_config.yaml deleted file mode 100644 index 90cc3b0..0000000 --- a/quarry/default_config.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# The OAUTH tokens are callbacks to a localhost domain -# it is ok that they are public. -OAUTH_CONSUMER_TOKEN: "e9bfbacf0b29a8ea64712b209c24ee36" -OAUTH_SECRET_TOKEN: "83ed4ac72176286f5c7798459f0a6b16a96853c4" -SECRET_KEY: 'aaarggghhpleaserescuemeiamstuckinarandomnumbergeneratorfactory' -DEBUG: True -DB_HOST: 'db' -DB_NAME: 'quarry' -DB_USER: 'quarry' -DB_PASSWORD: 'quarry' -broker_url: 'redis://redis' -result_backend: 'redis://redis' -worker_concurrency: 24 # Since all tasks are IO bound -task_acks_late: True # Tasks are idempotent! -task_track_started: True -worker_prefetch_multiplier: 1 # Tasks can run for a long time - -# Run queries against the live wikimedia replica databases. This requires a -# toolforge account, so that you can retrieve your credentials from -# ~/replica.my.cnf (and log into toolforge to establish the SOCKS5 proxy). These -# lines should remain commented if you're not using a SOCKS5 proxy. The IP -# address of the host running the docker container. On Windows or macOS this -# might be `host.docker.internal`. -# REPLICA_SOCKS5_PROXY_HOST: '172.17.0.1' -# REPLICA_SOCKS5_PROXY_PORT: 1080 - - -# Run queries against a fake wiki database -# Change these 3 lines if you're using the live replicas. -REPLICA_DOMAIN: '' # Change to `analytics.db.svc.wikimedia.cloud` for live replicas -REPLICA_USER: 'repl' # For live replicas, your replica.my.cnf username -REPLICA_PASSWORD: 'repl' # For live replicas, your replica.my.cnf password - -REPLICA_PORT: 3306 -OUTPUT_PATH_TEMPLATE: '/results/%s/%s/%s.sqlite' -REDIS_HOST: 'redis' -REDIS_PORT: 6379 -REDIS_DB: 0 -QUERY_TIME_LIMIT: 60 # 1 minute -QUERY_RESULTS_PER_PAGE: 50 -KILLER_LOG_PATH: 'killer.log' -MAINTENANCE_MSG: 'This is your local development environment.' - -# https://flask-caching.readthedocs.io/en/latest/#configuring-flask-caching -CACHE_TYPE: 'RedisCache' -CACHE_DEFAULT_TIMEOUT: 30 -CACHE_REDIS_HOST: 'redis' -CACHE_REDIS_PORT: 6379 -CACHE_REDIS_DB: 1 diff --git a/quarry/default_config.yaml b/quarry/default_config.yaml new file mode 120000 index 0000000..3823d66 --- /dev/null +++ b/quarry/default_config.yaml @@ -0,0 +1 @@ +../helm-quarry/dev_config.yaml \ No newline at end of file diff --git a/quarry/web/app.py b/quarry/web/app.py index db64c93..1da807a 100644 --- a/quarry/web/app.py +++ b/quarry/web/app.py @@ -25,15 +25,8 @@ def get_config(): conf = {} conf.update( - yaml.safe_load(open(os.path.join(__dir__, "../default_config.yaml"))) + yaml.safe_load(open(os.path.join(__dir__, "/config/config.yaml"))) ) - try: - conf.update( - yaml.safe_load(open(os.path.join(__dir__, "../config.yaml"))) - ) - except IOError: - # Is ok if we can't load config.yaml - pass return conf diff --git a/quarry/web/killer.py b/quarry/web/killer.py index 248879a..5c72ec7 100755 --- a/quarry/web/killer.py +++ b/quarry/web/killer.py @@ -6,12 +6,7 @@ from connections import Connections __dir__ = os.path.dirname(__file__) -config = yaml.safe_load(open(os.path.join(__dir__, "../default_config.yaml"))) -try: - config.update(yaml.safe_load(open(os.path.join(__dir__, "../config.yaml")))) -except IOError: - # is ok if we do not have config.yaml - pass +config = yaml.safe_load(open(os.path.join(__dir__, "/config/config.yaml"))) logging.basicConfig( filename=config["KILLER_LOG_PATH"], diff --git a/quarry/web/worker.py b/quarry/web/worker.py index 871384e..bbd4fdc 100644 --- a/quarry/web/worker.py +++ b/quarry/web/worker.py @@ -22,17 +22,18 @@ celery_log = get_task_logger(__name__) celery = Celery("quarry.web.worker") -celery.conf.update( - yaml.safe_load(open(os.path.join(__dir__, "../default_config.yaml"))) -) try: celery.conf.update( - yaml.safe_load(open(os.path.join(__dir__, "../config.yaml"))) + yaml.safe_load(open(os.path.join(__dir__, "/config/config.yaml"))) ) except IOError: - # Is ok if we can not load config.yaml + # for pytest + celery.conf.update( + yaml.safe_load(open(os.path.join(__dir__, "../default_config.yaml"))) + ) pass + conn = None