Pre-release v1.1.0 (#57)

Migrated away from using lein-env parameters in favor of lein-environ plugin (#56)
Renamed snapshots to higlights
Imported join
Upgraded dependencies, added lein plugin
Suggest configurable parameters
Rename list symbol
Make arguments optional when none is required
Renamed aggregate to members list
Updated korma (SQL library)
Migrated away from using lein-env parameters in favor of lein-environ plugin
Print stack trace in debug mode only
Upgraded all deps
Move dependencies links to markdown document
Updated tests workflow
Removed distributed lein-env examples
Removed extra step dedicated to password placeholder substitution
Revised git exclude rules
Removed org.slf4j/slf4j-simple dependency
Revised dependencies reference doc
Exported Timbre vars
Removed unused import
Use timbre error
Configured Timbre vars
Replaced logging library
Preserve log capture
Improve throttling (#58)
Added documentation about database requirements
Stop copying lein-env configuration file (#59)
Bugfix: Caught rate limite exceeded exception earlier on (#60)
Bugfix: Caught bad authentication exception (#61)

.env.dist

@@ -7,3 +7,29 @@ LIST_NAME=''
 WORKER='highlights.example.org'
 WORKER_UID=1000
 WORKER_GID=101
+DATABASE_CHARSET=''
+DATABASE_HOST=''
+DATABASE_NAME=''
+DATABASE_USER=''
+DATABASE_PASSWORD=''
+DATABASE_PORT=''
+DATABASE_READ_CHARSET=''
+DATABASE_READ_HOST=''
+DATABASE_READ_NAME=''
+DATABASE_READ_USER=''
+DATABASE_READ_PASSWORD=''
+DATABASE_READ_PORT=''
+DATABASE_ARCHIVE_CHARSET=''
+DATABASE_ARCHIVE_HOST=''
+DATABASE_ARCHIVE_NAME=''
+DATABASE_ARCHIVE_USER=''
+DATABASE_ARCHIVE_PASSWORD=''
+DATABASE_ARCHIVE_PORT=''
+RABBITMQ_USER=''
+RABBITMQ_PASSWORD=''
+RABBITMQ_HOST=''
+RABBITMQ_VHOST=''
+RABBITMQ_QUEUE_PUBLICATIONS=''
+LIST_MAIN=''
+TAOENSSO_TIMBRE_MIN_LEVEL_EDN=':info'
+TAOENSSO_TIMBRE_NS_PATTERN_EDN='{:allow #{"highlights.*"}}'

.env.test.dist

@@ -0,0 +1,28 @@
+DEBUG=1
+WORKER='highlights.example.org'
+WORKER_UID=1000
+WORKER_GID=101
+DATABASE_CHARSET='utf8'
+DATABASE_HOST='127.0.0.1'
+DATABASE_NAME='highlights_test'
+DATABASE_USER='highlights'
+DATABASE_PASSWORD='test'
+DATABASE_PORT='5433'
+DATABASE_READ_CHARSET='utf8'
+DATABASE_READ_HOST='127.0.0.1'
+DATABASE_READ_NAME='highlights_test'
+DATABASE_READ_USER='highlights'
+DATABASE_READ_PASSWORD='test'
+DATABASE_READ_PORT='5433'
+DATABASE_ARCHIVE_CHARSET='utf8'
+DATABASE_ARCHIVE_HOST='127.0.0.1'
+DATABASE_ARCHIVE_NAME='highlights_test'
+DATABASE_ARCHIVE_USER='highlights'
+DATABASE_ARCHIVE_PASSWORD='test'
+DATABASE_ARCHIVE_PORT='5433'
+RABBITMQ_USER='update_me'
+RABBITMQ_PASSWORD='update_me'
+RABBITMQ_HOST='update_me'
+RABBITMQ_VHOST='update_me'
+RABBITMQ_QUEUE_PUBLICATIONS='update_me'
+LIST_MAIN='update_me'

.github/workflows/continuous-integration.yml → .github/workflows/tests.yml

@@ -1,4 +1,4 @@
-name: highlights.revue-de-presse.org
+name: tests
 
 on: [push, pull_request]
 
@@ -25,23 +25,16 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Copy test environment variables configuration file
-        run: |
-          cat .lein-env.test.dist | sed -E "s/__password__/test/g" > .lein-env
-
       - name: Create test database schema
         run: |
           PGPASSWORD=test psql \
-          --host 127.0.0.1 \
-          --port 5433 \
-          --username highlights \
+          --host=127.0.0.1 \
+          --port=5433 \
+          --username=highlights \
           --dbname=highlights_test \
           --echo-all \
           --quiet \
           --file $GITHUB_WORKSPACE/build/create_schema.sql
 
-      - name: Install dependencies
-        run: lein deps
-
       - name: Run tests
-        run: lein test
+        run: make test

.gitignore

@@ -1,3 +1,9 @@
+/\.env*
+!/\.env.dist
+*.iml
+/.lein-*
+!/.lein-env.dist
+/.nrepl-port
 provisioning/containers/docker-compose.override.yaml
 resources/*edn
 node_modules
@@ -7,13 +13,6 @@ pom.xml.asc
 .rebel*
 *.jar
 *.class
-.hgignore
-.hg/
-*.iml
-/.env
-/.lein-*
-/!.lein-env.dist
-/.nrepl-port
 /classes
 /checkouts
 /target

Makefile

@@ -1,6 +1,10 @@
 SHELL:=/bin/bash
 
-.PHONY: doc build clean help install restart start stop test
+.PHONY: doc help
+
+.PHONY: build clean deps install
+
+.PHONY: restart start stop test
 
 WORKER ?= 'highlights.example.org'
 TMP_DIR ?= '/tmp/tmp_${WORKER}'
@@ -17,12 +21,18 @@ build: ## Build worker image
 clean: ## Remove worker container
 	@/bin/bash -c 'source fun.sh && clean "${TMP_DIR}"'
 
+deps: ## Install dependencies
+	@lein deps
+
 install: build ## Install requirements
 	@/bin/bash -c 'source fun.sh && install'
 
 start: ## Run worker e.g. COMMAND=''
 	@/bin/bash -c 'source fun.sh && start'
 
+test: deps ## Run tests
+	@/bin/bash -c 'source fun.sh && test'
+
 run-clojure-container: ## Run Clojure container
 	@/bin/bash -c "source ./bin/console.sh && run_clojure_container"
 

doc/dependencies.md

@@ -0,0 +1,18 @@
+# Dependencies
+
+ - [adamwynne/twitter-api](https://github.com/adamwynne/twitter-api)
+ - [clojure.org](https://clojure.org/releases/downloads)
+ - [clojure/data.json](https://github.com/clojure/data.json)
+ - [korma/Korma/tree/v0.4.3](https://github.com/korma/Korma/tree/v0.4.3)
+ - [clojure/math.numeric-tower](https://github.com/clojure/math.numeric-tower)
+ - [michaelklishin/langohr](https://github.com/michaelklishin/langohr)
+ - [mudge/php-clj](https://github.com/mudge/php-clj)
+ - [ptaoussanis/timbre](https://github.com/ptaoussanis/timbre)
+ - [weavejester/environ](https://github.com/weavejester/environ)
+ - [xsc/pandect](https://github.com/xsc/pandect)
+ - [https://danlentz.github.io/clj-uuid/](https://danlentz.github.io/clj-uuid/)
+ - [https://clojure.github.io/tools.logging/](https://clojure.github.io/tools.logging/)
+ - [https://search.maven.org/artifact/org.postgresql/postgresql/42.4.0/jar](https://search.maven.org/artifact/org.postgresql/postgresql/42.4.0/jar)
+ - [https://mvnrepository.com/artifact/org.slf4j/slf4j-api](https://mvnrepository.com/artifact/org.slf4j/slf4j-api)
+ - [https://mvnrepository.com/artifact/org.slf4j/slf4j-reload4j](https://mvnrepository.com/artifact/org.slf4j/slf4j-reload4j)
+ - [https://mvnrepository.com/artifact/org.slf4j/slf4j-simple](https://mvnrepository.com/artifact/org.slf4j/slf4j-simple)

doc/requirements.md

@@ -0,0 +1,32 @@
+## Requirements
+
+Make sure the targeted public members list exists in the database
+
+```SQL
+INSERT INTO public.publishers_list (
+    name,
+    screen_name,
+    locked,
+    locked_at,
+    unlocked_at,
+    list_id,
+    public_id,
+    total_members,
+    total_statuses,
+    deleted_at,
+    created_at
+)
+VALUES (
+    'MEMBERS_LIST_NAME',
+    '',
+    false,
+    null,
+    now(),
+    '1',
+    gen_random_uuid(),
+    0,
+    0,
+    null,
+    now()
+)
+```

fun.sh

@@ -6,10 +6,6 @@ function _set_up_configuration_files() {
         cp --verbose ./.env{.dist,}
     fi
 
-    if [ ! -e ./.lein-env ]; then
-        cp --verbose ./.lein-env{.dist,}
-    fi
-
     if [ ! -e ./provisioning/containers/docker-compose.override.yaml ]; then
         cp ./provisioning/containers/docker-compose.override.yaml{.dist,}
     fi
@@ -351,6 +347,17 @@ function stop() {
     remove_running_container_and_image_in_debug_mode 'worker'
 }
 
+function test() {
+  (
+    for param in $(\cat ./.env.test.dist | sed -E "s/='([^']*)'/=\1/g");
+        do export $param;
+    done
+
+    env | grep -E '^DATABASE_' | grep -v 'PASSWORD'
+    lein test
+  )
+}
+
 function validate_docker_compose_configuration() {
     docker compose \
         -f ./provisioning/containers/docker-compose.yaml \

project.clj

@@ -7,22 +7,24 @@
   :license {:name "AGPL-3.0"
             :url "https://opensource.org/licenses/AGPL-3.0"}
 
-  :dependencies [[org.clojure/clojure "1.9.0"]
-                 [clj-time "0.15.0"]
-                 [pandect "0.6.1"]
-                 [environ "1.1.0"]
-                 [com.novemberain/langohr "5.0.0" :exclusions [org.slf4j/slf4j-api]]
-                 [org.postgresql/postgresql "42.2.18.jre7"]
-                 [korma "0.4.0"]
-                 [org.clojure/data.json "0.2.6"]
-                 [twitter-api "1.8.0"]
-                 [danlentz/clj-uuid "0.1.7"]
-                 [org.clojure/tools.logging "0.4.1"]
-                 [org.clojure/math.numeric-tower "0.0.4"]
-                 [org.slf4j/slf4j-api "1.6.2"]
-                 [org.slf4j/slf4j-log4j12 "1.6.2"]
+
+  :dependencies [[clj-time "0.15.2"]
+                 [com.fzakaria/slf4j-timbre "0.3.21"]
+                 [com.novemberain/langohr "5.4.0" :exclusions [org.slf4j/slf4j-api]]
+                 [com.taoensso/timbre "5.2.1"]
+                 [danlentz/clj-uuid "0.1.9"]
+                 [environ "1.2.0"]
+                 [korma "0.4.3"]
+                 [pandect "1.0.2"]
+                 [org.clojure/clojure "1.11.1"]
+                 [org.clojure/data.json "2.4.0"]
+                 [org.clojure/math.numeric-tower "0.0.5"]
+                 [org.clojure/tools.logging "1.1.0"]
+                 [org.postgresql/postgresql "42.4.0"]
                  [php-clj "0.4.1"]
-                 [org.slf4j/slf4j-simple "1.8.0-beta2" :exclusions [org.slf4j/slf4j-api]]]
+                 [twitter-api "1.8.0"]]
+
+  :plugins [[lein-environ "1.2.0"]]
 
   :aot :all
 

resources/log4j.properties

@@ -1,14 +1,8 @@
-# @see https://stackoverflow.com/a/38841004/282073
 # Root logger option
 log4j.rootLogger=INFO, stdout
 
 # Direct log messages to stdout
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.Target=System.out
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-log4j.appender.FILE.File=./var/log/highlights.log
-log4j.appender.FILE.ImmediateFlush=true
-log4j.appender.FILE.Append=true
-
-
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n