Add smoke tests (#54)

* Update to new standard

* Add scripts to run tests

* Add workflow to run smoke tests in CI

* Exclude more files from Docker build context

* Add smoke tests

.dockerignore

@@ -8,3 +8,7 @@
 .gitattributes
 .dockerignore
 Dockerfile
+bin/
+!bin/run.sh
+snippets/
+tests/

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Tests
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/rust-analyzer
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh

Dockerfile

@@ -16,4 +16,4 @@ WORKDIR /opt/analyzer
 
 COPY --from=base /analyzer/bin/* ./bin/
 
-ENTRYPOINT ["bin/analyze.sh"]
+ENTRYPOINT ["bin/run.sh"]

bin/run-in-docker.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Run the analyzer on a solution using the analyzer Docker image.
+# The analyzer Docker image is built automatically.
+
+# Arguments:
+# $1: exercise slug
+# $2: path to solution folder
+# $3: path to output directory
+
+# Output:
+# Run the analyzer on a solution using the analyzer Docker image.
+# The analyzer Docker image is built automatically.
+
+# Example:
+# ./bin/run-in-docker.sh two-fer path/to/solution/folder/ path/to/output/directory/
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# If any required arguments is missing, print the usage and exit
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
+    exit 1
+fi
+
+slug="$1"
+solution_dir=$(realpath "${2%/}")
+output_dir=$(realpath "${3%/}")
+
+# Create the output directory if it doesn't exist
+mkdir -p "${output_dir}"
+
+# Build the Docker image
+docker build --rm -t exercism/rust-analyzer .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${solution_dir}",dst=/solution \
+    --mount type=bind,src="${output_dir}",dst=/output \
+    --mount type=tmpfs,dst=/tmp \
+    exercism/rust-analyzer "${slug}" /solution /output 

bin/run-tests-in-docker.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the analyzer Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The analyzer Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected analysis against the actual analysis
+# generated by the analyzer Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# Build the Docker image
+docker build --rm -t exercism/rust-analyzer .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${PWD}/snippets",dst=/opt/analyzer/snippets \
+    --mount type=tmpfs,dst=/tmp \
+    --volume "${PWD}/bin/run-tests.sh:/opt/analyzer/bin/run-tests.sh" \
+    --workdir /opt/analyzer \
+    --entrypoint /opt/analyzer/bin/run-tests.sh \
+    exercism/rust-analyzer

bin/run-tests.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the analyzer by running it against a predefined set of solutions 
+# with an expected output.
+
+# Output:
+# Outputs the diff of the expected analysis against the actual analysis
+# generated by the analyzer.
+
+# Example:
+# ./bin/run-tests.sh
+
+exit_code=0
+
+# Iterate over all test directories
+for analysis_file in $(find snippets -name expected_analysis.json); do
+    test_dir=$(dirname "${analysis_file}")
+    test_dir_name=$(basename "${test_dir}")
+    test_dir_path=$(realpath "${test_dir}")
+    slug=$(echo $test_dir | awk -F '/' '{ print $2 }')
+
+    bin/run.sh "${slug}" "${test_dir_path}" "${test_dir_path}"
+
+    file="analysis.json"
+    expected_file="expected_${file}"
+    echo "${test_dir_name}: comparing ${file} to ${expected_file}"
+
+    if ! diff "${test_dir_path}/${file}" "${test_dir_path}/${expected_file}"; then
+        exit_code=1
+    fi
+done
+
+exit ${exit_code}

snippets/reverse-string/approve_1/expected_analysis.json

@@ -0,0 +1,4 @@
+{
+  "status": "approve",
+  "comments": []
+}

snippets/reverse-string/approve_2/expected_analysis.json

@@ -0,0 +1,4 @@
+{
+  "status": "approve",
+  "comments": []
+}

snippets/reverse-string/approve_with_suggest_bonus_1/expected_analysis.json

@@ -0,0 +1,6 @@
+{
+  "status": "approve",
+  "comments": [
+    "rust.reverse-string.suggest_doing_bonus_test"
+  ]
+}

snippets/reverse-string/approve_with_suggest_bonus_2/expected_analysis.json

@@ -0,0 +1,6 @@
+{
+  "status": "approve",
+  "comments": [
+    "rust.reverse-string.suggest_doing_bonus_test"
+  ]
+}

snippets/reverse-string/approve_with_suggest_removing_extern_crate_1/expected_analysis.json

@@ -0,0 +1,6 @@
+{
+  "status": "approve",
+  "comments": [
+    "rust.reverse-string.suggest_removing_extern_crate"
+  ]
+}

snippets/reverse-string/approve_with_suggest_removing_extern_crate_2/expected_analysis.json

@@ -0,0 +1,6 @@
+{
+  "status": "approve",
+  "comments": [
+    "rust.reverse-string.suggest_removing_extern_crate"
+  ]
+}

snippets/reverse-string/disapprove_with_solution_not_found/expected_analysis.json

@@ -0,0 +1,6 @@
+{
+  "status": "disapprove",
+  "comments": [
+    "rust.reverse-string.solution_function_not_found"
+  ]
+}