diff --git a/.dockerignore b/.dockerignore index f3ebc2d2..10740692 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,6 @@ node_modules .gitattributes .dockerignore Dockerfile +test/ +bin/ +!bin/run.sh diff --git a/.github/workflows/ci.js.yml b/.github/workflows/ci.js.yml index 5be6049e..9dac21d7 100644 --- a/.github/workflows/ci.js.yml +++ b/.github/workflows/ci.js.yml @@ -24,7 +24,7 @@ jobs: - name: Run exercism/javascript-analyzer ci precheck (lint code) run: yarn lint - ci: + unit_tests: runs-on: ubuntu-latest strategy: @@ -46,3 +46,27 @@ jobs: - name: Run end-to-end tests run: yarn test:e2e + + smoke_tests: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 + with: + install: true + + - name: Build Docker image and store in cache + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 + with: + context: . + push: false + load: true + tags: exercism/javascript-analyzer + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Run Tests in Docker + run: bin/run-tests-in-docker.sh diff --git a/.github/workflows/pr.ci.js.yml b/.github/workflows/pr.ci.js.yml index 9674868c..8f0f7595 100644 --- a/.github/workflows/pr.ci.js.yml +++ b/.github/workflows/pr.ci.js.yml @@ -24,7 +24,7 @@ jobs: - name: Run exercism/javascript ci precheck (lint code) run: yarn lint - ci: + unit_tests: runs-on: ubuntu-latest strategy: @@ -48,3 +48,27 @@ jobs: - name: Run end-to-end tests run: yarn test:e2e + + smoke_tests: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 + with: + install: true + + - name: Build Docker image and store in cache + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 + with: + context: . + push: false + load: true + tags: exercism/javascript-analyzer + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Run Tests in Docker + run: bin/run-tests-in-docker.sh diff --git a/bin/run-in-docker.sh b/bin/run-in-docker.sh new file mode 100755 index 00000000..785ef37b --- /dev/null +++ b/bin/run-in-docker.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env sh + +# Synopsis: +# Run the analyzer on a solution using the analyzer Docker image. +# The analyzer Docker image is built automatically. + +# Arguments: +# $1: exercise slug +# $2: path to solution folder +# $3: path to output directory + +# Output: +# Run the analyzer on a solution using the analyzer Docker image. +# The analyzer Docker image is built automatically. + +# Example: +# ./bin/run-in-docker.sh two-fer path/to/solution/folder/ path/to/output/directory/ + +# Stop executing when a command returns a non-zero return code +set -e + +# If any required arguments is missing, print the usage and exit +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/" + exit 1 +fi + +slug="$1" +solution_dir=$(realpath "${2%/}") +output_dir=$(realpath "${3%/}") + +# Create the output directory if it doesn't exist +mkdir -p "${output_dir}" + +# Build the Docker image +docker build --rm -t exercism/javascript-analyzer . + +# Run the Docker image using the settings mimicking the production environment +docker run \ + --rm \ + --network none \ + --read-only \ + --mount type=bind,src="${solution_dir}",dst=/solution \ + --mount type=bind,src="${output_dir}",dst=/output \ + --mount type=tmpfs,dst=/tmp \ + exercism/javascript-analyzer "${slug}" /solution /output diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh new file mode 100755 index 00000000..19186132 --- /dev/null +++ b/bin/run-tests-in-docker.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env sh + +# Synopsis: +# Test the analyzer Docker image by running it against a predefined set of +# solutions with an expected output. +# The analyzer Docker image is built automatically. + +# Output: +# Outputs the diff of the expected analysis against the actual analysis +# generated by the analyzer Docker image. + +# Example: +# ./bin/run-tests-in-docker.sh + +# Stop executing when a command returns a non-zero return code +set -e + +# Build the Docker image +docker build --rm -t exercism/javascript-analyzer . + +# Run the Docker image using the settings mimicking the production environment +docker run \ + --rm \ + --network none \ + --read-only \ + --mount type=bind,src="${PWD}/test",dst=/opt/analyzer/test \ + --mount type=tmpfs,dst=/tmp \ + --volume "${PWD}/bin/run-tests.sh:/opt/analyzer/bin/run-tests.sh" \ + --workdir /opt/analyzer \ + --entrypoint /opt/analyzer/bin/run-tests.sh \ + exercism/javascript-analyzer diff --git a/bin/run-tests.sh b/bin/run-tests.sh new file mode 100755 index 00000000..2d515324 --- /dev/null +++ b/bin/run-tests.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env sh + +# Synopsis: +# Test the analyzer by running it against a predefined set of solutions +# with an expected output. + +# Output: +# Outputs the diff of the expected analysis against the actual analysis +# generated by the analyzer. + +# Example: +# ./bin/run-tests.sh + +exit_code=0 + +# We need to copy the fixtures to a temp directory as the user +# running within the Docker container does not have permissions +# to run the sed command on the fixtures directory +fixtures_dir="test/fixtures" +tmp_fixtures_dir="/tmp/test/fixtures" +rm -rf "${tmp_fixtures_dir}" +mkdir -p "${tmp_fixtures_dir}" +cp -R ${fixtures_dir}/* "${tmp_fixtures_dir}" + +# Iterate over all test directories +for analysis_file in $(find "${tmp_fixtures_dir}" -name expected_analysis.json); do + slug=$(echo "${analysis_file:${#tmp_fixtures_dir}+1}" | cut -d / -f 1) + test_dir=$(dirname "${analysis_file}") + test_dir_name=$(basename "${analysis_file}") + test_dir_path=$(realpath "${test_dir}") + bin/run.sh "${slug}" "${test_dir_path}" "${test_dir_path}" + + file="analysis.json" + expected_file="expected_${file}" + echo "${test_dir}: comparing ${file} to ${expected_file}" + + if ! diff "${test_dir}/${file}" "${test_dir}/${expected_file}"; then + exit_code=1 + fi +done + +exit ${exit_code} diff --git a/test/fixtures/amusement-park/exemplar/expected_analysis.json b/test/fixtures/amusement-park/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/amusement-park/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/annalyns-infiltration/exemplar/expected_analysis.json b/test/fixtures/annalyns-infiltration/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/annalyns-infiltration/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/annalyns-infiltration/optimal/expected_analysis.json b/test/fixtures/annalyns-infiltration/optimal/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/annalyns-infiltration/optimal/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/bird-watcher/exemplar/expected_analysis.json b/test/fixtures/bird-watcher/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/bird-watcher/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/coordinate-transformation/exemplar/expected_analysis.json b/test/fixtures/coordinate-transformation/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/coordinate-transformation/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/elyses-analytic-enchantments/exemplar/expected_analysis.json b/test/fixtures/elyses-analytic-enchantments/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/elyses-analytic-enchantments/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/elyses-destructured-enchantments/exemplar/expected_analysis.json b/test/fixtures/elyses-destructured-enchantments/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/elyses-destructured-enchantments/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/elyses-enchantments/exemplar/expected_analysis.json b/test/fixtures/elyses-enchantments/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/elyses-enchantments/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/elyses-looping-enchantments/exemplar/expected_analysis.json b/test/fixtures/elyses-looping-enchantments/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/elyses-looping-enchantments/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/elyses-transformative-enchantments/exemplar/expected_analysis.json b/test/fixtures/elyses-transformative-enchantments/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/elyses-transformative-enchantments/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/factory-sensors/exemplar/expected_analysis.json b/test/fixtures/factory-sensors/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/factory-sensors/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/freelancer-rates/exemplar/expected_analysis.json b/test/fixtures/freelancer-rates/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/freelancer-rates/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/high-score-board/exemplar/expected_analysis.json b/test/fixtures/high-score-board/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/high-score-board/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/lasagna-master/exemplar/expected_analysis.json b/test/fixtures/lasagna-master/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/lasagna-master/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/lucky-numbers/exemplar/expected_analysis.json b/test/fixtures/lucky-numbers/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/lucky-numbers/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/mixed-juices/exemplar/expected_analysis.json b/test/fixtures/mixed-juices/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/mixed-juices/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/nullability/exemplar/expected_analysis.json b/test/fixtures/nullability/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/nullability/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/ozans-playlist/exemplar/expected_analysis.json b/test/fixtures/ozans-playlist/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/ozans-playlist/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/pizza-order/exemplar/expected_analysis.json b/test/fixtures/pizza-order/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/pizza-order/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/poetry-club-door-policy/exemplar/expected_analysis.json b/test/fixtures/poetry-club-door-policy/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/poetry-club-door-policy/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/translation-service/exemplar/expected_analysis.json b/test/fixtures/translation-service/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/translation-service/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/vehicle-purchase/exemplar/expected_analysis.json b/test/fixtures/vehicle-purchase/exemplar/expected_analysis.json new file mode 100644 index 00000000..ae91b571 --- /dev/null +++ b/test/fixtures/vehicle-purchase/exemplar/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]} \ No newline at end of file diff --git a/test/fixtures/vehicle-purchase/uses-if-else/expected_analysis.json b/test/fixtures/vehicle-purchase/uses-if-else/expected_analysis.json new file mode 100644 index 00000000..652a2929 --- /dev/null +++ b/test/fixtures/vehicle-purchase/uses-if-else/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.vehicle-purchase.use_if_else","params":{"function":"chooseVehicle"},"type":"informative"},{"comment":"javascript.vehicle-purchase.use_if_else","params":{"function":"calculateResellPrice"},"type":"informative"}]} \ No newline at end of file diff --git a/test/fixtures/vehicle-purchase/uses-if/expected_analysis.json b/test/fixtures/vehicle-purchase/uses-if/expected_analysis.json new file mode 100644 index 00000000..6d65a4d1 --- /dev/null +++ b/test/fixtures/vehicle-purchase/uses-if/expected_analysis.json @@ -0,0 +1 @@ +{"comments":[{"comment":"javascript.vehicle-purchase.unnecessary_if_statement","type":"actionable"}]} \ No newline at end of file