mirror of
https://github.com/UniversityOfHelsinkiCS/oodikone.git
synced 2026-10-12 19:03:39 +03:00
557 lines
18 KiB
YAML
557 lines
18 KiB
YAML
name: 'Lint and test'
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
skip_db_update:
|
|
description: 'When true, skip committing/pushing anon DB images (used for PR runs)'
|
|
required: false
|
|
type: string
|
|
default: 'false'
|
|
secrets:
|
|
username:
|
|
required: true
|
|
description: 'Toska registry username'
|
|
password:
|
|
required: true
|
|
description: 'Toska registry password'
|
|
|
|
jobs:
|
|
# === Static tests / linting ===
|
|
lint_packages:
|
|
name: 'Lint packages'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Lint backend, frontend and shared
|
|
uses: ./.github/actions/lint
|
|
|
|
# === Build database images ===
|
|
build_backend:
|
|
name: 'Build backend'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Build backend
|
|
uses: ./.github/actions/build/backend
|
|
with:
|
|
environment: 'staging'
|
|
release_version: ${{ github.sha }}
|
|
push: false
|
|
|
|
build_frontend:
|
|
name: 'Build frontend'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Build frontend
|
|
uses: ./.github/actions/build/frontend
|
|
with:
|
|
environment: 'staging'
|
|
release_version: ${{ github.sha }}
|
|
push: false
|
|
|
|
# === Cache docker images ===
|
|
pull_docker_images:
|
|
name: Pull all needed docker images and cache them (if dbs have not changed)
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
(vars.RUNS_ON == 'ubuntu-latest' || vars.RUNS_ON == '') &&
|
|
!(
|
|
needs.check_updater_changed.outputs.has_updater_changed == 'true' ||
|
|
needs.check_migration_files.outputs.kone_db_migrations == 'true' ||
|
|
needs.check_migration_files.outputs.user_db_migrations == 'true' ||
|
|
needs.check_migration_files.outputs.sis_db_migrations == 'true'
|
|
)
|
|
needs:
|
|
- check_updater_changed
|
|
- check_migration_files
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Login to toska docker registry
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: Set up cache for pulled docker images (saved in Post-step)
|
|
id: cache-images
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: /tmp/docker-images.tar.zst
|
|
key: docker-images-${{ github.sha }}
|
|
|
|
- name: Pull required images
|
|
if: steps.cache-images.outputs.cache-hit != 'true'
|
|
run: docker compose --file docker-compose.ci.yml --profile oodikone --profile updater pull --ignore-buildable --quiet
|
|
|
|
- name: Save Docker images for downstream jobs
|
|
if: steps.cache-images.outputs.cache-hit != 'true'
|
|
run: |
|
|
docker save \
|
|
registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/kone-db \
|
|
registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/user-db \
|
|
registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-db \
|
|
registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi/sis-importer-db \
|
|
"quay.io/toska/jami:production" \
|
|
redis:7.2.6 \
|
|
postgres:15.10 \
|
|
nginx:latest \
|
|
| zstd -T0 -3 -o /tmp/docker-images.tar.zst
|
|
|
|
# === Update databases ===
|
|
check_migration_files:
|
|
name: Check if new migrations have been added
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
outputs:
|
|
kone_db_migrations: ${{ steps.check_migrations.outputs.kone_db_migrations }}
|
|
user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
|
|
sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Check if new migrations have been added
|
|
id: check_migrations
|
|
uses: dorny/paths-filter@v4
|
|
with:
|
|
filters: |
|
|
kone_db_migrations:
|
|
- 'services/backend/src/database/migrations_kone/**'
|
|
- '.github/workflows/test.yaml'
|
|
user_db_migrations:
|
|
- 'services/backend/src/database/migrations_user/**'
|
|
- '.github/workflows/test.yaml'
|
|
sis_db_migrations:
|
|
- 'updater/sis-updater-worker/src/db/migrations/**'
|
|
- '.github/workflows/test.yaml'
|
|
|
|
build_and_push_anon_sis_db:
|
|
name: 'Build anon sis-db and push to toska hub if new migrations have been added'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
needs.check_migration_files.outputs.sis_db_migrations == 'true' &&
|
|
inputs.skip_db_update != 'true'
|
|
needs:
|
|
- check_migration_files
|
|
- build_backend
|
|
- build_frontend
|
|
- lint_packages
|
|
- test_unit
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
|
|
run: |
|
|
docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
|
|
sleep 30 # wait for sis-updater-worker to start and run migrations
|
|
# TODO: Disable this until fixed
|
|
# ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
|
|
|
|
- name: Teardown containers started by compose
|
|
if: always()
|
|
uses: ./.github/actions/teardown
|
|
|
|
build_and_push_anon_kone_db:
|
|
name: 'Build anon kone-db and push to toska hub if new migrations have been added'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
needs.check_migration_files.outputs.kone_db_migrations == 'true' &&
|
|
inputs.skip_db_update != 'true'
|
|
needs:
|
|
- check_migration_files
|
|
- build_backend
|
|
- build_frontend
|
|
- lint_packages
|
|
- test_unit
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: 'Let backend run migrations, then push new version of image to toska hub'
|
|
run: |
|
|
docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
|
|
sleep 30 # wait for backend to start and run migrations
|
|
# TODO: Disable this until fixed
|
|
# ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
|
|
|
|
- name: Teardown containers started by compose
|
|
if: always()
|
|
uses: ./.github/actions/teardown
|
|
|
|
build_and_push_anon_user_db:
|
|
name: 'Build anon user-db and push to toska hub if new migrations have been added'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
needs.check_migration_files.outputs.user_db_migrations == 'true' &&
|
|
inputs.skip_db_update != 'true'
|
|
needs:
|
|
- check_migration_files
|
|
- build_backend
|
|
- build_frontend
|
|
- lint_packages
|
|
- test_unit
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: 'Let backend run migrations, then push new version of image to toska hub'
|
|
run: |
|
|
docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
|
|
sleep 30 # wait for backend to start and run migrations
|
|
# TODO: Disable this until fixed
|
|
# ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
|
|
|
|
- name: Teardown containers started by compose
|
|
if: always()
|
|
uses: ./.github/actions/teardown
|
|
|
|
# === Update sis-db with updater ===
|
|
check_updater_changed:
|
|
name: 'Check if updater has been updated'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure()
|
|
outputs:
|
|
has_updater_changed: ${{ steps.check_updater_changed.outputs.has_updater_changed }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- uses: dorny/paths-filter@v4
|
|
id: check_updater_changed
|
|
with:
|
|
filters: |
|
|
has_updater_changed:
|
|
- 'updater/**'
|
|
- '.github/workflows/test.yaml'
|
|
- '.github/workflows/staging.yaml'
|
|
- 'docker-compose.ci.yml'
|
|
|
|
run_updater_updates:
|
|
name: 'Run updater tests and updates'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
(needs.check_updater_changed.outputs.has_updater_changed == 'true' ||
|
|
needs.check_migration_files.outputs.sis_db_migrations == 'true')
|
|
needs:
|
|
- check_migration_files
|
|
- check_updater_changed
|
|
- build_and_push_anon_sis_db
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: Start worker containers
|
|
run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-worker --scale sis-updater-worker=3 --quiet-pull --detach --wait
|
|
|
|
- name: Run hourly jobs
|
|
run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
|
|
env:
|
|
SCHEDULE_IMMEDIATE: hourly
|
|
EXIT_AFTER_IMMEDIATES: yes
|
|
|
|
- name: Run weekly jobs
|
|
run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
|
|
env:
|
|
SCHEDULE_IMMEDIATE: weekly
|
|
EXIT_AFTER_IMMEDIATES: yes
|
|
|
|
# TODO: Push the updated sis-db to toska-hub when updater is deterministic with test-data
|
|
# - name: Push newly build db to toska hub
|
|
# if: |
|
|
# success() && inputs.skip_db_update != 'true'
|
|
# run: ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
|
|
|
|
- name: Dump logs on failure
|
|
if: failure()
|
|
run: docker compose --file docker-compose.ci.yml --profile updater logs
|
|
|
|
- name: Teardown containers started by compose
|
|
if: always()
|
|
uses: ./.github/actions/teardown
|
|
|
|
# === Backend api-tests ===
|
|
test_api:
|
|
name: 'Run backend api tests'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- pull_docker_images
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
strategy:
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run api tests
|
|
uses: ./.github/actions/test/api
|
|
with:
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
#
|
|
# === Backend service-tests ===
|
|
test_services:
|
|
name: 'Run backend service-tests'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- pull_docker_images
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
strategy:
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run service-tests
|
|
uses: ./.github/actions/test/services
|
|
with:
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
# === Unit testing ===
|
|
test_unit:
|
|
name: 'Run unit tests'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure()
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run unit tests (frontend and backend)
|
|
uses: ./.github/actions/test/unit
|
|
|
|
# === Frontend component testing ===
|
|
test_components:
|
|
name: 'Test frontend components'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- pull_docker_images
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run frontend component tests
|
|
uses: ./.github/actions/test/component
|
|
|
|
# === End-to-end tests ===
|
|
test_end_to_end_playwright:
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- pull_docker_images
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- name: 'Test population statistics'
|
|
spec: e2e/Population_statistics.spec.ts
|
|
- name: 'Test course statistics'
|
|
spec: e2e/Course_statistics.spec.ts
|
|
- name: 'Test degree programme overview'
|
|
spec: e2e/Degree_programme_overview.spec.ts
|
|
- name: 'Test Faculty statistics'
|
|
spec: e2e/Faculty_statistics.spec.ts
|
|
- name: 'Test teachers'
|
|
spec: e2e/Teachers.spec.ts
|
|
- name: 'Test users'
|
|
spec: e2e/Users.spec.ts
|
|
- name: 'Test access rights'
|
|
spec: e2e/Routes.spec.ts
|
|
- name: 'Test university'
|
|
spec: e2e/University.spec.ts
|
|
- name: 'Test language center view'
|
|
spec: e2e/Language_center.spec.ts
|
|
- name: 'Test study guidance groups'
|
|
spec: e2e/Study_guidance_groups.spec.ts
|
|
- name: 'Test custom population'
|
|
spec: e2e/Custom_population.spec.ts
|
|
- name: 'Test filters'
|
|
spec: e2e/Filters.spec.ts
|
|
- name: 'Test students'
|
|
spec: e2e/Students.spec.ts
|
|
name: ${{ matrix.name }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Install node-24 for playwright
|
|
uses: actions/setup-node@v7
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
npm ci
|
|
|
|
- name: Install Playwright Browsers
|
|
run: npx playwright install firefox --with-deps
|
|
|
|
- name: Restore cached docker images
|
|
id: cache-images-restore
|
|
uses: actions/cache/restore@v6
|
|
with:
|
|
path: |
|
|
/tmp/docker-images.tar.zst
|
|
key: docker-images-${{ github.sha }}
|
|
|
|
- name: Load Docker images (or skip if missing)
|
|
run: |
|
|
if [ -f /tmp/docker-images.tar.zst ]; then
|
|
zstd -d /tmp/docker-images.tar.zst --stdout | docker load
|
|
fi
|
|
|
|
- name: Run tests
|
|
uses: ./.github/actions/test/e2e-playwright
|
|
with:
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
spec: ${{ matrix.spec }}
|
|
test_name: ${{ matrix.name }}
|
|
|
|
test_end_to_end:
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- pull_docker_images
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- name: 'Test completed courses'
|
|
spec: cypress/e2e/Completed_courses.js
|
|
- name: 'Test numbers match'
|
|
spec: cypress/e2e/Numbers_match.js
|
|
name: ${{ matrix.name }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Install node-24 for cypress
|
|
uses: actions/setup-node@v7
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Restore cached docker images
|
|
id: cache-images-restore
|
|
uses: actions/cache/restore@v6
|
|
with:
|
|
path: |
|
|
/tmp/docker-images.tar.zst
|
|
key: docker-images-${{ github.sha }}
|
|
|
|
- name: Load Docker images (or skip if missing)
|
|
run: |
|
|
if [ -f /tmp/docker-images.tar.zst ]; then
|
|
zstd -d /tmp/docker-images.tar.zst --stdout | docker load
|
|
fi
|
|
|
|
- name: Run tests
|
|
uses: ./.github/actions/test/e2e
|
|
with:
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
spec: ${{ matrix.spec }}
|
|
test_name: ${{ matrix.name }}
|
|
|
|
# === Other tests ===
|
|
test_service_exit_statuses:
|
|
name: 'Check services run correctly'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- pull_docker_images
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: Restore cached docker images
|
|
id: cache-images-restore
|
|
uses: actions/cache/restore@v6
|
|
with:
|
|
path: |
|
|
/tmp/docker-images.tar.zst
|
|
key: docker-images-${{ github.sha }}
|
|
|
|
- name: Load Docker images (or skip if missing)
|
|
run: |
|
|
if [ -f /tmp/docker-images.tar.zst ]; then
|
|
zstd -d /tmp/docker-images.tar.zst --stdout | docker load
|
|
fi
|
|
|
|
- name: 'Run services, build rest of the needed images'
|
|
run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull --wait
|
|
|
|
- name: 'Check that exit statuses are not errors'
|
|
run: |
|
|
docker ps -a
|
|
if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
|
|
echo 'Some process had nonzero exit code';
|
|
exit 1;
|
|
fi
|
|
|
|
- name: 'Print logs on failure'
|
|
if: failure()
|
|
run: npm run logs:ci && sleep 15
|