mirror of
https://github.com/UniversityOfHelsinkiCS/oodikone.git
synced 2026-10-12 19:03:39 +03:00
475 lines
15 KiB
YAML
475 lines
15 KiB
YAML
name: 'Lint and test'
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
skip_db_update:
|
|
description: 'When true, skip committing/pushing anon DB images (used for PR runs)'
|
|
required: false
|
|
type: string
|
|
default: 'false'
|
|
secrets:
|
|
username:
|
|
required: true
|
|
description: 'Toska registry username'
|
|
password:
|
|
required: true
|
|
description: 'Toska registry password'
|
|
s3_access_key:
|
|
required: true
|
|
description: 'S3 access key for the test database dumps'
|
|
s3_secret_key:
|
|
required: true
|
|
description: 'S3 secret key for the test database dumps'
|
|
|
|
jobs:
|
|
# === Static tests / linting ===
|
|
lint_packages:
|
|
name: 'Lint packages'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Lint backend, frontend and shared
|
|
uses: ./.github/actions/lint
|
|
|
|
# === Build database images ===
|
|
build_backend:
|
|
name: 'Build backend'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Build backend
|
|
uses: ./.github/actions/build/backend
|
|
with:
|
|
environment: 'staging'
|
|
release_version: ${{ github.sha }}
|
|
push: false
|
|
|
|
build_frontend:
|
|
name: 'Build frontend'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Build frontend
|
|
uses: ./.github/actions/build/frontend
|
|
with:
|
|
environment: 'staging'
|
|
release_version: ${{ github.sha }}
|
|
push: false
|
|
|
|
# === Update databases ===
|
|
check_migration_files:
|
|
name: Check if new migrations have been added
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
outputs:
|
|
kone_db_migrations: ${{ steps.check_migrations.outputs.kone_db_migrations }}
|
|
user_db_migrations: ${{ steps.check_migrations.outputs.user_db_migrations }}
|
|
sis_db_migrations: ${{ steps.check_migrations.outputs.sis_db_migrations }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Check if new migrations have been added
|
|
id: check_migrations
|
|
uses: dorny/paths-filter@v4
|
|
with:
|
|
filters: |
|
|
kone_db_migrations:
|
|
- 'services/backend/src/database/migrations_kone/**'
|
|
- '.github/workflows/test.yaml'
|
|
user_db_migrations:
|
|
- 'services/backend/src/database/migrations_user/**'
|
|
- '.github/workflows/test.yaml'
|
|
sis_db_migrations:
|
|
- 'updater/sis-updater-worker/src/db/migrations/**'
|
|
- '.github/workflows/test.yaml'
|
|
|
|
build_and_push_anon_sis_db:
|
|
name: 'Build anon sis-db and push to toska hub if new migrations have been added'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
needs.check_migration_files.outputs.sis_db_migrations == 'true' &&
|
|
inputs.skip_db_update != 'true'
|
|
needs:
|
|
- check_migration_files
|
|
- build_backend
|
|
- build_frontend
|
|
- lint_packages
|
|
- test_unit
|
|
- test_updater_unit
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
|
|
env:
|
|
ACCESS_KEY: ${{ secrets.s3_access_key }}
|
|
SECRET_KEY: ${{ secrets.s3_secret_key }}
|
|
run: |
|
|
docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker
|
|
sleep 30 # wait for sis-updater-worker to start and run migrations
|
|
sudo apt update -y && sudo apt install s3cmd -y
|
|
./anonyymioodi/manage_test_dbs.sh release sis-db
|
|
|
|
- name: Teardown containers started by compose
|
|
if: always()
|
|
uses: ./.github/actions/teardown
|
|
|
|
build_and_push_anon_kone_db:
|
|
name: 'Build anon kone-db and push to toska hub if new migrations have been added'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
needs.check_migration_files.outputs.kone_db_migrations == 'true' &&
|
|
inputs.skip_db_update != 'true'
|
|
needs:
|
|
- check_migration_files
|
|
- build_backend
|
|
- build_frontend
|
|
- lint_packages
|
|
- test_unit
|
|
- test_updater_unit
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: 'Let backend run migrations, then push new version of image to toska hub'
|
|
env:
|
|
ACCESS_KEY: ${{ secrets.s3_access_key }}
|
|
SECRET_KEY: ${{ secrets.s3_secret_key }}
|
|
run: |
|
|
docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
|
|
sleep 30 # wait for backend to start and run migrations
|
|
sudo apt update -y && sudo apt install s3cmd -y
|
|
./anonyymioodi/manage_test_dbs.sh release kone-db
|
|
|
|
- name: Teardown containers started by compose
|
|
if: always()
|
|
uses: ./.github/actions/teardown
|
|
|
|
build_and_push_anon_user_db:
|
|
name: 'Build anon user-db and push to toska hub if new migrations have been added'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
needs.check_migration_files.outputs.user_db_migrations == 'true' &&
|
|
inputs.skip_db_update != 'true'
|
|
needs:
|
|
- check_migration_files
|
|
- build_backend
|
|
- build_frontend
|
|
- lint_packages
|
|
- test_unit
|
|
- test_updater_unit
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: 'Let backend run migrations, then push new version of image to toska hub'
|
|
env:
|
|
ACCESS_KEY: ${{ secrets.s3_access_key }}
|
|
SECRET_KEY: ${{ secrets.s3_secret_key }}
|
|
run: |
|
|
docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait
|
|
sleep 30 # wait for backend to start and run migrations
|
|
sudo apt update -y && sudo apt install s3cmd -y
|
|
./anonyymioodi/manage_test_dbs.sh release user-db
|
|
|
|
- name: Teardown containers started by compose
|
|
if: always()
|
|
uses: ./.github/actions/teardown
|
|
|
|
# === Update sis-db with updater ===
|
|
check_updater_changed:
|
|
name: 'Check if updater has been updated'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure()
|
|
outputs:
|
|
has_updater_changed: ${{ steps.check_updater_changed.outputs.has_updater_changed }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- uses: dorny/paths-filter@v4
|
|
id: check_updater_changed
|
|
with:
|
|
filters: |
|
|
has_updater_changed:
|
|
- 'updater/**'
|
|
- '.github/workflows/test.yaml'
|
|
- '.github/workflows/staging.yaml'
|
|
- 'docker-compose.ci.yml'
|
|
|
|
run_updater_updates:
|
|
name: 'Run updater tests and updates'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure() &&
|
|
(needs.check_updater_changed.outputs.has_updater_changed == 'true' ||
|
|
needs.check_migration_files.outputs.sis_db_migrations == 'true')
|
|
needs:
|
|
- check_migration_files
|
|
- check_updater_changed
|
|
- build_and_push_anon_sis_db
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: Start worker containers
|
|
run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-worker --scale sis-updater-worker=3 --quiet-pull --detach --wait
|
|
|
|
- name: Run hourly jobs
|
|
run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
|
|
env:
|
|
SCHEDULE_IMMEDIATE: hourly
|
|
EXIT_AFTER_IMMEDIATES: yes
|
|
|
|
- name: Run weekly jobs
|
|
run: docker compose -f docker-compose.ci.yml --profile updater up sis-updater-scheduler --exit-code-from sis-updater-scheduler
|
|
env:
|
|
SCHEDULE_IMMEDIATE: weekly
|
|
EXIT_AFTER_IMMEDIATES: yes
|
|
|
|
# TODO: Push the updated sis-db to toska-hub when updater is deterministic with test-data
|
|
- name: Push newly build db to toska hub
|
|
if: |
|
|
success() && inputs.skip_db_update != 'true'
|
|
env:
|
|
ACCESS_KEY: ${{ secrets.s3_access_key }}
|
|
SECRET_KEY: ${{ secrets.s3_secret_key }}
|
|
run: |
|
|
sudo apt update -y && sudo apt install s3cmd -y
|
|
./anonyymioodi/manage_test_dbs.sh release sis-db
|
|
|
|
- name: Dump logs on failure
|
|
if: failure()
|
|
run: docker compose --file docker-compose.ci.yml --profile updater logs
|
|
|
|
- name: Teardown containers started by compose
|
|
if: always()
|
|
uses: ./.github/actions/teardown
|
|
|
|
# === Backend api-tests ===
|
|
test_api:
|
|
name: 'Run backend api tests'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run api tests
|
|
uses: ./.github/actions/test/api
|
|
with:
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
#
|
|
# === Backend service-tests ===
|
|
test_services:
|
|
name: 'Run backend service-tests'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run service-tests
|
|
uses: ./.github/actions/test/services
|
|
with:
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
# === Unit testing ===
|
|
test_unit:
|
|
name: 'Run unit tests'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure()
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run unit tests (frontend and backend)
|
|
uses: ./.github/actions/test/unit
|
|
|
|
# === Updater unit testing ===
|
|
test_updater_unit:
|
|
name: 'Run updater unit tests'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure()
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run updater unit tests
|
|
uses: ./.github/actions/test/updater-unit
|
|
|
|
# === Frontend component testing ===
|
|
test_components:
|
|
name: 'Test frontend components'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Run frontend component tests
|
|
uses: ./.github/actions/test/component
|
|
|
|
# === End-to-end tests ===
|
|
test_end_to_end_playwright:
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- name: 'Population statistics'
|
|
spec: e2e/Population_statistics.spec.ts
|
|
- name: 'Course statistics'
|
|
spec: e2e/Course_statistics.spec.ts
|
|
- name: 'Degree programme overview'
|
|
spec: e2e/Degree_programme_overview.spec.ts
|
|
- name: 'faculty statistics'
|
|
spec: e2e/Faculty_statistics.spec.ts
|
|
- name: 'Teachers'
|
|
spec: e2e/Teachers.spec.ts
|
|
- name: 'Users'
|
|
spec: e2e/Users.spec.ts
|
|
- name: 'Access rights'
|
|
spec: e2e/Routes.spec.ts
|
|
- name: 'University'
|
|
spec: e2e/University.spec.ts
|
|
- name: 'Language center view'
|
|
spec: e2e/Language_center.spec.ts
|
|
- name: 'Study guidance groups'
|
|
spec: e2e/Study_guidance_groups.spec.ts
|
|
- name: 'Custom population'
|
|
spec: e2e/Custom_population.spec.ts
|
|
- name: 'Filters'
|
|
spec: e2e/Filters.spec.ts
|
|
- name: 'Students'
|
|
spec: e2e/Students.spec.ts
|
|
- name: 'Completed courses'
|
|
spec: e2e/Completed_courses.spec.ts
|
|
- name: 'Numbers match'
|
|
spec: e2e/Numbers_match.spec.ts
|
|
name: ${{ matrix.name }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Install node-24 for playwright
|
|
uses: actions/setup-node@v7
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
npm ci
|
|
|
|
- name: Install Playwright Browsers
|
|
run: npx playwright install firefox --with-deps
|
|
|
|
- name: Run tests
|
|
uses: ./.github/actions/test/e2e-playwright
|
|
with:
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
spec: ${{ matrix.spec }}
|
|
test_name: ${{ matrix.name }}
|
|
|
|
# === Other tests ===
|
|
test_service_exit_statuses:
|
|
name: 'Check services run correctly'
|
|
runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }}
|
|
timeout-minutes: 15
|
|
if: |
|
|
!cancelled() && !failure()
|
|
needs:
|
|
- run_updater_updates
|
|
- build_and_push_anon_user_db
|
|
- build_and_push_anon_kone_db
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: 'Login to toska docker registry'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi
|
|
username: ${{ secrets.username }}
|
|
password: ${{ secrets.password }}
|
|
|
|
- name: 'Run services, build rest of the needed images'
|
|
run: docker compose --file docker-compose.ci.yml --profile oodikone up --detach --quiet-pull --wait
|
|
|
|
- name: 'Check that exit statuses are not errors'
|
|
run: |
|
|
docker ps -a
|
|
if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
|
|
echo 'Some process had nonzero exit code';
|
|
exit 1;
|
|
fi
|
|
|
|
- name: 'Print logs on failure'
|
|
if: failure()
|
|
run: npm run logs:ci && sleep 15
|