From 84e8015192d04c2d354c0fc2db5521b372d72b43 Mon Sep 17 00:00:00 2001 From: Otto Ahoniemi Date: Thu, 9 Dec 2021 01:31:29 +0200 Subject: [PATCH] Move anonyymioodi scripts and readme to this main repo --- README.md | 2 +- anonyymioodi/.gitignore | 2 + anonyymioodi/README.md | 65 +++++++++ anonyymioodi/commit_and_push_to_toska_hub.sh | 49 +++++++ anonyymioodi/create_container_from_dump.sh | 64 +++++++++ anonyymioodi/docker-compose.yml | 50 +++++++ anonyymioodi/enrich_sampledata.sql | 133 +++++++++++++++++++ 7 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 anonyymioodi/.gitignore create mode 100644 anonyymioodi/README.md create mode 100755 anonyymioodi/commit_and_push_to_toska_hub.sh create mode 100755 anonyymioodi/create_container_from_dump.sh create mode 100644 anonyymioodi/docker-compose.yml create mode 100644 anonyymioodi/enrich_sampledata.sql diff --git a/README.md b/README.md index 0be4dc3ec..464d9f52b 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ Make sure you have your VPN on ### I made a change to sis-db models aand created a migration, but tests are failing in CI! -Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Head to [anonyymioodi repository](https://github.com/UniversityOfHelsinkiCS/anonyymioodi) to find more about how to to update the anon sis-db. +Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db. ### Everything is broken, can't get oodikone running, data is not there etc. diff --git a/anonyymioodi/.gitignore b/anonyymioodi/.gitignore new file mode 100644 index 000000000..fd0830dca --- /dev/null +++ b/anonyymioodi/.gitignore @@ -0,0 +1,2 @@ +*.sqz + diff --git a/anonyymioodi/README.md b/anonyymioodi/README.md new file mode 100644 index 000000000..798a16289 --- /dev/null +++ b/anonyymioodi/README.md @@ -0,0 +1,65 @@ +# Anonyymioodi - anonymous database setup for oodikone + +(name anonyymioodi for :culture: reasons) + +This repository contains a guide and scripts to create docker images from database dumps and/or push database images to Toska Hub docker registry. Images are mostly used by oodikone. + +## How to update database images + +Updating images varies a bit depending on the db you're trying to update. Complete steps as defined below and run `./commit_and_push_to_toska_hub.sh ` to publish the image you've created. + +### Sis-importer-db + +The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db. + +- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump +- Move `sis-importer-db.sqz` to this folder +- Run `./create_container_from_dump.sh sis-importer-db` +- Enrich data according to what's in `enrich_data.sql` file + +### Sis-db: + +The following steps are for building a clean sis-importer-db image from scratch using the sis-updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db. + +- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml +- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo +- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null` +- Start updater, see oodikone readme for more info. +- Populate sis-db from sis-importer-db: + +``` +curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \ +curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \ +curl --request GET --url 'http://localhost:8082/v1/students?token=dev' +``` + +(It might be a good idea to follow sis-updater-worker logs while updater is running) + +- Clean up any extra stuff you don't want to keep in sis-db. +- (Bonus: check that cypress tests run okay) + +### User-db / kone-db + +- Temporarily replace user-db/kone-db -service in oodikone's docker-compose.yml with the same service from this repo's docker-compose.yml +- Ensure the newest user-db/kone-db image is pulled from the registry with `docker pull` +- Start oodikone, let it run migrations +- Modify db to wanted state. Tips: + - Easy way to create new users is to modify dev user details in the frontend API config. Userservice creates a new user during login if the given uid is not present in user-db. + +By default, previous user-db/kone-db images are used as a base. If you want to build the image from a clean Postgres image, just comment out toska hub image and uncomment the clean Postgres image. + +## How to modify database images + +If you need to clean up / modify / do whatever with dbs, either use `psql` directly in docker by running `docker exec -it psql -U postgres ` or start adminer with oodikone's docker-compose. + +## How to update Postgres versions + +When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following: + +- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull `. +- Start database with docker-compose +- Create dump from database by running `docker exec -i pg_dump -Fc -U postgres > .sqz` +- Close container with docker-compose down. +- Update the Postgres version in docker-compose.yml in this folder. +- Uncomment row with Postgres image and comment row with Toskahub image. +- Run `./create_container_from_dump.sh `, followed by `./commit_and_push_to_toska_hub.sh ` diff --git a/anonyymioodi/commit_and_push_to_toska_hub.sh b/anonyymioodi/commit_and_push_to_toska_hub.sh new file mode 100755 index 000000000..1127b2d13 --- /dev/null +++ b/anonyymioodi/commit_and_push_to_toska_hub.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# === Config === + +# Quit if failing +set -euo pipefail + +# Print messages and logs that are not script output to stderr +msg() { + echo >&2 -e "${1-}" +} + +# Some special types of messages with colours +successmsg() { + msg "${GREEN}$1${NOFORMAT} + " +} +errormsg() { + msg "${RED}$1${NOFORMAT} + " +} +infomsg() { + msg "${BLUE}$1${NOFORMAT} + " +} +warningmsg() { + msg "${ORANGE}$1${NOFORMAT} + " +} + +# Setup colors for messages if running interactive shell +if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then + NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m' +else + NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE='' +fi + +args=("$@") +if [[ ${#args[@]} -ne 1 ]]; then + errormsg "No database given, exiting" + exit 1 +fi + +infomsg "Committing and pushing to registry" +database=${args[0]} +docker commit "$database" "registry.toska.cs.helsinki.fi/$database" +docker push "registry.toska.cs.helsinki.fi/$database" +successmsg "Succesfully pushed" +docker-compose down --remove-orphans diff --git a/anonyymioodi/create_container_from_dump.sh b/anonyymioodi/create_container_from_dump.sh new file mode 100755 index 000000000..9601f7504 --- /dev/null +++ b/anonyymioodi/create_container_from_dump.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# === Config === + +# Quit if failing +set -euo pipefail + +# Print messages and logs that are not script output to stderr +msg() { + echo >&2 -e "${1-}" +} + +# Some special types of messages with colours +successmsg() { + msg "${GREEN}$1${NOFORMAT} + " +} +errormsg() { + msg "${RED}$1${NOFORMAT} + " +} +infomsg() { + msg "${BLUE}$1${NOFORMAT} + " +} +warningmsg() { + msg "${ORANGE}$1${NOFORMAT} + " +} + +# Setup colors for messages if running interactive shell +if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then + NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m' +else + NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE='' +fi + +# === Create images === + +retry () { + sleep 5 + for i in {1..60}; do + "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5; + done +} + +args=("$@") +if [[ ${#args[@]} -ne 1 ]]; then + errormsg "No database given, exiting" + exit 1 +fi + + +infomsg "Restoring PostgreSQL dumps from backups. This might take a while." +database=${args[0]} +docker-compose down --remove-orphans +docker-compose up -d "$database" + +database_dump="$database.sqz" +infomsg "Attempting to restore database $database from dump $database_dump" +retry docker-compose exec "$database" pg_isready --dbname="$database" +docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \ +--format=custom --dbname=$database --no-owner" < "$database_dump" +successmsg "Database $database succesfully created" diff --git a/anonyymioodi/docker-compose.yml b/anonyymioodi/docker-compose.yml new file mode 100644 index 000000000..fd9ab2cbc --- /dev/null +++ b/anonyymioodi/docker-compose.yml @@ -0,0 +1,50 @@ +x-defaults: &defaults + restart: unless-stopped + +services: + kone-db: + <<: *defaults + container_name: kone-db + environment: + PGDATA: /data + POSTGRES_DB: kone-db + POSTGRES_HOST_AUTH_METHOD: trust + image: registry.toska.cs.helsinki.fi/kone-db + # image: postgres:13.4-alpine + profiles: + - oodikone + + sis-db: + <<: *defaults + container_name: sis-db + environment: + PGDATA: /data + POSTGRES_DB: sis-db + POSTGRES_HOST_AUTH_METHOD: trust + # image: registry.toska.cs.helsinki.fi/sis-db + image: postgres:13.2 + shm_size: 2gb + + sis-importer-db: + <<: *defaults + container_name: sis-importer-db + environment: + PGDATA: /data + POSTGRES_DB: sis-importer-db + POSTGRES_HOST_AUTH_METHOD: trust + # image: registry.toska.cs.helsinki.fi/sis-importer-db + image: postgres:13.1 + profiles: + - updater + + user-db: + <<: *defaults + container_name: user-db + environment: + PGDATA: /data + POSTGRES_DB: user-db + POSTGRES_HOST_AUTH_METHOD: trust + image: registry.toska.cs.helsinki.fi/user-db + # image: postgres:13.4-alpine + profiles: + - oodikone diff --git a/anonyymioodi/enrich_sampledata.sql b/anonyymioodi/enrich_sampledata.sql new file mode 100644 index 000000000..7384af648 --- /dev/null +++ b/anonyymioodi/enrich_sampledata.sql @@ -0,0 +1,133 @@ +-- add these to sis-importer db by running: +-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql + +-- if you need to run updater for these students only, see studentnumberlist in the end +-- this file + +-- Add admission types (valintaperusteet) to studyrights for some of the computer +-- science 2018-2019 population students. See person_ids / student_number below +-- (in case you need to update dumps). + +-- TODO: this should be moved into the sampledata creator script in +-- importer repo, but since this file contains studentnumbers and person ids, it's +-- better to keep this here + +-- KP (koepisteet) +-- hy-hlo-1412620 | 013683126 +-- hy-hlo-1485246 | 014340523 +-- hy-hlo-32230770 | 010589388 +-- hy-hlo-35626233 | 010775888 +-- hy-hlo-40150144 | 011065995 +UPDATE studyrights +SET admission_type_urn = 'urn:code:admission-type:kp' +WHERE person_id IN ( +'hy-hlo-1412620', +'hy-hlo-1485246', +'hy-hlo-32230770', +'hy-hlo-35626233', +'hy-hlo-40150144' +); + +-- TV (todistusvalinta) +-- hy-hlo-45783433 | 011321150 +-- hy-hlo-48537946 | 010764653 +-- hy-hlo-49801905 | 011194400 +-- hy-hlo-53321302 | 011338871 +-- hy-hlo-51223300 | 013951984 +-- hy-hlo-53310137 | 012520688 +-- hy-hlo-53621583 | 011196796 +-- hy-hlo-53863064 | 011736884 +-- hy-hlo-57711592 | 010262566 +-- hy-hlo-57732872 | 013670647 +UPDATE studyrights +SET admission_type_urn = 'urn:code:admission-type:tv' +WHERE person_id IN ( +'hy-hlo-45783433', +'hy-hlo-48537946', +'hy-hlo-49801905', +'hy-hlo-53321302', +'hy-hlo-51223300', +'hy-hlo-53310137', +'hy-hlo-53621583', +'hy-hlo-53863064', +'hy-hlo-57711592', +'hy-hlo-57732872' +); + +-- YP: +-- hy-hlo-67768022 | 011952325 +-- hy-hlo-73740025 | 011736606 +-- hy-hlo-74780917 | 012775433 +-- hy-hlo-79615371 | 014113941 +-- hy-hlo-78118455 | 014300615 +UPDATE studyrights +SET admission_type_urn = 'urn:code:admission-type:yp' +WHERE person_id IN ( +'hy-hlo-67768022', +'hy-hlo-73740025', +'hy-hlo-74780917', +'hy-hlo-79615371', +'hy-hlo-78118455' +); +-- AV: +-- hy-hlo-78777265 | 010779392 +-- hy-hlo-81909550 | 011894153 +-- hy-hlo-84571890 | 013897158 +-- hy-hlo-80330834 | 015085926 +-- hy-hlo-84775722 | 011791272 +UPDATE studyrights +SET admission_type_urn = 'urn:code:admission-type:av' +WHERE person_id IN ( +'hy-hlo-78777265', +'hy-hlo-81909550', +'hy-hlo-84571890', +'hy-hlo-80330834', +'hy-hlo-84775722' +); +-- M: +-- hy-hlo-87439827 | 011928456 +-- hy-hlo-96612371 | 014418361 +-- hy-hlo-84840261 | 010975895 +-- hy-hlo-90330681 | 010522688 +-- hy-hlo-96594603 | 013951573 +UPDATE studyrights +SET admission_type_urn = 'urn:code:admission-type:m' +WHERE person_id IN ( +'hy-hlo-87439827', +'hy-hlo-96612371', +'hy-hlo-84840261', +'hy-hlo-90330681', +'hy-hlo-96594603' +); + +-- all studentnumbers for running updater: +013683126 +014340523 +010589388 +010775888 +011065995 +011321150 +010764653 +011194400 +011338871 +013951984 +012520688 +011196796 +011736884 +010262566 +013670647 +011952325 +011736606 +012775433 +014113941 +014300615 +010779392 +011894153 +013897158 +015085926 +011791272 +011928456 +014418361 +010975895 +010522688 +013951573