diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index d43432563..938310ae0 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -26,3 +26,5 @@ jobs: secrets: username: ${{ secrets.TOSKAREGISTRY_USERNAME }} password: ${{ secrets.TOSKAREGISTRY_PASSWORD }} + s3_access_key: ${{ secrets.S3_ACCESS_KEY }} + s3_secret_key: ${{ secrets.S3_SECRET_KEY }} diff --git a/.github/workflows/staging.yaml b/.github/workflows/staging.yaml index 89ca8b870..4d3ef10bc 100644 --- a/.github/workflows/staging.yaml +++ b/.github/workflows/staging.yaml @@ -23,6 +23,8 @@ jobs: secrets: username: ${{ secrets.TOSKAREGISTRY_USERNAME }} password: ${{ secrets.TOSKAREGISTRY_PASSWORD }} + s3_access_key: ${{ secrets.S3_ACCESS_KEY }} + s3_secret_key: ${{ secrets.S3_SECRET_KEY }} release_backend: name: 'Build and release backend' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5f6d9ff77..4ed3b7ac7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,6 +15,12 @@ on: password: required: true description: 'Toska registry password' + s3_access_key: + required: true + description: 'S3 access key for the test database dumps' + s3_secret_key: + required: true + description: 'S3 secret key for the test database dumps' jobs: # === Static tests / linting === @@ -107,10 +113,13 @@ jobs: password: ${{ secrets.password }} - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub' + env: + ACCESS_KEY: ${{ secrets.s3_access_key }} + SECRET_KEY: ${{ secrets.s3_secret_key }} run: | docker compose --file docker-compose.ci.yml up --detach --quiet-pull sis-updater-worker sleep 30 # wait for sis-updater-worker to start and run migrations - # ./anonyymioodi/db_utils.sh local sis-db + # ./anonyymioodi/manage_test_dbs.sh release sis-db - name: Teardown containers started by compose if: always() @@ -143,10 +152,13 @@ jobs: password: ${{ secrets.password }} - name: 'Let backend run migrations, then push new version of image to toska hub' + env: + ACCESS_KEY: ${{ secrets.s3_access_key }} + SECRET_KEY: ${{ secrets.s3_secret_key }} run: | docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait sleep 30 # wait for backend to start and run migrations - # ./anonyymioodi/db_utils.sh local kone-db + # ./anonyymioodi/manage_test_dbs.sh release kone-db - name: Teardown containers started by compose if: always() @@ -179,10 +191,13 @@ jobs: password: ${{ secrets.password }} - name: 'Let backend run migrations, then push new version of image to toska hub' + env: + ACCESS_KEY: ${{ secrets.s3_access_key }} + SECRET_KEY: ${{ secrets.s3_secret_key }} run: | docker compose --file docker-compose.ci.yml up --detach --quiet-pull backend --wait sleep 30 # wait for backend to start and run migrations - # ./anonyymioodi/db_utils.sh local user-db + # ./anonyymioodi/manage_test_dbs.sh release user-db - name: Teardown containers started by compose if: always() @@ -252,7 +267,10 @@ jobs: # - name: Push newly build db to toska hub # if: | # success() && inputs.skip_db_update != 'true' - # run: ./anonyymioodi/db_utils.sh local sis-db + # env: + # ACCESS_KEY: ${{ secrets.s3_access_key }} + # SECRET_KEY: ${{ secrets.s3_secret_key }} + # run: ./anonyymioodi/manage_test_dbs.sh release sis-db - name: Dump logs on failure if: failure() diff --git a/anonyymioodi/README.md b/anonyymioodi/README.md index 3bfc857d1..91c1e22b5 100644 --- a/anonyymioodi/README.md +++ b/anonyymioodi/README.md @@ -4,6 +4,29 @@ This folder contains a guide and scripts to modify or create anonymous database The quick guide should be enough for most purposes: updating schema, data, or the PostgreSQL version. If you need to generate completely new `sis-importer-db` and `sis-db` images (which shouldn't happen too often), follow the instructions in the section [Generate new sis-importer-db and sis-db completely from scratch](#generate-new-sis-importer-db-and-sis-db-completely-from-scratch). +## Scripts + +`manage_test_dbs.sh` is the front end for the test database tooling. Run it without arguments for an interactive menu, or give it an action and one or more databases (or `all`) to run it directly: + +```bash +./manage_test_dbs.sh dump sis-db # dump sis-db from its running container +./manage_test_dbs.sh upload sis-db # upload the current dump to s3 +./manage_test_dbs.sh release all # dump, upload and push images for every database +``` + +Actions are `download`, `dump`, `upload`, `push`, `release` (dump + upload + push) and `rebuild` (download + push). Each one is a thin call to a script in this folder, which can also be used on its own: + +| Script | What it does | +| -------------------------- | ----------------------------------------------------------------------------------------------------- | +| `dump_local_db.sh` | `pg_dump`s the databases from the running containers into `.databasedumps/` | +| `download_dump_from_s3.sh` | Downloads the newest dump of each database from s3 (`-b ` to pick dumps from a single branch) | +| `upload_dump_to_s3.sh` | Uploads the dumps in `.databasedumps/test` to s3 as `--.sql` | +| `build_and_push_image.sh` | Builds database images from the current dumps and pushes them to the Toska registry | +| `create_s3_bucket.sh` | Creates the s3 bucket if it doesn't exist yet, run automatically before an upload | +| `lib/common.sh` | Shared configuration (dump directories, s3 bucket, registry, database names) | + +Dumps end up in `.databasedumps/test/.sql`, which is where `db.Dockerfile` reads them from when an image is built. The s3 commands read their credentials from the `ACCESS_KEY` and `SECRET_KEY` environment variables (e.g. in CI). If neither is set, they fall back to `~/.s3cfg`, which can be found on the toska/dokumentaatio repo. + ## Quick guide to updating a test database 1. Ensure you have the latest test-db. Just pulling images might not replace your existing volume, so you may need to remove it manually. This command might work: `docker volume rm oodikone-sis-db-data` (replace `oodikone-sis-db-data` with the correct volume name). The new image, including data, should be pulled the next time you launch Oodikone normally. diff --git a/anonyymioodi/build_and_push_image.sh b/anonyymioodi/build_and_push_image.sh new file mode 100755 index 000000000..a466d6ced --- /dev/null +++ b/anonyymioodi/build_and_push_image.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Build database images from the dumps in .databasedumps/test and push them to +# the Toska registry. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source-path=SCRIPTDIR +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR"/lib/common.sh + +usage() { + cat >&2 <... | all + +Builds an image of each given database and pushes it to $REGISTRY. +The dump is read from .databasedumps/test/.sql, so run +dump_local_db.sh or download_dump_from_s3.sh first. + +Databases: ${DATABASES[*]} +EOF +} + +if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then + usage + exit 0 +fi + +if [[ $# -eq 0 ]]; then + usage + die "No database given" +fi + +build_and_push_image() { + local database="$1" + local image="$REGISTRY/$database:latest" + local dump_file="$TEST_DUMP_DIR/$database.sql" + + [[ -f "$dump_file" ]] || die "No dump found at $dump_file, run dump_local_db.sh or download_dump_from_s3.sh first" + + infomsg "Building $image" + docker build "$PROJECT_ROOT" -f "$PROJECT_ROOT/db.Dockerfile" -t "$image" --build-arg DB_NAME="$database" + + infomsg "Pushing $image" + docker push "$image" + successmsg "Pushed $image" +} + +resolve_databases "$@" + +for database in "${selected_databases[@]}"; do + build_and_push_image "$database" +done diff --git a/anonyymioodi/create_s3_bucket.sh b/anonyymioodi/create_s3_bucket.sh new file mode 100755 index 000000000..c8933307f --- /dev/null +++ b/anonyymioodi/create_s3_bucket.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Create the s3 bucket used for the test database dumps, if it doesn't exist yet. +# Doing nothing when the bucket is already there, so this is safe to run every time. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source-path=SCRIPTDIR +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR"/lib/common.sh + +usage() { + cat >&2 < /dev/null 2>&1; then + infomsg "Bucket $S3_BUCKET already exists" + exit 0 +fi + +warningmsg "Bucket $S3_BUCKET doesn't exist, creating it" +s3 mb "$S3_BUCKET" +successmsg "Created bucket $S3_BUCKET" diff --git a/anonyymioodi/db_util.sh b/anonyymioodi/db_util.sh deleted file mode 100755 index 860ae1f21..000000000 --- a/anonyymioodi/db_util.sh +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env bash - -# === Config === - -# Quit if failing -set -euo pipefail - -# Print messages and logs that are not script output to stderr -msg() { - echo >&2 -e "${1-}" -} - -# Some special types of messages with colours -successmsg() { - msg "${GREEN}$1${NOFORMAT}" -} -errormsg() { - msg "${RED}$1${NOFORMAT}" -} -infomsg() { - msg "${BLUE}$1${NOFORMAT}" -} -warningmsg() { - msg "${ORANGE}$1${NOFORMAT}" -} - -# Setup colors for messages if running interactive shell -if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then - NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m' -else - NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE='' -fi - - -# === # - -PROJECT_ROOT="$(git rev-parse --show-toplevel)" - -get_project_relative_path () { - local absolute_path="$1" - local root="$2" - realpath -s --relative-to="${root:-$PROJECT_ROOT}" "$absolute_path" -} - -LOCAL_DUMP_DIR="$(get_project_relative_path ".databasedumps/local_containers" "$PWD")" # Dumps from running docker containers -S3_DUMP_DIR="$(get_project_relative_path ".databasedumps/s3" "$PWD")" # Dumps from s3 -TEST_DUMP_DIR="$(get_project_relative_path ".databasedumps/test" "$PWD")" # Location where .sql files are pulled while building the new image - -S3_CONFIG_FILE=~/.s3cfg -S3_BUCKET="s3://oodikone-test" - -REGISTRY="registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi" - -## Following the naming convention in docker compose, these are names for database -## services. The real data databases inside service will have names suffixed by "-real". -KONE_DB_NAME="kone-db" -SIS_DB_NAME="sis-db" -SIS_IMPORTER_DB_NAME="sis-importer-db" -USER_DB_NAME="user-db" -JAMI_DB_NAME="jami-db" -DATABASES=("$USER_DB_NAME" "$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$JAMI_DB_NAME") - -mkdir -p "$TEST_DUMP_DIR" -mkdir -p "$S3_DUMP_DIR" -mkdir -p "$LOCAL_DUMP_DIR" - - -# === Handle s3 bucket === -# Use s3 bucket as the dump save/pull location -download_s3_db () { - local database="$1" - - # TODO: Decompress compressed files - s3cmd -c "$S3_CONFIG_FILE" get "$S3_BUCKET/$database.sql" "$S3_DUMP_DIR/$database.sql" - cp --remove-destination -r "$S3_DUMP_DIR" "$TEST_DUMP_DIR" # Move dumps to the place where docker build pulls .sql files from -} - -upload_s3_db () { - local database="$1" - - if [[ $(s3cmd ls s3:// | grep -q oodikone-test; echo $?) -eq 0 ]] - then - infomsg "OODIKONE-TEST bucket exists, skipping creation"; - else - warningmsg "OODIKONE-TEST bucket doesn't exist, creating new bucket..."; - s3cmd -c "$S3_CONFIG_FILE" mb "s3://oodikone-test" - fi - - infomsg "Pushing $TEST_DUMP_DIR/$database.sql to $S3_BUCKET" - # TODO: Also compress files - s3cmd -c "$S3_CONFIG_FILE" put "$TEST_DUMP_DIR/$database.sql" "$S3_BUCKET/$database.sql" -} - - -# === Build images === -# Update registry with an existing dump located in .databasedumps/test/$1.sql -build_and_push_image () { - local database="$1" - - infomsg "Building docker image for $database\n" - docker build "$(get_project_relative_path "$PROJECT_ROOT" "$PWD")" -t "$REGISTRY/$database:latest" -f db.Dockerfile --build-arg DB_NAME="$database" - successmsg "$database docker image built" - infomsg "Pushing $database to $REGISTRY\n" - infomsg "$REGISTRY/$database" - docker push "$REGISTRY/$database" - successmsg "Image $REGISTRY/$database pushed successfully" -} - - -# === Handle dumping local dbs === -# To conveniently update all dbs, build all with docker-compose.db.yml and then run dump_all_local_dbs -dump_local_db () { - local database="$1" - local container_name="oodikone-$1" - local database_name="$database" - - if [[ $database == "jami-db" ]]; then database_name="postgres"; fi - - infomsg "Trying to dump $database to $LOCAL_DUMP_DIR/$database.sql" - docker exec -i "$container_name" pg_dump -Fp -U postgres -d "$database_name" > "$LOCAL_DUMP_DIR/$database.sql" - successmsg "Dumped $database to $LOCAL_DUMP_DIR/$database.sql" - cp --remove-destination -r "$LOCAL_DUMP_DIR/$database.sql" "$TEST_DUMP_DIR" # Move dump to the place where docker build pulls the .sql file from -} - - -# === Helpers === -do_for_all () { - local cmd="$1" - shift # Removes $1 from "$@" - - infomsg "Running $cmd for all databases" - for database in "${DATABASES[@]}"; do - "$cmd" "$database" "$@" - done - successmsg "$cmd ran successfully" -} - -ask_for_input () { - local cmd="$1" - shift # Removes $1 from "$@" - - echo - - init_database_menu - select database in "${options[@]}"; do - "$cmd" "$database" "$@" - break - done -} - - -init_action_menu () { - PS3="Please enter your choice: " - - options=( - "Pull dumps from s3" - "Dump from running docker images" - "Update dumps in s3" - "Push new images to registry" - ) -} - -init_mode_menu () { - PS3="Select command mode: " - options=( - "Run command for all databases" - "Select single database" - ) -} - -init_database_menu () { - PS3="Select database: " - options=("${DATABASES[@]}") -} - - -# === RUN === - -# For running as non-interactive CI-mode -# $1: Command to be run -# $2: Database name -case "$1" in - "local") - if [ "$#" -lt 2 ]; then echo "Missing database name"; exit 1; fi - dump_local_db "$2" - build_and_push_image "$2" - exit 0;; - - # TODO: NOT YET IMPLEMENTED, should use s3 storage - "ci") - exit 1 - # if [ "$#" -lt 2 ]; then echo "Missing database name"; exit 1; fi - # download_s3_db "$2" - # build_and_push_image "$2" - # exit 0;; -esac - -# For interactive running -while true; do - init_mode_menu - select mode in "${options[@]}"; do - case "$mode" in - "Run command for all databases") - cmd="do_for_all" - break;; - "Select single database") - cmd="ask_for_input" - break;; - esac - done - - echo - - echo - - init_action_menu - select opt in "${options[@]}"; do - case "$opt" in - "Pull dumps from s3") - "$cmd" download_s3_db "$S3_DUMP_DIR" - break;; - "Dump from running docker images") - "$cmd" dump_local_db "$LOCAL_DUMP_DIR" - break;; - "Update dumps in s3") - "$cmd" upload_s3_db - break;; - "Push new images to registry") - "$cmd" build_and_push_image - break;; - esac - done - - echo -done diff --git a/anonyymioodi/download_dump_from_s3.sh b/anonyymioodi/download_dump_from_s3.sh new file mode 100755 index 000000000..ea3727ed3 --- /dev/null +++ b/anonyymioodi/download_dump_from_s3.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +# Download database dumps from s3 into .databasedumps/s3, and copy them to +# .databasedumps/test where db.Dockerfile reads them from. +# +# Dumps are uploaded as --.sql (see +# upload_dump_to_s3.sh), so the newest matching dump is the one downloaded. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source-path=SCRIPTDIR +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR"/lib/common.sh + +usage() { + cat >&2 <] ... | all + +Downloads the newest dump of each given database from $S3_TARGET. +Databases: ${DATABASES[*]} + +Options: + -b, --branch Only consider dumps uploaded from this git branch + -h, --help Show this help +EOF +} + +branch="" +arguments=() + +while [[ $# -gt 0 ]]; do + case "$1" in + -b | --branch) + branch="${2-}" + [[ -n "$branch" ]] || { + usage + die "Missing value for $1" + } + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + -*) + usage + die "Unknown option: $1" + ;; + *) + arguments+=("$1") + shift + ;; + esac +done + +# Newest object in s3 for the given database, or empty if there are none +newest_dump_in_s3() { + local database="$1" + local prefix="$S3_TARGET/$database-" + if [[ -n "$branch" ]]; then prefix="$prefix$(sanitize_branch_name "$branch")-"; fi + + # s3cmd ls prints "