[Scripts] Add dump cleaner so that images don't accumulate over time too much

anonyymioodi/README.md

@@ -14,16 +14,17 @@ The quick guide should be enough for most purposes: updating schema, data, or th
 ./manage_test_dbs.sh release all      # dump, upload and push images for every database
 ```
 
-Actions are `download`, `dump`, `upload`, `push`, `release` (dump + upload + push) and `rebuild` (download + push). Each one is a thin call to a script in this folder, which can also be used on its own:
+Actions are `download`, `dump`, `upload` (upload + prune), `push`, `prune`, `release` (dump + upload + prune + push) and `rebuild` (download + push). Each one is a thin call to a script in this folder, which can also be used on its own:
 
-| Script                     | What it does                                                                                          |
+| Script                     | What it does                                                                                                           |
-| -------------------------- | ----------------------------------------------------------------------------------------------------- |
+| -------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
-| `dump_local_db.sh`         | `pg_dump`s the databases from the running containers into `.databasedumps/`                           |
+| `dump_local_db.sh`         | `pg_dump`s the databases from the running containers into `.databasedumps/`                                            |
-| `download_dump_from_s3.sh` | Downloads the newest dump of each database from s3 (`-b <branch>` to pick dumps from a single branch) |
+| `download_dump_from_s3.sh` | Downloads the newest dump of each database from s3 (`-b <branch>` to pick dumps from a single branch)                  |
-| `upload_dump_to_s3.sh`     | Uploads the dumps in `.databasedumps/test` to s3 as `<database>-<git_branch>-<datetime>.sql`          |
+| `upload_dump_to_s3.sh`     | Uploads the dumps in `.databasedumps/test` to s3 as `<database>-<git_branch>-<datetime>.sql`                           |
-| `build_and_push_image.sh`  | Builds database images from the current dumps and pushes them to the Toska registry                   |
+| `prune_s3_dumps.sh`        | Deletes dumps older than 30 days from s3, always keeping the newest `master` dump of each database (`-d <days>`, `-n`) |
-| `create_s3_bucket.sh`      | Creates the s3 bucket if it doesn't exist yet, run automatically before an upload                     |
+| `build_and_push_image.sh`  | Builds database images from the current dumps and pushes them to the Toska registry                                    |
-| `lib/common.sh`            | Shared configuration (dump directories, s3 bucket, registry, database names)                          |
+| `create_s3_bucket.sh`      | Creates the s3 bucket if it doesn't exist yet, run automatically before an upload                                      |
+| `lib/common.sh`            | Shared configuration (dump directories, s3 bucket, registry, database names)                                           |
 
 Dumps end up in `.databasedumps/test/<database>.sql`, which is where `db.Dockerfile` reads them from when an image is built. The s3 commands read their credentials from the `ACCESS_KEY` and `SECRET_KEY` environment variables (e.g. in CI). If neither is set, they fall back to `~/.s3cfg`, which can be found on the toska/dokumentaatio repo.
 

anonyymioodi/lib/common.sh

@@ -22,6 +22,8 @@ S3_HOST="s3.datacloud.helsinki.fi"
 S3_BUCKET="s3://oodikone"
 # Dumps live under a prefix of their own inside the bucket
 S3_TARGET="$S3_BUCKET/test-dbs"
+# Dumps older than this are pruned from s3, except for the newest dump of each database
+S3_DUMP_RETENTION_DAYS=30
 
 REGISTRY="registry-toska.ext.ocp-prod-0.k8s.it.helsinki.fi"
 

anonyymioodi/manage_test_dbs.sh

@@ -16,7 +16,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=lib/common.sh
 source "$SCRIPT_DIR"/lib/common.sh
 
-ACTIONS=("download" "dump" "upload" "push" "release" "rebuild")
+ACTIONS=("download" "dump" "upload" "push" "prune" "release" "rebuild")
 
 usage() {
   cat >&2 <<EOF
@@ -28,10 +28,11 @@ Actions:
   download  Download the newest dumps from s3
   dump      Dump the databases from the running docker containers
 
-  upload    Upload the current dumps to s3
+  upload    Upload the current dumps to s3, then prune old dumps
   push      Build images from the current dumps and push them to the registry
+  prune     Delete dumps older than $S3_DUMP_RETENTION_DAYS days from s3, keeping the newest one
 
-  release   Dump from the running containers, then upload and push (shorthand)
+  release   Dump from the running containers, then upload, prune and push (shorthand)
   rebuild   Download the dumps from s3, then push (shorthand)
 
 Databases: ${DATABASES[*]} all
@@ -52,6 +53,10 @@ upload_dump() {
   "$SCRIPT_DIR"/upload_dump_to_s3.sh "$1"
 }
 
+prune_dumps() {
+  "$SCRIPT_DIR"/prune_s3_dumps.sh "$1"
+}
+
 push_image() {
   "$SCRIPT_DIR"/build_and_push_image.sh "$1"
 }
@@ -63,11 +68,16 @@ run_action() {
   case "$action" in
     download) download_dump "$database" ;;
     dump) dump_database "$database" ;;
-    upload) upload_dump "$database" ;;
+    upload)
+      upload_dump "$database"
+      prune_dumps "$database"
+      ;;
     push) push_image "$database" ;;
+    prune) prune_dumps "$database" ;;
     release)
       dump_database "$database"
       upload_dump "$database"
+      prune_dumps "$database"
       push_image "$database"
       ;;
     rebuild)

anonyymioodi/prune_s3_dumps.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Delete old database dumps from s3. Dumps older than the retention period are
+# removed, but the newest master dump of each database is always kept, so that a
+# database which hasn't changed in a while can still be downloaded. Dumps from
+# other branches aren't kept, so that old branches don't accumulate dumps.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source-path=SCRIPTDIR
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR"/lib/common.sh
+
+# The newest dump of each database from this branch is never pruned
+MASTER_BRANCH="master"
+
+usage() {
+  cat >&2 <<EOF
+Usage: $(basename "$0") [-n|--dry-run] [-d|--days <days>] <database>... | all
+
+Deletes the dumps in $S3_TARGET older than the given number of days
+(default $S3_DUMP_RETENTION_DAYS). The newest $MASTER_BRANCH dump of each database is
+always kept, or the newest dump of any branch if the database has no $MASTER_BRANCH dumps.
+
+Databases: ${DATABASES[*]}
+
+Options:
+  -n, --dry-run      Print the dumps that would be deleted instead of deleting them
+  -d, --days <days>  Delete dumps older than this many days
+  -h, --help         Show this help
+EOF
+}
+
+dry_run=false
+days="$S3_DUMP_RETENTION_DAYS"
+arguments=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -n | --dry-run)
+      dry_run=true
+      shift
+      ;;
+    -d | --days)
+      days="${2-}"
+      [[ "$days" =~ ^[0-9]+$ ]] || {
+        usage
+        die "Expected a number of days for $1, got: $days"
+      }
+      shift 2
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    -*)
+      usage
+      die "Unknown option: $1"
+      ;;
+    *)
+      arguments+=("$1")
+      shift
+      ;;
+  esac
+done
+
+if [[ ${#arguments[@]} -eq 0 ]]; then
+  usage
+  die "No database given"
+fi
+
+prune_database() {
+  local database="$1"
+  local deleted=0
+
+  # s3cmd ls prints "<date> <time> <size> <object>", so sorting by the first two
+  # columns in reverse puts the newest dump first
+  local dumps
+  dumps="$(s3 ls "$S3_TARGET/$database-" | sort -rk1,2)"
+
+  # Find the newest master dump or in the case of it missing, the newest dump in general
+  local kept
+  kept="$(awk -v pattern="/$database-$MASTER_BRANCH-[0-9]+-[0-9]+[.]sql$" '$4 ~ pattern { print $4; exit }' <<< "$dumps")"
+  [[ -n "$kept" ]] || kept="$(awk 'NR == 1 { print $4 }' <<< "$dumps")"
+
+  # Remove any dumps older than the dump retention date (or -d)
+  # except for master for which at least one dump is saved
+  while read -r date time _ dump; do
+    [[ -n "$dump" && "$dump" != "$kept" && "$date $time" < "$cutoff" ]] || continue
+    if [[ "$dry_run" == true ]]; then
+      infomsg "Dry run, would delete $dump"
+    else
+      s3 del "$dump"
+    fi
+    deleted=$((deleted + 1))
+  done <<< "$dumps"
+
+  if [[ "$dry_run" == true ]]; then
+    successmsg "Would have pruned $deleted dump(s) of $database older than $days days"
+  else
+    successmsg "Pruned $deleted dump(s) of $database older than $days days"
+  fi
+}
+
+resolve_databases "${arguments[@]}"
+require_s3_config
+
+cutoff="$(date -u -d "-$days days" '+%Y-%m-%d %H:%M')"
+
+for database in "${selected_databases[@]}"; do
+  prune_database "$database"
+done