CLI full setup to create anon and real datasets

README.md

@@ -28,6 +28,8 @@ bash run.sh
 
 For local development with anonymized data use `2) Anon setup` (you can run e2e tests with this).
 
+To get also the full real dataset use `3) Full setup`
+
 > `1) e2e setup` is designed for use in travis. But you can still try it locally if you want to.
 
 Run E2E tests in with `npm run cypress:run` or `npm run cypress:open` with visual browser.
@@ -51,10 +53,20 @@ The development environment is entirely configured in the docker-compose.yml fil
 The mapping of ports and environment variables are also defined in the docker-compose file. You can start, stop and manage the development environment by running the following commands from a terminal shell in this directory.
 
 ### Start the development environment
+
+#### With anonymized data:
+
 ```
 docker-compose up -d
 ```
 
+#### With real data:
+
+```
+docker-compose -f docker-compose.yml -f docker-compose.real.yml up -d
+```
+
+
 ### Stop the development environment
 ```
 docker-compose down

docker-compose.real.yml

@@ -0,0 +1,9 @@
+version: '3'
+
+services:
+  backend:
+    environment:
+      DB_URL: postgres://postgres@db:5432/tkt_oodi_real
+  userservice:
+    environment:
+      DB_URL: postgres://postgres@user_db:5432/user_db_real

docker-compose.yml

@@ -44,7 +44,7 @@ services:
     container_name: oodi_user_db
     environment:
       - POSTGRES_USER=postgres
-      - POSTGRES_DB=user_db
+      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
   usage_db:
     image: postgres:9.6.3
     ports:
@@ -64,7 +64,7 @@ services:
       - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
     container_name: oodi_db
     environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
       - POSTGRES_USER=postgres
   redis:
     image: redis

scripts.sh

@@ -2,8 +2,10 @@
 
 DIR_PATH=$(dirname "$0")
 BACKUP_DIR=backups
-PSQL_DB_BACKUP="$BACKUP_DIR/latest-pg.bak"
-USER_DB_BACKUP="$BACKUP_DIR/latest-user-pg.bak"
+PSQL_DB_BACKUP="$BACKUP_DIR/anon.bak"
+USER_DB_BACKUP="$BACKUP_DIR/user-dump.bak"
+PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.bak"
+USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.bak"
 
 retry () {
     for i in {1..10}
@@ -48,8 +50,8 @@ get_anon_oodikone() {
       chmod 400 private.key
     fi
     GIT_SSH_COMMAND='ssh -i private.key' git clone git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
-    mv anonyymioodi/anon.bak.bz2 ./$BACKUP_DIR/latest-pg.bak.bz2
-    mv anonyymioodi/user-dump.bak.bz2 ./$BACKUP_DIR/latest-user-pg.bak.bz2
+    mv anonyymioodi/anon.bak.bz2 ./$BACKUP_DIR/anon.bak.bz2
+    mv anonyymioodi/user-dump.bak.bz2 ./$BACKUP_DIR/user-dump.bak.bz2
 
 }
 
@@ -61,10 +63,20 @@ restore_psql_from_backup () {
     cat $PSQL_DB_BACKUP | docker exec -i -u postgres oodi_db psql -d tkt_oodi
 }
 
+restore_real_psql_from_backup () {
+    cat $PSQL_REAL_DB_BACKUP | docker exec -i -u postgres oodi_db psql -d tkt_oodi_real
+}
+
 restore_userdb_from_backup () {
+    docker exec -u postgres oodi_user_db psql -c "CREATE DATABASE user_db"
     cat $USER_DB_BACKUP | docker exec -i -u postgres oodi_user_db psql -d user_db
 }
 
+restore_real_userdb_from_backup () {
+    docker exec -u postgres oodi_user_db psql -c "CREATE DATABASE user_db_real"
+    cat $USER_REAL_DB_BACKUP | docker exec -i -u postgres oodi_user_db psql -d user_db_real
+}
+
 restore_mongodb_from_backup () {
     docker exec -t mongo_db mongorestore -d oodilearn "/dump"
 }
@@ -79,6 +91,15 @@ ping_psql () {
     echo "Pinging psql in container $1 with db name $2"
     docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi"
     retry docker exec -u postgres $1 pg_isready
+    echo "Pinging psql in container $1 with db name tkt_oodi_test"
+    docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi_test"
+    retry docker exec -u postgres $1 pg_isready
+}
+
+ping_psql_real () {
+    echo "Pinging psql in container $1 with db name $2"
+    docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi_real"
+    retry docker exec -u postgres $1 pg_isready
 }
 
 db_setup_full () {
@@ -87,13 +108,13 @@ db_setup_full () {
     echo "Unpacking compressed files"
     unpack_oodikone_server_backup
     echo "Restoring PostgreSQL from backup"
-    ping_psql "oodi_db" "tkt_oodi"
-    retry restore_psql_from_backup
+    ping_psql_real "oodi_db" "tkt_oodi_real"
+    retry restore_real_psql_from_backup
     echo "Restoring MongoDB from backup"
     retry restore_mongodb_from_backup
     echo "Restore user db from backup"
-    ping_psql "oodi_user_db" "user_db"
-    retry restore_userdb_from_backup
+    ping_psql "oodi_user_db" "user_db_real"
+    retry restore_real_userdb_from_backup
     echo "Database setup finished"
 }
 
@@ -138,6 +159,7 @@ run_full_setup () {
     docker_build
     echo "Setup oodikone db from dump, this will prompt you for your password."
     db_setup_full
+    db_anon_setup_full
     echo "Restarting Docker backend containers to run migrations, etc."
     docker_restart_backend
     show_instructions