Use extend in travis docker-compose files

.travis.yml

@@ -38,7 +38,7 @@ jobs:
       deploy: # deploy is always skipped for PR
         provider: script
         skip_cleanup: true
-        script: "docker-compose -f docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
+        script: "docker-compose -f docker-compose.yml -f docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
         on:
           branch: master
     - # in parallel
@@ -59,7 +59,7 @@ jobs:
       deploy: # deploy is always skipped for PR
         provider: script
         skip_cleanup: true
-        script: "docker-compose -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
+        script: "docker-compose -f docker-compose.yml -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
         on:
           tags: true
 

docker-compose.lateste2e.production.yml

@@ -1,76 +1,13 @@
 version: '3'
 
 services:
-  nats:
-    image: nats-streaming
-    command: -cid updaterNATS
-    expose:
-      - "4222"
-    container_name: nats
-
-  analytics_db:
-    image: postgres:9.6.3
-    ports:
-      - "5423:5432"
-    volumes:
-      - ./analytics_pgdata:/var/lib/postgresql/data
-    container_name: oodi_analytics_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=analytics_db
-
   user_db:
-    image: postgres:9.6.3
-    ports:
-      - "5422:5432"
-    volumes:
-      - ./user_pgdata:/var/lib/postgresql/data
-    container_name: oodi_user_db
     environment:
-      - POSTGRES_USER=postgres
       - POSTGRES_DB=user_db
 
   db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
     environment:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
-      - POSTGRES_USER=postgres
-
-  db_kone:
-    image: postgres:9.6.3
-    ports:
-      - "5425:5432"
-    volumes:
-      - ./pgdata_kone:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: db_kone
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_kone
-      - POSTGRES_USER=postgres
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-  analytics_redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6380:6379"
-    volumes:
-      - ./analytics-redis-data:/data
-    container_name: analytics_redis
 
   backend:
     image: toska/oodikone2-backend:latest
@@ -78,30 +15,11 @@ services:
       context: services/backend
       dockerfile: oodikone2-backend/Dockerfile
     ports:
-      - "8080:8080"
       - "9229:9229"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222
-      REDIS: redis
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
       FRONT_URL: http://localhost:1337
       TOKEN_SECRET: IWannaBeTheVeryBest
-      OODILEARN_URL: http://oodilearn:5000
-      USERSERVICE_URL: http://userservice:4567
-      USAGESERVICE_URL: http://usageservice:545
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      ANALYTICS_URL: http://analytics:4568
-      NATS_URI: nats://nats:4222
-    container_name: backend
-    depends_on:
-      - db
-      - redis
-      - nats
 
   frontend:
     image: toska/oodikone2-frontend:latest
@@ -113,122 +31,39 @@ services:
       - "5000:5000"
     environment:
       WAIT_HOSTS: user_db:5432
-      BACKEND_ADDR: backend
-      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-    container_name: frontend
 
   userservice:
     image: toska/oodikone2-userservice:latest
     build:
       context: services/oodikone2-userservice
-    ports:
-      - 4567:4567
-    environment:
-      - WAIT_HOSTS=user_db:5432
-      - DB_URL=postgres://postgres@user_db:5432/user_db
-      - TEST_DB=postgres://postgres@user_db:5432/user_db_test
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      - TOKEN_SECRET=IWannaBeTheVeryBest
-    depends_on:
-      - user_db
-    container_name: userservice
 
   usageservice:
     image: toska/oodikone2-usageservice:latest
     build:
       context: services/oodikone2-usageservice
-    ports:
-      - 545:545
-    environment:
-      - WAIT_HOSTS=usage_db:5432
-      - PORT=545
-      - DB_URL=postgres://postgres@usage_db:5432/usage_db
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    depends_on:
-      - usage_db
-    container_name: usageservice
-
-  usage_db:
-    image: postgres:9.6.3
-    ports:
-      - "5424:5432"
-    volumes:
-      - ./usage_pgdata:/var/lib/postgresql/data
-    container_name: oodi_usage_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=usage_db
-
-  analytics:
-    image: toska/oodikone2-analytics:latest
-    build:
-      context: services/oodikone2-analytics
-    ports:
-      - 4568:4568
-    environment:
-      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
-      - REDIS=analytics_redis
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
-    depends_on:
-      - analytics_db
-      - analytics_redis
-    container_name: analytics
 
   updater_api:
     image: toska/updater_api:latest
     build:
       context: services/updater_api
-    depends_on:
-      - nats
-    environment:
-      - WAIT_HOSTS=nats:4222
-      - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_api
 
   updater_writer:
     image: toska/updater_writer:latest
     build:
       context: services/backend
       dockerfile: updater_writer/Dockerfile
-    depends_on:
-      - nats
-      - db
-    environment:
-      WAIT_HOSTS: nats:4222,db:5432
-      NATS_URI: nats://nats:4222
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
-
-  scheduler_mongo:
-    image: mongo
-    volumes:
-      - ./mongodata:/data/db
-    expose:
-      - "27017"
-    container_name: scheduler_db
 
   updater_scheduler:
     image: toska/updater_scheduler:latest
     build:
       context: services/updater_scheduler
     command: ["echo", "mluukkai on orjuuttaja"]
-    depends_on:
-      - nats
-      - scheduler_mongo
-    environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
-      - NATS_URI=nats://nats:4222
-      - MONGO_URI=mongodb://scheduler_mongo:27017
-      - STUDENT_NUMBERS=./all_student_numbers.txt
-      - ACTIVE_STUDENTS=./active_student_numbers.txt
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_scheduler
+
+  analytics:
+    image: toska/oodikone2-analytics:latest
+    build:
+      context: services/oodikone2-analytics
+    container_name: analytics
 
   nginx:
     image: nginx:latest
@@ -245,5 +80,4 @@ services:
       - backend
       - userservice
       - analytics
-
     container_name: nginx

docker-compose.lateste2e.yml

@@ -1,79 +1,16 @@
 version: '3'
 
 services:
-  nats:
-    image: nats-streaming
-    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
-    expose:
-      - "4222"
-    volumes:
-      - ./datastore:/datastore
-    container_name: nats
-
-  analytics_db:
-    image: postgres:9.6.3
-    ports:
-      - "5423:5432"
-    volumes:
-      - ./analytics_pgdata:/var/lib/postgresql/data
-    container_name: oodi_analytics_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=analytics_db
-
   user_db:
-    image: postgres:9.6.3
-    ports:
-      - "5422:5432"
-    volumes:
-      - ./user_pgdata:/var/lib/postgresql/data
-    container_name: oodi_user_db
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_DB=user_db
 
   db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
     environment:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
       - POSTGRES_USER=postgres
 
-  db_kone:
-    image: postgres:9.6.3
-    ports:
-      - "5425:5432"
-    volumes:
-      - ./pgdata_kone:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: db_kone
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_kone
-      - POSTGRES_USER=postgres
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-  analytics_redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6380:6379"
-    volumes:
-      - ./analytics-redis-data:/data
-    container_name: analytics_redis
-
   backend:
     image: toska/oodikone2-backend:staging
     build:
@@ -83,27 +20,8 @@ services:
       - "8080:8080"
       - "9229:9229"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222
-      REDIS: redis
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
       FRONT_URL: http://localhost:1337
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      OODILEARN_URL: http://oodilearn:5000
-      USERSERVICE_URL: http://userservice:4567
-      USAGESERVICE_URL: http://usageservice:545
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      ANALYTICS_URL: http://analytics:4568
-      NATS_URI: nats://nats:4222
-    container_name: backend
-    depends_on:
-      - db
-      - redis
-      - nats
 
   frontend:
     image: toska/oodikone2-frontend:staging
@@ -115,122 +33,39 @@ services:
       - "5000:5000"
     environment:
       WAIT_HOSTS: user_db:5432
-      BACKEND_ADDR: backend
-      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-    container_name: frontend
 
   userservice:
     image: toska/oodikone2-userservice:staging
     build:
       context: services/oodikone2-userservice
-    ports:
-      - 4567:4567
-    environment:
-      - WAIT_HOSTS=user_db:5432
-      - DB_URL=postgres://postgres@user_db:5432/user_db
-      - TEST_DB=postgres://postgres@user_db:5432/user_db_test
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      - TOKEN_SECRET=IWannaBeTheVeryBest
-    depends_on:
-      - user_db
-    container_name: userservice
 
   usageservice:
     image: toska/oodikone2-usageservice:staging
     build:
       context: services/oodikone2-usageservice
-    ports:
-      - 545:545
-    environment:
-      - WAIT_HOSTS=usage_db:5432
-      - PORT=545
-      - DB_URL=postgres://postgres@usage_db:5432/usage_db
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    depends_on:
-      - usage_db
-    container_name: usageservice
-
-  usage_db:
-    image: postgres:9.6.3
-    ports:
-      - "5424:5432"
-    volumes:
-      - ./usage_pgdata:/var/lib/postgresql/data
-    container_name: oodi_usage_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=usage_db
-
-  analytics:
-    image: toska/oodikone2-analytics:staging
-    build:
-      context: services/oodikone2-analytics
-    ports:
-      - 4568:4568
-    environment:
-      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
-      - REDIS=analytics_redis
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
-    depends_on:
-      - analytics_db
-      - analytics_redis
-    container_name: analytics
 
   updater_api:
     image: toska/updater_api:staging
     build:
       context: services/updater_api
-    depends_on:
-      - nats
-    environment:
-      - WAIT_HOSTS=nats:4222
-      - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_api
 
   updater_writer:
     image: toska/updater_writer:staging
     build:
       context: services/backend
       dockerfile: updater_writer/Dockerfile
-    depends_on:
-      - nats
-      - db
-    environment:
-      WAIT_HOSTS: nats:4222,db:5432
-      NATS_URI: nats://nats:4222
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
-
-  scheduler_mongo:
-    image: mongo
-    volumes:
-      - ./mongodata:/data/db
-    expose:
-      - "27017"
-    container_name: scheduler_db
 
   updater_scheduler:
     image: toska/updater_scheduler:staging
     build:
       context: services/updater_scheduler
     command: ["echo", "mluukkai on orjuuttaja"]
-    depends_on:
-      - nats
-      - scheduler_mongo
-    environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
-      - NATS_URI=nats://nats:4222
-      - MONGO_URI=mongodb://scheduler_mongo:27017
-      - STUDENT_NUMBERS=./all_student_numbers.txt
-      - ACTIVE_STUDENTS=./active_student_numbers.txt
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_scheduler
+
+  analytics:
+    image: toska/oodikone2-analytics:staging
+    build:
+      context: services/oodikone2-analytics
+    container_name: analytics
 
   nginx:
     image: nginx:latest
@@ -247,5 +82,4 @@ services:
       - backend
       - userservice
       - analytics
-
     container_name: nginx

docker-compose.real.yml

@@ -5,6 +5,7 @@ services:
     environment:
       DB_URL: postgres://postgres@db:5432/tkt_oodi_real
       DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
+
   userservice:
     environment:
       DB_URL: postgres://postgres@user_db:5432/user_db_real

docker-compose.yml

@@ -29,7 +29,7 @@ services:
 
   nats:
     image: nats-streaming
-    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
+    command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
     expose:
       - "4222"
     volumes:
@@ -146,7 +146,6 @@ services:
       - ./services/backend/shared/models:/usr/src/app/src/models
       - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
       - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
-
     container_name: backend
     depends_on:
       - db
@@ -262,7 +261,6 @@ services:
       - ./services/backend/shared/models:/usr/src/app/models
       - ./services/backend/shared/migrations:/usr/src/app/database/migrations
       - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
-
     depends_on:
       - nats
       - db
@@ -273,8 +271,7 @@ services:
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
       TEST_DB_SCHEMA: updater_test
       DB_SCHEMA_KONE: kone_data
-
-
+    container_name: updater_writer
 
   scheduler_mongo:
     image: mongo
@@ -308,10 +305,8 @@ services:
       - LOG_HOST=toska.cs.helsinki.fi
       - LOG_PORT=9501
       - LOG_HOSTNAME=updater_scheduler
-
     container_name: updater_scheduler
 
-
   analytics:
     build:
       context: services/oodikone2-analytics

scripts.sh

@@ -160,10 +160,10 @@ run_e2e_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    docker-compose -f $1 build
+    docker-compose -f docker-compose.yml -f $1 build
     echo "Setup oodikone db from dump."
-    docker-compose -f $1 up -d db user_db db_kone
+    docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone
     db_anon_setup_full
     echo "Starting services."
-    docker-compose -f $1 up -d
+    docker-compose -f docker-compose.yml -f $1 up -d
 }