separate updater from oodikone

.gitignore

@@ -1,3 +1,4 @@
+**/node_modules
 data
 pgdata
 pgdata_kone

dco.data.yml

@@ -0,0 +1,52 @@
+version: "3.2"
+
+services:
+  nats:
+    image: nats-streaming:0.15.1
+    command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
+    expose:
+      - "4222"
+    ports:
+      - "8222:8222"
+      - "4222:4222"
+    container_name: nats
+    volumes:
+      - datastore:/datastore
+
+  db:
+    image: postgres:10-alpine
+    ports:
+      - "5421:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: oodi_db
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_HOST_AUTH_METHOD=trust
+
+  db_sis:
+    image: postgres:9.6.3
+    ports:
+      - "5426:5432"
+    volumes:
+      - pgdata_sis:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_sis
+    environment:
+      - POSTGRES_USER=postgres
+
+  redis:
+    image: redis
+    command: ["redis-server", "--appendonly", "yes"]
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis-data:/data
+    container_name: redis
+
+volumes:
+  datastore:
+  pgdata:
+  pgdata_sis:
+  redis-data:

docker-compose.yml

@@ -1,257 +0,0 @@
-version: "3.2"
-
-services:
-  nats:
-    image: nats-streaming:0.15.1
-    command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
-    expose:
-      - "4222"
-    ports:
-      - "8222:8222"
-      - "4222:4222"
-    container_name: nats
-    volumes:
-      - datastore:/datastore
-
-  analytics_db:
-    image: postgres:9.6.3
-    ports:
-      - "5423:5432"
-    volumes:
-      - analytics_pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_analytics_db
-    environment:
-      - POSTGRES_USER=postgres
-
-  user_db:
-    image: postgres:9.6.3
-    ports:
-      - "5422:5432"
-    volumes:
-      - user_pgdata:/var/lib/postgresql/data
-    container_name: oodi_user_db
-    environment:
-      - POSTGRES_USER=postgres
-
-  db:
-    image: postgres:10-alpine
-    ports:
-      - "5421:5432"
-    volumes:
-      - pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_HOST_AUTH_METHOD=trust
-
-  db_kone:
-    image: postgres:9.6.3
-    ports:
-      - "5425:5432"
-    volumes:
-      - pgdata_kone:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: db_kone
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_kone
-      - POSTGRES_USER=postgres
-
-  db_sis:
-    image: postgres:9.6.3
-    ports:
-      - "5426:5432"
-    volumes:
-      - pgdata_sis:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: db_sis
-    environment:
-      - POSTGRES_USER=postgres
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - redis-data:/data
-    container_name: redis
-
-  backend:
-    build:
-      context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
-    ports:
-      - "8080:8080"
-    environment:
-      REDIS: redis
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      # DB_MAX_CONNECTIONS: 90 # defaults to 5
-      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
-      SIS_DB_URL: postgres://postgres@db_sis:5432/db_sis
-      # if NODE_ENV === 'test', these are used instead
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
-      # same but for NODE_ENV === 'anon'
-      # ANON_DB:
-      # ANON_DB_KONE:
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      SECRET_TOKEN: dev
-      USERSERVICE_URL: http://userservice:4567
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      ANALYTICS_URL: http://analytics:4568
-      NATS_URI: nats://nats:4222
-      UPDATER_URL: http://updater_scheduler:3678
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      # Set these in production:
-      # ANALYTICS_INFLUXDB_URL
-      # ANALYTICS_INFLUXDB_DB
-      # ANALYTICS_INFLUXDB_PASSWORD
-      # ANALYTICS_INFLUXDB_USER
-      # MATOMO_SITE_ID
-      # MATOMO_URL
-    container_name: backend
-    depends_on:
-      - db
-      - redis
-      - nats
-
-  frontend:
-    build:
-      context: services/oodikone2-frontend
-    environment:
-      BACKEND_ADDR: backend
-      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-    container_name: frontend
-
-  userservice:
-    build:
-      context: services/oodikone2-userservice
-    ports:
-      - 4567:4567
-    environment:
-      DB_URL: postgres://postgres@user_db:5432/user_db
-      TEST_DB: postgres://postgres@user_db:5432/user_db_test
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    depends_on:
-      - user_db
-    container_name: userservice
-
-  updater_api:
-    build:
-      context: services/updater_api
-    depends_on:
-      - nats
-    environment:
-      - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi/test
-      - TOKEN=jotaindiibadaabaa
-    container_name: updater_api
-
-  updater_writer:
-    build:
-      context: services/backend
-      dockerfile: updater_writer/Dockerfile
-    depends_on:
-      - nats
-      - db
-    environment:
-      NATS_URI: nats://nats:4222
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
-
-  scheduler_mongo:
-    image: mongo:4.0.10
-    volumes:
-      - mongodata:/data/db
-    expose:
-      - "27017"
-    container_name: scheduler_mongo
-
-  updater_scheduler:
-    build:
-      context: services/updater_scheduler
-    depends_on:
-      - nats
-      - scheduler_mongo
-    environment:
-      - NATS_URI=nats://nats:4222
-      - MONGO_URI=mongodb://scheduler_mongo:27017
-      - PORT=3678
-    container_name: updater_scheduler
-
-  analytics:
-    build:
-      context: services/oodikone2-analytics
-    ports:
-      - 4568:4568
-    environment:
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
-    depends_on:
-      - analytics_db
-    container_name: analytics
-
-  sis-updater-nats:
-    image: nats-streaming:0.16.2
-    command: -cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore
-    container_name: sis-updater-nats
-    volumes:
-      - sis-updater-datastore:/datastore
-
-  sis-updater-scheduler:
-    build:
-      context: services/sis-updater-scheduler
-    environment:
-      REDIS_URI: //redis:6379
-      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
-      SIS_NATS_TOKEN: dev
-      SIS_IMPORTER_HOST: importer-db
-      SIS_IMPORTER_USER: dev
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_DATABASE: importer-db
-      SECRET_TOKEN: dev
-      # LOG_HOST: host.docker.internal
-      # LOG_HOSTNAME: sis-updater-scheduler
-      # LOG_PORT: 9501
-    
-    ports:
-      - "8082:8082"
-    container_name: sis-updater-scheduler
-    depends_on:
-      - redis
-      - sis-updater-nats
-
-  sis-updater-worker:
-    build:
-      context: services/sis-updater-worker
-    environment:
-      REDIS_URI: //redis:6379
-      DB_URL: postgres://postgres@db_sis:5432/db_sis
-      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
-      SIS_NATS_TOKEN: dev
-      SIS_IMPORTER_HOST: importer-db
-      SIS_IMPORTER_USER: dev
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_DATABASE: importer-db
-      # LOG_HOST: host.docker.internal
-      # LOG_HOSTNAME: sis-updater-worker
-      # LOG_PORT: 9501
-    depends_on:
-      - redis
-      - sis-updater-nats
-
-volumes:
-  datastore:
-  analytics_pgdata:
-  user_pgdata:
-  pgdata:
-  pgdata_kone:
-  pgdata_sis:
-  redis-data:
-  mongodata:
-  sis-updater-datastore:

docker/dco.data.ci.yml

@@ -0,0 +1,15 @@
+version: '3.2'
+
+services:
+  db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
+
+  db_sis:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_sis
+
+volumes:
+  datastore-real:
+  redis-data-real:
+  mongodata-real:

docker/dco.data.dev.real.yml

@@ -0,0 +1,14 @@
+version: '3.2'
+
+services:
+  nats:
+    volumes:
+      - datastore-real:/datastore
+
+  redis:
+    volumes:
+      - redis-data-real:/data
+
+volumes:
+  datastore-real:
+  redis-data-real:

docker/dco.data.dev.yml

@@ -0,0 +1,31 @@
+version: '3.2'
+
+services:
+  db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
+    restart: unless-stopped
+    shm_size: '2gb'
+
+  db_sis:
+    command:
+      - "postgres"
+      - "-c"
+      - "max_wal_size=8GB"
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_sis
+    restart: unless-stopped
+
+  adminer:
+    image: adminer:4.6.2
+    environment:
+      - ADMINER_DESIGN=pepa-linha
+    volumes:
+      - ./adminer-plugins:/var/www/html/plugins-enabled
+    ports:
+      - 5050:8080
+    container_name: adminer
+    restart: unless-stopped
+
+networks:
+  default:

docker/docker-compose.dev.real.yml

@@ -1,40 +0,0 @@
-version: '3.2'
-
-services:
-  nats:
-    volumes:
-      - datastore-real:/datastore
-
-  backend:
-    environment:
-      DB_URL: postgres://postgres@db:5432/tkt_oodi_real
-      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
-
-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user_db:5432/user_db_real
-
-  analytics:
-    environment:
-      DB_URL: postgres://postgres@analytics_db:5432/analytics_db_real
-
-  updater_writer:
-    environment:
-      DB_URL: postgres://postgres@db:5432/tkt_oodi_real
-  updater_api:
-    environment:
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN
-
-  redis:
-    volumes:
-      - redis-data-real:/data
-
-  scheduler_mongo:
-    volumes:
-      - mongodata-real:/data/db
-
-volumes:
-  datastore-real:
-  redis-data-real:
-  mongodata-real:

docker/docker-compose.dev.yml

@@ -1,176 +0,0 @@
-version: '3.2'
-
-services:
-  user_db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
-    restart: unless-stopped
-
-  analytics_db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=analytics_db,analytics_db_real
-    restart: unless-stopped
-
-  db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
-    restart: unless-stopped
-    shm_size: '2gb'
-
-  db_sis:
-    command:
-      - "postgres"
-      - "-c"
-      - "max_wal_size=8GB"
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_sis
-    restart: unless-stopped
-
-  backend:
-    build:
-      args:
-        NODE_ENV: 'development'
-        TAG: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    environment:
-      FRONT_URL: http://localhost:8000
-    volumes:
-      - ./services/backend/oodikone2-backend/:/usr/src/app
-      - ./services/backend/shared/models:/usr/src/app/src/models
-      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
-      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
-    restart: unless-stopped
-
-  frontend:
-    build:
-      args:
-        NODE_ENV: 'development'
-        TAG: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run docker']
-    ports:
-      - "8081:8081"
-      - "7777:7777"
-    environment:
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
-      SIS_ADMINER_URL: http://localhost:5050/?pgsql=db_sis&username=postgres
-    volumes:
-      - ./services/oodikone2-frontend/:/usr/src/app
-    restart: unless-stopped
-
-  userservice:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/oodikone2-userservice/:/usr/src/app
-    restart: unless-stopped
-
-  updater_api:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/updater_api/:/usr/src/app
-    restart: unless-stopped
-
-  updater_writer:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/backend/updater_writer/:/usr/src/app
-      - ./services/backend/shared/models:/usr/src/app/models
-      - ./services/backend/shared/migrations:/usr/src/app/database/migrations
-      - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
-    environment:
-      DB_SCHEMA_KONE: kone_data
-    restart: unless-stopped
-
-  updater_scheduler:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/updater_scheduler/:/usr/src/app
-    restart: unless-stopped
-
-  analytics:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/oodikone2-analytics/:/usr/src/app
-    restart: unless-stopped
-
-  adminer:
-    image: adminer:4.6.2
-    environment:
-      - ADMINER_DESIGN=pepa-linha
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-    ports:
-      - 5050:8080
-    container_name: adminer
-    restart: unless-stopped
-
-  sis-updater-scheduler:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/sis-updater-scheduler/:/usr/src/app
-    restart: unless-stopped
-
-  sis-updater-worker:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: 'npm run dev'
-    volumes:
-      - ./services/sis-updater-worker/:/usr/src/app
-    restart: unless-stopped
-
-networks:
-  default:
-
-  # zookeeper:
-  #   image: wurstmeister/zookeeper
-  #   ports:
-  #     - "2181:2181"
-  #   environment:
-  #     - KAFKA_BROKER_ID=1
-  #   container_name: zookeeper
-
-  # kafka:
-  #   image: wurstmeister/kafka
-  #   container_name: kafka
-  #   depends_on:
-  #     - zookeeper
-  #   ports:
-  #     - "9092:9092"
-  #   environment:
-  #     - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}'
-  #     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
-  #     - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
-  #     - KAFKA_DELETE_TOPIC_ENABLE=true
-  #     - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1
-  #     - KAFKA_LOG_DIRS=/kafka/kafka-logs-1
-  #   volumes:
-  #     - ./kafka/docker.sock:/var/run/docker.sock
-  #     - ./kafka/logs:/kafka/kafka-logs-1
-
-  # mongo_db:
-  #   image: mongo
-  #   volumes:
-  #     - ./mongodata:/data/db
-  #   ports:
-  #     - "27017:27017"
-  #   container_name: mongo_db

package.json

@@ -1,7 +1,7 @@
 {
   "name": "oodikone",
   "version": "1.0.0",
-  "description": "Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the docker-compose.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.",
+  "description": "Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the dco.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.",
   "main": "./scripts/run.sh",
   "scripts": {
     "cli": "./scripts/run.sh",
@@ -17,22 +17,31 @@
     "test_services": "npm run services -- -e -c test",
     "test": "./scripts/test.sh",
     "test:smoke": "./scripts/test.sh -s",
-    "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml",
-    "docker:cypress": "npm run docker -- -f ./docker/docker-compose.cypress.yml run --rm cypress",
-    "docker:real": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml",
-    "docker:ci": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml",
-    "docker:ci:cypress": "npm run docker:ci -- -f ./docker/docker-compose.cypress.yml run --rm cypress",
-    "docker:build": "npm run docker -- build",
-    "docker:up": "npm run docker -- up -d",
-    "docker:up:real": "npm run docker:real -- up -d",
-    "docker:down": "npm run docker -- down",
-    "docker:logs": "npm run docker -- logs -f",
-    "docker:logs:backend": "npm run docker -- logs -f backend",
-    "docker:restart": "npm run docker -- restart",
-    "docker:restart:backend": "npm run docker -- restart backend",
+    "docker:updater:base": "docker-compose -f ./dco.data.yml -f ./updater/dco.updater.yml",
+    "docker:updater:dev": "npm run docker:updater:base -- -f ./docker/dco.data.dev.yml -f ./updater/docker/dco.dev.yml",
+    "docker:updater:real": "npm run docker:updater:dev -- -f ./docker/dco.data.dev.real.yml -f ./updater/docker/dco.dev.real.yml",
+    "docker:oodikone:base": "docker-compose -f ./dco.data.yml -f ./services/dco.oodikone.yml",
+    "docker:oodikone:dev": "npm run docker:oodikone:base -- -f ./docker/dco.data.dev.yml -f ./services/docker/dco.dev.yml",
+    "docker:oodikone:real": "npm run docker:oodikone:dev -- -f ./docker/dco.data.dev.real.yml -f ./services/docker/dco.dev.real.yml",
+    "docker:both:base": "docker-compose -f ./dco.data.yml -f ./updater/dco.updater.yml -f ./services/dco.oodikone.yml",
+    "docker:both:dev": "npm run docker:both:base -- -f ./docker/dco.data.dev.yml -f ./updater/docker/dco.dev.yml -f ./services/docker/dco.dev.yml",
+    "docker:both:real": "npm run docker:both:dev -- -f ./docker/dco.data.dev.real.yml  -f ./updater/docker/dco.dev.real.yml -f ./services/docker/dco.dev.real.yml",
+    "docker:cypress": "npm run docker:oodikone:dev -- -f ./docker/dco.cypress.yml run --rm cypress",
+    "docker:ci": "npm run docker:both:base -- -f ./docker/dco.data.ci.yml -f ./services/docker/dco.ci.yml -f ./updater/dco.ci.yml",
+    "docker:ci:cypress": "npm run docker:ci -- -f ./docker/dco.cypress.yml run --rm cypress",
+    "docker:build": "npm run docker:oodikone:dev -- build",
+    "docker:mac:up": "npm run docker:oodikone:dev -- up -d backend && npm run docker:up",
+    "docker:mac:up:real": "npm run docker:oodikone:real -- up -d backend && npm run docker:up:real",
+    "docker:up": "npm run docker:oodikone:dev -- up -d",
+    "docker:up:real": "npm run docker:oodikone:real -- up -d",
+    "docker:down": "npm run docker:oodikone:dev -- down",
+    "docker:logs": "npm run docker:oodikone:dev -- logs -f",
+    "docker:logs:backend": "npm run docker:oodikone:dev -- logs -f backend",
+    "docker:restart": "npm run docker:oodikone:dev -- restart",
+    "docker:restart:backend": "npm run docker:oodikone:dev -- restart backend",
     "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune -a",
     "start": "npm run docker:up && npm run docker:logs",
-    "start:real": "npm run docker:real -- up -d && npm run docker:real -- logs -f",
+    "start:real": "npm run docker:oodikone:real -- up -d && npm run docker:oodikone:real -- logs -f",
     "start:updater": "npm run docker:up -- --scale sis-updater-worker=4 sis-updater-scheduler sis-updater-worker db_sis sis-updater-nats db adminer redis && npm run docker:logs -- --tail=\"1000\""
   },
   "husky": {

scripts/exec_command_for_services.sh

@@ -43,13 +43,8 @@ else
   services=(
     backend
     frontend
-    updater_writer
-    updater_api
-    updater_scheduler
     analytics
     userservice
-    sis-updater-scheduler
-    oodikone_sis-updater-worker_1
   )
 fi
 

scripts/scripts.sh

@@ -15,7 +15,7 @@ USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
 ANALYTICS_REAL_DB_BACKUP="$BACKUP_DIR/latest-analytics-pg.sqz"
 
 docker-compose-dev () {
-    docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml "$@"
+    npm run docker:oodikone:dev -- "$@"
 }
 
 retry () {

services/backend/oodikone2-backend/Dockerfile

@@ -4,17 +4,14 @@ WORKDIR /usr/src/app
 EXPOSE 8080
 CMD npm start
 
-COPY ./oodikone2-backend/package.json /usr/src/app/package.json
-COPY ./oodikone2-backend/package-lock.json /usr/src/app/package-lock.json
-COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
-COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
+COPY package* ./
+
 RUN npm ci
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
 ARG TAG=local
 ENV TAG=${TAG}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
 COPY ./oodikone2-backend /usr/src/app
 COPY ./shared/models /usr/src/app/src/models

services/dco.oodikone.yml

@@ -0,0 +1,114 @@
+version: "3.2"
+
+services:
+
+  frontend:
+    build:
+      context: services/oodikone2-frontend
+    environment:
+      BACKEND_ADDR: backend
+      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
+    container_name: frontend
+
+  backend:
+    build:
+      context: services/backend
+      dockerfile: oodikone2-backend/Dockerfile
+    ports:
+      - "8080:8080"
+    environment:
+      REDIS: redis
+      DB_URL: postgres://postgres@db:5432/tkt_oodi
+      # DB_MAX_CONNECTIONS: 90 # defaults to 5
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
+      SIS_DB_URL: postgres://postgres@db_sis:5432/db_sis
+      # if NODE_ENV === 'test', these are used instead
+      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
+      # same but for NODE_ENV === 'anon'
+      # ANON_DB:
+      # ANON_DB_KONE:
+      TOKEN_SECRET: IWannaBeTheVeryBest
+      SECRET_TOKEN: dev
+      USERSERVICE_URL: http://userservice:4567
+      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      ANALYTICS_URL: http://analytics:4568
+      NATS_URI: nats://nats:4222
+      UPDATER_URL: http://updater_scheduler:3678
+      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      # Set these in production:
+      # ANALYTICS_INFLUXDB_URL
+      # ANALYTICS_INFLUXDB_DB
+      # ANALYTICS_INFLUXDB_PASSWORD
+      # ANALYTICS_INFLUXDB_USER
+      # MATOMO_SITE_ID
+      # MATOMO_URL
+    container_name: backend
+    depends_on:
+      - db
+      - redis
+      - nats
+
+  db_kone:
+    image: postgres:9.6.3
+    ports:
+      - "5425:5432"
+    volumes:
+      - pgdata_kone:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_kone
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_kone
+      - POSTGRES_USER=postgres
+
+  userservice:
+    build:
+      context: services/oodikone2-userservice
+    ports:
+      - 4567:4567
+    environment:
+      DB_URL: postgres://postgres@user_db:5432/user_db
+      TEST_DB: postgres://postgres@user_db:5432/user_db_test
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      TOKEN_SECRET: IWannaBeTheVeryBest
+    depends_on:
+      - user_db
+    container_name: userservice
+
+  user_db:
+    image: postgres:9.6.3
+    ports:
+      - "5422:5432"
+    volumes:
+      - user_pgdata:/var/lib/postgresql/data
+    container_name: oodi_user_db
+    environment:
+      - POSTGRES_USER=postgres
+
+  analytics:
+    build:
+      context: services/oodikone2-analytics
+    ports:
+      - 4568:4568
+    environment:
+      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
+      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
+    depends_on:
+      - analytics_db
+    container_name: analytics
+
+  analytics_db:
+    image: postgres:9.6.3
+    ports:
+      - "5423:5432"
+    volumes:
+      - analytics_pgdata:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: oodi_analytics_db
+    environment:
+      - POSTGRES_USER=postgres
+
+volumes:
+  analytics_pgdata:
+  user_pgdata:
+  pgdata_kone:

services/docker/dco.ci.yml

@@ -5,18 +5,10 @@ services:
     environment:
       - POSTGRES_DB=user_db
 
-  db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
-
   analytics_db:
     environment:
       - POSTGRES_MULTIPLE_DATABASES=analytics_db
 
-  db_sis:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_sis
-
   backend:
     image: toska/oodikone2-backend:${TAG}
     build:
@@ -54,30 +46,6 @@ services:
         - toska/oodikone2-userservice:staging
         - toska/oodikone2-userservice:latest
 
-  updater_api:
-    image: toska/updater_api:${TAG}
-    build:
-      cache_from:
-        - toska/updater_api:trunk
-        - toska/updater_api:staging
-        - toska/updater_api:latest
-
-  updater_writer:
-    image: toska/updater_writer:${TAG}
-    build:
-      cache_from:
-        - toska/updater_writer:trunk
-        - toska/updater_writer:staging
-        - toska/updater_writer:latest
-
-  updater_scheduler:
-    image: toska/updater_scheduler:${TAG}
-    build:
-      cache_from:
-        - toska/updater_scheduler:trunk
-        - toska/updater_scheduler:staging
-        - toska/updater_scheduler:latest
-
   analytics:
     image: toska/oodikone2-analytics:${TAG}
     build:
@@ -86,27 +54,6 @@ services:
         - toska/oodikone2-analytics:staging
         - toska/oodikone2-analytics:latest
 
-  sis-updater-scheduler:
-    image: toska/sis-updater-scheduler:${TAG}
-    environment:
-      CI: 1
-    build:
-      cache_from:
-        - toska/sis-updater-scheduler:trunk
-        - toska/sis-updater-scheduler:staging
-        - toska/sis-updater-scheduler:latest
-
-  sis-updater-worker:
-    image: toska/sis-updater-worker:${TAG}
-    environment:
-      CI: 1
-    build:
-      cache_from:
-        - toska/sis-updater-worker:trunk
-        - toska/sis-updater-worker:staging
-        - toska/sis-updater-worker:latest
-    container_name: oodikone_sis-updater-worker_1
-
   nginx:
     image: nginx:latest
     volumes:

services/docker/dco.dev.real.yml

@@ -0,0 +1,15 @@
+version: '3.2'
+
+services:
+  backend:
+    environment:
+      DB_URL: postgres://postgres@db:5432/tkt_oodi_real
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
+
+  userservice:
+    environment:
+      DB_URL: postgres://postgres@user_db:5432/user_db_real
+
+  analytics:
+    environment:
+      DB_URL: postgres://postgres@analytics_db:5432/analytics_db_real

services/docker/dco.dev.yml

@@ -0,0 +1,63 @@
+version: '3.2'
+
+services:
+  user_db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
+    restart: unless-stopped
+
+  analytics_db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=analytics_db,analytics_db_real
+    restart: unless-stopped
+
+  backend:
+    build:
+      args:
+        NODE_ENV: 'development'
+        TAG: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    environment:
+      FRONT_URL: http://localhost:8000
+    volumes:
+      - ./services/backend/oodikone2-backend/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/src/models
+      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
+    restart: unless-stopped
+
+  frontend:
+    build:
+      args:
+        NODE_ENV: 'development'
+        TAG: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run docker']
+    ports:
+      - "8081:8081"
+      - "7777:7777"
+    environment:
+      USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
+      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
+      KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
+      SIS_ADMINER_URL: http://localhost:5050/?pgsql=db_sis&username=postgres
+    volumes:
+      - ./services/oodikone2-frontend/:/usr/src/app
+    restart: unless-stopped
+
+  userservice:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/oodikone2-userservice/:/usr/src/app
+    restart: unless-stopped
+
+  analytics:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/oodikone2-analytics/:/usr/src/app
+    restart: unless-stopped

services/oodikone2-analytics/Dockerfile

@@ -1,18 +1,16 @@
 FROM node:10.15-alpine
 
 WORKDIR /usr/src/app
-EXPOSE 4568
-CMD npm start
 
-COPY package.json /usr/src/app/package.json
-COPY package-lock.json /usr/src/app/package-lock.json
-COPY package.json /usr/src/cache/package.json
-COPY package-lock.json /usr/src/cache/package-lock.json
-RUN npm ci
+EXPOSE 4568
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
-COPY . /usr/src/app
+COPY package* ./
 
+RUN npm ci
+
+COPY . .
+
+CMD npm start

services/oodikone2-userservice/Dockerfile

@@ -2,17 +2,14 @@ FROM node:10.15-alpine
 
 WORKDIR /usr/src/app
 EXPOSE 4567
-CMD npm start
-
-COPY package.json /usr/src/app/package.json
-COPY package-lock.json /usr/src/app/package-lock.json
-COPY package.json /usr/src/cache/package.json
-COPY package-lock.json /usr/src/cache/package-lock.json
-RUN npm ci
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
-COPY . /usr/src/app
+COPY package* ./
 
+RUN npm ci
+
+COPY . .
+
+CMD npm start

services/sis-updater-scheduler/Dockerfile

@@ -1,18 +0,0 @@
-FROM node:10.15-alpine
-
-WORKDIR /usr/src/app
-EXPOSE 8082
-CMD npm start
-
-COPY package.json /usr/src/app/package.json
-COPY package-lock.json /usr/src/app/package-lock.json
-COPY package.json /usr/src/cache/package.json
-COPY package-lock.json /usr/src/cache/package-lock.json
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
-
-COPY . /usr/src/app
-

services/sis-updater-worker/Dockerfile

@@ -1,17 +0,0 @@
-FROM node:10.15-alpine
-
-WORKDIR /usr/src/app
-
-CMD npm start
-
-COPY package.json /usr/src/app/package.json
-COPY package-lock.json /usr/src/app/package-lock.json
-COPY package.json /usr/src/cache/package.json
-COPY package-lock.json /usr/src/cache/package-lock.json
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
-
-COPY . /usr/src/app

services/updater_api/Dockerfile

@@ -1,17 +0,0 @@
-FROM node:10.15-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package.json /usr/src/app/package.json
-COPY package-lock.json /usr/src/app/package-lock.json
-COPY package.json /usr/src/cache/package.json
-COPY package-lock.json /usr/src/cache/package-lock.json
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
-
-COPY . /usr/src/app
-

services/updater_scheduler/Dockerfile

@@ -1,16 +0,0 @@
-FROM node:10.15-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package.json /usr/src/app/package.json
-COPY package-lock.json /usr/src/app/package-lock.json
-COPY package.json /usr/src/cache/package.json
-COPY package-lock.json /usr/src/cache/package-lock.json
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
-
-COPY . /usr/src/app

updater/dco.updater.yml

@@ -0,0 +1,100 @@
+version: "3.2"
+
+services:
+  updater_api:
+    build:
+      context: updater/updater_api
+    depends_on:
+      - nats
+    environment:
+      - NATS_URI=nats://nats:4222
+      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi/test
+      - TOKEN=jotaindiibadaabaa
+    container_name: updater_api
+
+  updater_writer:
+    build:
+      context: services/backend
+      dockerfile: updater_writer/Dockerfile
+    depends_on:
+      - nats
+      - db
+    environment:
+      NATS_URI: nats://nats:4222
+      DB_URL: postgres://postgres@db:5432/tkt_oodi
+      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_SCHEMA: updater_test
+    container_name: updater_writer
+
+  scheduler_mongo:
+    image: mongo:4.0.10
+    volumes:
+      - mongodata:/data/db
+    expose:
+      - "27017"
+    container_name: scheduler_mongo
+
+  updater_scheduler:
+    build:
+      context: updater/updater_scheduler
+    depends_on:
+      - nats
+      - scheduler_mongo
+    environment:
+      - NATS_URI=nats://nats:4222
+      - MONGO_URI=mongodb://scheduler_mongo:27017
+      - PORT=3678
+    container_name: updater_scheduler
+
+  sis-updater-nats:
+    image: nats-streaming:0.16.2
+    command: -cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore
+    container_name: sis-updater-nats
+    volumes:
+      - sis-updater-datastore:/datastore
+
+  sis-updater-scheduler:
+    build:
+      context: updater/sis-updater-scheduler
+    environment:
+      REDIS_URI: //redis:6379
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SIS_NATS_TOKEN: dev
+      SIS_IMPORTER_HOST: importer-db
+      SIS_IMPORTER_USER: dev
+      SIS_IMPORTER_PASSWORD: dev
+      SIS_IMPORTER_DATABASE: importer-db
+      SECRET_TOKEN: dev
+      # LOG_HOST: host.docker.internal
+      # LOG_HOSTNAME: sis-updater-scheduler
+      # LOG_PORT: 9501
+    
+    ports:
+      - "8082:8082"
+    container_name: sis-updater-scheduler
+    depends_on:
+      - redis
+      - sis-updater-nats
+
+  sis-updater-worker:
+    build:
+      context: updater/sis-updater-worker
+    environment:
+      REDIS_URI: //redis:6379
+      DB_URL: postgres://postgres@db_sis:5432/db_sis
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SIS_NATS_TOKEN: dev
+      SIS_IMPORTER_HOST: importer-db
+      SIS_IMPORTER_USER: dev
+      SIS_IMPORTER_PASSWORD: dev
+      SIS_IMPORTER_DATABASE: importer-db
+      # LOG_HOST: host.docker.internal
+      # LOG_HOSTNAME: sis-updater-worker
+      # LOG_PORT: 9501
+    depends_on:
+      - redis
+      - sis-updater-nats
+
+volumes:
+  mongodata:
+  sis-updater-datastore:

updater/docker/dco.ci.yml

@@ -0,0 +1,47 @@
+version: '3.2'
+
+services:
+  updater_api:
+    image: toska/updater_api:${TAG}
+    build:
+      cache_from:
+        - toska/updater_api:trunk
+        - toska/updater_api:staging
+        - toska/updater_api:latest
+
+  updater_writer:
+    image: toska/updater_writer:${TAG}
+    build:
+      cache_from:
+        - toska/updater_writer:trunk
+        - toska/updater_writer:staging
+        - toska/updater_writer:latest
+
+  updater_scheduler:
+    image: toska/updater_scheduler:${TAG}
+    build:
+      cache_from:
+        - toska/updater_scheduler:trunk
+        - toska/updater_scheduler:staging
+        - toska/updater_scheduler:latest
+
+  sis-updater-scheduler:
+    image: toska/sis-updater-scheduler:${TAG}
+    environment:
+      CI: 1
+    build:
+      cache_from:
+        - toska/sis-updater-scheduler:trunk
+        - toska/sis-updater-scheduler:staging
+        - toska/sis-updater-scheduler:latest
+
+  sis-updater-worker:
+    image: toska/sis-updater-worker:${TAG}
+    environment:
+      CI: 1
+    build:
+      cache_from:
+        - toska/sis-updater-worker:trunk
+        - toska/sis-updater-worker:staging
+        - toska/sis-updater-worker:latest
+    container_name: oodikone_sis-updater-worker_1

updater/docker/dco.dev.real.yml

@@ -0,0 +1,18 @@
+version: '3.2'
+
+services:
+  updater_writer:
+    environment:
+      DB_URL: postgres://postgres@db:5432/tkt_oodi_real
+
+  updater_api:
+    environment:
+      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
+      - TOKEN
+
+  scheduler_mongo:
+    volumes:
+      - mongodata-real:/data/db
+
+volumes:
+  mongodata-real:

updater/docker/dco.dev.yml

@@ -0,0 +1,52 @@
+version: '3.2'
+
+services:
+  updater_api:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./updater/updater_api/:/usr/src/app
+    restart: unless-stopped
+
+  updater_writer:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/backend/updater_writer/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/models
+      - ./services/backend/shared/migrations:/usr/src/app/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
+    environment:
+      DB_SCHEMA_KONE: kone_data
+    restart: unless-stopped
+
+  updater_scheduler:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./updater/updater_scheduler/:/usr/src/app
+    restart: unless-stopped
+
+  sis-updater-scheduler:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./updater/sis-updater-scheduler/:/usr/src/app
+    restart: unless-stopped
+
+  sis-updater-worker:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: 'npm run dev'
+    volumes:
+      - ./updater/sis-updater-worker/:/usr/src/app
+    restart: unless-stopped

updater/sis-updater-scheduler/Dockerfile

@@ -0,0 +1,15 @@
+FROM node:10.15-alpine
+
+WORKDIR /usr/src/app
+EXPOSE 8082
+CMD npm start
+
+COPY package* ./
+
+RUN npm ci
+
+ARG NODE_ENV=production
+ENV NODE_ENV=${NODE_ENV}
+
+COPY . /usr/src/app
+

updater/sis-updater-worker/Dockerfile

@@ -0,0 +1,14 @@
+FROM node:10.15-alpine
+
+WORKDIR /usr/src/app
+
+CMD npm start
+
+COPY package* ./
+
+RUN npm ci
+
+ARG NODE_ENV=production
+ENV NODE_ENV=${NODE_ENV}
+
+COPY . .