add optional dpp_module and snapshots

- dpp_module becomes optional
  - docker compose for simple devicehub and with the dpp module
  - changed logic in entrypoint so different parts are configured or
  - not depending on the new DPP_MODULE env var
- optional snapshots
  - new default directory for SNAPSHOTS_PATH
  - new env var IMPORT_SNAPSHOTS to optionally import the snapshots or not
- new Make file targets:
  - dc_up_devicehub: docker compose for simple devicehub
  - dc_up_devicehub_dpp: docker compose for devicehub with DPP_module

.gitignore

@@ -135,6 +135,7 @@ env*
 examples/create-db2.sh
 package-lock.json
 snapshots/
+!examples/snapshots
 modules/
 
 # emacs

Makefile

@@ -32,3 +32,17 @@ docker:
 	$(MAKE) docker_build
 	$(MAKE) docker_publish
 	@printf "\ndocker images published\n"
+
+# manage 2 kinds of deployments with docker compose
+
+dc_up_devicehub:
+	docker compose -f docker-compose_devicehub.yml up || true
+
+dc_down_devicehub:
+	docker compose -f docker-compose_devicehub.yml down -v || true
+
+dc_up_devicehub_dpp:
+	docker compose -f docker-compose_devicehub-dpp.yml up || true
+
+dc_down_devicehub_dpp:
+	docker compose -f docker-compose_devicehub-dpp.yml down -v || true

docker-compose.yml

@@ -1,95 +0,0 @@
-version: "3.9"
-services:
-
-  devicehub-id-server:
-    init: true
-    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp_docker__2c4b0006
-    environment:
-      - DB_USER=${DB_USER}
-      - DB_PASSWORD=${DB_PASSWORD}
-      - DB_HOST=postgres-id-server
-      - DB_DATABASE=${DB_DATABASE}
-      - HOST=${HOST}
-      - EMAIL_DEMO=user5000@dhub.com
-      - PASSWORD_DEMO=${PASSWORD_DEMO}
-      - JWT_PASS=${JWT_PASS}
-      - SECRET_KEY=${SECRET_KEY}
-      - API_DLT=${API_DLT}
-      - API_RESOLVER=${API_RESOLVER}
-      - API_DLT_TOKEN=${API_DLT_TOKEN}
-      - DEVICEHUB_HOST=${SERVER_ID_DEVICEHUB_HOST}
-      - ID_FEDERATED=${SERVER_ID_FEDERATED}
-      - URL_MANUALS=${URL_MANUALS}
-      - ID_SERVICE=${SERVER_ID_SERVICE}
-      - AUTHORIZED_CLIENT_URL=${CLIENT_ID_DEVICEHUB_HOST}
-    ports:
-      - 5000:5000
-    volumes:
-      - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro
-      - shared:/shared:rw
-
-  postgres-id-server:
-    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp_docker__2c4b0006
-    # 4. To create the database.
-    # 5. Give permissions to the corresponding users in the database.
-    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
-    environment:
-      - POSTGRES_PASSWORD=${DB_PASSWORD}
-      - POSTGRES_USER=${DB_USER}
-      - POSTGRES_DB=${DB_DATABASE}
-    # DEBUG
-    #ports:
-    #  - 5432:5432
-    # TODO persistence
-    #volumes:
-    #  - pg_data:/var/lib/postgresql/data
-
-  devicehub-id-client:
-    init: true
-    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp_docker__2c4b0006
-    environment:
-      - DB_USER=${DB_USER}
-      - DB_PASSWORD=${DB_PASSWORD}
-      - DB_HOST=postgres-id-client
-      - DB_DATABASE=${DB_DATABASE}
-      - HOST=${HOST}
-      - EMAIL_DEMO=user5001@dhub.com
-      - PASSWORD_DEMO=${PASSWORD_DEMO}
-      - JWT_PASS=${JWT_PASS}
-      - SECRET_KEY=${SECRET_KEY}
-      - API_DLT=${API_DLT}
-      - API_RESOLVER=${API_RESOLVER}
-      - API_DLT_TOKEN=${API_DLT_TOKEN}
-      - DEVICEHUB_HOST=${CLIENT_ID_DEVICEHUB_HOST}
-      - SERVER_ID_HOST=${SERVER_ID_DEVICEHUB_HOST}
-      - ID_FEDERATED=${CLIENT_ID_FEDERATED}
-      - URL_MANUALS=${URL_MANUALS}
-      - ID_SERVICE=${CLIENT_ID_SERVICE}
-    ports:
-      - 5001:5000
-    volumes:
-      - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro
-      - shared:/shared:ro
-
-  postgres-id-client:
-    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp_docker__2c4b0006
-    # 4. To create the database.
-    # 5. Give permissions to the corresponding users in the database.
-    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
-    environment:
-      - POSTGRES_PASSWORD=${DB_PASSWORD}
-      - POSTGRES_USER=${DB_USER}
-      - POSTGRES_DB=${DB_DATABASE}
-    # DEBUG
-    #ports:
-    #  - 5432:5432
-    # TODO persistence
-    #volumes:
-    #  - pg_data:/var/lib/postgresql/data
-
-
-    # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
-    #nginx
-
-volumes:
-  shared:

docker-compose.yml

@@ -0,0 +1 @@
+docker-compose_devicehub-dpp.yml

docker-compose_devicehub-dpp.yml

@@ -0,0 +1,99 @@
+version: "3.9"
+services:
+
+  devicehub-id-server:
+    init: true
+    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp__285635b7
+    environment:
+      - DB_USER=${DB_USER}
+      - DB_PASSWORD=${DB_PASSWORD}
+      - DB_HOST=postgres-id-server
+      - DB_DATABASE=${DB_DATABASE}
+      - HOST=${HOST}
+      - EMAIL_DEMO=${SERVER_ID_EMAIL_DEMO}
+      - PASSWORD_DEMO=${PASSWORD_DEMO}
+      - JWT_PASS=${JWT_PASS}
+      - SECRET_KEY=${SECRET_KEY}
+      - API_DLT=${API_DLT}
+      - API_RESOLVER=${API_RESOLVER}
+      - API_DLT_TOKEN=${API_DLT_TOKEN}
+      - DEVICEHUB_HOST=${SERVER_ID_DEVICEHUB_HOST}
+      - ID_FEDERATED=${SERVER_ID_FEDERATED}
+      - URL_MANUALS=${URL_MANUALS}
+      - ID_SERVICE=${SERVER_ID_SERVICE}
+      - AUTHORIZED_CLIENT_URL=${CLIENT_ID_DEVICEHUB_HOST}
+      - DPP_MODULE=y
+      - IMPORT_SNAPSHOTS=${IMPORT_SNAPSHOTS}
+    ports:
+      - 5000:5000
+    volumes:
+      - ${SNAPSHOTS_PATH:-./examples/snapshots}:/mnt/snapshots:ro
+      - shared:/shared:rw
+
+  postgres-id-server:
+    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp__285635b7
+    # 4. To create the database.
+    # 5. Give permissions to the corresponding users in the database.
+    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
+    environment:
+      - POSTGRES_PASSWORD=${DB_PASSWORD}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_DB=${DB_DATABASE}
+    # DEBUG
+    #ports:
+    #  - 5432:5432
+    # TODO persistence
+    #volumes:
+    #  - pg_data:/var/lib/postgresql/data
+
+  devicehub-id-client:
+    init: true
+    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp__285635b7
+    environment:
+      - DB_USER=${DB_USER}
+      - DB_PASSWORD=${DB_PASSWORD}
+      - DB_HOST=postgres-id-client
+      - DB_DATABASE=${DB_DATABASE}
+      - HOST=${HOST}
+      - EMAIL_DEMO=${CLIENT_ID_EMAIL_DEMO}
+      - PASSWORD_DEMO=${PASSWORD_DEMO}
+      - JWT_PASS=${JWT_PASS}
+      - SECRET_KEY=${SECRET_KEY}
+      - API_DLT=${API_DLT}
+      - API_RESOLVER=${API_RESOLVER}
+      - API_DLT_TOKEN=${API_DLT_TOKEN}
+      - DEVICEHUB_HOST=${CLIENT_ID_DEVICEHUB_HOST}
+      - SERVER_ID_HOST=${SERVER_ID_DEVICEHUB_HOST}
+      - ID_FEDERATED=${CLIENT_ID_FEDERATED}
+      - URL_MANUALS=${URL_MANUALS}
+      - ID_SERVICE=${CLIENT_ID_SERVICE}
+      - DPP_MODULE=y
+      - IMPORT_SNAPSHOTS=${IMPORT_SNAPSHOTS}
+    ports:
+      - 5001:5000
+    volumes:
+      - ${SNAPSHOTS_PATH:-./examples/snapshots}:/mnt/snapshots:ro
+      - shared:/shared:ro
+
+  postgres-id-client:
+    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp__285635b7
+    # 4. To create the database.
+    # 5. Give permissions to the corresponding users in the database.
+    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
+    environment:
+      - POSTGRES_PASSWORD=${DB_PASSWORD}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_DB=${DB_DATABASE}
+    # DEBUG
+    #ports:
+    #  - 5432:5432
+    # TODO persistence
+    #volumes:
+    #  - pg_data:/var/lib/postgresql/data
+
+
+    # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
+    #nginx
+
+volumes:
+  shared:

docker-compose_devicehub.yml

@@ -0,0 +1,48 @@
+version: "3.9"
+services:
+
+  devicehub:
+    init: true
+    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp__285635b7
+    environment:
+      - DB_USER=${DB_USER}
+      - DB_PASSWORD=${DB_PASSWORD}
+      - DB_HOST=postgres
+      - DB_DATABASE=${DB_DATABASE}
+      - HOST=${HOST}
+      - EMAIL_DEMO=${EMAIL_DEMO}
+      - PASSWORD_DEMO=${PASSWORD_DEMO}
+      - JWT_PASS=${JWT_PASS}
+      - SECRET_KEY=${SECRET_KEY}
+      - DEVICEHUB_HOST=${DEVICEHUB_HOST}
+      - URL_MANUALS=${URL_MANUALS}
+      - AUTHORIZED_CLIENT_URL=${CLIENT_ID_DEVICEHUB_HOST}
+      - DPP_MODULE=n
+      - IMPORT_SNAPSHOTS=${IMPORT_SNAPSHOTS}
+    ports:
+      - 5000:5000
+    volumes:
+      - ${SNAPSHOTS_PATH:-./examples/snapshots}:/mnt/snapshots:ro
+      - shared:/shared:rw
+
+  postgres:
+    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp__285635b7
+    # 4. To create the database.
+    # 5. Give permissions to the corresponding users in the database.
+    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
+    environment:
+      - POSTGRES_PASSWORD=${DB_PASSWORD}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_DB=${DB_DATABASE}
+    # DEBUG
+    #ports:
+    #  - 5432:5432
+    # TODO persistence
+    #volumes:
+    #  - pg_data:/var/lib/postgresql/data
+
+    # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
+    #nginx
+
+volumes:
+  shared:

docker/devicehub.entrypoint.sh

@@ -7,16 +7,24 @@ set -x
 
 # 3. Generate an environment .env file.
 gen_env_vars() {
-        # generate config using env vars from docker
+        # specific dpp env vars
-        cat > .env <<END
+        if [ "${DPP_MODULE}" = 'y' ]; then
-DB_USER='${DB_USER}'
+                dpp_env_vars="$(cat <<END
-DB_PASSWORD='${DB_PASSWORD}'
-DB_HOST='${DB_HOST}'
-DB_DATABASE='${DB_DATABASE}'
 API_DLT='${API_DLT}'
 API_DLT_TOKEN='${API_DLT_TOKEN}'
 API_RESOLVER='${API_RESOLVER}'
 ID_FEDERATED='${ID_FEDERATED}'
+END
+)"
+        fi
+
+        # generate config using env vars from docker
+        cat > .env <<END
+${dpp_env_vars:-}
+DB_USER='${DB_USER}'
+DB_PASSWORD='${DB_PASSWORD}'
+DB_HOST='${DB_HOST}'
+DB_DATABASE='${DB_DATABASE}'
 URL_MANUALS='${URL_MANUALS}'
 
 HOST='${HOST}'
@@ -145,33 +153,48 @@ config_oidc() {
         fi
 }
 
+config_dpp_part1() {
+        # 12. Add a new server to the 'api resolver'
+        handle_federated_id
+
+        # 13. Do a rsync api resolve
+        flask dlt_rsync_members
+
+        # 14. Register a new user to the DLT
+        flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator
+}
+
+config_dpp_part2() {
+        # 16.
+        flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+        # 20. config server or client ID
+        config_oidc
+}
+
 config_phase() {
         init_flagfile='/already_configured'
         if [ ! -f "${init_flagfile}" ]; then
                 # 7, 8, 9, 11
                 init_data
 
-                # 12. Add a new server to the 'api resolver'
+                if [ "${DPP_MODULE}" = 'y' ]; then
-                handle_federated_id
+                        # 12, 13, 14
-
+                        config_dpp_part1
-                # 13. Do a rsync api resolve
+                fi
-                flask dlt_rsync_members
-
-                # 14. Register a new user to the DLT
-                flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator
 
                 # non DL user (only for the inventory)
                 #   flask adduser user2@dhub.com ${PASSWORD_DEMO}
 
                 # # 15. Add inventory snapshots for user "${EMAIL_DEMO}".
-                cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files
+                if [ "${IMPORT_SNAPSHOTS}" = 'y' ]; then
-                /usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+                        cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files
+                        /usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+                fi
 
-                # # 16.
+                if [ "${DPP_MODULE}" = 'y' ]; then
-                flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+                        # 16, 20
-
+                        config_dpp_part2
-                # config server or client ID
+                fi
-                config_oidc
 
                 # remain next command as the last operation for this if conditional
                 touch "${init_flagfile}"

examples/env.example

@@ -4,6 +4,9 @@ API_DLT_TOKEN=$TOKEN
 API_RESOLVER='http://$IP_API_RESOLVER'
 # you might change or register ID_FEDERATED if you change DEVICEHUB_HOST
 ID_FEDERATED='DH12'
+# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED
+SERVER_ID_FEDERATED='DH12'
+CLIENT_ID_FEDERATED='DH20'
 
 # Database Variables
 DB_USER='dhub'
@@ -13,10 +16,7 @@ DB_DATABASE='dpp'
 SCHEMA='dbtest'
 DB_SCHEMA='dbtest'
 
-# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED
+DEVICEHUB_HOST='http://localhost:5000'
-SERVER_ID_FEDERATED='DH12'
-CLIENT_ID_FEDERATED='DH20'
-
 #SERVER_ID_DEVICEHUB_HOST='http://devicehub-server-id.example.com'
 SERVER_ID_DEVICEHUB_HOST='http://localhost:5000'
 #CLIENT_ID_DEVICEHUB_HOST='http://devicehub-client-id.example.com'
@@ -26,6 +26,8 @@ CLIENT_ID_SERVICE='client_id'
 HOST='localhost'
 
 EMAIL_DEMO='user@example.com'
+SERVER_ID_EMAIL_DEMO='user5000@example.com'
+CLIENT_ID_EMAIL_DEMO='user5001@example.com'
 PASSWORD_DEMO='1234'
 
 JWT_PASS='aaaa'
@@ -33,9 +35,10 @@ SECRET_KEY='aaaa'
 
 # important to import snapshots (step 15)
 #   rel path starts with ./
-#SNAPSHOTS_PATH='./relpath/to/snapshots'
+SNAPSHOTS_PATH='./examples/snapshots'
 #   full path starts with /
-SNAPSHOTS_PATH='/tmp/dhub_docker/snapshots'
+#SNAPSHOTS_PATH='/tmp/dhub_docker/snapshots'
+IMPORT_SNAPSHOTS='y'
 
 # If you have a URL_MANUALS implementation, please change this url
 URL_MANUALS='http://localhost:4000'