Bootup fixes

redirect http to https

.env.local

@@ -0,0 +1,26 @@
+# General
+
+CLUSTERED_MODE=false
+
+# Log
+
+DEBUG=0
+BASHLOG_FILE=0
+BASHLOG_FILE_PATH=platform.log
+
+# Data Mapper - Logstash
+
+LOGSTASH_DEV_MOUNT=false
+LOGSTASH_PACKAGE_PATH=
+
+# Dashboard Visualiser - JS Report
+
+## !NOTE: MAKE SURE YOU HAVE RUN 'set-permissions.sh' SCRIPT BEFORE AND AFTER RUNNING JS REPORT
+JS_REPORT_DEV_MOUNT=false
+JS_REPORT_PACKAGE_PATH=
+
+# Message Bus - Kafka
+
+#  !NOTE: Topics should comma seperated, optional include partion and repliction values
+#   e.g. <topic>:<partions>:<replicationFactor> -> test:3:2 (defaults to <topics>:3:1)
+# KAFKA_TOPICS=2xx,reprocess,3xx,metrics:3:1

.github/workflows/main.yml

@@ -0,0 +1,61 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  release:
+    types: [published]
+  workflow_dispatch:
+env:
+  IPLUS_DB_PW: mysqlpw
+jobs:
+  CI:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Pull containers
+        run: docker-compose pull
+      
+      - name: Cache containers
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+
+      - name: Start all containers
+        run: docker-compose up -d
+      
+      - name: Sleep for 120 seconds
+        run: sleep 120
+        shell: bash
+
+      - name: Display docker logs for openhim config
+        run: docker-compose logs openhim-config
+
+      - name: Show docker logs for isanteplus
+        run: docker logs isanteplus
+
+      - name: Wait For OpenMRS
+        run: docker-compose -f test.docker-compose.yml up --exit-code-from wait wait
+
+      - name: Restart streaming pipeline
+        run: docker-compose up -d --force-recreate streaming-pipeline
+      
+      - name: Sleep for 30 seconds
+        run: sleep 30
+        shell: bash
+      
+      - name: Run Basic API Tests (https://www.postman.com/itechuw/workspace/haiti-sedish/collection/1525496-192e3c9f-caea-4056-8077-b67af442f01c)
+        env:
+          POSTMAN_COLLECTION: https://www.getpostman.com/collections/22647ce67595a103f9c1
+        run: docker-compose -f test.docker-compose.yml up --exit-code-from newman newman
+               
+      - name: Run iSantePlus QA Framework (https://github.com/IsantePlus/isanteplus-qaframework)
+        run: docker-compose -f test.docker-compose.yml up --exit-code-from qa qa
+        continue-on-error: false
+      

.gitignore

@@ -0,0 +1,4 @@
+instant
+instant-linux
+instant-macosx
+instant.exe

.postman/env.json

@@ -0,0 +1,11 @@
+{
+	"name": "ci",
+	"values": [
+		{
+			"key": "isanteplus-url",
+			"value": "http://isanteplus:8080",
+			"enabled": true
+		}
+	],
+	"_postman_variable_scope": "environment"
+}

Dockerfile

@@ -0,0 +1,12 @@
+FROM openhie/package-base:2.1.3
+
+# Install yq
+RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.23.1/yq_linux_amd64 -o /usr/bin/yq
+RUN chmod +x /usr/bin/yq
+
+# Install jq
+RUN apt-get install jq -y 1>/dev/null
+# Install envsubst
+RUN apt-get install gettext-base -y 1>/dev/null
+
+ADD . .

README.md

@@ -1,2 +1,73 @@
 # HaitiHealthBox
 An all-inclusive health information system setup for health facilities in Haiti
+
+# Setup Instructions
+
+## Step One: Install Docker
+See https://docs.docker.com/engine/install/
+
+Test to make sure both of these commands work:
+```sh
+docker -v
+docker compose version
+```
+
+**Note: If your docker version does not support the `docker compose` syntax, 
+install `docker-compose` using the instructions here: https://docs.docker.com/compose/install/**
+
+
+## Step Two: Get HaitiHealthBox from GitHub
+
+Run the following command:
+```sh
+git clone https://github.com/I-TECH-UW/HaitiHealthBox.git
+```
+
+If you want to use a specific version from the [version list](https://github.com/I-TECH-UW/HaitiHealthBox/releases), you can check out the corresponding tag. For example, to use version `0.0.0-alpha`, run:
+```sh
+git checkout v0.0.0-alpha
+```
+
+Otherwise, you can use the latest `main` branch that is the default. 
+
+## Step Three: Set some configurations
+Set some temporary variables to store your desired iSantePlus Database and iSantePlus admin passwords:
+
+```sh
+export IPLUS_ADMIN_PW=<your-password>
+export IPLUS_DB_PW=<your-password>
+```
+## Step Four: Boot up Docker Enviornment
+Navigate to the `HaitiHealthBox` directory on your machine and run the following command:
+```sh
+docker-compose up -d
+```
+
+## Local setup
+
+Spin up the services
+
+```
+docker-compose -f docker-compose.local.yml up -d
+```
+
+Acces the services at 
+
+| Instance  |     URL       | credentials (user : password)|
+|---------- |:-------------:|------:                       |
+| Isanteplus   |  https://localhost/openmrs/login.htm  | admin : Admin123 |
+| OpenHIM   |    https://localhost/  |  root@openhim.org : admin |
+| OpenELIS | https://localhost/OpenELIS-Global/ |    admin : adminADMIN!| 
+
+## Instructions 
+1. Add the necesary Lab Unit permisions to the User      
+Go To `Admin -> User Management -> Select User -> Under "Roles" ,Choose "ALL Lab Units" and select  "All Permissions"`
+
+2. Enable Importing electronic orders       
+Go To `Admin -> Order Entry Configuration -> Select "external orders" -> Click "Modify" -> select "True"`
+
+3. Add the Right Test Catalogue ie tests with `Loinc Codes`
+
+see [more](https://i-tech-uw.github.io/healthinformationexchange/lis-workflows/lis-workflows.html#tutorial-lab-order-communication-between-openmrs-and-openelis) for the EMR-LIS communication
+
+

build-image.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+TAG_NAME=${1:-latest}
+docker build --platform linux/x86_64 -t itechuw/haiti-health-box:"$TAG_NAME" .

config.yaml

@@ -0,0 +1,12 @@
+projectName: Haiti Health Box
+image: itechuw/haiti-health-box
+logPath: /tmp/logs
+
+packages:
+    - identity-access-manager-keycloak
+    - interoperability-layer-openhim
+    - lis-openelis-global
+    - monitoring
+    - emr-isanteplus
+    - openhim-mediator-data-stream
+    - reverse-proxy-caddy

docker-compose.yml

@@ -1,112 +0,0 @@
-version: "3.8"
-
-services:
-  ###
-  # reverse proxy
-  ###
-  traefik:
-    image: traefik:v2.6.1
-    container_name: traefik
-    ports:
-      - '80:80'
-      - '443:443'
-      - '8080:8080'
-    volumes:
-      - ./certs/:/etc/certs/       
-      - /var/run/docker.sock:/var/run/docker.sock
-      - ./configs/conf.yaml:/conf/conf.yaml  
-      - ./configs/traefik.yaml:/etc/traefik/traefik.yaml
-    
-  ###
-  # iSantePlus Server
-  ###
-  isanteplus:
-    container_name: isanteplus
-    hostname: isanteplus
-    image: ghcr.io/isanteplus/docker-isanteplus-server:v2.3.0
-    restart: unless-stopped
-    env_file:
-      - ./isanteplus.env
-    volumes:
-      - /openmrs/data
-      - ./custom_modules:/custom_modules
-      - ./dist:/openmrs/data/frontend
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "10m"
-
-  isanteplus-mysql:
-    image: ghcr.io/isanteplus/docker-isanteplus-db:v2.2.1
-    command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci --sql_mode=""
-    restart: unless-stopped
-    container_name: isanteplus-mysql
-    hostname: isanteplus-mysql
-    healthcheck:
-      test: "exit 0"
-    environment:
-      - MYSQL_DATABASE=openmrs
-      - MYSQL_ROOT_PASSWORD=debezium
-      - MYSQL_USER=mysqluser
-      - MYSQL_PASSWORD=mysqlpw
-      - MYSQL_ROOT_HOST=% # Allow docker containers to connect to mysql
-    volumes:
-      - ./mysql.cnf:/etc/mysql/conf.d/custom.cnf # mysql config preconfigured to allow binlog/debezium
-      - /var/lib/mysql
-    ports:
-    - "3306:3306"
-
-  ###
-  # OpenHIM
-  ###
-  openhim-core:
-    container_name: openhim-core
-    hostname: openhim-core
-    image: jembi/openhim-core:latest
-    restart: unless-stopped
-    environment:
-      mongo_url: "mongodb://mongo-db/openhim"
-      mongo_atnaUrl: "mongodb://mongo-db/openhim"
-      NODE_ENV: "development"
-    healthcheck:
-      test: "node /healthcheck.js"
-      interval: 20s
-      timeout: 20s
-      retries: 2
-    networks:
-      - hie
-    volumes:
-      - ./configs/openhim/healthcheck.js:/healthcheck.js
-
-  openhim-console:
-    container_name: openhim-console
-    hostname: openhim-console
-    image: jembi/openhim-console:latest
-    restart: unless-stopped
-    volumes:
-      - ./configs/openhim-console/default.json:/usr/share/nginx/html/config/default.json
-    healthcheck:
-      test: "curl -sS http://openhim-console || exit 1"
-      interval: 10s
-      timeout: 60s
-      retries: 3
-    networks:
-      - hie
-
-  # Loads Default OpenHIM Config
-  openhim-config:
-    container_name: openhim-config
-    image: ghcr.io/i-tech-uw/openhim-config:v0.0.0
-    volumes:
-      - ./configs/openhim/test-openhim-config.json:/app/test-openhim-config.json
-    networks:
-      - hie
-
-  mongo-db:
-    container_name: mongo-db
-    image: mongo:3.4
-    volumes:
-      - /data/db
-    restart: unless-stopped
-    networks:
-      - hie

emr-isanteplus/config/mysql.cnf

@@ -0,0 +1,62 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a pre-configuration for Docker MySQL to allow BinLog and replication necessary configurations for Debezium to work
+# see ./openmrs-compose.yaml
+# REF: https://github.com/debezium/debezium/blob/master/debezium-connector-mysql/src/test/docker/server-gtids/my.cnf
+
+# For advice on how to change settings please see
+# http://dev.mysql.com/doc/refman/5.7/en/server-configuration-defaults.html
+
+[mysqld]
+#
+# Remove leading # and set to the amount of RAM for the most important data
+# cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%.
+# innodb_buffer_pool_size = 128M
+#
+# Remove leading # to turn on a very important data integrity option: logging
+# changes to the binary log between backups.
+# log_bin
+#
+# Remove leading # to set options mainly useful for reporting servers.
+# The server defaults are faster for transactions and fast SELECTs.
+# Adjust sizes as needed, experiment to find the optimal values.
+# join_buffer_size = 128M
+# sort_buffer_size = 2M
+# read_rnd_buffer_size = 2M
+skip-host-cache
+skip-name-resolve
+#datadir=/var/lib/mysql
+#socket=/var/lib/mysql/mysql.sock
+#secure-file-priv=/var/lib/mysql-files
+user=mysql
+
+# Disabling symbolic-links is recommended to prevent assorted security risks
+symbolic-links=0
+
+#log-error=/var/log/mysqld.log
+#pid-file=/var/run/mysqld/mysqld.pid
+
+# ----------------------------------------------
+# Enable the binlog for replication & CDC
+# ----------------------------------------------
+
+# Enable binary replication log and set the prefix, expiration, and log format.
+# The prefix is arbitrary, expiration can be short for integration tests but would
+# be longer on a production system. Row-level info is required for ingest to work.
+# Server ID is required, but this will vary on production systems
+server-id         = 223344
+log_bin           = mysql-bin
+expire_logs_days  = 1
+binlog_format     = row

emr-isanteplus/docker-compose-mysql.yml

@@ -0,0 +1,34 @@
+  isanteplus-mysql:
+    image: ghcr.io/isanteplus/docker-isanteplus-db:v2.2.1
+    command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci --sql_mode=""
+    restart: unless-stopped
+    container_name: isanteplus-mysql
+    hostname: isanteplus-mysql
+    healthcheck:
+      test: "exit 0"
+    environment:
+      - MYSQL_DATABASE=openmrs
+      - MYSQL_ROOT_PASSWORD=debezium
+      - MYSQL_USER=mysqluser
+      - MYSQL_PASSWORD=${IPLUS_DB_PW-mysqlpw}
+      - MYSQL_ROOT_HOST=% # Allow docker containers to connect to mysql
+    volumes:
+      - /var/lib/mysql
+    configs:
+      - source: mysql.cnf
+        target: /etc/mysql/conf.d/mysql.cnf
+        
+    networks:
+      - isanteplus
+      - default
+  configs: 
+    mysql.cnf:
+      file: ./config/mysql.cnf
+      name: mysql.cnf-${mysql_cnf_DIGEST:?err}
+      labels:
+        name: isanteplus
+  networks:
+    isanteplus:
+      external: true
+      name: emr-isanteplus
+  default:

emr-isanteplus/docker-compose.dev.yml

@@ -0,0 +1,10 @@
+version: '3.9'
+
+services:
+  emr-isanteplus:
+    ports:
+      - target: 1044
+        published: 1044
+        mode: host
+    environment:
+      - OMRS_DEV_DEBUG_PORT=1044

emr-isanteplus/docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3.9'
+
+services:
+  isanteplus:
+    image: ghcr.io/isanteplus/docker-isanteplus-server:v2.3.7
+    restart: unless-stopped
+    environment:
+      - OMRS_JAVA_MEMORY_OPTS="-Xmx8192m -Xms4096m -XX:NewSize=1024m"
+      - OMRS_CONFIG_CONNECTION_SERVER=${OMRS_CONFIG_CONNECTION_SERVER:-isanteplus-mysql}
+      - OMRS_CONFIG_CREATE_DATABASE_USER=false
+      - OMRS_CONFIG_CREATE_TABLES=false
+      - OMRS_CONFIG_ADD_DEMO_DATA=false
+      - OMRS_CONFIG_CONNECTION_URL=jdbc:mysql://${OMRS_CONFIG_CONNECTION_SERVER:-isanteplus-mysql}:3306/openmrs?autoReconnect=true
+      - OMRS_CONFIG_HAS_CURRENT_OPENMRS_DATABASE=true
+      - OMRS_JAVA_SERVER_OPTS="-Dfile.encoding=UTF-8 -server -Djava.security.egd=file:/dev/./urandom -Djava.awt.headless=true -Djava.awt.headlesslib=true"
+      - OMRS_CONFIG_CONNECTION_USERNAME=${OMRS_CONFIG_CONNECTION_USERNAME:-mysqluser}
+      - OMRS_CONFIG_CONNECTION_PASSWORD=${OMRS_CONFIG_CONNECTION_PASSWORD:-mysqlpw}
+    volumes:
+      - /openmrs/data
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+    networks:
+      - public    
+      - isanteplus
+      - default
+
+networks:
+  public:
+    name: reverse-proxy_public
+    external: true
+  networks:
+    isanteplus:
+      external: true
+      name: emr-isanteplus
+  default:

emr-isanteplus/package-metadata.json

@@ -0,0 +1,9 @@
+{
+  "id": "emr-isanteplus",
+  "name": "iSantePlus Haiti EMR",
+  "description": "OpenMRS-based EMR for Haiti",
+  "type": "infrastructure",
+  "version": "0.0.1",
+  "dependencies": [],
+  "environmentVariables": {}
+}

emr-isanteplus/swarm.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare SERVICE_NAMES=()
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  SERVICE_NAMES=(
+    "emr-isanteplus"
+  )
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+  readonly SERVICE_NAMES
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function initialize_package() {
+  local package_dev_compose_filename=""
+  if [[ "${MODE}" == "dev" ]]; then
+    log info "Running package in DEV mode"
+    package_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  (
+    docker::deploy_service "${COMPOSE_FILE_PATH}" "$package_dev_compose_filename"
+    docker::deploy_sanity "${SERVICE_NAMES[@]}"
+  ) || {
+    log error "Failed to deploy package"
+    exit 1
+  }
+}
+
+function destroy_package() {
+  docker::service_destroy "${SERVICE_NAMES[@]}"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    log info "Running package in Single node mode"
+
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+
+    docker::scale_services_down "${SERVICE_NAMES[@]}"
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"

get-cli.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+option="${1:-all}"
+cli_version=${2:-latest}
+
+case ${option} in
+linux)
+  echo "Downloading linux binary version: ${cli_version}"
+  curl -L https://github.com/openhie/package-starter-kit/releases/download/"$cli_version"/instant-linux -o instant
+  chmod +x ./instant
+  exit 0
+  ;;
+macos)
+  echo "Downloading macos binary version: ${cli_version}"
+  curl -L https://github.com/openhie/package-starter-kit/releases/download/"$cli_version"/instant-macos -o instant
+  chmod +x ./instant
+  exit 0
+  ;;
+windows)
+  echo "Downloading windows binary version: ${cli_version}"
+  curl -L https://github.com/openhie/package-starter-kit/releases/download/"$cli_version"/instant.exe -o instant.exe
+  chmod +x ./instant.exe
+  exit 0
+  ;;
+all)
+  echo "Downloading all binaries, version: ${cli_version}"
+  curl -L https://github.com/openhie/package-starter-kit/releases/download/"$cli_version"/instant-linux -o instant-linux
+  curl -L https://github.com/openhie/package-starter-kit/releases/download/"$cli_version"/instant-macos -o instant-macos
+  curl -L https://github.com/openhie/package-starter-kit/releases/download/"$cli_version"/instant.exe -o instant.exe
+  chmod +x ./instant-linux
+  chmod +x ./instant-macos
+  chmod +x ./instant.exe
+  exit 0
+  ;;
+--help)
+  echo "Usage: get-cli.sh [linux|macos|windows|all] {cli_version}"
+  exit 0
+  ;;
+esac

identity-access-manager-keycloak/config/grafana.json

@@ -0,0 +1,50 @@
+{
+  "clientId": "${KC_GRAFANA_CLIENT_ID}",
+  "name": "grafana",
+  "description": "",
+  "rootUrl": "${KC_GRAFANA_ROOT_URL}",
+  "adminUrl": "${KC_GRAFANA_ROOT_URL}",
+  "baseUrl": "${KC_GRAFANA_ROOT_URL}",
+  "surrogateAuthRequired": false,
+  "enabled": true,
+  "alwaysDisplayInConsole": false,
+  "clientAuthenticatorType": "client-secret",
+  "secret": "${KC_GRAFANA_CLIENT_SECRET}",
+  "redirectUris": ["${KC_GRAFANA_ROOT_URL}/login/generic_oauth"],
+  "webOrigins": ["${KC_GRAFANA_ROOT_URL}"],
+  "notBefore": 0,
+  "bearerOnly": false,
+  "consentRequired": false,
+  "standardFlowEnabled": true,
+  "implicitFlowEnabled": false,
+  "directAccessGrantsEnabled": true,
+  "serviceAccountsEnabled": false,
+  "publicClient": false,
+  "frontchannelLogout": true,
+  "protocol": "openid-connect",
+  "attributes": {
+    "oidc.ciba.grant.enabled": "false",
+    "post.logout.redirect.uris": "${KC_GRAFANA_ROOT_URL}/login",
+    "client.secret.creation.time": "1672390081",
+    "backchannel.logout.session.required": "true",
+    "oauth2.device.authorization.grant.enabled": "false",
+    "display.on.consent.screen": "false",
+    "backchannel.logout.revoke.offline.tokens": "false",
+    "frontchannel.logout.url": "${KC_GRAFANA_ROOT_URL}/logout"
+  },
+  "authenticationFlowBindingOverrides": {},
+  "fullScopeAllowed": true,
+  "nodeReRegistrationTimeout": -1,
+  "defaultClientScopes": ["web-origins", "acr", "roles", "profile", "email"],
+  "optionalClientScopes": [
+    "address",
+    "phone",
+    "offline_access",
+    "microprofile-jwt"
+  ],
+  "access": {
+    "view": true,
+    "configure": true,
+    "manage": true
+  }
+}

identity-access-manager-keycloak/config/jempi.json

@@ -0,0 +1,49 @@
+{
+  "clientId": "${KC_JEMPI_CLIENT_ID}",
+  "name": "JeMPI",
+  "description": "",
+  "rootUrl": "${KC_JEMPI_ROOT_URL}",
+  "adminUrl": "${KC_JEMPI_ROOT_URL}",
+  "baseUrl": "${KC_JEMPI_ROOT_URL}",
+  "surrogateAuthRequired": false,
+  "enabled": true,
+  "alwaysDisplayInConsole": false,
+  "clientAuthenticatorType": "client-secret",
+  "secret": "${KC_JEMPI_CLIENT_SECRET}",
+  "redirectUris": ["${KC_JEMPI_ROOT_URL}/login"],
+  "webOrigins": ["${KC_JEMPI_ROOT_URL}"],
+  "notBefore": 0,
+  "bearerOnly": false,
+  "consentRequired": false,
+  "standardFlowEnabled": true,
+  "implicitFlowEnabled": false,
+  "directAccessGrantsEnabled": true,
+  "serviceAccountsEnabled": false,
+  "publicClient": false,
+  "frontchannelLogout": true,
+  "protocol": "openid-connect",
+  "attributes": {
+    "oidc.ciba.grant.enabled": "false",
+    "client.secret.creation.time": "1674028783",
+    "backchannel.logout.session.required": "true",
+    "post.logout.redirect.uris": "${KC_JEMPI_ROOT_URL}",
+    "display.on.consent.screen": "false",
+    "oauth2.device.authorization.grant.enabled": "false",
+    "backchannel.logout.revoke.offline.tokens": "false"
+  },
+  "authenticationFlowBindingOverrides": {},
+  "fullScopeAllowed": true,
+  "nodeReRegistrationTimeout": -1,
+  "defaultClientScopes": ["web-origins", "acr", "roles", "profile", "email"],
+  "optionalClientScopes": [
+    "address",
+    "phone",
+    "offline_access",
+    "microprofile-jwt"
+  ],
+  "access": {
+    "view": true,
+    "configure": true,
+    "manage": true
+  }
+}

identity-access-manager-keycloak/config/openhim.json

@@ -0,0 +1,50 @@
+{
+  "clientId": "${KC_OPENHIM_CLIENT_ID}",
+  "name": "OpenHIM",
+  "description": "",
+  "rootUrl": "${KC_OPENHIM_ROOT_URL}",
+  "adminUrl": "${KC_OPENHIM_ROOT_URL}",
+  "baseUrl": "${KC_OPENHIM_ROOT_URL}",
+  "surrogateAuthRequired": false,
+  "enabled": true,
+  "alwaysDisplayInConsole": false,
+  "clientAuthenticatorType": "client-secret",
+  "secret": "${KC_OPENHIM_CLIENT_SECRET}",
+  "redirectUris": ["${KC_OPENHIM_ROOT_URL}"],
+  "webOrigins": ["${KC_OPENHIM_ROOT_URL}"],
+  "notBefore": 0,
+  "bearerOnly": false,
+  "consentRequired": false,
+  "standardFlowEnabled": true,
+  "implicitFlowEnabled": false,
+  "directAccessGrantsEnabled": true,
+  "serviceAccountsEnabled": false,
+  "publicClient": false,
+  "frontchannelLogout": true,
+  "protocol": "openid-connect",
+  "attributes": {
+    "oidc.ciba.grant.enabled": "false",
+    "client.secret.creation.time": "1674028783",
+    "backchannel.logout.session.required": "true",
+    "post.logout.redirect.uris": "${KC_OPENHIM_ROOT_URL}",
+    "display.on.consent.screen": "false",
+    "oauth2.device.authorization.grant.enabled": "false",
+    "backchannel.logout.revoke.offline.tokens": "false",
+    "frontchannel.logout.url": "${KC_OPENHIM_ROOT_URL}/#!/logout"
+  },
+  "authenticationFlowBindingOverrides": {},
+  "fullScopeAllowed": true,
+  "nodeReRegistrationTimeout": -1,
+  "defaultClientScopes": ["web-origins", "acr", "roles", "profile", "email"],
+  "optionalClientScopes": [
+    "address",
+    "phone",
+    "offline_access",
+    "microprofile-jwt"
+  ],
+  "access": {
+    "view": true,
+    "configure": true,
+    "manage": true
+  }
+}

identity-access-manager-keycloak/config/realm.json

@@ -0,0 +1,23 @@
+{
+  "id": "${KC_REALM_NAME}",
+  "realm": "${KC_REALM_NAME}",
+  "displayNameHtml": "<div class=\"kc-logo-text\"><span>Keycloak</span></div>",
+  "enabled": true,
+  "clients": [],
+  "users": [
+    {
+      "email": "test@jembi.org",
+      "username": "test",
+      "enabled": true,
+      "credentials": [
+        {
+          "temporary": false,
+          "type": "password",
+          "value": "dev_password_only"
+        }
+      ],
+      "realmRoles": ["default-roles-${KC_REALM_NAME}"],
+      "clientRoles": {}
+    }
+  ]
+}

identity-access-manager-keycloak/config/superset.json

@@ -0,0 +1,46 @@
+{
+  "clientId": "${KC_SUPERSET_CLIENT_ID}",
+  "name": "superset",
+  "description": "",
+  "rootUrl": "${KC_SUPERSET_ROOT_URL}",
+  "adminUrl": "${KC_SUPERSET_ROOT_URL}",
+  "baseUrl": "${KC_SUPERSET_ROOT_URL}",
+  "surrogateAuthRequired": false,
+  "enabled": true,
+  "alwaysDisplayInConsole": false,
+  "clientAuthenticatorType": "client-secret",
+  "secret": "${KC_SUPERSET_CLIENT_SECRET}",
+  "redirectUris": ["${KC_SUPERSET_ROOT_URL}/oidc_callback"],
+  "webOrigins": ["${KC_SUPERSET_ROOT_URL}"],
+  "notBefore": 0,
+  "bearerOnly": false,
+  "consentRequired": false,
+  "standardFlowEnabled": true,
+  "implicitFlowEnabled": false,
+  "directAccessGrantsEnabled": true,
+  "serviceAccountsEnabled": false,
+  "publicClient": false,
+  "frontchannelLogout": true,
+  "protocol": "openid-connect",
+  "attributes": {
+    "oidc.ciba.grant.enabled": "false",
+    "client.secret.creation.time": "1674028783",
+    "backchannel.logout.session.required": "true",
+    "post.logout.redirect.uris": "${KC_SUPERSET_ROOT_URL}/login/",
+    "display.on.consent.screen": "false",
+    "oauth2.device.authorization.grant.enabled": "false",
+    "backchannel.logout.revoke.offline.tokens": "false",
+    "frontchannel.logout.url": "${KC_SUPERSET_ROOT_URL}/backchannel-logout/"
+  },
+  "authenticationFlowBindingOverrides": {},
+  "fullScopeAllowed": true,
+  "nodeReRegistrationTimeout": -1,
+  "defaultClientScopes": ["web-origins", "acr", "roles", "profile", "email"],
+  "optionalClientScopes": [
+    "address",
+    "phone",
+    "offline_access",
+    "microprofile-jwt"
+  ],
+  "access": { "view": true, "configure": true, "manage": true }
+}

identity-access-manager-keycloak/docker-compose-postgres.cluster.yml

@@ -0,0 +1,75 @@
+version: '3.9'
+
+services:
+  keycloak-postgres-1:
+    environment:
+      REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES}
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-1"
+
+  keycloak-postgres-2:
+    image: bitnami/postgresql-repmgr:14
+    environment:
+      POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD}
+      POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME}
+      POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE}
+      REPMGR_NODE_NETWORK_NAME: keycloak-postgres-2
+      REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD}
+      REPMGR_RECONNECT_INTERVAL: 3
+      REPMGR_NODE_NAME: keycloak-postgres-2
+      REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST}
+      REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES}
+    volumes:
+      - 'keycloak-postgres-2-data:/bitnami/postgresql'
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-2"
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${KC_POSTGRES_CPU_LIMIT}
+          memory: ${KC_POSTGRES_MEMORY_LIMIT}
+        reservations:
+          cpus: ${KC_POSTGRES_CPU_RESERVE}
+          memory: ${KC_POSTGRES_MEMORY_RESERVE}
+    networks:
+      default:
+      keycloak_backup_net: {}
+
+
+  keycloak-postgres-3:
+    image: bitnami/postgresql-repmgr:14
+    environment:
+      POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD}
+      POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME}
+      POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE}
+      REPMGR_NODE_NETWORK_NAME: keycloak-postgres-3
+      REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD}
+      REPMGR_RECONNECT_INTERVAL: 3
+      REPMGR_NODE_NAME: keycloak-postgres-3
+      REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST}
+      REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES}
+    volumes:
+      - 'keycloak-postgres-3-data:/bitnami/postgresql'
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-3"
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${KC_POSTGRES_CPU_LIMIT}
+          memory: ${KC_POSTGRES_MEMORY_LIMIT}
+        reservations:
+          cpus: ${KC_POSTGRES_CPU_RESERVE}
+          memory: ${KC_POSTGRES_MEMORY_RESERVE}
+    networks:
+      default:
+      keycloak_backup_net: {}
+
+volumes:
+  keycloak-postgres-2-data:
+  keycloak-postgres-3-data:

identity-access-manager-keycloak/docker-compose-postgres.dev.yml

@@ -0,0 +1,8 @@
+version: '3.9'
+
+services:
+  keycloak-postgres-1:
+    ports:
+      - target: 5432
+        published: 5434
+        mode: host

identity-access-manager-keycloak/docker-compose-postgres.yml

@@ -0,0 +1,40 @@
+version: "3.9"
+
+services:
+  keycloak-postgres-1:
+    image: bitnami/postgresql-repmgr:14
+    environment:
+      POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD}
+      POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME}
+      POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE}
+      POSTGRESQL_POSTGRES_PASSWORD: ${KC_POSTGRESQL_PASSWORD}
+      REPMGR_NODE_NETWORK_NAME: keycloak-postgres-1
+      REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD}
+      REPMGR_RECONNECT_INTERVAL: 3
+      REPMGR_NODE_NAME: keycloak-postgres-1
+      REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST}
+      REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES}
+    volumes:
+      - "keycloak-postgres-1-data:/bitnami/postgresql"
+    deploy:
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${KC_POSTGRES_CPU_LIMIT}
+          memory: ${KC_POSTGRES_MEMORY_LIMIT}
+        reservations:
+          cpus: ${KC_POSTGRES_CPU_RESERVE}
+          memory: ${KC_POSTGRES_MEMORY_RESERVE}
+    networks:
+      default:
+      keycloak_backup_net: {}
+
+volumes:
+  keycloak-postgres-1-data:
+
+networks:
+  default:
+  keycloak_backup_net:
+    name: keycloak_backup
+    driver: overlay
+    attachable: true

identity-access-manager-keycloak/docker-compose.dev.yml

@@ -0,0 +1,8 @@
+version: '3.9'
+
+services:
+  identity-access-manager-keycloak:
+    ports:
+      - target: 8080
+        published: 9088
+        mode: host

identity-access-manager-keycloak/docker-compose.yml

@@ -0,0 +1,67 @@
+version: '3.9'
+
+services:
+  identity-access-manager-keycloak:
+    image: keycloak/keycloak:20.0
+    command:
+      [
+        "start",
+        "--proxy=edge",
+        "--hostname-url=${KC_FRONTEND_URL}",
+        "--import-realm"
+      ]
+    hostname: identity-access-manager-keycloak
+    healthcheck:
+      test: curl --fail http://localhost:8080/health/ready || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    configs:
+      - target: /opt/keycloak/data/import/realm.json
+        source: realm.json
+    environment:
+      KC_DB: postgres
+      KC_DB_USERNAME: ${KC_POSTGRESQL_USERNAME}
+      KC_DB_PASSWORD: ${KC_POSTGRESQL_PASSWORD}
+      KC_DB_URL: "jdbc:postgresql://${KC_POSTGRES_REPLICA_SET}/${KC_POSTGRESQL_DATABASE}?targetServerType=primary"
+      KC_METRICS_ENABLED: "true"
+      KC_HEALTH_ENABLED: "true"
+      KC_REALM_NAME: ${KC_REALM_NAME}
+      KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN}
+      KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
+      KC_GRAFANA_CLIENT_ID: ${KC_GRAFANA_CLIENT_ID}
+      KC_GRAFANA_CLIENT_SECRET: ${KC_GRAFANA_CLIENT_SECRET}
+      KC_GRAFANA_ROOT_URL: ${KC_GRAFANA_ROOT_URL}
+      KC_JEMPI_CLIENT_ID: ${KC_JEMPI_CLIENT_ID}
+      KC_JEMPI_CLIENT_SECRET: ${KC_JEMPI_CLIENT_SECRET}
+      KC_JEMPI_ROOT_URL: ${KC_JEMPI_ROOT_URL}
+      KC_SUPERSET_CLIENT_ID: ${KC_SUPERSET_CLIENT_ID}
+      KC_SUPERSET_CLIENT_SECRET: ${KC_SUPERSET_CLIENT_SECRET}
+      KC_SUPERSET_ROOT_URL: ${KC_SUPERSET_ROOT_URL}
+      KC_OPENHIM_CLIENT_ID: ${KC_OPENHIM_CLIENT_ID}
+      KC_OPENHIM_CLIENT_SECRET: ${KC_OPENHIM_CLIENT_SECRET}
+      KC_OPENHIM_ROOT_URL: ${KC_OPENHIM_ROOT_URL}
+    deploy:
+      placement:
+        max_replicas_per_node: 1
+    networks:
+      reverse-proxy:
+      public:
+      default:
+
+configs:
+  realm.json:
+    file: ./config/realm.json
+    name: realm.json-${realm_json_DIGEST:?err}
+    labels:
+      name: keycloak
+
+networks:
+  reverse-proxy:
+    name: reverse-proxy_public
+    external: true
+  public:
+    name: keycloak_public
+    external: true
+  default:

identity-access-manager-keycloak/package-metadata.json

@@ -0,0 +1,45 @@
+{
+  "id": "identity-access-manager-keycloak",
+  "name": "Identity Access Manager Keycloak",
+  "description": "An identity and access management solution",
+  "type": "infrastructure",
+  "version": "0.0.1",
+  "dependencies": [],
+  "environmentVariables": {
+    "KEYCLOAK_ADMIN": "admin",
+    "KEYCLOAK_ADMIN_PASSWORD": "dev_password_only",
+    "KC_FRONTEND_URL": "http://localhost:9088",
+    "KC_REALM_NAME": "platform-realm",
+    "KC_REPMGR_PRIMARY_HOST": "keycloak-postgres-1",
+    "KC_REPMGR_PARTNER_NODES": "keycloak-postgres-1",
+    "KC_REPMGR_PASSWORD": "instant101",
+    "KC_POSTGRES_REPLICA_SET": "keycloak-postgres-1:5432",
+    "KC_POSTGRES_CPU_LIMIT": "0",
+    "KC_POSTGRES_CPU_RESERVE": "0.05",
+    "KC_POSTGRES_MEMORY_LIMIT": "3G",
+    "KC_POSTGRES_MEMORY_RESERVE": "500M",
+    "KC_POSTGRESQL_PASSWORD": "instant101",
+    "KC_POSTGRESQL_USERNAME": "admin",
+    "KC_POSTGRESQL_DATABASE": "keycloak",
+    "KC_GRAFANA_SSO_ENABLED": "false",
+    "KC_GRAFANA_CLIENT_ID": "grafana-oauth",
+    "KC_GRAFANA_CLIENT_SECRET": "CV14QfwnpYFj1IH5dK5lScPNCYAIYP1c",
+    "KC_GRAFANA_ROOT_URL": "http://localhost:3000",
+    "KC_GRAFANA_CLIENT_ROLES": "admin,editor,viewer",
+    "KC_JEMPI_SSO_ENABLED": "false",
+    "KC_JEMPI_CLIENT_ID": "jempi-oauth",
+    "KC_JEMPI_CLIENT_SECRET": "Tbe3llP5OJIlqUjz7K1wPp8YDAdCOEMn",
+    "KC_JEMPI_ROOT_URL": "http://localhost:3033",
+    "KC_JEMPI_CLIENT_ROLES": "admin",
+    "KC_SUPERSET_SSO_ENABLED": "false",
+    "KC_SUPERSET_CLIENT_ID": "superset-oauth",
+    "KC_SUPERSET_CLIENT_SECRET": "g0J7oLbX69dL3CS8HVjRYlhRYVsPoDbQ",
+    "KC_SUPERSET_ROOT_URL": "http://localhost:8089",
+    "KC_SUPERSET_CLIENT_ROLES": "admin",
+    "KC_OPENHIM_SSO_ENABLED": "true",
+    "KC_OPENHIM_CLIENT_ID": "openhim-oauth",
+    "KC_OPENHIM_CLIENT_SECRET": "tZKfEbWf0Ka5HBNZwFrdSyQH2xT1sNMR",
+    "KC_OPENHIM_ROOT_URL": "http://localhost:9000",
+    "KC_OPENHIM_CLIENT_ROLES": "admin"
+  }
+}

identity-access-manager-keycloak/swarm.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare STACK="keycloak"
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/config-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function append_client_config() {
+  local -r CONFIG_NAME="${1:?$(missing_param "append_client_config" "CONFIG_NAME")}"
+  local -r CLIENT_ID_ENV_NAME="${2:?$(missing_param "append_client_config" "CLIENT_ID_ENV_NAME")}"
+  local -r CLIENT_ROLES_ENV_NAME="${3:?$(missing_param "append_client_config" "CLIENT_ROLES_ENV_NAME")}"
+
+  # Comma separate env var and quote the values
+  IFS=',' read -r -a client_roles_array <<<"$CLIENT_ROLES_ENV_NAME"
+  client_roles_quoted=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${client_roles_array[@]}")
+  # Append clients configs
+  yq ".clients += [load(\"${COMPOSE_FILE_PATH}/config/$CONFIG_NAME.json\")]" "${COMPOSE_FILE_PATH}/config/realm.json" >tmp.json
+  # Append clients roles
+  jq ".users[0].clientRoles += {\"$CLIENT_ID_ENV_NAME\": ${client_roles_quoted[*]}}" tmp.json >"${COMPOSE_FILE_PATH}/config/realm.json"
+  rm -f tmp.json
+}
+
+function append_config_sso_enabled() {
+  if [[ "${KC_GRAFANA_SSO_ENABLED}" == "true" ]]; then
+    append_client_config "grafana" "$KC_GRAFANA_CLIENT_ID" "$KC_GRAFANA_CLIENT_ROLES"
+  fi
+  if [[ "${KC_SUPERSET_SSO_ENABLED}" == "true" ]]; then
+    append_client_config "superset" "$KC_SUPERSET_CLIENT_ID" "$KC_SUPERSET_CLIENT_ROLES"
+  fi
+  if [[ "${KC_JEMPI_SSO_ENABLED}" == "true" ]]; then
+    append_client_config "jempi" "$KC_JEMPI_CLIENT_ID" "$KC_JEMPI_CLIENT_ROLES"
+  fi
+  if [[ "${KC_OPENHIM_SSO_ENABLED}" == "true" ]]; then
+    append_client_config "openhim" "$KC_OPENHIM_CLIENT_ID" "$KC_OPENHIM_CLIENT_ROLES"
+  fi
+}
+
+function initialize_package() {
+  local postgres_cluster_compose_filename=""
+  local postgres_dev_compose_filename=""
+  local keycloak_dev_compose_filename=""
+
+  if [ "${MODE}" == "dev" ]; then
+    log info "Running package in DEV mode"
+    postgres_dev_compose_filename="docker-compose-postgres.dev.yml"
+    keycloak_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  if [ "${CLUSTERED_MODE}" == "true" ]; then
+    postgres_cluster_compose_filename="docker-compose-postgres.cluster.yml"
+  fi
+
+  append_config_sso_enabled
+
+  (
+    docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose-postgres.yml" "$postgres_cluster_compose_filename" "$postgres_dev_compose_filename"
+    docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$keycloak_dev_compose_filename"
+  ) ||
+    {
+      log error "Failed to deploy package"
+      exit 1
+    }
+}
+
+function destroy_package() {
+  docker::stack_destroy $STACK
+
+  if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+    log warn "Volumes are only deleted on the host on which the command is run. Postgres volumes on other nodes are not deleted"
+  fi
+
+  docker::prune_configs "keycloak"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+      log info "Running package in Cluster node mode"
+    else
+      log info "Running package in Single node mode"
+    fi
+
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+
+    docker::scale_services $STACK 0
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"

interoperability-layer-openhim/README.md

@@ -0,0 +1,66 @@
+
+# Interoperability Layer - OpenHIM
+
+The Interoperability Layer is the base of the Platform architecture.
+
+This component consists of two services:
+
+* Interoperability Layer - [OpenHIM](http://openhim.org/)
+
+## Accessing the services
+
+### OpenHIM
+
+* Console: <http://localhost:9000>
+* Username: **root@openhim.org**
+* Password: **instant101**
+
+## Testing the Interoperability Component
+
+As part of the Interoperability Layer setup we also do some initial config import for connecting the services together.
+
+* OpenHIM: Import a channel configuration that routes requests to the Data Store - HAPI FHIR service
+
+## Backup restore
+
+### Single node
+
+[Single node restore docs](https://www.mongodb.com/docs/v4.2/tutorial/backup-and-restore-tools/)
+
+The following job may be used to set up a backup job for a single node mongo
+
+```ini
+[job-run "mongo-backup"]
+schedule= @every 24h
+image= mongo:4.2
+network= mongo_backup
+volume= /backups:/tmp/backups
+command= sh -c 'mongodump --uri=${OPENHIM_MONGO_URL} --gzip --archive=/tmp/backups/mongodump_$(date +%s).gz'
+delete= true
+```
+
+### Cluster
+
+[Cluster restore docs](https://www.mongodb.com/docs/v4.2/tutorial/restore-replica-set-from-backup/)
+
+The following job may be used to set up a backup job for clustered mongo
+
+```ini
+[job-run "mongo-backup"]
+schedule= @every 24h
+image= mongo:4.2
+network= mongo_backup
+volume= /backups:/tmp/backups
+command= sh -c 'mongodump --uri=${OPENHIM_MONGO_URL} --gzip --archive=/tmp/backups/mongodump_$(date +%s).gz'
+delete= true
+```
+
+### Restore
+
+In order to restore from a backup you would need to launch a mongo container with access to the backup file and the mongo_backup network:
+
+`docker run -d --network=mongo_backup --mount type=bind,source=/backups,target=/backups mongo:4.2`
+
+Then exec into the container and run mongorestore:
+
+`mongorestore --uri="mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhim?replicaSet=mongo-set" --gzip --archive=/backups/<NAME_OF_BACKUP_FILE>`

interoperability-layer-openhim/docker-compose-mongo.cluster.yml

@@ -0,0 +1,61 @@
+version: '3.9'
+
+services:
+  mongo-1:
+    command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5']
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-1"
+
+  mongo-2:
+    image: mongo:4.2
+    volumes:
+      - 'openhim-mongo-02:/data/db'
+      - 'openhim-mongo-02-config:/data/configdb'
+    command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5']
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-2"
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${OPENHIM_MONGO_CPU_LIMIT}
+          memory: ${OPENHIM_MONGO_MEMORY_LIMIT}
+        reservations:
+          cpus: ${OPENHIM_MONGO_CPU_RESERVE}
+          memory: ${OPENHIM_MONGO_MEMORY_RESERVE}
+    networks:
+      default:
+      openhim-mongo:
+      mongo_backup_net:
+
+  mongo-3:
+    image: mongo:4.2
+    volumes:
+      - 'openhim-mongo-03:/data/db'
+      - 'openhim-mongo-03-config:/data/configdb'
+    command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5']
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-3"
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${OPENHIM_MONGO_CPU_LIMIT}
+          memory: ${OPENHIM_MONGO_MEMORY_LIMIT}
+        reservations:
+          cpus: ${OPENHIM_MONGO_CPU_RESERVE}
+          memory: ${OPENHIM_MONGO_MEMORY_RESERVE}
+    networks:
+      default:
+      openhim-mongo:
+      mongo_backup_net:
+
+volumes:
+  openhim-mongo-02:
+  openhim-mongo-03:
+  openhim-mongo-02-config:
+  openhim-mongo-03-config:

interoperability-layer-openhim/docker-compose-mongo.dev.yml

@@ -0,0 +1,8 @@
+version: '3.9'
+
+services:
+  mongo-1:
+    ports:
+      - target: 27017
+        published: 27017
+        mode: host

interoperability-layer-openhim/docker-compose-mongo.yml

@@ -0,0 +1,35 @@
+version: "3.9"
+
+services:
+  mongo-1:
+    image: mongo:4.2
+    volumes:
+      - "openhim-mongo-01:/data/db"
+      - "openhim-mongo-01-config:/data/configdb"
+    deploy:
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${OPENHIM_MONGO_CPU_LIMIT}
+          memory: ${OPENHIM_MONGO_MEMORY_LIMIT}
+        reservations:
+          cpus: ${OPENHIM_MONGO_CPU_RESERVE}
+          memory: ${OPENHIM_MONGO_MEMORY_RESERVE}
+    networks:
+      default:
+      openhim-mongo:
+      mongo_backup_net:
+
+volumes:
+  openhim-mongo-01:
+  openhim-mongo-01-config:
+
+networks:
+  default:
+  openhim-mongo:
+    name: openhim_mongo_public
+    external: true
+  mongo_backup_net:
+    name: mongo_backup
+    driver: overlay
+    attachable: true

interoperability-layer-openhim/docker-compose.await-helper.yml

@@ -0,0 +1,10 @@
+version: '3.9'
+
+services:
+  await-helper:
+    image: jembi/await-helper:1.0.1
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: none
+    command: '-k https://openhim-core:8080/heartbeat'

interoperability-layer-openhim/docker-compose.dev.yml

@@ -0,0 +1,29 @@
+version: '3.9'
+
+services:
+  openhim-core:
+    ports:
+      - target: 8080
+        published: 8080
+        mode: host
+      - target: 5000
+        published: 5000
+        mode: host
+      - target: 5001
+        published: 5001
+        mode: host
+      - target: 5050
+        published: 5050
+        mode: host
+      - target: 5051
+        published: 5051
+        mode: host
+      - target: 5052
+        published: 5052
+        mode: host
+
+  openhim-console:
+    ports:
+      - target: 80
+        published: 9000
+        mode: host

interoperability-layer-openhim/docker-compose.yml

@@ -0,0 +1,77 @@
+version: '3.9'
+
+services:
+  openhim-core:
+    image: jembi/openhim-core:v8.0.1
+    networks:
+      kafka:
+      hapi-fhir:
+      reverse-proxy:
+      keycloak:
+      public:
+      default:
+    environment:
+      - mongo_url=${OPENHIM_MONGO_URL}
+      - mongo_atnaUrl=${OPENHIM_MONGO_ATNAURL}
+      - api_authenticationTypes=["token", "basic", "openid", "local"]
+      - authentication_enableCustomTokenAuthentication=true
+      - router_kafkaBrokers=${KAFKA_HOSTS}
+      - api_openid_url=${KC_FRONTEND_URL}/realms/${KC_REALM_NAME}
+      - api_openid_callbackUrl=${KC_OPENHIM_ROOT_URL}
+      - api_openid_clientId=${KC_OPENHIM_CLIENT_ID}
+      - api_openid_clientSecret=${KC_OPENHIM_CLIENT_SECRET}
+    deploy:
+      replicas: ${OPENHIM_CORE_INSTANCES}
+      placement:
+        max_replicas_per_node: 1
+      resources:
+        limits:
+          cpus: ${OPENHIM_CPU_LIMIT}
+          memory: ${OPENHIM_MEMORY_LIMIT}
+        reservations:
+          cpus: ${OPENHIM_CPU_RESERVE}
+          memory: ${OPENHIM_MEMORY_RESERVE}
+
+  openhim-console:
+    image: jembi/openhim-console:v1.18.2
+    environment:
+      OPENHIM_CORE_MEDIATOR_HOSTNAME: ${OPENHIM_CORE_MEDIATOR_HOSTNAME}
+      OPENHIM_MEDIATOR_API_PORT: ${OPENHIM_MEDIATOR_API_PORT}
+      KC_OPENHIM_SSO_ENABLED: ${KC_OPENHIM_SSO_ENABLED}
+      KC_OPENHIM_CLIENT_ID: ${KC_OPENHIM_CLIENT_ID}
+      KC_REALM_NAME: ${KC_REALM_NAME}
+      KC_FRONTEND_URL: ${KC_FRONTEND_URL}
+    networks:
+      reverse-proxy:
+      keycloak:
+      public:
+      default:
+    deploy:
+      replicas: ${OPENHIM_CONSOLE_INSTANCES}
+      placement:
+        max_replicas_per_node: 1
+      resources:
+        limits:
+          cpus: ${OPENHIM_CONSOLE_CPU_LIMIT}
+          memory: ${OPENHIM_CONSOLE_MEMORY_LIMIT}
+        reservations:
+          cpus: ${OPENHIM_CONSOLE_CPU_RESERVE}
+          memory: ${OPENHIM_CONSOLE_MEMORY_RESERVE}
+
+networks:
+  kafka:
+    name: kafka_public
+    external: true
+  hapi-fhir:
+    name: hapi-fhir_public
+    external: true
+  reverse-proxy:
+    name: reverse-proxy_public
+    external: true
+  keycloak:
+    name: keycloak_public
+    external: true
+  public:
+    name: openhim_public
+    external: true
+  default:

interoperability-layer-openhim/features/core.feature

@@ -0,0 +1,15 @@
+Feature: FHIR server available via OpenHIM
+
+  Rule: Authorised clients should be able to access the FHIR server through the OpenHIM
+
+    Scenario: An authorised client searches for a patient
+      Given a patient, Jane Doe, exists in the FHIR server
+      And an authorised client, Alice, exists in the OpenHIM
+      When Alice searches for a patient
+      Then Alice is able to get a result
+
+    Scenario: An un-authorised client searches for a patient
+      Given a patient, Jane Doe, exists in the FHIR server
+      And an authorised client, Alice, exists in the OpenHIM
+      When Malice searches for a patient
+      Then Malice is NOT able to get a result

interoperability-layer-openhim/features/core.js

@@ -0,0 +1,228 @@
+'use strict'
+
+const axios = require('axios')
+
+const { AfterAll, Given, Then, When } = require('@cucumber/cucumber')
+const { expect } = require('chai')
+
+const OPENHIM_PROTOCOL = process.env.OPENHIM_PROTOCOL || 'http'
+const OPENHIM_CORE_SERVICE_NAME = process.env.OPENHIM_CORE_SERVICE_NAME || 'localhost'
+const OPENHIM_TRANSACTION_API_PORT =
+  process.env.OPENHIM_TRANSACTION_API_PORT || '5001'
+const OPENHIM_MEDIATOR_API_PORT =
+  process.env.OPENHIM_MEDIATOR_API_PORT || '8080'
+const CUSTOM_TOKEN_ID = process.env.CUSTOM_TOKEN_ID || 'test'
+const BASIC_AUTH_HEADER =
+  process.env.BASIC_AUTH_HEADER || 'Basic cm9vdEBvcGVuaGltLm9yZzppbnN0YW50MTAx'
+
+// Save test Patient resource ID for post test cleanup
+let hapiFhirPatientID
+
+Given('a patient, Jane Doe, exists in the FHIR server', async function () {
+  const checkPatientExistsOptions = {
+    url: `${OPENHIM_PROTOCOL}://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_TRANSACTION_API_PORT}/fhir/Patient?identifier:value=test`,
+    method: 'GET',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Custom ${CUSTOM_TOKEN_ID}`,
+      'Cache-Control': 'no-cache'
+    }
+  }
+
+  const checkPatientExistsResponse = await axios(checkPatientExistsOptions)
+
+  if (checkPatientExistsResponse.data.total === 0) {
+    console.log(
+      `\nPatient record for Jane Doe does not exist. Creating Patient`
+    )
+    const options = {
+      url: `${OPENHIM_PROTOCOL}://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_TRANSACTION_API_PORT}/fhir/Patient`,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Custom ${CUSTOM_TOKEN_ID}`,
+        'Cache-Control': 'no-cache'
+      },
+      data: {
+        resourceType: 'Patient',
+        name: {
+          use: 'temp',
+          family: 'Doe',
+          given: ['Jane']
+        },
+        identifier: {
+          use: 'temp',
+          value: 'test'
+        },
+        gender: 'male'
+      }
+    }
+
+    const createPatientResponse = await axios(options)
+
+    expect(createPatientResponse.status).to.eql(201)
+
+    hapiFhirPatientID = createPatientResponse.data.id
+  } else if (checkPatientExistsResponse.data.total === 1) {
+    console.log(`\nPatient record for Jane Doe already exists`)
+
+    expect(
+      checkPatientExistsResponse.data.entry[0].resource.name[0].given
+    ).to.eql(['Jane'])
+    expect(
+      checkPatientExistsResponse.data.entry[0].resource.name[0].family
+    ).to.eql('Doe')
+
+    hapiFhirPatientID = checkPatientExistsResponse.data.entry[0].resource.id
+  } else {
+    // Previous test data should have been cleaned out
+    throw new Error(
+      `Multiple Patient records for Jane Doe exist: ${checkPatientExistsResponse.data.total}`
+    )
+  }
+})
+
+Given('an authorised client, Alice, exists in the OpenHIM', async function () {
+  process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0
+  const checkClientExistsOptions = {
+    url: `https://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_MEDIATOR_API_PORT}/clients`,
+    method: 'GET',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: BASIC_AUTH_HEADER
+    }
+  }
+
+  const checkClientExistsResponse = await axios(checkClientExistsOptions)
+
+  let createClient = true
+
+  for (let client of checkClientExistsResponse.data) {
+    if (client.clientID === 'test-harness-client') {
+      expect(client.name).to.eql('Alice')
+      createClient = false
+      break
+    }
+  }
+
+  if (createClient) {
+    console.log(`\nThe test Harness Client does not exist. Creating Client`)
+    const options = {
+      url: `https://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_MEDIATOR_API_PORT}/clients`,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: BASIC_AUTH_HEADER
+      },
+      data: {
+        roles: ['instant'],
+        clientID: 'test-harness-client',
+        name: 'Alice',
+        customTokenID: 'test-harness-token'
+      }
+    }
+
+    const response = await axios(options)
+    expect(response.status).to.eql(201)
+  } else {
+    console.log(`\nThe Test Harness Client (Alice) already exists`)
+  }
+})
+
+When('Alice searches for a patient', async function () {
+  const checkPatientExistsOptions = {
+    url: `${OPENHIM_PROTOCOL}://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_TRANSACTION_API_PORT}/fhir/Patient?identifier:value=test`,
+    method: 'GET',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Custom test-harness-token`,
+      'Cache-Control': 'no-cache'
+    }
+  }
+  const checkPatientExistsResponse = await axios(checkPatientExistsOptions)
+  expect(checkPatientExistsResponse.status).to.eql(200)
+  this.setTo(checkPatientExistsResponse.data.entry[0])
+})
+
+Then('Alice is able to get a result', function () {
+  expect(this.searchResults.resource.resourceType).equal('Patient')
+  expect(this.searchResults.resource.identifier[0].value).equal('test')
+  expect(this.searchResults.resource.name[0].given[0]).equal('Jane')
+  expect(this.searchResults.resource.name[0].family).equal('Doe')
+})
+
+When('Malice searches for a patient', async function () {
+  const checkPatientExistsOptions = {
+    url: `${OPENHIM_PROTOCOL}://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_TRANSACTION_API_PORT}/fhir/Patient?identifier:value=test`,
+    method: 'GET',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Custom invalid-token`,
+      'Cache-Control': 'no-cache'
+    },
+    validateStatus: (status) => {
+      return status >= 400
+    }
+  }
+  const checkPatientExistsResponse = await axios(checkPatientExistsOptions)
+  expect(checkPatientExistsResponse.status).to.eql(401)
+  this.setTo(checkPatientExistsResponse.data)
+})
+
+Then('Malice is NOT able to get a result', function () {
+  expect(this.searchResults).to.eql('')
+})
+
+AfterAll(async function () {
+  if (hapiFhirPatientID) {
+    console.log(`\nDeleting FHIR test Patient record`)
+    const deletePatientOptions = {
+      url: `${OPENHIM_PROTOCOL}://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_TRANSACTION_API_PORT}/fhir/Patient/${hapiFhirPatientID}`,
+      method: 'DELETE',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Custom ${CUSTOM_TOKEN_ID}`,
+        'Cache-Control': 'no-cache'
+      }
+    }
+
+    const deletePatientResponse = await axios(deletePatientOptions)
+    expect(deletePatientResponse.status).to.eql(200)
+  }
+
+  process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0
+  const checkClientExistsOptions = {
+    url: `https://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_MEDIATOR_API_PORT}/clients`,
+    method: 'GET',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: BASIC_AUTH_HEADER
+    }
+  }
+
+  const checkClientExistsResponse = await axios(checkClientExistsOptions)
+
+  let clientObjectId
+  // Previous test data should have been cleaned out
+  for (let client of checkClientExistsResponse.data) {
+    if (client.clientID === 'test-harness-client') {
+      clientObjectId = client._id
+      break
+    }
+  }
+
+  if (clientObjectId) {
+    console.log(`\nDeleting OpenHIM test Client record`)
+    const deleteClientOptions = {
+      url: `https://${OPENHIM_CORE_SERVICE_NAME}:${OPENHIM_MEDIATOR_API_PORT}/clients/${clientObjectId}`,
+      method: 'DELETE',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: BASIC_AUTH_HEADER
+      }
+    }
+
+    const deleteClientResponse = await axios(deleteClientOptions)
+    expect(deleteClientResponse.status).to.eql(200)
+  }
+})

interoperability-layer-openhim/features/support.js

@@ -0,0 +1,13 @@
+const { setWorldConstructor } = require('@cucumber/cucumber')
+
+class CoreWorld {
+  constructor() {
+    this.searchResults = null
+  }
+
+  setTo(result) {
+    this.searchResults = result
+  }
+}
+
+setWorldConstructor(CoreWorld)

interoperability-layer-openhim/importer/docker-compose.config.yml

@@ -0,0 +1,33 @@
+version: '3.9'
+
+services:
+  # container for executing config import scripts for setting up initial config
+  interoperability-layer-openhim-config-importer:
+    image: node:erbium-alpine
+    environment:
+      OPENHIM_API_USERNAME: 'root@openhim.org'
+      OPENHIM_API_PASSWORD: 'openhim-password'
+      # Reject unauthorised is only needed if the OpenHIM's SSL is not setup
+      NODE_TLS_REJECT_UNAUTHORIZED: 0
+    command: sh -c "node openhimConfig.js"
+    configs:
+      - source: openhim-config-importer-openhimConfig.js
+        target: /openhimConfig.js
+      - source: openhim-config-importer-openhim-import.json
+        target: /openhim-import.json
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: none
+
+configs:
+  openhim-config-importer-openhimConfig.js:
+    file: ./volume/openhimConfig.js
+    name: openhim-config-importer-openhimConfig.js-${openhim_config_importer_openhimConfig_js_DIGEST:?err}
+    labels:
+      name: openhim
+  openhim-config-importer-openhim-import.json:
+    file: ./volume/openhim-import.json
+    name: openhim-config-importer-openhim-import.json-${openhim_config_importer_openhim_import_js_DIGEST:?err}
+    labels:
+      name: openhim

interoperability-layer-openhim/importer/volume/new/console.json

@@ -0,0 +1,19 @@
+{
+  "version": "1.18.2",
+  "minimumCoreVersion": "5.2.0",
+  "protocol": "https",
+  "host": "localhost",
+  "hostPath": "openhim-api",
+  "port": "",
+  "title": "Admin Console",
+  "footerTitle": "OpenHIM Administration Console",
+  "footerPoweredBy": "<a href='http://openhim.org/' target='_blank'>Powered by OpenHIM</a>",
+  "loginBanner": "",
+  "mediatorLastHeartbeatWarningSeconds": 60,
+  "mediatorLastHeartbeatDangerSeconds": 120,
+  "showLoginForm": true,
+  "ssoEnabled": false,
+  "keyCloakUrl": "http://localhost:9088",
+  "keyCloakRealm": "platform-realm",
+  "keyCloakClientId": "openhim-oauth"
+}

interoperability-layer-openhim/importer/volume/new/initial-config.json

@@ -0,0 +1,85 @@
+{
+  "Users": [
+    {
+      "groups": [
+        "admin"
+      ],
+      "firstname": "Super",
+      "surname": "User",
+      "email": "root@openhim.org",
+      "passwordAlgorithm": "sha512",
+      "passwordHash": "e12ea7a5468925ab2b6d839e5b0566c8d8e68d9dd48241282ec1c448c0b11386b499be64011c88060b241ed5a143fea90992dabb0d91a334daecd96e20156c24",
+      "passwordSalt": "5f49ab7df7fac0fd7860d3b1be95cbf4",
+      "expiry": null,
+      "locked": false,
+      "tokenType": null,
+      "token": null
+    }
+  ],
+  "Clients": [],
+  "Channels": [
+    {
+      "methods": [
+        "GET",
+        "POST",
+        "DELETE",
+        "PUT",
+        "OPTIONS",
+        "HEAD",
+        "TRACE",
+        "CONNECT",
+        "PATCH"
+      ],
+      "type": "http",
+      "allow": [],
+      "whitelist": [],
+      "authType": "public",
+      "matchContentTypes": [],
+      "properties": [],
+      "txViewAcl": [],
+      "txViewFullAcl": [],
+      "txRerunAcl": [],
+      "status": "enabled",
+      "rewriteUrls": false,
+      "addAutoRewriteRules": true,
+      "autoRetryEnabled": false,
+      "autoRetryPeriodMinutes": 60,
+      "routes": [
+        {
+          "type": "http",
+          "status": "enabled",
+          "forwardAuthHeader": false,
+          "name": "Central FHIR server",
+          "secured": false,
+          "host": "fhir.openelis.org.remote",
+          "port": 8080,
+          "path": "",
+          "pathTransform": "",
+          "primary": true,
+          "username": "",
+          "password": ""
+        }
+      ],
+      "requestBody": true,
+      "responseBody": true,
+      "rewriteUrlsConfig": [],
+      "name": "central fhir server",
+      "description": "Access to the central FHIR server",
+      "pollingSchedule": null,
+      "matchContentRegex": null,
+      "matchContentXpath": null,
+      "matchContentValue": null,
+      "matchContentJson": null,
+      "urlPattern": "^/fhir/.*$",
+      "tcpHost": null,
+      "tcpPort": null,
+      "updatedBy": {
+        "id": "6374bbabba26d10013784480",
+        "name": "Super User"
+      },
+      "alerts": []
+    }
+  ],
+  "Mediators": [],
+  "ContactGroups": []
+}

interoperability-layer-openhim/importer/volume/openhim-import-temp.json

@@ -0,0 +1,29 @@
+{
+  "Users": [
+    {
+      "groups": [
+        "admin"
+      ],
+      "firstname": "Super",
+      "surname": "User",
+      "email": "root@openhim.org",
+      "passwordAlgorithm": "sha512",
+      "passwordHash": "{{PASSWORD_HASH}}",
+      "passwordSalt": "{{PASSWORD_SALT}}",
+      "expiry": null,
+      "locked": false,
+      "token": null,
+      "tokenType": null
+    }
+  ],
+  "Clients": [
+    {
+      "roles": [
+        "{{TEST_ROLE}}"
+      ],
+      "customTokenID": "{{OPENHIM_CLIENT_TOKEN}}",
+      "clientID": "{{TEST_CLIENT_ID}}",
+      "name": "{{TEST_CLIENT_NAME}}"
+    }
+  ]
+}

interoperability-layer-openhim/importer/volume/openhim-import.json

@@ -0,0 +1,37 @@
+{
+  "Users": [
+    {
+      "provider": "token",
+      "groups": ["admin"],
+      "firstname": "Super",
+      "surname": "User",
+      "email": "root@openhim.org",
+      "expiry": null,
+      "locked": false,
+      "token": null,
+      "tokenType": null
+    }
+  ],
+  "Passports": [
+    {
+      "protocol": "local",
+      "email": "root@openhim.org",
+      "password": "$2a$10$qvja1lAaItmcZolkoKCPn.9EXY2b8s9dTlUFx36Q7ey0rieXXnxVC"
+    },
+    {
+      "protocol": "token",
+      "email": "root@openhim.org",
+      "passwordHash": "c5e14f0062bfac45bf20f0fcf0403e758a2233756e40e7b328e1f10dfef3d8ba57c2bf230ee9fd627956fbb4e57fab7e32efe108db76d515f1f61f6b853ffba3",
+      "passwordAlgorithm": "sha512",
+      "passwordSalt": "c5f2a9fd7907e1a1c6658841426e6f5d"
+    }
+  ],
+  "Clients": [
+    {
+      "roles": ["instant"],
+      "customTokenID": "test",
+      "clientID": "test",
+      "name": "Test Client"
+    }
+  ]
+}

interoperability-layer-openhim/importer/volume/openhimConfig.js

@@ -0,0 +1,54 @@
+'use strict'
+
+const fs = require('fs')
+const https = require('https')
+const path = require('path')
+
+const OPENHIM_CORE_SERVICE_NAME = 'openhim-core'
+const OPENHIM_MEDIATOR_API_PORT = 8080
+const OPENHIM_API_PASSWORD =
+  process.env.OPENHIM_API_PASSWORD || 'openhim-password'
+const OPENHIM_API_USERNAME =
+  process.env.OPENHIM_API_USERNAME || 'root@openhim.org'
+
+const authHeader = new Buffer.from(
+  `${OPENHIM_API_USERNAME}:${OPENHIM_API_PASSWORD}`
+).toString('base64')
+
+const jsonData = JSON.parse(
+  fs.readFileSync(path.resolve(__dirname, 'openhim-import.json'))
+)
+
+const data = JSON.stringify(jsonData)
+
+const options = {
+  protocol: 'https:',
+  hostname: OPENHIM_CORE_SERVICE_NAME,
+  port: OPENHIM_MEDIATOR_API_PORT,
+  path: '/metadata',
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    'Content-Length': data.length,
+    Authorization: `Basic ${authHeader}`
+  }
+}
+
+const req = https.request(options, res => {
+  if (res.statusCode == 401) {
+    throw new Error(`Incorrect OpenHIM API credentials`)
+  }
+
+  if (res.statusCode != 201) {
+    throw new Error(`Failed to import OpenHIM config: ${res.statusCode}`)
+  }
+
+  console.log('Successfully Imported OpenHIM Config')
+})
+
+req.on('error', error => {
+  console.error('Failed to import OpenHIM config: ', error)
+})
+
+req.write(data)
+req.end()

interoperability-layer-openhim/initiate-replica-set.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# - $1 : Stack name that the services are grouped under
+STACK_NAME="${1:-"instant"}"
+
+COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+)
+
+ROOT_PATH="${COMPOSE_FILE_PATH}/.."
+. "${ROOT_PATH}/utils/config-utils.sh"
+. "${ROOT_PATH}/utils/log.sh"
+
+log info 'Initiating the mongo replica set'
+
+MONGO_SET_COUNT=${MONGO_SET_COUNT:-3}
+config='{"_id":"mongo-set","members":['
+priority="1"
+for i in $(seq 1 "$MONGO_SET_COUNT"); do
+    config=$(printf '%s{"_id":%s,"priority":%s,"host":"mongo-%s:27017"}' "$config" $((i - 1)) $priority "$i")
+    if [[ $i != $MONGO_SET_COUNT ]]; then
+        config=$(printf '%s,' "$config")
+    fi
+    priority="0.5"
+done
+config=$(printf '%s]}' "$config")
+
+log info 'Waiting to ensure all the mongo instances for the replica set are up and running'
+running_instance_count=0
+start_time=$(date +%s)
+until [[ $running_instance_count -eq $MONGO_SET_COUNT ]]; do
+    config::timeout_check "$start_time" "mongo replica set to run"
+    sleep 1
+
+    running_instance_count=0
+    for i in $(docker service ls -f name=${STACK_NAME}_mongo --format "{{.Replicas}}"); do
+        if [[ $i = "1/1" ]]; then
+            running_instance_count=$((running_instance_count + 1))
+        fi
+    done
+done
+
+# Ensures that the replica sets are reachable
+reachable_instance_count=1
+until [[ $reachable_instance_count -eq $((MONGO_SET_COUNT + 1)) ]]; do
+    config::await_service_reachable "mongo-$reachable_instance_count" $STACK_NAME "waiting for connections on port"
+    reachable_instance_count=$((reachable_instance_count + 1))
+done
+
+# TODO (PLAT-256): only works if deploying to node-1 labeled node
+# With docker swarm any manager can be the target but this bit of code only work if we target node-1 specifically.
+# Which is generally what we do, but if node-1 is down or we choose to target another node this won't work.
+container_name=""
+if [[ "$(docker ps -f name=${STACK_NAME}_mongo-1 --format "{{.ID}}")" ]]; then
+    container_name="$(docker ps -f name=${STACK_NAME}_mongo-1 --format "{{.ID}}")"
+fi
+
+initiate_rep_set_response=$(docker exec -i "$container_name" mongo --eval "rs.initiate($config)")
+if [[ $initiate_rep_set_response == *"{ \"ok\" : 1 }"* ]] || [[ $initiate_rep_set_response == *"already initialized"* ]]; then
+    log info "Replica set successfully set up"
+else
+    log error "Fatal: Unable to set up replica set"
+    exit 1
+fi

interoperability-layer-openhim/package-metadata.json

@@ -0,0 +1,37 @@
+{
+  "id": "interoperability-layer-openhim",
+  "name": "Interoperability Layer Package - Openhim",
+  "description": "This is the interoperability layer that enables simpler data exchange between the different systems. It is also the security layer for the other systems",
+  "version": "1.0.0",
+  "type": "infrastructure",
+  "dependencies": [],
+  "environmentVariables": {
+    "MONGO_SET_COUNT": "1",
+    "OPENHIM_CORE_INSTANCES": "1",
+    "OPENHIM_CONSOLE_INSTANCES": "1",
+    "OPENHIM_CORE_MEDIATOR_HOSTNAME": "localhost",
+    "OPENHIM_MEDIATOR_API_PORT": "8080",
+    "OPENHIM_CPU_LIMIT": "0",
+    "OPENHIM_CPU_RESERVE": "0.05",
+    "OPENHIM_MEMORY_LIMIT": "3G",
+    "OPENHIM_MEMORY_RESERVE": "500M",
+    "OPENHIM_CONSOLE_CPU_LIMIT": "0",
+    "OPENHIM_CONSOLE_CPU_RESERVE": "0.05",
+    "OPENHIM_CONSOLE_MEMORY_LIMIT": "2G",
+    "OPENHIM_CONSOLE_MEMORY_RESERVE": "500M",
+    "OPENHIM_MONGO_CPU_LIMIT": "0",
+    "OPENHIM_MONGO_CPU_RESERVE": "0.05",
+    "OPENHIM_MONGO_MEMORY_LIMIT": "2G",
+    "OPENHIM_MONGO_MEMORY_RESERVE": "500M",
+    "OPENHIM_MONGO_URL": "mongodb://mongo-1:27017/openhim",
+    "OPENHIM_MONGO_ATNAURL": "mongodb://mongo-1:27017/openhim",
+    "KAFKA_HOSTS": "kafka-01:9092",
+    "KC_REALM_NAME": "platform-realm",
+    "KC_FRONTEND_URL": "http://localhost:9088",
+    "KC_OPENHIM_SSO_ENABLED": false,
+    "KC_OPENHIM_CLIENT_ID": "openhim-oauth",
+    "KC_OPENHIM_CLIENT_SECRET": "tZKfEbWf0Ka5HBNZwFrdSyQH2xT1sNMR",
+    "KC_OPENHIM_ROOT_URL": "http://localhost:9000",
+    "KC_REALM_FULL_URL": "http://localhost:9088/realms/platform-realm"
+  }
+}

interoperability-layer-openhim/swarm.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare STACK="openhim"
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+  readonly STACK
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/config-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function initialize_package() {
+  local mongo_cluster_compose_filename=""
+  local mongo_dev_compose_filename=""
+  local openhim_dev_compose_filename=""
+
+  if [[ "${MODE}" == "dev" ]]; then
+    log info "Running package in DEV mode"
+    mongo_dev_compose_filename="docker-compose-mongo.dev.yml"
+    openhim_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+    mongo_cluster_compose_filename="docker-compose-mongo.cluster.yml"
+  fi
+
+  (
+    docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose-mongo.yml" "$mongo_cluster_compose_filename" "$mongo_dev_compose_filename"
+
+    if [[ "${CLUSTERED_MODE}" == "true" ]] && [[ "${ACTION}" == "init" ]]; then
+      try "${COMPOSE_FILE_PATH}/initiate-replica-set.sh $STACK" throw "Fatal: Initiate Mongo replica set failed"
+    fi
+
+    docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$openhim_dev_compose_filename"
+
+    log info "Waiting OpenHIM Core to be running and responding"
+    config::await_service_running "openhim-core" "${COMPOSE_FILE_PATH}"/docker-compose.await-helper.yml "${OPENHIM_CORE_INSTANCES}" "$STACK"
+  ) ||
+    {
+      log error "Failed to deploy package"
+      exit 1
+    }
+
+  if [[ "${ACTION}" == "init" ]]; then
+    docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/docker-compose.config.yml" "interoperability-layer-openhim-config-importer" "openhim"
+  fi
+}
+
+function destroy_package() {
+  docker::stack_destroy "$STACK"
+
+  if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+    log warn "Volumes are only deleted on the host on which the command is run. Mongo volumes on other nodes are not deleted"
+  fi
+
+  docker::prune_configs "openhim"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+      log info "Running package in Cluster node mode"
+    else
+      log info "Running package in Single node mode"
+    fi
+
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+
+    docker::scale_services "$STACK" 0
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"

lis-openelis-global/config/common.properties

@@ -0,0 +1,35 @@
+server.ssl.key-store = file:/etc/openelis-global/keystore
+server.ssl.key-store-password = kspass
+server.ssl.key-password = kspass
+server.ssl.trust-store= file:/etc/openelis-global/truststore
+server.ssl.trust-store-password=tspass
+
+#/usr/local/tomcat/webapps/OpenELIS-Global/certs
+encryption.general.password=kspass
+
+spring.datasource.driver=org.postgresql.Driver
+spring.datasource.url= jdbc:postgresql://172.17.0.1:5432/clinlims?currentSchema=clinlims
+spring.datasource.username=clinlims
+spring.datasource.password=MIvfOkcLgzPA
+spring.jpa.hibernate.ddl-auto=update
+
+org.openelisglobal.fhirstore.uri=http://fhir.openelis.org:8080/fhir/
+
+org.openelisglobal.remote.source.uri=http://openhim-core:5001/fhir/
+org.openelisglobal.remote.source.updateStatus=true
+org.openelisglobal.remote.source.identifier=Practitioner/f9badd80-ab76-11e2-9e96-0800200c9a66
+org.openelisglobal.task.useBasedOn=true
+
+org.openelisglobal.fhir.subscriber=http://openhim-core:5001/fhir/
+org.openelisglobal.fhir.subscriber.resources=Task,Patient,ServiceRequest,DiagnosticReport,Observation,Specimen,Practitioner,Encounter
+
+#org.openelisglobal.facilitylist.fhirstore = http://localhost:4000/fhir/DEFAULT
+# org.openelisglobal.facilitylist.authurl= http://localhost:4000/auth/token
+# org.openelisglobal.facilitylist.username= root@gofr.org
+# org.openelisglobal.facilitylist.password= gofr
+# org.openelisglobal.facilitylist.auth= token
+facilitylist.schedule.fixedRate=864000000
+
+#provider List
+org.openelisglobal.providerlist.fhirstore=http://openhim-core:5001/fhir/
+org.openelisglobal.requester.identifier=Practitioner/f9badd80-ab76-11e2-9e96-0800200c9a66

lis-openelis-global/config/datasource.password

@@ -0,0 +1 @@
+clinlims

lis-openelis-global/config/oe_server.xml

@@ -0,0 +1,167 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!-- Note:  A "Server" is not itself a "Container", so you may not
+     define subcomponents such as "Valves" at this level.
+     Documentation at /docs/config/server.html
+ -->
+<Server port="-1" shutdown="SHUTDOWN">
+  <Listener className="org.apache.catalina.startup.VersionLoggerListener" />
+  <!-- Security listener. Documentation at /docs/config/listeners.html
+  <Listener className="org.apache.catalina.security.SecurityListener" />
+  -->
+  <!--APR library loader. Documentation at /docs/apr.html -->
+  <!--<Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />-->
+  <!-- Prevent memory leaks due to use of particular java/javax APIs-->
+  <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
+  <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
+  <Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />
+
+  <!-- Global JNDI resources
+       Documentation at /docs/jndi-resources-howto.html
+  -->
+  <GlobalNamingResources>
+    <!-- Editable user database that can also be used by
+         UserDatabaseRealm to authenticate users
+    -->
+    <Resource name="UserDatabase" auth="Container"
+              type="org.apache.catalina.UserDatabase"
+              description="User database that can be updated and saved"
+              factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
+              pathname="conf/tomcat-users.xml" />
+  </GlobalNamingResources>
+
+  <!-- A "Service" is a collection of one or more "Connectors" that share
+       a single "Container" Note:  A "Service" is not itself a "Container",
+       so you may not define subcomponents such as "Valves" at this level.
+       Documentation at /docs/config/service.html
+   -->
+  <Service name="Catalina">
+
+    <!--The connectors can use a shared executor, you can define one or more named thread pools-->
+    <!--
+    <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
+        maxThreads="150" minSpareThreads="4"/>
+    -->
+
+
+    <Connector port="8443" minProcessors="5" maxProcessors="75" 
+               enableLookups="true" disableUploadTimeout="true"
+               acceptCount="100" debug="0" scheme="https" secure="true"
+               sslProtocol="TLS" SSLEnabled="true" 
+               keystoreFile="/etc/openelis-global/keystore" keystoreType="PKCS12" keystorePass="kspass"
+               truststoreFile="/etc/openelis-global/truststore" truststoreType="PKCS12" truststorePass="tspass"
+               URIEncoding="UTF-8" />
+               
+    <!-- A "Connector" using the shared thread pool-->
+    <!--
+    <Connector executor="tomcatThreadPool"
+               port="8080" protocol="HTTP/1.1"
+               connectionTimeout="20000"
+               redirectPort="8443" />
+    -->
+    <!-- Define a SSL/TLS HTTP/1.1 Connector on port 8443
+         This connector uses the NIO implementation. The default
+         SSLImplementation will depend on the presence of the APR/native
+         library and the useOpenSSL attribute of the
+         AprLifecycleListener.
+         Either JSSE or OpenSSL style configuration may be used regardless of
+         the SSLImplementation selected. JSSE style configuration is used below.
+    -->
+    <!--
+    <Connector port="8443" protocol="org.apache.coyote.http11.Http11NioProtocol"
+               maxThreads="150" SSLEnabled="true">
+        <SSLHostConfig>
+            <Certificate certificateKeystoreFile="conf/localhost-rsa.jks"
+                         type="RSA" />
+        </SSLHostConfig>
+    </Connector>
+    -->
+    <!-- Define a SSL/TLS HTTP/1.1 Connector on port 8443 with HTTP/2
+         This connector uses the APR/native implementation which always uses
+         OpenSSL for TLS.
+         Either JSSE or OpenSSL style configuration may be used. OpenSSL style
+         configuration is used below.
+    -->
+    <!--
+    <Connector port="8443" protocol="org.apache.coyote.http11.Http11AprProtocol"
+               maxThreads="150" SSLEnabled="true" >
+        <UpgradeProtocol className="org.apache.coyote.http2.Http2Protocol" />
+        <SSLHostConfig>
+            <Certificate certificateKeyFile="conf/localhost-rsa-key.pem"
+                         certificateFile="conf/localhost-rsa-cert.pem"
+                         certificateChainFile="conf/localhost-rsa-chain.pem"
+                         type="RSA" />
+        </SSLHostConfig>
+    </Connector>
+    -->
+
+    <!-- Define an AJP 1.3 Connector on port 8009 -->
+    <!-- 
+    <Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
+    -->
+
+
+    <!-- An Engine represents the entry point (within Catalina) that processes
+         every request.  The Engine implementation for Tomcat stand alone
+         analyzes the HTTP headers included with the request, and passes them
+         on to the appropriate Host (virtual host).
+         Documentation at /docs/config/engine.html -->
+
+    <!-- You should set jvmRoute to support load-balancing via AJP ie :
+    <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
+    -->
+    <Engine name="Catalina" defaultHost="localhost">
+
+      <!--For clustering, please take a look at documentation at:
+          /docs/cluster-howto.html  (simple how to)
+          /docs/config/cluster.html (reference documentation) -->
+      <!--
+      <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
+      -->
+
+      <!-- Use the LockOutRealm to prevent attempts to guess user passwords
+           via a brute-force attack -->
+<!--       <Realm className="org.apache.catalina.realm.LockOutRealm"> -->
+        <!-- This Realm uses the UserDatabase configured in the global JNDI
+             resources under the key "UserDatabase".  Any edits
+             that are performed against this UserDatabase are immediately
+             available for use by the Realm.  -->
+<!--         <Realm className="org.apache.catalina.realm.UserDatabaseRealm" -->
+<!--                resourceName="UserDatabase"/> -->
+<!--       </Realm> -->
+
+      <Host name="localhost"  appBase="webapps"
+            unpackWARs="true" autoDeploy="true">
+            
+        <!-- SingleSignOn valve, share authentication between web applications
+             Documentation at: /docs/config/valve.html -->
+        <!--
+        <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
+        -->
+
+        <!-- Access log processes all example.
+             Documentation at: /docs/config/valve.html
+             Note: The pattern used is equivalent to using pattern="common" -->
+<!--         <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs" -->
+<!--                prefix="localhost_access_log" suffix=".txt" -->
+<!--                pattern="%h %l %u %t &quot;%r&quot; %s %b" /> -->
+
+      </Host>
+    </Engine>
+  </Service>
+</Server>

lis-openelis-global/docker-compose-certs.yml

@@ -0,0 +1,19 @@
+version: '3.9'
+
+services:
+  certs:
+    container_name: oe-certs
+    image: ghcr.io/i-tech-uw/certgen:main
+    restart: always
+    environment:
+      - KEYSTORE_PW="kspass"
+      - TRUSTSTORE_PW="tspass"
+    volumes:
+      - cert-volume:/etc/openelis-global
+    networks:
+      - default
+      - openelis
+  networks:
+    openelis:
+      external: true
+      name: openelis

lis-openelis-global/docker-compose-hapi.yml

@@ -0,0 +1,30 @@
+version: '3.9'
+services:
+  fhir.openelis.org:
+    container_name: external-fhir-api
+    image: hapiproject/hapi:v5.5.1
+    restart: always
+    environment:
+      - hapi.fhir.auto_create_placeholder_reference_targets=true
+      - hapi.fhir.allow_placeholder_references=true
+      - hapi.fhir.allow_external_references=true
+    networks:
+      - default
+      - openelis
+
+  fhir.openelis.org.remote:
+    container_name: openelis-remote-source
+    image: hapiproject/hapi:v5.5.1
+    restart: always
+    environment:
+      - hapi.fhir.auto_create_placeholder_reference_targets=true
+      - hapi.fhir.allow_placeholder_references=true
+      - hapi.fhir.allow_external_references=true
+    networks:
+      - default
+      - openelis
+
+networks:
+  openelis:
+    external: true
+    name: openelis

lis-openelis-global/docker-compose-postgres.yml

@@ -0,0 +1,28 @@
+ version: '3.9'
+ services:
+  database:
+    container_name: openelisglobal-database
+    image: postgres:9.5
+    ports:
+      - "15432:5432"
+    restart: always
+    env_file:
+      - ./volume/database/database.env
+    volumes:
+      # preserves the database between containers
+      - db-data:/var/lib/postgresql/data
+        # files here will run on install
+      - ./volume/database/dbInit:/docker-entrypoint-initdb.d
+    networks:
+      - default
+      - openelis
+    healthcheck:
+      test: ["CMD", "pg_isready", "-q", "-d", "clinlims", "-U", "clinlims"]
+      timeout: 45s
+      interval: 10s
+      retries: 10
+
+  networks:
+    openelis:
+      external: true
+      name: openelis

lis-openelis-global/docker-compose.dev.yml

@@ -0,0 +1,8 @@
+version: '3.9'
+
+services:
+  lis-openelis-global:
+    ports:
+      - target: 8080
+        published: 8091
+        mode: host

lis-openelis-global/docker-compose.yml

@@ -0,0 +1,44 @@
+version: '3.9'
+
+services:
+  oe.openelis.org:
+    container_name: openelisglobal-webapp
+    image: ghcr.io/i-tech-uw/openelis-global-2:develop
+    restart: always
+    environment:
+      - TZ=Africa/Nairobi
+        # context.xml doesn't seem to be able to pick up environment variables directly, so we are passing them in as CATALINA_OPTS
+      - CATALINA_OPTS= -Ddatasource.url=jdbc:postgresql://database:5432/clinlims -Ddatasource.username=clinlims -Ddatasource.password=clinlims
+    volumes:
+      - cert-volume:/etc/openelis-global
+      # -  openelis-plugins-data:/var/lib/openelis-global/plugins
+    secrets:
+      - source: datasource.password
+      - source: common.properties
+    networks:
+      - default
+      - openelis
+    configs:
+      - source: oe_server.xml
+        file: ./config/oe_server.xml
+        target: /usr/local/tomcat/conf/server.xml
+        
+  configs:
+    oe_server.xml:
+      file: ./oe_server.xml
+
+  secrets:
+    datasource.password:
+      file: ./config/datasource.password
+    common.properties:
+      file: ./config/common.properties
+  volumes:
+    cert-volume:
+      external: true
+      name: openelis-global-certs
+    openelis-plugins-data:
+  networks:
+    openelis:
+      external: true
+      name: openelis
+    default:

lis-openelis-global/importer/database/database.env

@@ -0,0 +1,4 @@
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=admin
+POSTGRES_DB=clinlims
+POSTGRES_INITDB_ARGS="--auth-host=md5"

lis-openelis-global/importer/database/dbInit/1-pgsqlPermissions.sql

@@ -0,0 +1,4 @@
+
+CREATE USER clinlims PASSWORD 'clinlims';
+CREATE USER admin SUPERUSER PASSWORD 'cPATaahXJlSj';
+ALTER DATABASE clinlims OWNER TO clinlims;

lis-openelis-global/importer/database/dbInit/siteInfo.sql

@@ -0,0 +1,6 @@
+DELETE from clinlims.site_information where name = 'siteNumber';
+INSERT INTO clinlims.site_information( id, lastupdated, "name", description, "value") VALUES ( nextval('clinlims.site_information_seq'), now(), 'siteNumber', 'The site number of the this lab', 'DEV01' );
+
+DELETE from clinlims.site_information where name = 'Accession number prefix';
+INSERT INTO clinlims.site_information( id, lastupdated, "name", description, "value") VALUES ( nextval('clinlims.site_information_seq'), now(), 'Accession number prefix', 'Prefix for SITEYEARNUM format.  Can not be changed if there are samples', 'DEV01' );
+

lis-openelis-global/package-metadata.json

@@ -0,0 +1,9 @@
+{
+  "id": "lis-openelis-global",
+  "name": "OpenELIS Global Lab Information System",
+  "description": "An open-source lab information management system designed for global health use.",
+  "type": "infrastructure",
+  "version": "0.0.1",
+  "dependencies": [],
+  "environmentVariables": {}
+}

lis-openelis-global/swarm.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare SERVICE_NAMES=()
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  SERVICE_NAMES=(
+    "lis-openelis-global"
+  )
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+  readonly SERVICE_NAMES
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function initialize_package() {
+  local package_dev_compose_filename=""
+  if [[ "${MODE}" == "dev" ]]; then
+    log info "Running package in DEV mode"
+    package_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  (
+    docker::deploy_service "${COMPOSE_FILE_PATH}" "$package_dev_compose_filename"
+    docker::deploy_sanity "${SERVICE_NAMES[@]}"
+  ) || {
+    log error "Failed to deploy package"
+    exit 1
+  }
+}
+
+function destroy_package() {
+  docker::service_destroy "${SERVICE_NAMES[@]}"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    log info "Running package in Single node mode"
+
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+
+    docker::scale_services_down "${SERVICE_NAMES[@]}"
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"

monitoring/docker-compose.cluster.yml

@@ -0,0 +1,132 @@
+version: '3.9'
+
+services:
+  prometheus:
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-1"
+      replicas: 1
+
+  prometheus_backup:
+    image: prom/prometheus:v2.38.0
+    user: root
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name!=node-1"
+      replicas: 1
+    volumes:
+      - prometheus_data_backup:/prometheus
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    configs:
+      - target: /etc/prometheus/prometheus.yml
+        source: prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+    networks:
+      public:
+      default:
+
+  minio-01:
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-1"
+
+  minio-02:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-02
+    volumes:
+      - minio-02-data1:/data1
+      - minio-02-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-1"
+      replicas: 1
+
+  minio-03:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-03
+    volumes:
+      - minio-03-data1:/data1
+      - minio-03-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-2"
+      replicas: 1
+
+  minio-04:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-04
+    volumes:
+      - minio-04-data1:/data1
+      - minio-04-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-3"
+      replicas: 1
+
+volumes:
+  prometheus_data_backup:
+  minio-02-data1:
+  minio-02-data2:
+  minio-03-data1:
+  minio-03-data2:
+  minio-04-data1:
+  minio-04-data2:

monitoring/docker-compose.dev.yml

@@ -0,0 +1,26 @@
+version: '3.9'
+
+services:
+  grafana:
+    ports:
+      - target: 3000
+        published: 3000
+        mode: host
+
+  prometheus:
+    ports:
+      - target: 9090
+        published: 9090
+        mode: host
+
+  loki:
+    ports:
+      - target: 3100
+        published: 3100
+        mode: host
+
+  minio-01:
+    ports:
+      - target: 9001
+        published: 9005
+        mode: host

monitoring/docker-compose.yml

@@ -0,0 +1,236 @@
+version: '3.9'
+
+services:
+  grafana:
+    image: grafana/grafana-oss:9.2.3
+    volumes:
+      - grafana-data:/var/lib/grafana
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER}
+      GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_SMTP_ENABLED: ${GF_SMTP_ENABLED}
+      GF_SMTP_HOST: ${GF_SMTP_HOST}
+      GF_SMTP_USER: ${GF_SMTP_USER}
+      GF_SMTP_PASSWORD: ${GF_SMTP_PASSWORD}
+      GF_AUTH_BASIC_ENABLED: ${KC_GRAFANA_SSO_ENABLED}
+      GF_AUTH_DISABLE_LOGIN_FORM: ${KC_GRAFANA_SSO_ENABLED}
+      GF_AUTH_GENERIC_OAUTH_TLS_SKIP_VERIFY_INSECURE: "true"
+      GF_AUTH_GENERIC_OAUTH_ENABLED: ${KC_GRAFANA_SSO_ENABLED}
+      GF_AUTH_GENERIC_OAUTH_NAME: "Keycloak"
+      GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: "true"
+      GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${KC_GRAFANA_CLIENT_ID}
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${KC_GRAFANA_CLIENT_SECRET}
+      GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_NAME: "email:primary"
+      GF_AUTH_GENERIC_OAUTH_SCOPES: "openid email profile offline_access roles"
+      GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH: "email"
+      GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH: "username"
+      GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH: "full_name"
+      GF_AUTH_GENERIC_OAUTH_AUTH_URL: "${KC_FRONTEND_URL}/realms/${KC_REALM_NAME}/protocol/openid-connect/auth"
+      GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "${KC_API_URL}/realms/${KC_REALM_NAME}/protocol/openid-connect/token"
+      GF_AUTH_GENERIC_OAUTH_API_URL: "${KC_API_URL}/realms/${KC_REALM_NAME}/protocol/openid-connect/userinfo"
+      GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: "contains(roles[*], 'admin') && 'Admin' || contains(roles[*], 'editor') && 'Editor' || 'Viewer'"
+      GF_SERVER_DOMAIN: ${GF_SERVER_DOMAIN}
+      GF_SERVER_ROOT_URL: ${KC_GRAFANA_ROOT_URL}
+      GF_AUTH_SIGNOUT_REDIRECT_URL: "${KC_FRONTEND_URL}/realms/${KC_REALM_NAME}/protocol/openid-connect/logout?client_id=${KC_GRAFANA_CLIENT_ID}&post_logout_redirect_uri=${KC_GRAFANA_ROOT_URL}/login"
+    configs:
+      - target: /etc/grafana/grafana.ini
+        source: grafana.ini
+      - target: /etc/grafana/provisioning/datasources/datasource.yml
+        source: datasource.yml
+      - target: /etc/grafana/provisioning/dashboards/dashboard.yml
+        source: dashboard.yml
+      - target: /etc/grafana/provisioning/dashboards/nodes/node-exporter-full_rev27.json
+        source: node-exporter-full_rev27.json
+      - target: /etc/grafana/provisioning/dashboards/containers/docker-swarm-monitor-v2_rev2.json
+        source: docker-swarm-monitor-v2_rev2.json
+      - target: /etc/grafana/provisioning/dashboards/applications/kminion-cluster_rev1.json
+        source: kminion-cluster_rev1.json
+      - target: /etc/grafana/provisioning/dashboards/applications/kminion-groups_rev1.json
+        source: kminion-groups_rev1.json
+      - target: /etc/grafana/provisioning/dashboards/applications/kminion-topic_rev1.json
+        source: kminion-topic_rev1.json
+      - target: /etc/grafana/provisioning/dashboards/containers/logging-universal-dashboard_rev1.json
+        source: logging-universal-dashboard_rev1.json
+    networks:
+      keycloak:
+      reverse-proxy:
+      default:
+
+  prometheus:
+    image: prom/prometheus:v2.38.0
+    user: root
+    volumes:
+      - prometheus-data:/prometheus
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    configs:
+      - target: /etc/prometheus/prometheus.yml
+        source: prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+    networks:
+      public:
+      default:
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.45.0
+    command: -docker_only
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker
+      - /dev/disk/:/dev/disk:ro
+    deploy:
+      mode: global
+
+  node-exporter:
+    image: quay.io/prometheus/node-exporter:v1.3.1
+    hostname: "{{.Node.ID}}"
+    command:
+      - '--path.rootfs=/host'
+    volumes:
+      - '/:/host:ro,rslave'
+    deploy:
+      mode: global
+
+  loki:
+    image: grafana/loki:2.6.1
+    volumes:
+      - loki-data:/tmp/loki
+    environment:
+      MO_SECURITY_ADMIN_USER: ${MO_SECURITY_ADMIN_USER}
+      MO_SECURITY_ADMIN_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    configs:
+      - target: /etc/loki/loki-config.yml
+        source: loki-config.yml
+    command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
+    deploy:
+      labels:
+        - prometheus-job-service=loki
+        - prometheus-address=loki:3100
+
+  promtail:
+    image: grafana/promtail:2.6.1
+    volumes:
+      - /var/lib/docker/containers:/host/containers
+      - /var/log:/var/log:ro
+    configs:
+      - target: /etc/promtail/promtail-config.yml
+        source: promtail-config.yml
+    command: -config.file=/etc/promtail/promtail-config.yml
+    deploy:
+      mode: global
+
+  minio-01:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...${NUM_MINIO_SERVERS}}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-01
+    volumes:
+      - minio-01-data1:/data1
+      - minio-01-data2:/data2
+    deploy:
+      replicas: 1
+    networks:
+      reverse-proxy:
+      default:
+
+configs:
+  grafana.ini:
+    file: ./grafana/grafana.ini
+    name: grafana.ini-${grafana_ini_DIGEST:?err}
+    labels:
+      name: grafana
+  datasource.yml:
+    file: ./grafana/datasource.yml
+    name: datasource.yml-${datasource_yml_DIGEST:?err}
+    labels:
+      name: grafana
+  dashboard.yml:
+    file: ./grafana/dashboards/dashboard.yml
+    name: dashboard.yml-${dashboard_yml_DIGEST:?err}
+    labels:
+      name: grafana
+  node-exporter-full_rev27.json:
+    file: ./grafana/dashboards/nodes/node-exporter-full_rev27.json
+    name: node-exporter-full_rev27.json-${node_exporter_full_rev27_json_DIGEST:?err}
+    labels:
+      name: grafana
+  docker-swarm-monitor-v2_rev2.json:
+    file: ./grafana/dashboards/containers/docker-swarm-monitor-v2_rev2.json
+    name: docker-swarm-monitor-v2_rev2.json-${docker_swarm_monitor_v2_rev2_json_DIGEST:?err}
+    labels:
+      name: grafana
+  kminion-cluster_rev1.json:
+    file: ./grafana/dashboards/applications/kminion-cluster_rev1.json
+    name: kminion-cluster_rev1.json-${kminion_cluster_rev1_json_DIGEST:?err}
+    labels:
+      name: grafana
+  kminion-groups_rev1.json:
+    file: ./grafana/dashboards/applications/kminion-groups_rev1.json
+    name: kminion-groups_rev1.json-${kminion_groups_rev1_json_DIGEST:?err}
+    labels:
+      name: grafana
+  kminion-topic_rev1.json:
+    file: ./grafana/dashboards/applications/kminion-topic_rev1.json
+    name: kminion-topic_rev1.json-${kminion_topic_rev1_json_DIGEST:?err}
+    labels:
+      name: grafana
+  logging-universal-dashboard_rev1.json:
+    file: ./grafana/dashboards/containers/logging-universal-dashboard_rev1.json
+    name: logging-universal-dashboard_rev1.json-${logging_universal_dashboard_rev1_json_DIGEST:?err}
+    labels:
+      name: grafana
+  prometheus.yml:
+    file: ./prometheus/prometheus.yml
+    name: prometheus.yml-${prometheus_yml_DIGEST:?err}
+    labels:
+      name: prometheus
+  loki-config.yml:
+    file: ./loki/loki-config.yml
+    name: loki-config.yml-${loki_config_yml_DIGEST:?err}
+    labels:
+      name: loki
+  promtail-config.yml:
+    file: ./promtail/promtail-config.yml
+    name: promtail-config.yml-${promtail_config_yml_DIGEST:?err}
+    labels:
+      name: promtail
+
+volumes:
+  prometheus-data:
+  grafana-data:
+  loki-data:
+  minio-01-data1:
+  minio-01-data2:
+
+networks:
+  keycloak:
+    name: keycloak_public
+    external: true
+  reverse-proxy:
+    name: reverse-proxy_public
+    external: true
+  public:
+    name: prometheus_public
+    external: true
+  default:

monitoring/grafana/dashboards/applications/kminion-cluster_rev1.json

@@ -0,0 +1,914 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_CORTEX",
+      "label": "Prometheus",
+      "description": "Prometheus Server",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.3.6"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "singlestat",
+      "name": "Singlestat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Cluster wide metrics exported by KMinion",
+  "editable": true,
+  "gnetId": 14012,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1614981106152,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "Summed delta between High and Low watermark of all partitions in the given topic. Therefore not accurate for compacted topics that contain compacted records.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 16,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum by (kubernetes_pod_name) (rate(kminion_kafka_topic_high_water_mark_sum[$__rate_interval]))",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Messages in/sec",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:545",
+          "decimals": 0,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:546",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 0
+      },
+      "id": 11,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "$$hashKey": "object:221",
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "$$hashKey": "object:222",
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "cluster_version",
+      "targets": [
+        {
+          "expr": "kminion_kafka_cluster_info",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Cluster Version",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "$$hashKey": "object:224",
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": null,
+            "filterable": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 10,
+      "options": {
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": false,
+            "displayName": "Address"
+          }
+        ]
+      },
+      "pluginVersion": "7.3.6",
+      "targets": [
+        {
+          "expr": "kminion_kafka_broker_info",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Brokers",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__cluster__": true,
+              "__name__": true,
+              "instance": true,
+              "job": true,
+              "kubernetes_namespace": true,
+              "kubernetes_pod_name": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 12,
+              "__cluster__": 1,
+              "__name__": 2,
+              "address": 4,
+              "broker_id": 3,
+              "instance": 6,
+              "is_controller": 8,
+              "job": 9,
+              "kubernetes_namespace": 10,
+              "kubernetes_pod_name": 11,
+              "port": 5,
+              "rack_id": 7
+            },
+            "renameByName": {
+              "address": "Address",
+              "broker_id": "BrokerID",
+              "instance": "",
+              "is_controller": "Is Controller",
+              "port": "Port",
+              "rack_id": "RackID"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 4
+      },
+      "id": 7,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "$$hashKey": "object:221",
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "$$hashKey": "object:222",
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "broker_count",
+      "targets": [
+        {
+          "expr": "kminion_kafka_cluster_info",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Brokers Online",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "$$hashKey": "object:224",
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "Log dir size across all log dirs, topics and partitions for a given broker.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 14,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (broker_id) (kminion_kafka_broker_log_dir_size_total_bytes)",
+          "interval": "",
+          "legendFormat": "Broker {{broker_id}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Log Dir Size by Broker",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:545",
+          "decimals": 0,
+          "format": "bytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:546",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": false,
+        "max": true,
+        "min": false,
+        "show": true,
+        "sort": "avg",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (topic_name) (rate(kminion_kafka_topic_high_water_mark_sum{topic_name=~\"$highThroughputTopics\"}[$__rate_interval]))",
+          "interval": "",
+          "legendFormat": "{{topic_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Messages in/sec by topic [Top 15]",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:39",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:40",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 17,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": false,
+        "max": true,
+        "min": false,
+        "show": true,
+        "sort": "avg",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (group_id) (rate(kminion_kafka_consumer_group_offset_commits_total{group_id=~\"$topOffsetCommitters\"}[$__rate_interval]))",
+          "interval": "",
+          "legendFormat": "{{group_id}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Top Offset Committers [Top 5]",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:39",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:40",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (kubernetes_pod_name) (rate(kminion_exporter_offset_consumer_records_consumed_total[$__rate_interval]))",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{kubernetes_pod_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Offset commits processed",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:163",
+          "format": "short",
+          "label": "Messages/s",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:164",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "Prometheus",
+        "definition": "query_result(topk(15, avg by (topic_name) (avg_over_time(kminion_kafka_topic_high_water_mark_sum[${__range_s}s]))))",
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "highThroughputTopics",
+        "options": [],
+        "query": "query_result(topk(15, avg by (topic_name) (avg_over_time(kminion_kafka_topic_high_water_mark_sum[${__range_s}s]))))",
+        "refresh": 1,
+        "regex": ".*topic_name=\"(.*?)\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "Prometheus",
+        "definition": "query_result(topk(5, avg by (group_id) (avg_over_time(kminion_kafka_consumer_group_offset_commits_total[${__range_s}s]))))",
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "topOffsetCommitters",
+        "options": [],
+        "query": "query_result(topk(5, avg by (group_id) (avg_over_time(kminion_kafka_consumer_group_offset_commits_total[${__range_s}s]))))",
+        "refresh": 1,
+        "regex": ".*group_id=\"(.*?)\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "KMinion Cluster Dashboard - Prometheus Exporter for Apache Kafka",
+  "uid": "Ax-9LJsGz",
+  "version": 25
+}

monitoring/grafana/dashboards/applications/kminion-groups_rev1.json

@@ -0,0 +1,528 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_CORTEX",
+      "label": "Prometheus",
+      "description": "Prometheus Server",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.3.6"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "singlestat",
+      "name": "Singlestat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": 14014,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1614981814739,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (group_id,topic_name) (kminion_kafka_consumer_group_topic_lag{group_id=\"$consumerGroup\", topic_name=\"$topic\"})",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{group_id}} => {{topic_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Consumergroup => Topic Lag",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:750",
+          "decimals": 0,
+          "format": "short",
+          "label": "Messages",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:751",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorPostfix": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 8,
+        "y": 0
+      },
+      "id": 4,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "$$hashKey": "object:896",
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "$$hashKey": "object:897",
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "commits / s",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": true
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "avg (sum by (kubernetes_pod_name) (rate(kminion_kafka_consumer_group_offset_commits_total{group_id=\"$consumerGroup\"}[1m])))",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "200, 1000",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Offset Commit Rate",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "$$hashKey": "object:899",
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": true,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "format": "locale",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 12,
+        "y": 0
+      },
+      "id": 6,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "$$hashKey": "object:1000",
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "$$hashKey": "object:1001",
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": true
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "avg by (group_id,topic_name) (kminion_kafka_consumer_group_topic_lag{group_id=\"$consumerGroup\", topic_name=\"$topic\"})",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "200, 1000",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Topic Lag",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "$$hashKey": "object:1003",
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 8,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (topic_name) (rate(kminion_kafka_topic_high_water_mark_sum{topic_name=\"$topic\"}[$__rate_interval]))",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incoming: {{topic_name}}",
+          "refId": "A"
+        },
+        {
+          "expr": "avg by (topic_name, group_id) (rate(kminion_kafka_consumer_group_topic_offset_sum{group_id=\"$consumerGroup\", topic_name=\"$topic\"}[$__rate_interval]))",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Consuming: {{topic_name}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Messages In vs Consumed",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:1187",
+          "decimals": 0,
+          "format": "short",
+          "label": "Messages/s",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:1188",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "Prometheus",
+        "definition": "label_values(kminion_kafka_consumer_group_topic_lag, group_id)",
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "Consumer Group",
+        "multi": false,
+        "name": "consumerGroup",
+        "options": [],
+        "query": "label_values(kminion_kafka_consumer_group_topic_lag, group_id)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "Prometheus",
+        "definition": "label_values(kminion_kafka_consumer_group_topic_lag{group_id=\"$consumerGroup\"}, topic_name)",
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "Topic",
+        "multi": false,
+        "name": "topic",
+        "options": [],
+        "query": "label_values(kminion_kafka_consumer_group_topic_lag{group_id=\"$consumerGroup\"}, topic_name)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "KMinion Consumer Group Dashboard - Prometheus Exporter for Apache Kafka",
+  "uid": "RS8RnbsGz",
+  "version": 7,
+  "description": "Kafka Topic metrics exported by KMinion"
+}

monitoring/grafana/dashboards/applications/kminion-topic_rev1.json

@@ -0,0 +1,878 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_CORTEX",
+      "label": "Prometheus",
+      "description": "Prometheus Server",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.3.6"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "singlestat",
+      "name": "Singlestat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": 14013,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1614981632034,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "Summed delta between High and Low watermark of all partitions in the given topic. Therefore not accurate for compacted topics that contain compacted records.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (topic_name) (kminion_kafka_topic_high_water_mark_sum{topic_name=\"$topic\"} - kminion_kafka_topic_low_water_mark_sum{topic_name=\"$topic\"})",
+          "interval": "",
+          "legendFormat": "{{topic_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Message count",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:246",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:247",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 4,
+        "x": 8,
+        "y": 0
+      },
+      "id": 4,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "$$hashKey": "object:571",
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "$$hashKey": "object:572",
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "partition_count",
+      "targets": [
+        {
+          "expr": "kminion_kafka_topic_info{topic_name=\"$topic\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Partition Count",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "$$hashKey": "object:574",
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 12,
+        "y": 0
+      },
+      "id": 13,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "$$hashKey": "object:571",
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "$$hashKey": "object:572",
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "cleanup_policy",
+      "targets": [
+        {
+          "expr": "kminion_kafka_topic_info{topic_name=\"$topic\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Cleanup Policy",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "$$hashKey": "object:574",
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "Summed delta between High and Low watermark of all partitions in the given topic. Therefore not accurate for compacted topics that contain compacted records.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 16,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (topic_name) (rate(kminion_kafka_topic_high_water_mark_sum{topic_name=\"$topic\"}[$__rate_interval]))",
+          "interval": "",
+          "legendFormat": "{{topic_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Messages in/sec",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:545",
+          "decimals": 0,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:546",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 4,
+        "x": 8,
+        "y": 3
+      },
+      "id": 12,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "$$hashKey": "object:571",
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "$$hashKey": "object:572",
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "replication_factor",
+      "targets": [
+        {
+          "expr": "kminion_kafka_topic_info{topic_name=\"$topic\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Replication Factor",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "$$hashKey": "object:574",
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": null,
+            "filterable": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 8,
+      "options": {
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": false,
+            "displayName": "Group ID"
+          }
+        ]
+      },
+      "pluginVersion": "7.3.6",
+      "targets": [
+        {
+          "expr": "kminion_kafka_consumer_group_topic_lag{topic_name=\"$topic\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Subscribed Consumer Groups",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "__cluster__": true,
+              "__name__": true,
+              "expires_at": false,
+              "instance": true,
+              "job": true,
+              "kubernetes_namespace": true,
+              "kubernetes_pod_name": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 10,
+              "__cluster__": 1,
+              "__name__": 2,
+              "expires_at": 9,
+              "group_id": 3,
+              "instance": 4,
+              "job": 5,
+              "kubernetes_namespace": 6,
+              "kubernetes_pod_name": 7,
+              "topic_name": 8
+            },
+            "renameByName": {
+              "Value": "Group -> Topic Lag",
+              "expires_at": "Last Offset Expires At",
+              "group_id": "Group ID",
+              "instance": "",
+              "job": "",
+              "topic_name": "Topic"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "Log dir size across all partitions and brokers for the given topic.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.3.6",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg by (topic_name) (kminion_kafka_topic_log_dir_size_total_bytes{topic_name=\"$topic\"})",
+          "interval": "",
+          "legendFormat": "{{topic_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Total Log Dir Size",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:545",
+          "decimals": 0,
+          "format": "bytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:546",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": null,
+            "filterable": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 13
+      },
+      "id": 11,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "7.3.6",
+      "targets": [
+        {
+          "expr": "avg by (partition_id) (kminion_kafka_topic_partition_low_water_mark{topic_name=\"$topic\"})",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        },
+        {
+          "expr": "avg by (partition_id) (kminion_kafka_topic_partition_high_water_mark{topic_name=\"$topic\"})",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "B"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Partitions",
+      "transformations": [
+        {
+          "id": "seriesToColumns",
+          "options": {
+            "byField": "partition_id"
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time 1": true,
+              "Time 2": true
+            },
+            "indexByName": {},
+            "renameByName": {
+              "Time 1": "",
+              "Time 2": "",
+              "Value #A": "Low Water Mark",
+              "Value #B": "High Water Mark",
+              "partition_id": "Partition ID"
+            }
+          }
+        },
+        {
+          "id": "calculateField",
+          "options": {
+            "alias": "Message Count",
+            "mode": "reduceRow",
+            "reduce": {
+              "include": [
+                "High Water Mark",
+                "Low Water Mark"
+              ],
+              "reducer": "delta"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "Prometheus",
+        "definition": "label_values(kminion_kafka_topic_low_water_mark_sum, topic_name)",
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "Topic",
+        "multi": false,
+        "name": "topic",
+        "options": [],
+        "query": "label_values(kminion_kafka_topic_low_water_mark_sum, topic_name)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "KMinion Topic Dashboard - Prometheus Exporter for Apache Kafka",
+  "uid": "kS6iVtyMk",
+  "version": 19,
+  "description": "Kafka Topic metrics exported by KMinion"
+}

monitoring/grafana/dashboards/containers/docker-swarm-monitor-v2_rev2.json

@@ -0,0 +1,688 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Docker Swarm Monitor Dashboad v2.0",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 1505,
+  "graphTooltip": 0,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "tags": [
+        "prometheus",
+        "cAdvisor",
+        "node-exporter",
+        "alertmanager"
+      ],
+      "type": "dashboards"
+    }
+  ],
+  "liveNow": false,
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 5,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": "270px",
+      "hiddenSeries": false,
+      "id": 1,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "hideEmpty": false,
+        "hideZero": true,
+        "max": true,
+        "min": true,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "9.1.0",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum by(name) (rate(container_cpu_usage_seconds_total{name=~\".+\", container_label_com_docker_swarm_node_id=~\"$node\"}[$interval])) * 100",
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "range": true,
+          "refId": "A",
+          "step": 2
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "CPU Usage per Container",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                1000000000
+              ],
+              "type": "gt"
+            },
+            "operator": {
+              "type": "and"
+            },
+            "query": {
+              "datasourceId": 1,
+              "model": {
+                "expr": "sum(container_memory_rss{name=~\".+\"}) by (name)",
+                "hide": false,
+                "intervalFactor": 2,
+                "legendFormat": "{{name}}",
+                "metric": "container_memory_rss",
+                "refId": "A",
+                "step": 2
+              },
+              "params": [
+                "A",
+                "5m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "executionErrorState": "alerting",
+        "frequency": "60s",
+        "handler": 1,
+        "name": "Memory Usage per Container alert",
+        "noDataState": "keep_state",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 3,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "height": "270px",
+      "hiddenSeries": false,
+      "id": 5,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "hideZero": true,
+        "max": true,
+        "min": true,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "9.1.0",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum by(name) (container_memory_rss{name=~\".+\", container_label_com_docker_swarm_node_id=~\"$node\"})",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "metric": "container_memory_rss",
+          "range": true,
+          "refId": "A",
+          "step": 2
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 1000000000
+        }
+      ],
+      "timeRegions": [],
+      "title": "Memory Usage per Container",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "height": "270px",
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "hideEmpty": false,
+        "hideZero": true,
+        "max": true,
+        "min": true,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "9.1.0",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(container_network_receive_bytes_total{name=~\".+\", container_label_com_docker_swarm_node_id=~\"$node\"}[$interval])) by (name)",
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "range": true,
+          "refId": "A",
+          "step": 2
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Incoming Network Traffic per Container",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "height": "270px",
+      "hiddenSeries": false,
+      "id": 8,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "hideEmpty": false,
+        "hideZero": true,
+        "max": true,
+        "min": true,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "9.1.0",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(container_network_transmit_bytes_total{name=~\".+\", container_label_com_docker_swarm_node_id=~\"$node\"}[$interval])) by (name)",
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "range": true,
+          "refId": "A",
+          "step": 2
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Outgoing Network Traffic per Container",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "schemaVersion": 37,
+  "style": "dark",
+  "tags": [
+    "prometheus",
+    "cAdvisor",
+    "node-exporter",
+    "alertmanager"
+  ],
+  "templating": {
+    "list": [
+      {
+        "auto": true,
+        "auto_count": 50,
+        "auto_min": "50s",
+        "current": {
+          "selected": false,
+          "text": "auto",
+          "value": "$__auto_interval_interval"
+        },
+        "hide": 0,
+        "label": "Interval",
+        "name": "interval",
+        "options": [
+          {
+            "selected": true,
+            "text": "auto",
+            "value": "$__auto_interval_interval"
+          },
+          {
+            "selected": false,
+            "text": "30s",
+            "value": "30s"
+          },
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": false,
+            "text": "2m",
+            "value": "2m"
+          },
+          {
+            "selected": false,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "7m",
+            "value": "7m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "30m",
+            "value": "30m"
+          },
+          {
+            "selected": false,
+            "text": "1h",
+            "value": "1h"
+          },
+          {
+            "selected": false,
+            "text": "6h",
+            "value": "6h"
+          },
+          {
+            "selected": false,
+            "text": "12h",
+            "value": "12h"
+          },
+          {
+            "selected": false,
+            "text": "1d",
+            "value": "1d"
+          },
+          {
+            "selected": false,
+            "text": "7d",
+            "value": "7d"
+          },
+          {
+            "selected": false,
+            "text": "14d",
+            "value": "14d"
+          },
+          {
+            "selected": false,
+            "text": "30d",
+            "value": "30d"
+          }
+        ],
+        "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+        "refresh": 2,
+        "skipUrlSync": false,
+        "type": "interval"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "10.0.1.70:8080",
+          "value": "10.0.1.70:8080"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(container_last_seen, instance)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Node",
+        "multi": true,
+        "name": "server",
+        "options": [],
+        "query": {
+          "query": "label_values(container_last_seen, instance)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "7hemy3416k1amfomh6ot9a5cv",
+            "rpsixil74u5g369hg4ikes6w0",
+            "p3uoxingm0oxszdeatytq1yyh"
+          ],
+          "value": [
+            "7hemy3416k1amfomh6ot9a5cv",
+            "rpsixil74u5g369hg4ikes6w0",
+            "p3uoxingm0oxszdeatytq1yyh"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(container_last_seen, container_label_com_docker_swarm_node_id)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Docker Node ID:",
+        "multi": true,
+        "name": "node",
+        "options": [],
+        "query": {
+          "query": "label_values(container_last_seen, container_label_com_docker_swarm_node_id)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Docker-Swarm-Monitor-v2",
+  "uid": "2gSjFKi4z",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/grafana/dashboards/containers/logging-universal-dashboard_rev1.json

@@ -0,0 +1,926 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Universal and flexible dashboard for logging",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 12611,
+  "graphTooltip": 0,
+  "id": 11,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "description": "Total count of log lines in the specified time range",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "color": "rgb(31, 255, 7)",
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgb(31, 255, 7)",
+                "value": null
+              },
+              {
+                "color": "rgb(31, 255, 7)",
+                "value": 10
+              },
+              {
+                "color": "rgb(31, 255, 7)",
+                "value": 50
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 11,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"})[$__interval]))",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Total count of logs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "description": "Total Count: of $searchable_pattern in the specified time range",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "color": "rgb(222, 15, 43)",
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgb(222, 15, 43)",
+                "value": null
+              },
+              {
+                "color": "rgb(222, 15, 43)",
+                "value": 10
+              },
+              {
+                "color": "rgb(222, 15, 43)",
+                "value": 50
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 6,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"} |~ \"(?i)$searchable_pattern\")[$__interval]))",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Count: of \"$searchable_pattern\"",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "description": "Live logs is a like 'tail -f | grep' in a real time",
+      "gridPos": {
+        "h": 22,
+        "w": 24,
+        "x": 0,
+        "y": 3
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": true,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "{swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"} |~ \"(?i)$searchable_pattern\"",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Live logs (filtered by \"$searchable_pattern\")",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 7,
+        "x": 0,
+        "y": 25
+      },
+      "id": 19,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "7.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"})[$__interval])) by (stream)",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Total count of stderr / stdout pie",
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 7,
+        "y": 25
+      },
+      "id": 20,
+      "interval": "1m",
+      "links": [],
+      "maxDataPoints": "",
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "pieType": "donut",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "6.4.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"} |~ \"(?i)$searchable_pattern\")[$__interval])) by (swarm_task_name)",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Matched word: \"$searchable_pattern\" donut",
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "color": "#299c46",
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "#299c46",
+                "value": null
+              },
+              {
+                "color": "rgba(237, 129, 40, 0.89)",
+                "value": 10
+              },
+              {
+                "color": "#C4162A",
+                "value": 50
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 5,
+        "x": 19,
+        "y": 25
+      },
+      "id": 9,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": false
+      },
+      "pluginVersion": "9.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"} |~ \"(?i)$searchable_pattern\")[$__interval])) * 100 / sum(count_over_time(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"})[$__interval]))",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "\"$searchable_pattern\" Percentage for specified time",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "Count",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 100,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 3,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "id": 18,
+      "interval": "1m",
+      "links": [],
+      "maxDataPoints": "",
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "9.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"} |~ \"(?i)$searchable_pattern\")[$__interval])) by (swarm_task_name)",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Matched word: \"$searchable_pattern\" historical",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 100,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "log": 10,
+              "type": "log"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "id": 10,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "9.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"} |~ \"(?i)$searchable_pattern\")[30s])) by (swarm_task_name)",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "\"$searchable_pattern\" Rate per Task",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P00201832B18B88C3"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 40,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 6,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "always",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "{stream=\"stderr\"} stderr"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#C4162A",
+                  "mode": "fixed"
+                }
+              },
+              {
+                "id": "custom.lineWidth",
+                "value": 2
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "{stream=\"stdout\"} stdout"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#56A64B",
+                  "mode": "fixed"
+                }
+              },
+              {
+                "id": "custom.lineWidth",
+                "value": 2
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 41
+      },
+      "id": 7,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "9.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P00201832B18B88C3"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time(({swarm_service_name=\"$service_name\", stream=~\"$stream\", swarm_task_name=~\"$task_name\"})[$__interval])) by (stream)",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Count of stderr / stdout historical",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 37,
+  "style": "dark",
+  "tags": [
+    "Loki",
+    "logging"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "monitoring_prometheus-kafka-adapter",
+          "value": "monitoring_prometheus-kafka-adapter"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "P00201832B18B88C3"
+        },
+        "definition": "label_values({swarm_service_name=~\".+\"}, swarm_service_name)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Service",
+        "multi": false,
+        "name": "service_name",
+        "options": [],
+        "query": "label_values({swarm_service_name=~\".+\"}, swarm_service_name)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "P00201832B18B88C3"
+        },
+        "definition": "label_values({swarm_service_name=\"$service_name\"}, swarm_task_name)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Task",
+        "multi": true,
+        "name": "task_name",
+        "options": [],
+        "query": "label_values({swarm_service_name=\"$service_name\"}, swarm_task_name)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "P00201832B18B88C3"
+        },
+        "definition": "label_values({swarm_service_name=\"$service_name\"}, stream)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Stream",
+        "multi": false,
+        "name": "stream",
+        "options": [],
+        "query": "label_values({swarm_service_name=\"$service_name\"}, stream)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "error",
+          "value": "error"
+        },
+        "hide": 0,
+        "label": "Search (case insensitive)",
+        "name": "searchable_pattern",
+        "options": [
+          {
+            "selected": true,
+            "text": "error",
+            "value": "error"
+          }
+        ],
+        "query": "error",
+        "skipUrlSync": false,
+        "type": "textbox"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "Container logs",
+  "uid": "fRIvzUZMf",
+  "version": 4,
+  "weekStart": ""
+}

monitoring/grafana/dashboards/dashboard.yml

@@ -0,0 +1,10 @@
+# config file version
+apiVersion: 1
+
+providers:
+  - name: dashboards
+    type: file
+    updateIntervalSeconds: 30
+    options:
+      path: /etc/grafana/provisioning/dashboards
+      foldersFromFilesStructure: true

monitoring/grafana/datasource.yml

@@ -0,0 +1,12 @@
+# config file version
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  access: proxy
+  url: http://prometheus:9090
+- name: LokiLogs
+  type: loki
+  access: direct
+  url: http://loki:3100

monitoring/grafana/grafana.ini

@@ -0,0 +1,22 @@
+[paths]
+provisioning = /etc/grafana/provisioning/
+
+[server]
+enable_gzip = true
+#protocol = https
+#;http_addr =	
+#http_port = 3000	
+#domain = localhost	
+#enforce_domain = false	
+#root_url = https://localhost:3000	
+#router_logging = false	
+#static_root_path = public	
+#cert_file = /etc/certs/cert.pem	
+#cert_key = /etc/certs/cert-key.pem
+
+[security]
+# If you want to embed grafana into an iframe for example
+allow_embedding = true
+
+[users]
+default_theme = dark

monitoring/loki/loki-config.yml

@@ -0,0 +1,63 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  grpc_server_max_concurrent_streams: 0
+
+ingester:
+  wal:
+    enabled: true
+    dir: /tmp/wal
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h
+  max_chunk_age: 1h
+  chunk_target_size: 1048576
+  chunk_retain_period: 30s
+  max_transfer_retries: 0
+
+schema_config:
+  configs:
+    - from: 2020-07-01
+      store: boltdb-shipper
+      object_store: aws
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/boltdb-shipper-active
+    cache_location: /loki/boltdb-shipper-cache
+    resync_interval: 30s
+    shared_store: s3
+  aws:
+    s3: http://${MO_SECURITY_ADMIN_USER}:${MO_SECURITY_ADMIN_PASSWORD}@minio-01.:9000/loki
+    s3forcepathstyle: true
+
+compactor:
+  working_directory: /loki/boltdb-shipper-compactor
+  shared_store: s3
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  ingestion_rate_mb: 100
+  ingestion_burst_size_mb: 150
+  max_concurrent_tail_requests: 200
+  max_cache_freshness_per_query: 10m
+  max_streams_per_user: 500
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s

monitoring/package-metadata.json

@@ -0,0 +1,26 @@
+{
+  "id": "monitoring",
+  "name": "Monitoring package",
+  "description": "A package for monitoring the platform services",
+  "type": "infrastructure",
+  "version": "0.0.1",
+  "dependencies": [],
+  "environmentVariables": {
+    "GF_SECURITY_ADMIN_USER": "admin",
+    "GF_SECURITY_ADMIN_PASSWORD": "dev_password_only",
+    "GF_SMTP_ENABLED": "false",
+    "GF_SMTP_HOST": "localhost:25",
+    "GF_SMTP_USER": "",
+    "GF_SMTP_PASSWORD": "",
+    "KC_GRAFANA_SSO_ENABLED": "false",
+    "KC_REALM_NAME": "platform-realm",
+    "KC_FRONTEND_URL": "http://localhost:9088",
+    "KC_API_URL": "http://identity-access-manager-keycloak:8080",
+    "KC_GRAFANA_CLIENT_ID": "grafana-oauth",
+    "KC_GRAFANA_CLIENT_SECRET": "CV14QfwnpYFj1IH5dK5lScPNCYAIYP1c",
+    "KC_GRAFANA_ROOT_URL": "http://localhost:3000",
+    "GF_SERVER_DOMAIN": "localhost",
+    "MO_SECURITY_ADMIN_USER": "admin",
+    "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only"
+  }
+}

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,76 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+    - targets: ["prometheus:9090"]
+
+  - job_name: "grafana"
+    static_configs:
+    - targets: ["grafana:3000"]
+
+  # Create a job for Docker daemons.
+  - job_name: 'docker'
+    dockerswarm_sd_configs:
+      - host: unix:///var/run/docker.sock
+        role: nodes
+    relabel_configs:
+      # Fetch metrics on port 9323.
+      - source_labels: [__meta_dockerswarm_node_address]
+        target_label: __address__
+        replacement: $1:9323
+      # Set hostname as instance label
+      - source_labels: [__meta_dockerswarm_node_hostname]
+        target_label: instance
+
+  # Create a job for reading metrics from nodes
+  - job_name: 'node-exporter'
+    dns_sd_configs:
+    - names:
+      - 'tasks.node-exporter'
+      type: 'A'
+      port: 9100
+
+  # Create a job for reading metrics from containers
+  - job_name: 'cadvisor'
+    dns_sd_configs:
+    - names:
+      - 'tasks.cadvisor'
+      type: 'A'
+      port: 8080
+
+  # Create a job for Docker Swarm services with particular labels (one target for the whole service, possibly multiple containers)
+  - job_name: 'dockerswarm-services'
+    dockerswarm_sd_configs:
+      - host: unix:///var/run/docker.sock
+        role: services
+    relabel_configs:
+      # Only keep services that have a `prometheus-job-service` label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_job_service]
+        regex: .+
+        action: keep
+      # Use the prometheus-job Swarm label as Prometheus job label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_job_service]
+        target_label: job
+      # Use the prometheus-address Swarm label as the target address for swarm service.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_address]
+        target_label: __address__
+
+  # Create a job for Docker Swarm container with particular labels (one target per container/task)
+  - job_name: 'dockerswarm-tasks'
+    dockerswarm_sd_configs:
+      - host: unix:///var/run/docker.sock
+        role: tasks
+    relabel_configs:
+      # Only keep containers that should be running.
+      - source_labels: [__meta_dockerswarm_task_desired_state]
+        regex: running
+        action: keep
+      # Only keep containers that have a `prometheus-job-task` label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_job_task]
+        regex: .+
+        action: keep
+      # Use the prometheus-job Swarm label as Prometheus job label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_job_task]
+        target_label: job

monitoring/promtail/promtail-config.yml

@@ -0,0 +1,47 @@
+server:
+  http_listen_address: 0.0.0.0
+  http_listen_port: 9080
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+- url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+
+- job_name: containers
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: containerlogs
+      __path__: /host/containers/*/*log
+
+  pipeline_stages:
+  - json:
+      expressions:
+        log: log
+        stream: stream
+        time: time
+        tag: attrs.tag
+        stack_name: attrs."com.docker.stack.namespace"
+        swarm_service_name: attrs."com.docker.swarm.service.name"
+        swarm_task_name: attrs."com.docker.swarm.task.name"
+        swarm_node_id: attrs."com.docker.swarm.node.id"
+  - regex:
+      expression: "^/host/containers/(?P<container_id>.{12}).+/.+-json.log$"
+      source: filename
+  - timestamp:
+      format: RFC3339Nano
+      source: time
+  - labels:
+      stream:
+      container_id:
+      tag:
+      stack_name:
+      swarm_service_name:
+      swarm_task_name:
+      swarm_node_id:
+  - output:
+      source: log

monitoring/swarm.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare STACK="monitoring"
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+  readonly STACK
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/config-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function initialize_package() {
+  local monitoring_dev_compose_filename=""
+  local monitoring_cluster_compose_filename=""
+
+  if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+    monitoring_cluster_compose_filename="docker-compose.cluster.yml"
+    export NUM_MINIO_SERVERS=4
+  else
+    export NUM_MINIO_SERVERS=1
+  fi
+
+  if [[ "${MODE}" == "dev" ]]; then
+    log info "Running package in DEV mode"
+    monitoring_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  (
+    docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$monitoring_dev_compose_filename" "$monitoring_cluster_compose_filename"
+  ) || {
+    log error "Failed to deploy package"
+    exit 1
+  }
+}
+
+function scale_services_down() {
+  docker::service_destroy $STACK "cadvisor" "node-exporter" "promtail"
+
+  docker::scale_services $STACK 0
+}
+
+function destroy_package() {
+  docker::stack_destroy $STACK
+
+  if [[ $CLUSTERED_MODE == "true" ]]; then
+    sleep 5
+    log warn "Volumes are only deleted on the host on which the command is run. Monitoring volumes on other nodes are not deleted"
+  fi
+
+  docker::prune_configs "grafana" "prometheus" "promtail" "loki"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+      log info "Running package in Cluster node mode"
+    else
+      log info "Running package in Single node mode"
+    fi
+
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+
+    scale_services_down
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"

openhim-mediator-data-stream/config/config.json

@@ -0,0 +1,246 @@
+{
+    "debeziumConfigurations": {
+      "databaseServerName" : "isanteplus-mysql",
+      "databaseHostName" : "isanteplus-mysql",
+      "databaseServerId" : "223344",
+      "databasePort" : "3306",
+      "databaseUser" : "root",
+      "databasePassword" : "debezium",
+      "databaseName" : "openmrs",
+      "databaseOffsetStorage" : "data/offset.dat",
+      "databaseHistory" : "data/dbhistory.dat",
+      "snapshotMode" : "schema_only"
+     },
+    "eventConfigurations": {
+      "location": {
+        "enabled": "false",
+        "title": "Location",
+        "parentTable": "location",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/location/{uuid}?v=full",
+          "fhir": "/Location/{uuid}"
+        }
+      },
+      "concept": {
+        "enabled": "false",
+        "title": "Concept",
+        "parentTable": "concept",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/concept/{uuid}?v=full"
+        }
+      },
+      "cohort": {
+        "enabled": "true",
+        "title": "Cohort",
+        "parentTable": "cohort",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/cohort/{uuid}?v=full",
+          "fhir": "/Group/{uuid}"
+        }
+      },
+      "visittype": {
+        "enabled": "false",
+        "title": "Visit Type",
+        "parentTable": "visittype",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/visittype/{uuid}?v=full"
+        }
+      },
+      "person_address": {
+        "enabled": "false",
+        "title": "PersonAddress",
+        "parentTable": "person_address",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/person/{parent-uuid}/address/{uuid}?v=full"
+        }
+      },
+      "person_name": {
+        "enabled": "false",
+        "title": "PersonName",
+        "parentTable": "person_name",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/person/{parent-uuid}/name/{uuid}?v=full"
+        }
+      },
+      "patient_identifier": {
+        "enabled": "false",
+        "title": "PatientIdentifier",
+        "parentTable": "patient_identifier",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/patient/{parent-uuid}/identifier/{uuid}?v=full"
+        }
+      },
+      "person_attribute": {
+        "enabled": "false",
+        "title": "PersonAttribute",
+        "parentTable": "person_attribute",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/person/{parent-uuid}/attribute/{uuid}?v=full"
+        }
+      },
+      "person": {
+        "enabled": "true",
+        "title": "Person",
+        "parentTable": "person",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/person/{uuid}?v=full",
+          "fhir": "/Person/{uuid}"
+        }
+      },
+      "provider": {
+        "enabled": "true",
+        "title": "Provider",
+        "parentTable": "provider",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/provider/{uuid}?v=full",
+          "fhir": "/Provider/{uuid}"
+        }
+      },
+      "user": {
+        "enabled": "false",
+        "title": "User",
+        "parentTable": "user",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/user/{uuid}?v=full"
+        }
+      },
+      "patient": {
+        "enabled": "true",
+        "title": "Patient",
+        "parentTable": "person",
+        "parentForeignKey": "person_id",
+        "childPrimaryKey":"patient_id",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/patient/{uuid}?v=full",
+          "fhir": "/Patient/{uuid}"
+        }
+      },
+      "relationship": {
+        "enabled": "true",
+        "title": "Relationship",
+        "parentTable": "relationship",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/relationship/{uuid}?v=full",
+          "fhir": "/Relationship/{uuid}"
+        }
+      },
+      "drug": {
+        "enabled": "true",
+        "title": "Drug",
+        "parentTable": "drug",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/drug/{uuid}?v=full",
+          "fhir": "/Drug/{uuid}"
+        }
+      },
+      "allergy": {
+        "enabled": "true",
+        "title": "Allergy",
+        "parentTable": "allergy",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/allergy/{uuid}?v=full",
+          "fhir": "/AllergyIntolerance/{uuid}"
+        }
+      },
+      "form":{
+        "enabled": "true",
+        "title": "Form",
+        "parentTable": "form",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/form/{uuid}?v=full"
+        }
+      },
+      "order": {
+        "enabled": "true",
+        "title": "Order",
+        "parentTable": "order",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/order/{uuid}?v=full",
+          "fhir": "/Order/{uuid}"
+        }
+      },
+      "drug_order":{
+        "enabled": "true",
+        "title": "DrugOrder",
+        "parentTable": "drug_order",
+        "parentForeignKey": "order_id",
+        "childPrimaryKey":"order_id",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/drug_order/{uuid}?v=full",
+          "fhir": "/MedicationRequest/{uuid}"
+        }
+      },
+      "test_order": {
+        "enabled": "true",
+        "title": "TestOrder",
+        "parentTable": "test_order",
+        "parentForeignKey": "order_id",
+        "childPrimaryKey":"order_id",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/test_order/{uuid}?v=full",
+          "fhir": "/ProcedureRequest/{uuid}"
+        }
+      },
+      "obs": {
+        "enabled": "true",
+        "title": "Observation",
+        "parentTable": "obs",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/obs/{uuid}?v=full",
+          "fhir": "/Observation/{uuid}"
+        }
+      },
+      "encounter":{
+        "enabled": "true",
+        "title": "Encounter",
+        "parentTable": "encounter",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/encounter/{uuid}?v=full",
+          "fhir": "/Encounter/{uuid}"
+        }
+      },
+      "visit":{
+        "enabled": "false",
+        "title": "Visit",
+        "parentTable": "visit",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/visit/{uuid}?v=full",
+          "fhir": "/Encounter/{uuid}"
+        }
+      },
+      "program":{
+        "enabled": "true",
+        "title": "Program",
+        "parentTable": "program",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/program/{uuid}?v=full",
+          "fhir": "/Program/{uuid}"
+        }
+      },
+      "programenrollment":{
+        "enabled": "true",
+        "title": "Program Enrollment",
+        "parentTable": "programenrollment",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/programenrollment/{uuid}?v=full",
+          "fhir": "/Programenrollment/{uuid}"
+        }
+      },
+      "privilege":{
+        "enabled": "true",
+        "title": "Privilege",
+        "parentTable": "privilege",
+        "linkTemplates": {
+          "rest": "/ws/rest/v1/privilege/{uuid}?v=full"
+        }
+      },
+      "audit_message":{
+        "enabled": "true",
+        "title": "AuditMessage",
+        "parentTable": "audit_message",
+        "linkTemplates": {
+          "rest": "/ws/rest/sync2/messages/{uuid}"
+        }
+      }
+    }
+  }

openhim-mediator-data-stream/docker-compose.yml

@@ -0,0 +1,41 @@
+version: '3.9'
+
+services:
+  openhim-mediator-data-stream:
+    image: ghcr.io/i-tech-uw/openmrs-fhir-analytics/streaming-binlog:latest
+    container_name: streaming-pipeline
+    healthcheck:
+      test: "exit 0"
+    #volumes:
+      # data is the directory which you want to persist the generated parquet files
+      # - ./configs/streaming-pipeline/parquet:/tmp
+    environment:
+      - OPENMRS_URL=http://isanteplus:8080/openmrs
+      - OPENMRS_USERNAME=${IPLUS_ADMIN_USER-Admin}
+      - OPENMRS_PASSWORD=${IPLUS_ADMIN_PW-Admin123}
+      - SINK_URL=${OPENHIM_STREAMING_SINK_URL-https://openhim-ssl.sedish-haiti.org/SHR/fhir}
+      - SINK_USERNAME=${OPENHIM_STREAMING_CLIENT-streaming}
+      - SINK_PASSWORD=${OPENHIM_STREAMING_PW-streaming}
+      - JDBC_FETCH_SIZE=10000
+      - JDBC_MAX_POOL_SIZE=50
+      - JDBC_INITIAL_POOL_SIZE=10
+      - JDBC_DRIVER_CLASS=com.mysql.cj.jdbc.Driver
+      # the 2 variable below should be same as volume mappings above
+      - PARQUET_PATH=
+      - FHIR_DEBEZIUM_CONFIG_PATH=/deployments/config.json
+    networks:
+      - default
+      - isanteplus
+      - openhim
+    configs:
+      - source: pipeline-config
+        target: /deployments/config.json
+
+networks:
+  isanteplus:
+    external: true
+    name: isanteplus
+  openhim:
+    name: openhim
+    external: true
+  default:

openhim-mediator-data-stream/package-metadata.json

@@ -0,0 +1,9 @@
+{
+  "id": "openhim-mediator-data-stream",
+  "name": "iSantePlus Data Stream",
+  "description": "Streaming service for data from iSantePlus to a FHIR store.",
+  "type": "use-case",
+  "version": "0.0.1",
+  "dependencies": [],
+  "environmentVariables": {}
+}

openhim-mediator-data-stream/swarm.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare SERVICE_NAMES=()
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  SERVICE_NAMES=(
+    "openhim-mediator-data-stream"
+  )
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+  readonly SERVICE_NAMES
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function initialize_package() {
+  local package_dev_compose_filename=""
+  if [[ "${MODE}" == "dev" ]]; then
+    log info "Running package in DEV mode"
+    package_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  (
+    docker::deploy_service "${COMPOSE_FILE_PATH}" "$package_dev_compose_filename"
+    docker::deploy_sanity "${SERVICE_NAMES[@]}"
+  ) || {
+    log error "Failed to deploy package"
+    exit 1
+  }
+}
+
+function destroy_package() {
+  docker::service_destroy "${SERVICE_NAMES[@]}"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    log info "Running package in Single node mode"
+
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+
+    docker::scale_services_down "${SERVICE_NAMES[@]}"
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"

platform-test-harness/test.docker-compose.yml

@@ -0,0 +1,48 @@
+version: "3.8"
+
+services:  
+  # QA Framework
+  qa:
+    image: ghcr.io/isanteplus/isanteplus-qaframework:latest
+    container_name: qa
+    environment:
+      - ISANTEPLUS_URL=http://isanteplus:8080/openmrs
+      - ISANTEPLUS_USER=admin
+      - ISANTEPLUS_PW=Admin123
+      - REMOTE_URL_CHROME=http://chrome:4444/wd/hub
+      - TEST_OPTIONS="-Dcucumber.filter.tags=@login"
+    networks:
+      - local_facility
+
+  chrome:
+    image: selenium/standalone-chrome:latest
+    hostname: chrome
+    container_name: chrome
+    ports:
+      - "4444:4444"
+      - "7900:7900"
+    networks:
+      - local_facility
+  # Newman Tests
+  newman:
+    image: postman/newman
+    volumes:
+      - ./.postman:/.postman
+    entrypoint: newman run $POSTMAN_COLLECTION -e /.postman/env.json --insecure --timeout-request 20000 --delay-request 1000
+    networks:
+      - local_facility
+
+  wait:
+    container_name: wait
+    image: ghcr.io/i-tech-uw/wait-for-openmrs-action:v0.4.0 
+    command:
+    - "http://isanteplus:8080/openmrs"
+    - "20" 
+    - "10000"
+    networks:
+      - local_facility
+
+networks:
+  local_facility:
+    external: true
+    name: haitihealthbox_local_facility

reverse-proxy-caddy/build/Dockerfile

@@ -0,0 +1,14 @@
+ARG CADDY_VERSION=2.6.4
+FROM caddy:${CADDY_VERSION}-builder AS builder
+
+RUN xcaddy build \
+    --with github.com/lucaslorentz/caddy-docker-proxy/v2 \
+    --with github.com/corazawaf/coraza-caddy/v2
+
+FROM caddy:${CADDY_VERSION}-alpine
+
+COPY --from=builder /usr/bin/caddy /usr/bin/caddy
+
+RUN touch /Caddyfile
+
+CMD ["caddy", "docker-proxy", "--caddyfile-path", "/Caddyfile"]

reverse-proxy-caddy/config/Caddyfile

@@ -0,0 +1,34 @@
+{
+        order coraza_waf first
+}
+
+:80 {
+    handle /openmrs/* {
+        reverse_proxy isanteplus:8080
+    }
+
+    handle_path /openhim/* {
+        reverse_proxy openhim-console:80
+    }
+
+    handle_path /openhim-core-http/* {
+        reverse_proxy openhim-core:5001
+    }
+    
+    handle_path /openhim-core-ssl/* {
+        reverse_proxy openhim-core:5000 {
+            transport http {
+                tls_insecure_skip_verify
+            }
+        }
+        
+    }
+
+    handle_path /openhim-core-api/* {
+        reverse_proxy openhim-core:8080 {
+            transport http {
+                tls_insecure_skip_verify
+            }
+        }
+    }
+}

reverse-proxy-caddy/docker-compose.yml

@@ -0,0 +1,27 @@
+version: '3.9'
+
+services:
+  caddy:
+    container_name: caddy
+    build:
+      context: ./build
+      dockerfile: Dockerfile
+    ports:
+      - 80:80
+      - 443:443
+    environment:
+      - CADDY_INGRESS_NETWORKS=reverse-proxy_public
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - caddy-data:/data
+      - ./ruleset:/ruleset
+      - audit:/audit
+      - ./caddy/Caddyfile:/Caddyfile
+    restart: unless-stopped
+    networks:
+      - public
+
+networks:
+  public:
+    name: reverse-proxy_public
+    external: true

reverse-proxy-caddy/package-metadata.json

@@ -0,0 +1,9 @@
+{
+  "id": "reverse-proxy-caddy",
+  "name": "Caddy Reverse Proxy with Coraza WAF",
+  "description": "A Caddy reverse proxy with docker integration and coraza WAF",
+  "type": "infrastructure",
+  "version": "0.0.1",
+  "dependencies": [],
+  "environmentVariables": {}
+}

reverse-proxy-caddy/swarm.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare SERVICE_NAMES=()
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  SERVICE_NAMES=(
+    "reverse-proxy-caddy"
+  )
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+  readonly SERVICE_NAMES
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function initialize_package() {
+  local package_dev_compose_filename=""
+  if [[ "${MODE}" == "dev" ]]; then
+    log info "Running package in DEV mode"
+    package_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  (
+    docker::deploy_service "${COMPOSE_FILE_PATH}" "$package_dev_compose_filename"
+    docker::deploy_sanity "${SERVICE_NAMES[@]}"
+  ) || {
+    log error "Failed to deploy package"
+    exit 1
+  }
+}
+
+function destroy_package() {
+  docker::service_destroy "${SERVICE_NAMES[@]}"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    log info "Running package in Single node mode"
+
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+
+    docker::scale_services_down "${SERVICE_NAMES[@]}"
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"

utils/config-utils.sh

@@ -0,0 +1,433 @@
+#!/bin/bash
+#
+# Library name: config
+# This is a library that contains functions to assist with docker configs
+#
+# For functions using `declare -n`, note the following explanation https://linuxhint.com/bash_declare_command/#:~:text=giving%20them%20attributes.-,Namerefs,-If%20you%20are
+
+. "$(pwd)/utils/log.sh"
+
+# Sets the digest variables for the conf raft files in the provided docker compose file
+#
+# Requirements:
+# - All configs must have a file and name property
+# - The name property must end in -${DIGEST_VAR_NAME:?err} (eg. name: my-file-${MY_FILE_DIGEST:?err})
+#
+# Arguments:
+# - $1 : docker compose directory path (eg. /home/user/project/docker-compose.yml)
+#
+# Exports:
+# As many digest environment variables as are declared in the provided docker compose file
+#
+config::set_config_digests() {
+    local -r DOCKER_COMPOSE_PATH="${1:?$(missing_param "set_config_digests")}"
+
+    # Get configs files and names from yml file
+    local -r files=($(yq '.configs."*.*".file' "${DOCKER_COMPOSE_PATH}"))
+    local -r names=($(yq '.configs."*.*".name' "${DOCKER_COMPOSE_PATH}"))
+    local -r compose_folder_path="${DOCKER_COMPOSE_PATH%/*}"
+
+    if [[ "${files[*]}" != *"null"* ]] && [[ "${names[*]}" != *"null"* ]]; then
+        log info "Setting config digests"
+
+        for ((i = 0; i < ${#files[@]}; i++)); do
+            file=${files[$i]}
+            name=${names[$i]}
+
+            file_name="${compose_folder_path}${file//\.\///}" # TODO: Throw an error if the file name is too long to allow for a unique enough digest
+            env_var_name=$(echo "${name}" | grep -P -o "{.*:?err}" | sed 's/[{}]//g' | sed 's/:?err//g')
+
+            if [[ -n "$env_var_name" ]]; then
+                # generate and truncate the digest to conform to the 64 character restriction on docker config names
+                env_declaration_characters=":?err" # '${:?err}' from setting an env variable
+                remainder=$((64 - (${#name} - ${#env_var_name} - ${#env_declaration_characters})))
+                export "${env_var_name}"="$(cksum "${file_name}" | awk '{print $1}' | cut -c -${remainder})"
+            fi
+        done
+    elif [[ "${files[*]}" == *"null"* ]]; then
+        log error "No files found to set the digest in:\n $DOCKER_COMPOSE_PATH"
+        exit 1
+    else
+        log error "You should specify names for the files in:\n $DOCKER_COMPOSE_PATH"
+        exit 1
+    fi
+}
+
+# Removes stale docker configs based on the provided docker-compose file
+#
+# Requirements:
+# - All configs must have a file and name property
+# - The name property must end in -${DIGEST_VAR_NAME:?err} (eg. name: my-file-${MY_FILE_DIGEST:?err})
+#
+# Arguments:
+# - $1 : docker compose directory path (eg. /home/user/project/docker-compose.yml)
+# - $2 : config label (eg. logstash)
+#
+config::remove_stale_service_configs() {
+    local -r DOCKER_COMPOSE_PATH="${1:?$(missing_param "remove_stale_service_configs" "DOCKER_COMPOSE_PATH")}"
+    local -r CONFIG_LABEL="${2:?$(missing_param "remove_stale_service_configs" "CONFIG_LABEL")}"
+
+    local -r compose_names=($(yq '.configs."*.*".name' "${DOCKER_COMPOSE_PATH}"))
+    local configs_to_remove=()
+
+    if [[ "${compose_names[*]}" != "null" ]]; then
+        for compose_name in "${compose_names[@]}"; do
+            compose_name_without_env=$(echo "${compose_name}" | sed 's/-\${.*//g')
+
+            compose_name_occurences=$(for word in "${compose_names[@]}"; do echo "${word}"; done | grep -c "${compose_name_without_env}")
+            if [[ $compose_name_occurences -gt "1" ]]; then
+                log warn "Warning: Duplicate config name (${compose_name_without_env}) was found in ${DOCKER_COMPOSE_PATH}"
+            fi
+
+            raft_ids=($(docker config ls -f "label=name=${CONFIG_LABEL}" -f "name=${compose_name_without_env}" --format "{{.ID}}"))
+            # Only keep the most recent of all configs with the same name
+            if [[ ${#raft_ids[@]} -gt 1 ]]; then
+                most_recent_raft_id="${raft_ids[0]}"
+                for ((i = 1; i < ${#raft_ids[@]}; i++)); do
+                    raft_id=${raft_ids[$i]}
+                    most_recent_raft_created_date=$(docker config inspect -f "{{.CreatedAt}}" "${most_recent_raft_id}")
+                    raft_created_date=$(docker config inspect -f "{{.CreatedAt}}" "${raft_id}")
+                    if [[ $raft_created_date > $most_recent_raft_created_date ]]; then
+                        configs_to_remove+=("${most_recent_raft_id}")
+                        most_recent_raft_id="${raft_id}"
+                    else
+                        configs_to_remove+=("${raft_id}")
+                    fi
+                done
+            fi
+        done
+    else
+        log warn "No name files found in the compose config to be removed"
+    fi
+
+    if [[ "${#configs_to_remove[@]}" -gt 0 ]]; then
+        try \
+            "docker config rm ${configs_to_remove[*]}" \
+            catch \
+            "Failed to remove configs: ${configs_to_remove[*]}"
+    fi
+}
+
+# A function that exists in a loop to see how long that loop has run for, providing a warning
+# at the time specified in argument $3, and exits with code 124 after the time specified in argument $4.
+#
+# Arguments:
+# - $1 : start time of the timeout check
+# - $2 : a message containing reference to the loop that timed out
+# - $3 : timeout time in seconds, default is 300 seconds
+# - $4 : elapsed time to issue running-for-longer-than-expected warning (in seconds), default is 60 seconds
+#
+config::timeout_check() {
+    local start_time=$(($1))
+    local message=$2
+    local exit_time="${3:-300}"
+    local warning_time="${4:-60}"
+
+    local timeDiff=$(($(date +%s) - $start_time))
+    if [[ $timeDiff -ge $warning_time ]] && [[ $timeDiff -lt $(($warning_time + 1)) ]]; then
+        log warn "Warning: Waited $warning_time seconds for $message. This is taking longer than it should..."
+    elif [[ $timeDiff -ge $exit_time ]]; then
+        log error "Fatal: Waited $exit_time seconds for $message. Exiting..."
+        exit 124
+    fi
+}
+
+# A generic function confirming whether or not a containerized api is reachable
+#
+# Requirements:
+# - The function attempts to start up a helper container using the jembi/await-helper image. It is therefore necessary
+#   to specify the docker-compose file to deploy the await-helper container which the await_service_running function
+#   relies on. Details on configuring the await-helper can be found at https://github.com/jembi/platform-await-helper.
+#
+# Arguments:
+# - $1 : the service being awaited
+# - $2 : path to await-helper compose.yml file (eg. ~/projects/platform/dashboard-visualiser-jsreport/docker-compose.await-helper.yml)
+# - $3 : desired number of instances of the awaited-service
+# - $4 : stack name that the service falls under (eg. openhim)
+# - $5 : (optional) the max time allowed to wait for a service's response, defaults to 300 seconds
+# - $6 : (optional) elapsed time to throw a warning, defaults to 60 seconds
+#
+config::await_service_running() {
+    local -r SERVICE_NAME="${1:?$(missing_param "await_service_running" "SERVICE_NAME")}"
+    local -r AWAIT_HELPER_FILE_PATH="${2:?$(missing_param "await_service_running" "AWAIT_HELPER_FILE_PATH")}"
+    local -r SERVICE_INSTANCES="${3:?$(missing_param "await_service_running" "SERVICE_INSTANCES")}"
+    local -r STACK_NAME="${4:?$(missing_param "await_service_running" "STACK_NAME")}"
+    local -r exit_time="${5:-}"
+    local -r warning_time="${6:-}"
+    local start_time
+    start_time=$(date +%s)
+
+    docker service rm "$STACK_NAME"_await-helper &>/dev/null
+
+    try "docker stack deploy -c $AWAIT_HELPER_FILE_PATH $STACK_NAME" throw "Failed to deploy await helper"
+    until [[ $(docker service ls -f name="$STACK_NAME"_"$SERVICE_NAME" --format "{{.Replicas}}") == *"$SERVICE_INSTANCES/$SERVICE_INSTANCES"* ]]; do
+        config::timeout_check "$start_time" "$SERVICE_NAME to start" "$exit_time" "$warning_time"
+        sleep 1
+    done
+
+    start_time=$(date +%s) # Reintialize for the second loop
+    local await_helper_state
+    await_helper_state=$(docker service ps "$STACK_NAME"_await-helper --format "{{.CurrentState}}")
+    until [[ $await_helper_state == *"Complete"* ]]; do
+        config::timeout_check "$start_time" "$SERVICE_NAME status check" "$exit_time" "$warning_time"
+        sleep 1
+
+        await_helper_state=$(docker service ps "$STACK_NAME"_await-helper --format "{{.CurrentState}}")
+        if [[ $await_helper_state == *"Failed"* ]] || [[ $await_helper_state == *"Rejected"* ]]; then
+            log error "Fatal: Received error when trying to verify state of $SERVICE_NAME. Error:
+       $(docker service ps "$STACK_NAME"_await-helper --no-trunc --format '{{.Error}}')"
+            exit 1
+        fi
+    done
+
+    try "docker service rm "$STACK_NAME"_await-helper" catch "Failed to remove await-helper"
+}
+
+# A function which removes a config importing service on successful completion, and exits with an error otherwise
+#
+# Arguments:
+# - $1 : stack name that the service falls under (eg. openhim)
+# - $2 : the name of the config importer
+# - $3 : (optional) the timeout time for the config importer to run, defaults to 300 seconds
+# - $4 : (optional) elapsed time to throw a warning, defaults to 60 seconds
+#
+config::remove_config_importer() {
+    local -r STACK_NAME="${1:?$(missing_param "remove_config_importer" "STACK_NAME")}"
+    local -r CONFIG_IMPORTER_SERVICE_NAME="${2:?$(missing_param "remove_config_importer" "CONFIG_IMPORTER_SERVICE_NAME")}"
+    local -r exit_time="${3:-}"
+    local -r warning_time="${4:-}"
+    local -r start_time=$(date +%s)
+
+    local config_importer_state
+
+    if [[ -z $(docker service ps "$STACK_NAME"_"$CONFIG_IMPORTER_SERVICE_NAME") ]]; then
+        log info "${STACK_NAME}_$CONFIG_IMPORTER_SERVICE_NAME service cannot be removed as it does not exist!"
+        exit 0
+    fi
+
+    config_importer_state=$(docker service ps "$STACK_NAME"_"$CONFIG_IMPORTER_SERVICE_NAME" --format "{{.CurrentState}}")
+    until [[ $config_importer_state == *"Complete"* ]]; do
+        config::timeout_check "$start_time" "$CONFIG_IMPORTER_SERVICE_NAME to run" "$exit_time" "$warning_time"
+        sleep 1
+
+        config_importer_state=$(docker service ps "$STACK_NAME"_"$CONFIG_IMPORTER_SERVICE_NAME" --format "{{.CurrentState}}")
+        if [[ $config_importer_state == *"Failed"* ]] || [[ $config_importer_state == *"Rejected"* ]]; then
+            log error "Fatal: $CONFIG_IMPORTER_SERVICE_NAME failed with error:
+       $(docker service ps ${STACK_NAME}_"$CONFIG_IMPORTER_SERVICE_NAME" --no-trunc --format '{{.Error}}')"
+            exit 1
+        fi
+    done
+
+    try "docker service rm "$STACK_NAME"_$CONFIG_IMPORTER_SERVICE_NAME" catch "Failed to remove config importer"
+}
+
+# Waits for the provided service to be removed
+#
+# Arguments:
+# - $1 : stack name that the service falls under (eg. openhim)
+# - $2 : service name (eg. analytics-datastore-elastic-search)
+#
+config::await_service_removed() {
+    local -r STACK_NAME="${1:?$(missing_param "await_service_removed", "STACK_NAME")}"
+    local -r SERVICE_NAME="${2:?$(missing_param "await_service_removed", "SERVICE_NAME")}"
+    local start_time=$(date +%s)
+
+    until [[ -z $(docker stack ps $STACK_NAME -qf name="${STACK_NAME}_${SERVICE_NAME}" 2>/dev/null) ]]; do
+        config::timeout_check "$start_time" "${SERVICE_NAME} to be removed"
+        sleep 1
+    done
+    log info "Service $SERVICE_NAME successfully removed"
+}
+
+# Generates configs for a service from a folder and adds them to a temp docker-compose file
+#
+# Arguments:
+# - $1 : service name (eg. data-mapper-logstash)
+# - $2 : target base (eg. /usr/share/logstash/)
+# - $3 : target folder path in absolute format (eg. "$PATH_TO_FILE"/pipeline)
+# - $4 : compose file path (eg. "$PATH_TO_FILE")
+#
+# Exports:
+# All exports are required for yq to process the values and are not intended for external use
+# - service_config_query
+# - config_target
+# - config_source
+# - config_query
+# - config_file
+# - config_label_name
+# - config_service_name
+#
+config::generate_service_configs() {
+    local -r SERVICE_NAME=${1:?$(missing_param "generate_service_configs" "SERVICE_NAME")}
+    local -r TARGET_BASE=${2:?$(missing_param "generate_service_configs" "TARGET_BASE")}
+    local -r TARGET_FOLDER_PATH=${3:?$(missing_param "generate_service_configs" "TARGET_FOLDER_PATH")}
+    local -r COMPOSE_PATH=${4:?$(missing_param "generate_service_configs" "COMPOSE_PATH")}
+    local -r LABEL_NAME=${5:?$(missing_param "generate_service_configs" "LABEL_NAME")}
+    local -r TARGET_FOLDER_NAME=$(basename "${TARGET_FOLDER_PATH}")
+    local count=0
+
+    try \
+        "touch ${COMPOSE_PATH}/docker-compose.tmp.yml" \
+        throw \
+        "Failed to create temp service config compose file"
+
+    find "${TARGET_FOLDER_PATH}" -maxdepth 10 -mindepth 1 -type f | while read -r file; do
+        file_name=${file/"${TARGET_FOLDER_PATH%/}"/}
+        file_name=${file_name:1}
+        file_hash=$(cksum "${file}" | awk '{print $1}')
+
+        # for these variables to be visible by yq they need to be exported
+        export service_config_query=".services.${SERVICE_NAME}.configs[${count}]"
+        export config_target="${TARGET_BASE%/}/${TARGET_FOLDER_NAME}/${file_name}"
+        export config_source="${SERVICE_NAME}-${file_hash}"
+
+        export config_query=".configs.${config_source}"
+        export config_file="./${TARGET_FOLDER_NAME}/${file_name}"
+        export config_label_name=$LABEL_NAME
+        export config_service_name=$SERVICE_NAME
+
+        yq -i '
+        .version = "3.9" |
+        eval(strenv(service_config_query)).target = env(config_target) |
+        eval(strenv(service_config_query)).source = strenv(config_source) |
+        eval(strenv(config_query)).file = strenv(config_file) |
+        eval(strenv(config_query)).name = strenv(config_source) |
+        eval(strenv(config_query)).labels.name = strenv(config_label_name) |
+        eval(strenv(config_query)).labels.service = strenv(config_service_name)
+        ' "${COMPOSE_PATH}/docker-compose.tmp.yml"
+
+        count=$((count + 1))
+    done
+}
+
+# Replaces all environment variables in a file with the environment variable value
+#
+# Arguments:
+# - $1 : the path to the file that you wish to substitute env vars into (eg. "${COMPOSE_FILE_PATH}"/config.ini)
+#
+config::substitute_env_vars() {
+    local -r FILE_PATH="${1:?$(missing_param "substitute_env_vars")}"
+    config_with_env=$(envsubst <"${FILE_PATH}")
+    echo "" >"${FILE_PATH}"
+    echo "$config_with_env" >>"${FILE_PATH}"
+}
+
+# Modify a variable to contain the necessary `--config-rm` and `--config-add` arguments to update a service's
+# configs based off newly created docker configs for a provided folder. The modified variable must then be
+# used in a `docker service update` command, like follows:
+# ```
+#   service_update_args=""
+#   config::update_service_configs service_update_args /usr/share/logstash/ "$PATH_TO_FILE"/pipeline cares
+#   docker service update $service_update_args instant_data-mapper-logstash
+# ```
+# Reference arguments:
+# - $1 : config update variable name (eg. service_update_args)
+#
+# Arguments:
+# - $2 : target base (eg. /usr/share/logstash/)
+# - $3 : target folder path in absolute format (eg. "$PATH_TO_FILE"/pipeline)
+# - $4 : config label name (eg. cares)
+#
+config::update_service_configs() {
+    declare -n REF_config_update_var="${1:?$(missing_param "update_service_configs" "REF_config_update_var")}"
+    local -r TARGET_BASE=${2:?$(missing_param "update_service_configs" "TARGET_BASE")}
+    local -r TARGET_FOLDER_PATH=${3:?$(missing_param "update_service_configs" "TARGET_FOLDER_PATH")}
+    local -r CONFIG_LABEL_NAME="${4:?$(missing_param "update_service_configs" "CONFIG_LABEL_NAME")}"
+    local config_rm_string=""
+    local config_add_string=""
+
+    files=$(find "${TARGET_FOLDER_PATH}" -maxdepth 10 -mindepth 1 -type f)
+
+    for file in $files; do
+        file_name=${file/"${TARGET_FOLDER_PATH%/}"/}
+        file_name=${file_name:1}
+        file_hash=$(md5sum "${file}" | awk '{print $1}')
+        config_file="${TARGET_FOLDER_PATH}/${file_name}"
+        config_target="${TARGET_BASE%/}/${file_name}"
+        config_name=$(basename "$file_name")-$file_hash
+        old_config_name=$(docker config inspect --format="{{.Spec.Name}}" "$(docker config ls -qf name="$(basename "$file_name")")" 2>/dev/null)
+
+        if [[ "$config_name" != "$old_config_name" ]]; then
+            if [[ -n $old_config_name ]]; then
+                config_rm_string+="--config-rm $old_config_name "
+            fi
+            config_add_string+="--config-add source=$config_name,target=$config_target "
+
+            try \
+                "docker config create --label name=$CONFIG_LABEL_NAME $config_name $config_file" \
+                catch \
+                "Failed to create config"
+        fi
+    done
+
+    REF_config_update_var+="$config_rm_string $config_add_string"
+}
+
+# Modify a variable to contain the necessary `--env-add` arguments to update a service's
+# environment specified in a .env file. The modified variable must then be
+# used in a `docker service update` command, like follows:
+# ```
+#   service_update_args=""
+#   config::env_var_add_from_file service_update_args "$PATH_TO_FILE"/.env.add
+#   docker service update $service_update_args instant_data-mapper-logstash
+# ```
+# Reference arguments:
+# - $1 : service update variable name (eg. service_update_args)
+#
+# Arguments:
+# - $2 : .env file (eg. "$PATH_TO_FILE"/.env.add)
+#
+config::env_var_add_from_file() {
+    declare -n REF_service_update_var="${1:?$(missing_param "env_var_add_from_file" "REF_service_update_var")}"
+    local -r ENV_FILE=${2:?$(missing_param "env_var_add_from_file" "ENV_FILE")}
+
+    if [[ ! -f $ENV_FILE ]]; then
+        log error "$ENV_FILE: No such file or directory. Exiting..."
+        return 1
+    fi
+
+    readarray -t env_vars <"$ENV_FILE"
+    for env_var in "${env_vars[@]}"; do
+        REF_service_update_var+=" --env-add $env_var"
+    done
+}
+
+# Modify a variable to contain the necessary `--env-add` arguments to update a service's
+# environment based on the provided env var. The modified variable must then be
+# used in a `docker service update` command, like follows:
+# ```
+#   service_update_args=""
+#   config::env_var_add service_update_args MY_ENV_VAR=my_value
+#   docker service update $service_update_args instant_data-mapper-logstash
+# ```
+# Reference arguments:
+# - $1 : service update variable name (eg. service_update_args)
+#
+# Arguments:
+# - $2 : env var (eg. MY_ENV_VAR=my_value)
+#
+config::env_var_add() {
+    declare -n REF_service_update_var="${1:?$(missing_param "env_var_add" "REF_service_update_var")}"
+    local -r ENV_VAR=${2:?$(missing_param "env_var_add" "ENV_VAR")}
+
+    REF_service_update_var+=" --env-add $ENV_VAR"
+}
+
+# Waits for the provided service to be reachable by checking logs
+#
+# Arguments:
+# $1 : service name (eg. analytics-datastore-elastic-search)
+# $2 : stack name that the service falls under (eg. openhim)
+# $3 : log string to be checked (eg. Starting)
+#
+config::await_service_reachable() {
+    local -r SERVICE_NAME=${1:?$(missing_param "await_service_reachable" "SERVICE_NAME")}
+    local -r STACK_NAME=${2:?$(missing_param "await_service_reachable" "STACK_NAME")}
+    local -r LOG_MESSAGE=${3:?$(missing_param "await_service_reachable" "LOG_MESSAGE")}
+    local -r start_time=$(date +%s)
+
+    until [[ $(docker service logs --tail all "${STACK_NAME}"_"${SERVICE_NAME}" 2>/dev/null | grep -c "${LOG_MESSAGE}") -gt 0 ]]; do
+        config::timeout_check "$start_time" "${STACK_NAME}_$SERVICE_NAME to be reachable"
+        sleep 1
+    done
+}

utils/docker-utils.sh

@@ -0,0 +1,574 @@
+#!/bin/bash
+#
+# Library name: docker
+# This is a library that contains functions to assist with docker actions
+
+. "$(pwd)/utils/config-utils.sh"
+. "$(pwd)/utils/log.sh"
+
+# Gets current status of the provided service
+#
+# Arguments:
+# - $1 : service name (eg. analytics-datastore-elastic-search)
+#
+docker::get_current_service_status() {
+    local -r SERVICE_NAME=${1:?$(missing_param "get_current_service_status")}
+    docker service ps "${SERVICE_NAME}" --format "{{.CurrentState}}" 2>/dev/null
+}
+
+# Gets unique errors from the provided service
+#
+# Arguments:
+# - $1 : service name (eg. analytics-datastore-elastic-search)
+#
+docker::get_service_unique_errors() {
+    local -r SERVICE_NAME=${1:?$(missing_param "get_service_unique_errors")}
+
+    # Get unique error messages using sort -u
+    docker service ps "${SERVICE_NAME}" --no-trunc --format '{{ .Error }}' 2>&1 | sort -u
+}
+
+# Waits for a container to be up
+#
+# Arguments:
+# - $1 : stack name that the service falls under (eg. elastic)
+# - $2 : service name (eg. analytics-datastore-elastic-search)
+#
+docker::await_container_startup() {
+    local -r STACK_NAME=${1:?$(missing_param "await_container_startup", "STACK_NAME")}
+    local -r SERVICE_NAME=${2:?$(missing_param "await_container_startup", "SERVICE_NAME")}
+
+    log info "Waiting for ${SERVICE_NAME} to start up..."
+    local start_time
+    start_time=$(date +%s)
+    until [[ -n $(docker service ls -qf name="${STACK_NAME}"_"${SERVICE_NAME}") ]]; do
+        config::timeout_check "${start_time}" "${SERVICE_NAME} to start"
+        sleep 1
+    done
+    overwrite "Waiting for ${SERVICE_NAME} to start up... Done"
+}
+
+# Waits for a container to be up
+#
+# Arguments:
+# - $1 : stack name that the service falls under (eg. elastic)
+# - $2 : service name (eg. analytics-datastore-elastic-search)
+# - $3 : service status (eg. running)
+#
+docker::await_service_status() {
+    local -r STACK_NAME=${1:?$(missing_param "await_service_status" "STACK_NAME")}
+    local -r SERVICE_NAME=${2:?$(missing_param "await_service_status" "SERVICE_NAME")}
+    local -r SERVICE_STATUS=${3:?$(missing_param "await_service_status" "SERVICE_STATUS")}
+    local -r start_time=$(date +%s)
+    local error_message=()
+
+    log info "Waiting for ${STACK_NAME}_${SERVICE_NAME} to be ${SERVICE_STATUS}..."
+    until [[ $(docker::get_current_service_status ${STACK_NAME}_${SERVICE_NAME}) == *"${SERVICE_STATUS}"* ]]; do
+        config::timeout_check "${start_time}" "${STACK_NAME}_${SERVICE_NAME} to start"
+        sleep 1
+
+        # Get unique error messages using sort -u
+        new_error_message=($(docker::get_service_unique_errors ${STACK_NAME}_$SERVICE_NAME))
+        if [[ -n ${new_error_message[*]} ]]; then
+            # To prevent logging the same error
+            if [[ "${error_message[*]}" != "${new_error_message[*]}" ]]; then
+                error_message=(${new_error_message[*]})
+                log error "Deploy error in service ${STACK_NAME}_$SERVICE_NAME: ${error_message[*]}"
+            fi
+
+            # To exit in case the error is not having the image
+            if [[ "${new_error_message[*]}" == *"No such image"* ]]; then
+                log error "Do you have access to pull the image?"
+                exit 124
+            fi
+        fi
+    done
+    overwrite "Waiting for ${STACK_NAME}_${SERVICE_NAME} to be ${SERVICE_STATUS}... Done"
+}
+
+# Waits for a container to be destroyed
+#
+# Arguments:
+# - $1 : stack name that the service container falls under (eg. elastic)
+# - $2 : service name (eg. analytics-datastore-elastic-search)
+#
+docker::await_container_destroy() {
+    local -r STACK_NAME=${1:?$(missing_param "await_container_destroy", "STACK_NAME")}
+    local -r SERVICE_NAME=${2:?$(missing_param "await_container_destroy", "SERVICE_NAME")}
+
+    log info "Waiting for ${STACK_NAME}_${SERVICE_NAME} to be destroyed..."
+    local start_time
+    start_time=$(date +%s)
+    until [[ -z $(docker ps -qlf name="${STACK_NAME}_${SERVICE_NAME}") ]]; do
+        config::timeout_check "${start_time}" "${SERVICE_NAME} to be destroyed"
+        sleep 1
+    done
+    overwrite "Waiting for ${STACK_NAME}_${SERVICE_NAME} to be destroyed... Done"
+}
+
+# Waits for a service to be destroyed
+#
+# Arguments:
+# - $1 : service name (eg. analytics-datastore-elastic-search)
+# - $2 : stack name that the service falls under (eg. elastic)
+#
+docker::await_service_destroy() {
+    local -r SERVICE_NAME=${1:?$(missing_param "await_service_destroy", "SERVICE_NAME")}
+    local -r STACK_NAME=${2:?$(missing_param "await_service_destroy", "STACK_NAME")}
+    local start_time
+    start_time=$(date +%s)
+
+    while docker service ls | grep -q "\s${STACK_NAME}_${SERVICE_NAME}\s"; do
+        config::timeout_check "${start_time}" "${SERVICE_NAME} to be destroyed"
+        sleep 1
+    done
+}
+
+# Removes services containers then the service itself
+# This was created to aid in removing volumes,
+# since volumes being removed were still attached to some lingering containers after container remove
+#
+# NB: Global services can't be scale down
+#
+# Arguments:
+# - $1 : stack name that the services fall under (eg. elasticsearch)
+# - $@ : service names list (eg. analytics-datastore-elastic-search)
+#
+docker::service_destroy() {
+    local -r STACK_NAME=${1:?$(missing_param "service_destroy", "STACK_NAME")}
+    shift
+
+    if [[ -z "$*" ]]; then
+        log error "$(missing_param "service_destroy", "[SERVICE_NAMES]")"
+        exit 1
+    fi
+
+    for service_name in "$@"; do
+        local service="${STACK_NAME}"_$service_name
+        log info "Waiting for service $service to be removed ... "
+        if [[ -n $(docker service ls -qf name=$service) ]]; then
+            if [[ $(docker service ls --format "{{.Mode}}" -f name=$service) != "global" ]]; then
+                try "docker service scale $service=0" catch "Failed to scale down ${service_name}"
+            fi
+            try "docker service rm $service" catch "Failed to remove service ${service_name}"
+            docker::await_service_destroy "$service_name" "$STACK_NAME"
+        fi
+        overwrite "Waiting for service $service_name to be removed ... Done"
+    done
+}
+
+# Removes the stack and awaits for each service in the stack to be removed
+#
+# Arguments:
+# - $1 : stack name to be removed
+#
+docker::stack_destroy() {
+    local -r STACK_NAME=${1:?$(missing_param "stack_destroy")}
+    log info "Waiting for stack $STACK_NAME to be removed ..."
+    try "docker stack rm \
+        $STACK_NAME" \
+        throw \
+        "Failed to remove $STACK_NAME"
+
+    local start_time=$(date +%s)
+    while [[ -n "$(docker stack ps $STACK_NAME 2>/dev/null)" ]] ; do
+        config::timeout_check "${start_time}" "${STACK_NAME} to be destroyed"
+        sleep 1
+    done
+
+    overwrite "Waiting for stack $STACK_NAME to be removed ... Done"
+
+    log info "Pruning networks ... "
+    try "docker network prune -f" catch "Failed to prune networks"
+    overwrite "Pruning networks ... done"
+
+    docker::prune_volumes
+}
+
+# Loops through all current services and builds up a dictionary of volume names currently in use
+# (this also considers downed services, as you don't want to prune volumes for downed services)
+# It then loops through all volumes and removes any that do not have a service definition attached to it
+#
+docker::prune_volumes() {
+    # Create an associative array to act as the dictionary to hold service volume names
+    # Need to add instant, which the gocli uses but is not defined as a service
+    declare -A referenced_volumes=(['instant']=true)
+
+    log info "Pruning volumes ... "
+
+    for service in $(docker service ls -q); do
+        for volume in $(docker service inspect $service --format '{{range .Spec.TaskTemplate.ContainerSpec.Mounts}}{{println .Source}}{{end}}'); do
+            referenced_volumes[$volume]=true
+        done
+    done
+
+    for volume in $(docker volume ls --format {{.Name}}); do
+        # Check to see if the key (which is the volume name) exists
+        if [[ -v referenced_volumes[$volume] ]]; then
+            continue
+        fi
+
+        # Ignore volumes attached to a container but are not apart of a service definition
+        local start_time=$(date +%s)
+        local should_ignore=true
+        if [[ -n $(docker ps -a -q --filter volume=$volume) ]]; then
+            local timeDiff=$(($(date +%s) - $start_time))
+            until [[ $timeDiff -ge 10 ]]; do
+                timeDiff=$(($(date +%s) - $start_time))
+                if [[ -n $(docker ps -a -q --filter volume=$volume) ]]; then 
+                    sleep 1
+                else
+                    should_ignore=false
+                fi
+            done
+            if $should_ignore; then
+                continue
+            fi
+        fi
+
+        log info "Waiting for volume $volume to be removed..."
+        start_time=$(date +%s)
+        until [[ -z "$(docker volume ls -q --filter name=^$volume$ 2>/dev/null)" ]]; do
+            docker volume rm $volume >/dev/null 2>&1
+            config::timeout_check "${start_time}" "$volume to be removed" "60" "10"
+            sleep 1
+        done
+        overwrite "Waiting for volume $volume to be removed... Done"
+    done
+
+    overwrite "Pruning volumes ... done"
+}
+
+# Prunes configs based on a label
+#
+# Arguments:
+# - $@ : configs label list (eg. logstash)
+#
+docker::prune_configs() {
+    if [[ -z "$*" ]]; then
+        log error "$(missing_param "prune_configs", "[CONFIG_LABELS]")"
+        exit 1
+    fi
+
+    for config_name in "$@"; do
+        # shellcheck disable=SC2046
+        if [[ -n $(docker config ls -qf label=name="$config_name") ]]; then
+            log info "Waiting for configs to be removed..."
+
+            docker config rm $(docker config ls -qf label=name="$config_name") &>/dev/null
+
+            overwrite "Waiting for configs to be removed... Done"
+        fi
+    done
+}
+
+# Checks if the image exists, if not it will pull it from docker
+#
+# Arguments:
+# - $@ : images list (eg. bitnami/kafka:3.3.1)
+#
+docker::check_images_existence() {
+    if [[ -z "$*" ]]; then
+        log error "$(missing_param "check_images_existence", "[IMAGES]")"
+        exit 1
+    fi
+
+    local timeout_pull_image
+    timeout_pull_image=300
+    for image_name in "$@"; do
+        image_name=$(eval echo "$image_name")
+        if [[ -z $(docker image inspect "$image_name" --format "{{.Id}}" 2>/dev/null) ]]; then
+            log info "The image $image_name is not found, Pulling from docker..."
+            try \
+                "timeout $timeout_pull_image docker pull $image_name 1>/dev/null" \
+                throw \
+                "An error occured while pulling the image $image_name"
+
+            overwrite "The image $image_name is not found, Pulling from docker... Done"
+        fi
+    done
+}
+
+# Deploys a service
+# It will pull images if they don't exist in the local docker hub registry
+# It will set config digests (in case a config is defined in the compose file)
+# It will remove stale configs
+#
+# Arguments:
+# - $1 : docker stack name to group the service under
+# - $2 : docker compose path (eg. /instant/monitoring)
+# - $3 : docker compose file (eg. docker-compose.yml or docker-compose.cluster.yml)
+# - $@ : (optional) list of docker compose files (eg. docker-compose.cluster.yml docker-compose.dev.yml)
+# - $@:4:n : (optional) a marker 'defer-sanity' used to defer deploy::sanity to the caller, can appear anywhere in the optional list 
+#
+docker::deploy_service() {
+    local -r STACK_NAME="${1:?$(missing_param "deploy_service" "STACK_NAME")}"
+    local -r DOCKER_COMPOSE_PATH="${2:?$(missing_param "deploy_service" "DOCKER_COMPOSE_PATH")}"
+    local -r DOCKER_COMPOSE_FILE="${3:?$(missing_param "deploy_service" "DOCKER_COMPOSE_FILE")}"
+    local docker_compose_param=""
+
+    # Check for the existance of the images
+    local -r images=($(yq '.services."*".image' "${DOCKER_COMPOSE_PATH}/$DOCKER_COMPOSE_FILE"))
+    if [[ "${images[*]}" != "null" ]]; then
+        docker::check_images_existence "${images[@]}"
+    fi
+
+    local defer_sanity=false
+    for optional_config in "${@:4}"; do
+        if [[ -n $optional_config ]]; then
+            if [[ $optional_config == "defer-sanity" ]]; then
+                defer_sanity=true
+            else
+                docker_compose_param="$docker_compose_param -c ${DOCKER_COMPOSE_PATH}/$optional_config"
+            fi
+        fi
+    done
+
+    docker::prepare_config_digests "$DOCKER_COMPOSE_PATH/$DOCKER_COMPOSE_FILE" ${docker_compose_param//-c /}
+    docker::ensure_external_networks_existence "$DOCKER_COMPOSE_PATH/$DOCKER_COMPOSE_FILE" ${docker_compose_param//-c /}
+
+    try "docker stack deploy \
+        -c ${DOCKER_COMPOSE_PATH}/$DOCKER_COMPOSE_FILE \
+        $docker_compose_param \
+        --with-registry-auth \
+        ${STACK_NAME}" \
+        throw \
+        "Wrong configuration in ${DOCKER_COMPOSE_PATH}/$DOCKER_COMPOSE_FILE or in the other supplied compose files"
+
+    docker::cleanup_stale_configs "$DOCKER_COMPOSE_PATH/$DOCKER_COMPOSE_FILE" ${docker_compose_param//-c /}
+
+    if [[ $defer_sanity != true ]]; then
+        docker::deploy_sanity "$STACK_NAME" "$DOCKER_COMPOSE_PATH/$DOCKER_COMPOSE_FILE" ${docker_compose_param//-c /}
+    fi
+}
+
+# Deploys a config importer
+# Sets the config digests, deploys the config importer, removes it and removes the stale configs
+#
+# Arguments:
+# - $1 : stack name that the service falls under
+# - $2 : docker compose path (eg. /instant/monitoring/importer/docker-compose.config.yml)
+# - $3 : services name (eg. clickhouse-config-importer)
+# - $4 : config label (eg. clickhouse kibana)
+#
+docker::deploy_config_importer() {
+    local -r STACK_NAME="${1:?$(missing_param "deploy_config_importer" "STACK_NAME")}"
+    local -r CONFIG_COMPOSE_PATH="${2:?$(missing_param "deploy_config_importer" "CONFIG_COMPOSE_PATH")}"
+    local -r SERVICE_NAME="${3:?$(missing_param "deploy_config_importer" "SERVICE_NAME")}"
+    local -r CONFIG_LABEL="${4:?$(missing_param "deploy_config_importer" "CONFIG_LABEL")}"
+
+    log info "Waiting for config importer $SERVICE_NAME to start ..."
+    (
+        if [[ ! -f "$CONFIG_COMPOSE_PATH" ]]; then
+            log error "No such file: $CONFIG_COMPOSE_PATH"
+            exit 1
+        fi
+
+        config::set_config_digests "$CONFIG_COMPOSE_PATH"
+
+        try \
+            "docker stack deploy -c ${CONFIG_COMPOSE_PATH} ${STACK_NAME}" \
+            throw \
+            "Wrong configuration in $CONFIG_COMPOSE_PATH"
+
+        log info "Waiting to give core config importer time to run before cleaning up service"
+
+        config::remove_config_importer "$STACK_NAME" "$SERVICE_NAME"
+        config::await_service_removed "$STACK_NAME" "$SERVICE_NAME"
+
+        log info "Removing stale configs..."
+        config::remove_stale_service_configs "$CONFIG_COMPOSE_PATH" "$CONFIG_LABEL"
+        overwrite "Removing stale configs... Done"
+    ) || {
+        log error "Failed to deploy the config importer: $SERVICE_NAME"
+        exit 1
+    }
+}
+
+# Checks for errors when deploying
+#
+# Arguments:
+# - $1 : stack name that the services falls under
+# - $@ : fully qualified path to the compose file(s) with service definitions (eg. /instant/interoperability-layer-openhim/docker-compose.yml)
+#
+docker::deploy_sanity() {
+    local -r STACK_NAME="${1:?$(missing_param "deploy_sanity" "STACK_NAME")}"
+    # shift off the stack name to get the subset of services to check  
+    shift
+
+    if [[ -z "$*" ]]; then
+        log error "$(missing_param "deploy_sanity" "[COMPOSE_FILES]")"
+        exit 1
+    fi
+
+    local services=()
+    for compose_file in "$@"; do
+    # yq 'keys' returns:"- foo - bar" if you have yml with a foo: and bar: service definition
+    # which is why we remove the "- " before looping
+    # it will also return '#' as a key if you have a comment, so we clean them with ' ... comments="" ' first
+        local compose_services=$(yq '... comments="" | .services | keys' $compose_file)
+        compose_services=${compose_services//- /}
+        for service in ${compose_services[@]}; do
+            # only append unique service to services
+            if [[ ! ${services[*]} =~ $service ]]; then
+                services+=($service)
+            fi
+        done
+    done
+
+    for service_name in ${services[@]}; do
+        docker::await_service_status $STACK_NAME "$service_name" "Running"
+    done
+}
+
+# Scales services to the passed in replica count
+#
+# Arguments:
+# - $1 : stack name that the services falls under
+# - $2 : replicas number (eg. 0 (to scale down) or 1 (to scale up) or 2 (to scale up more))
+#
+docker::scale_services() {
+    local -r STACK_NAME="${1:?$(missing_param "scale_services" "STACK_NAME")}"
+    local -r REPLICAS="${2:?$(missing_param "scale_services" "REPLICAS")}"
+    local services=($(docker stack services $STACK_NAME | awk '{print $2}' | tail -n +2))
+    for service_name in "${services[@]}"; do
+        log info "Waiting for $service_name to scale to $REPLICAS ..."
+        try \
+            "docker service scale $service_name=$REPLICAS" \
+            catch \
+            "Failed to scale $service_name to $REPLICAS"
+        overwrite "Waiting for $service_name to scale to $REPLICAS ... Done"
+    done
+}
+
+# Checks if the external networks exist and tries to create them if they do not
+#
+# Arguments:
+# - $@ : fully qualified path to the docker compose file(s) with the possible network definitions (eg. /instant/interoperability-layer-openhim/docker-compose.yml)
+#
+docker::ensure_external_networks_existence() {
+    if [[ -z "$*" ]]; then
+        log error "$(missing_param "ensure_external_networks_existence", "[COMPOSE_FILES]")"
+        exit 1
+    fi
+
+    for compose_file in "$@"; do
+        if [[ $(yq '.networks' $compose_file) == "null" ]]; then
+            continue
+        fi
+        
+        local network_keys=$(yq '... comments="" | .networks | keys' $compose_file)
+        local networks=(${network_keys//- /})
+        if [[ "${networks[*]}" != "null" ]]; then
+            for network_name in "${networks[@]}"; do
+                # check if the property external is both present and set to true for the current network
+                # then pull the necessary properties to create the network
+                if [[ $(name=$network_name yq '.networks.[env(name)] | select(has("external")) | .external' $compose_file) == true ]]; then
+                    local name=$(name=$network_name yq '.networks.[env(name)] | .name' $compose_file)
+                    if [[ $name == "null" ]]; then
+                        name=$network_name
+                    fi
+                    
+                    # network with the name already exists so no need to create it
+                    if docker network ls | awk '{print $2}' | grep -q -w "$name"; then
+                        continue
+                    fi
+
+                    local driver=$(name=$network_name yq '.networks.[env(name)] | .driver' $compose_file)
+                    if [[ $driver == "null" ]]; then
+                        driver="overlay"
+                    fi
+
+                    local attachable=""
+                    if [[ $(name=$network_name yq '.networks.[env(name)] | .attachable' $compose_file) == true ]]; then
+                        attachable="--attachable"
+                    fi
+
+                    log info "Waiting to create external network $name ..."
+                    try \
+                        "docker network create --scope=swarm \
+                        -d $driver \
+                        $attachable \
+                        $name" \
+                        throw \
+                        "Failed to create network $name"
+                    overwrite "Waiting to create external network $name ... Done"
+                fi
+            done
+        fi
+    done
+}
+
+# Joins a service to a network by updating the service spec to include the network.
+#
+# Note: Do not remove if not used in the Platform as this is mainly used by
+# custom packages that cannot overwrite the docker compose file to add the network connection required.
+#
+# Arguments:
+# - $1 : service name that needs to join the network (eg. analytics-datastore-elastic-search)
+# - $2 : network name to join (eg. elastic_public)
+#
+docker::join_network() {
+    local -r SERVICE_NAME="${1:?$(missing_param "join_network" "SERVICE_NAME")}"
+    local -r NETWORK_NAME="${2:?$(missing_param "join_network" "NETWORK_NAME")}"
+    local network_id
+    network_id=$(docker network ls --filter name="$NETWORK_NAME$" --format '{{.ID}}')
+    if [[ -n "${network_id}" ]]; then
+        if docker service inspect "$SERVICE_NAME" --format "{{.Spec.TaskTemplate.Networks}}" | grep -q "$network_id"; then
+            log info "Service $SERVICE_NAME is already connected to network $NETWORK_NAME."
+        else
+            log info "Waiting to join $SERVICE_NAME to external network $NETWORK_NAME ..."
+            try \
+                "docker service update  \
+              --network-add name=$NETWORK_NAME \
+              $SERVICE_NAME" \
+                throw \
+                "Failed to join network $NETWORK_NAME"
+        fi
+    else
+        log error "Network $NETWORK_NAME does not exist, cannot join $SERVICE_NAME to it ..."
+    fi
+}
+
+# Checks the compose file(s) passed in for the existance of a config.file definition to pass to config::set_config_digests
+#
+# Arguments:
+# - $@ : fully qualified path to the compose file(s) to check (eg. /instant/interoperability-layer-openhim/docker-compose.yml)
+#
+docker::prepare_config_digests()
+{
+    if [[ -z "$*" ]]; then
+        log error "$(missing_param "prepare_config_digests", "[COMPOSE_FILES]")"
+        exit 1
+    fi
+
+    for compose_file in "$@"; do
+        local files=($(yq '.configs."*.*".file' "$compose_file"))
+        if [[ "${files[*]}" != "null" ]]; then
+            config::set_config_digests "$compose_file"
+        fi
+    done
+}
+
+# Checks the compose file(s) passed in for the existance of a config.lables.name definition to pass to config::remove_stale_service_configs
+# To ensure that the service has the most up to date config digest
+#
+# Arguments:
+# - $@ : fully qualified path to the compose file(s) to check (eg. /instant/interoperability-layer-openhim/docker-compose.yml)
+#
+docker::cleanup_stale_configs()
+{
+    if [[ -z "$*" ]]; then
+        log error "$(missing_param "cleanup_stale_configs", "[COMPOSE_FILES]")"
+        exit 1
+    fi
+
+    for compose_file in "$@"; do
+        local label_names=($(yq '.configs."*.*".labels.name' "$compose_file" | sort -u))
+        if [[ "${label_names[*]}" != "null" ]]; then
+            for label_name in "${label_names[@]}"; do
+                config::remove_stale_service_configs "$compose_file" "${label_name}"
+            done
+        fi
+    done
+}

utils/log.sh

@@ -0,0 +1,228 @@
+#!/bin/bash
+
+set -uo pipefail
+
+# Global constants
+PREV_LINE="\e[1A" # moves cursor to previous line
+CLEAR_LINE="\e[K" # clears the current line the cursor is on
+CLEAR_PREV_LINE="${PREV_LINE}${PREV_LINE}${CLEAR_LINE}"
+
+# Defaults
+DEBUG="${DEBUG:-0}"
+BASHLOG_FILE="${BASHLOG_FILE:-0}"
+
+root_log_file_path="/tmp/logs"
+LOG_FILE_PATH="${root_log_file_path}/${BASHLOG_FILE_PATH:-platform.log}"
+
+function _log_exception() {
+    (
+        BASHLOG_FILE=0
+        BASHLOG_JSON=0
+        BASHLOG_SYSLOG=0
+
+        log 'error' "Logging Exception: ${@}"
+    )
+}
+
+function log() {
+    local date_format="${BASHLOG_DATE_FORMAT:-+%F %T}"
+    local date="$(date "${date_format}")"
+    local date_s="$(date "+%s")"
+
+    local file="${BASHLOG_FILE:-0}"
+    local file_path="${LOG_FILE_PATH:-/tmp/$(basename "${0}").log}"
+
+    local json="${BASHLOG_JSON:-0}"
+    local json_path="${BASHLOG_JSON_PATH:-/tmp/$(basename "${0}").log.json}"
+
+    local syslog="${BASHLOG_SYSLOG:-0}"
+    local tag="${BASHLOG_SYSLOG_TAG:-$(basename "${0}")}"
+    local facility="${BASHLOG_SYSLOG_FACILITY:-local0}"
+    local pid="${$}"
+
+    local level="${1}"
+    local upper="$(echo "${level}" | awk '{print toupper($0)}')"
+    local debug_level="${DEBUG:-0}"
+
+    shift 1
+
+    local line="${@}"
+
+    # RFC 5424
+    #
+    # Numerical         Severity
+    #   Code
+    #
+    #    0       Emergency: system is unusable
+    #    1       Alert: action must be taken immediately
+    #    2       Critical: critical conditions
+    #    3       Error: error conditions
+    #    4       Warning: warning conditions
+    #    5       Notice: normal but significant condition
+    #    6       Informational: informational messages
+    #    7       Debug: debug-level messages
+
+    local -A severities
+    severities['DEBUG']=7
+    severities['INFO']=6
+    severities['NOTICE']=5 # Unused
+    severities['WARN']=4
+    severities['ERROR']=3
+    severities['CRIT']=2  # Unused
+    severities['ALERT']=1 # Unused
+    severities['EMERG']=0 # Unused
+
+    local severity="${severities[${upper}]:-3}"
+
+    if [ "${debug_level}" -gt 0 ] || [ "${severity}" -lt 7 ]; then
+
+        if [ "${syslog}" -eq 1 ]; then
+            local syslog_line="${upper}: ${line}"
+
+            logger \
+                --id="${pid}" \
+                -t "${tag}" \
+                -p "${facility}.${severity}" \
+                "${syslog_line}" ||
+                _log_exception "logger --id=\"${pid}\" -t \"${tag}\" -p \"${facility}.${severity}\" \"${syslog_line}\""
+        fi
+
+        if [ "${file}" -eq 1 ]; then
+            clean_line="${line//\\e[1A/}"
+            clean_line="${clean_line//\\e[K/}"
+            local file_line="${date} [${upper}] ${clean_line}"
+            echo -e "${file_line}" >>"${file_path}" ||
+                _log_exception "echo -e \"${file_line}\" >> \"${file_path}\""
+        fi
+
+        if [ "${json}" -eq 1 ]; then
+            local json_line="$(printf '{"timestamp":"%s","level":"%s","message":"%s"}' "${date_s}" "${level}" "${line}")"
+            echo -e "${json_line}" >>"${json_path}" ||
+                _log_exception "echo -e \"${json_line}\" >> \"${json_path}\""
+        fi
+
+    fi
+
+    local -A colours
+    colours['DEBUG']='\033[34m'  # Blue
+    colours['INFO']='\033[32m'   # Green
+    colours['NOTICE']=''         # Unused
+    colours['WARN']='\033[33m'   # Yellow
+    colours['ERROR']='\033[31m'  # Red
+    colours['CRIT']=''           # Unused
+    colours['ALERT']=''          # Unused
+    colours['EMERG']=''          # Unused
+    colours['DEFAULT']='\033[0m' # Default
+
+    local -A emoticons
+    emoticons['DEBUG']='🔷'
+    emoticons['INFO']='❕'
+    emoticons['NOTICE']='💡'
+    emoticons['WARN']='🔶'
+    emoticons['ERROR']='❌'
+    emoticons['CRIT']='⛔'
+    emoticons['ALERT']='❗❗'
+    emoticons['EMERG']='🚨'
+    emoticons['DEFAULT']=''
+
+    local norm="${colours['DEFAULT']}"
+    local colour="${colours[${upper}]:-\033[31m}"
+
+    if [[ "${line}" == *"${CLEAR_PREV_LINE}"* ]]; then
+        # Append package name dynamically when override
+        line="${CLEAR_PREV_LINE}[$(dirname -- "$0" | sed -e 's/-/ /g' -e 's/\b\(.\)/\u\1/g')] ${line#*"$CLEAR_PREV_LINE"}"
+    else
+        line="[$(dirname -- "$0" | sed -e 's/-/ /g' -e 's/\b\(.\)/\u\1/g')] ${line}"
+    fi
+
+    local std_line="${colour} ${emoticons[${upper}]} ${line}${norm}"
+
+    # Standard Output (Pretty)
+    case "${level}" in
+    'default' | 'info' | 'warn')
+        echo -e "${std_line}"
+        ;;
+    'debug')
+        if [ "${debug_level}" -gt 0 ]; then
+            echo -e "${std_line}"
+        fi
+        ;;
+    'error')
+        echo -e "${std_line}" >&2
+        ;;
+    *)
+        log 'error' "Undefined log level trying to log: ${@}"
+        ;;
+    esac
+}
+
+# This is an option if you want to log every single command executed,
+# but it will significantly impact script performance and unit tests will fail
+if [[ $DEBUG -eq 1 ]]; then
+    declare -g prev_cmd="null"
+    declare -g this_cmd="null"
+
+    trap 'prev_cmd=$this_cmd; this_cmd=$BASH_COMMAND; log debug $this_cmd' DEBUG
+fi
+
+# A function that will return a message called when of parameter not provided
+#
+# Arguments:
+# - $1 : optional - function name missing the parameter
+# - $2 : optional - name of the parameter missing
+missing_param() {
+    local FUNC_NAME=${1:-""}
+    local ARG_NAME=${2:-""}
+
+    echo "FATAL: ${FUNC_NAME} parameter ${ARG_NAME} not provided"
+}
+
+# Overwrites the last echo'd command with what is provided
+#
+# Arguments:
+# - $1 : message (eg. "Setting passwords... Done")
+overwrite() {
+    local -r MESSAGE=${1:?$(missing_param "overwrite")}
+    if [ "${DEBUG}" -eq 1 ]; then
+        log info "${MESSAGE}"
+    else
+        log info "${CLEAR_PREV_LINE}${MESSAGE}"
+    fi
+}
+
+# Execute a command handle logging of the output
+#
+# Arguments:
+# - $1 : command (eg. "docker service rm elastic-search")
+# - $2 : throw or catch (eg. "throw", "catch")
+# - $3 : error message (eg. "Failed to remove elastic-search service")
+try() {
+    local -r COMMAND=${1:?$(missing_param "try" "COMMAND")}
+    local -r SHOULD_THROW=${2:-"throw"}
+    local -r ERROR_MESSAGE=${3:?$(missing_param "try" "ERROR_MESSAGE")}
+
+    if [ "${BASHLOG_FILE}" -eq 1 ]; then
+        if ! eval "$COMMAND" >>"$LOG_FILE_PATH" 2>&1; then
+            log error "$ERROR_MESSAGE"
+            if [[ "$SHOULD_THROW" == "throw" ]]; then
+                exit 1
+            fi
+        fi
+    else
+        if [ "${DEBUG}" -eq 1 ]; then
+            if ! eval "$COMMAND"; then
+                log error "$ERROR_MESSAGE"
+                if [[ "$SHOULD_THROW" == "throw" ]]; then
+                    exit 1
+                fi
+            fi
+        else
+            if ! eval "$COMMAND" 1>/dev/null; then
+                log error "$ERROR_MESSAGE"
+                if [[ "$SHOULD_THROW" == "throw" ]]; then
+                    exit 1
+                fi
+            fi
+        fi
+    fi
+}