Provision BetaCrew databases and encrypted backups

README.md

@@ -16,11 +16,14 @@ This repository owns the shared PostgreSQL server. Application repositories conn
 - `bootstrap/keycloak-runtrace-app.sql`: targeted idempotent bootstrap for the Runtrace Keycloak database
 - `bootstrap/runtrace-app.sql`: idempotent SQL bootstrap for the Runtrace application database
 - `bootstrap/jotwink-databases.sql`: idempotent bootstrap for the isolated Jotwink application and identity databases
+- `bootstrap/betacrew-databases.sql`: idempotent bootstrap for the isolated BetaCrew application and identity databases
 - `bootstrap/openpanel-app.sql`: idempotent SQL bootstrap for the OpenPanel application database
 - `scripts/run-runtrace-backup.sh`: certificate-verified logical backup for Runtrace app and identity data
 - `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets
 - `scripts/run-jotwink-backup.sh`: certificate-verified logical backup for Jotwink application and identity data
 - `scripts/verify-jotwink-restore.sh`: destructive Jotwink restore verification against explicit non-production targets
+- `scripts/run-betacrew-backup.sh`: six-hour encrypted BetaCrew application and identity backup
+- `scripts/verify-betacrew-restore.sh`: destructive BetaCrew restore verification against explicit non-production targets
 
 ## Networks
 
@@ -72,7 +75,8 @@ Keycloak while requiring TLS and SCRAM for `runtrace` and
 `keycloak_runtrace`. Jotwink is stricter: `jotwink_app` can reach `jotwink`
 only from the application WireGuard peer `10.80.0.1`, and
 `keycloak_jotwink_app` can reach `keycloak_jotwink` only from the dedicated
-Keycloak host. Other databases keep their existing SCRAM transport policy.
+Keycloak host. BetaCrew uses the same source isolation with independent roles
+and databases. Other databases keep their existing SCRAM transport policy.
 
 Required environment secrets:
 
@@ -172,6 +176,29 @@ scripts/verify-jotwink-restore.sh /var/lib/makepad/postgres-backups/jotwink/<tim
 Record the artifact checksum, elapsed time, recovery-point age, and operator.
 The verifier checks both the app migration table and Keycloak realm table.
 
+## BetaCrew Backup And Restore
+
+BetaCrew has an independent six-hour backup service for `betacrew` and
+`keycloak_betacrew`. Dumps are validated, encrypted with OpenSSL CMS
+AES-256-GCM before persistent storage, checksummed, and retained for 35 days.
+Provision `/var/lib/makepad/postgres-backups/betacrew` for uid 70 and install
+the public recipient certificate at
+`/etc/makepad/postgres-backup/betacrew-recipient.pem`; keep its private key
+offline. Replicate completed backup directories to independently administered
+storage.
+
+Run a restore drill against empty non-production databases with:
+
+```bash
+export PGSERVICEFILE=/etc/makepad/postgres-restore-services.conf
+export BETACREW_RESTORE_SERVICE=betacrew_restore_test
+export KEYCLOAK_BETACREW_RESTORE_SERVICE=keycloak_betacrew_restore_test
+export BETACREW_RESTORE_CONFIRM=replace-nonproduction-restore-targets
+export BETACREW_BACKUP_DECRYPTION_CERT=/secure/betacrew-recipient.pem
+export BETACREW_BACKUP_DECRYPTION_KEY=/secure/betacrew-recipient.key
+scripts/verify-betacrew-restore.sh /var/lib/makepad/postgres-backups/betacrew/<timestamp>
+```
+
 ## Application Databases
 
 Create one database and one dedicated user per application.
@@ -197,6 +224,8 @@ Jotwink application and identity persistence use separate roles:
 | --- | --- | --- |
 | Jotwink app | `jotwink` | `jotwink_app` |
 | Jotwink Keycloak | `keycloak_jotwink` | `keycloak_jotwink_app` |
+| BetaCrew app | `betacrew` | `betacrew_app` |
+| BetaCrew Keycloak | `keycloak_betacrew` | `keycloak_betacrew_app` |
 
 OpenPanel application persistence uses:
 
@@ -216,6 +245,8 @@ Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must
 : "${OPENPANEL_DB_PASSWORD:?set OPENPANEL_DB_PASSWORD to a generated password}"
 : "${JOTWINK_DB_PASSWORD:?set JOTWINK_DB_PASSWORD to a generated password}"
 : "${KEYCLOAK_JOTWINK_DB_PASSWORD:?set KEYCLOAK_JOTWINK_DB_PASSWORD to a generated password}"
+: "${BETACREW_DB_PASSWORD:?set BETACREW_DB_PASSWORD to a generated password}"
+: "${KEYCLOAK_BETACREW_DB_PASSWORD:?set KEYCLOAK_BETACREW_DB_PASSWORD to a generated password}"
 
 psql "$POSTGRES_ADMIN_URL" \
   -v keycloak_vif_app_password="$KEYCLOAK_VIF_DB_PASSWORD" \
@@ -241,6 +272,11 @@ psql "$POSTGRES_ADMIN_URL" \
   -v jotwink_app_password="$JOTWINK_DB_PASSWORD" \
   -v keycloak_jotwink_app_password="$KEYCLOAK_JOTWINK_DB_PASSWORD" \
   -f bootstrap/jotwink-databases.sql
+
+psql "$POSTGRES_ADMIN_URL" \
+  -v betacrew_app_password="$BETACREW_DB_PASSWORD" \
+  -v keycloak_betacrew_app_password="$KEYCLOAK_BETACREW_DB_PASSWORD" \
+  -f bootstrap/betacrew-databases.sql
 ```
 
 Jotwink application traffic follows the existing WireGuard path from

bootstrap/betacrew-databases.sql

@@ -0,0 +1,53 @@
+\set ON_ERROR_STOP on
+
+-- Run with a PostgreSQL superuser connection. Passwords are supplied as psql
+-- variables so credentials never enter source control.
+\if :{?betacrew_app_password}
+\else
+  \echo 'missing required psql variable: betacrew_app_password'
+  \quit 1
+\endif
+\if :{?keycloak_betacrew_app_password}
+\else
+  \echo 'missing required psql variable: keycloak_betacrew_app_password'
+  \quit 1
+\endif
+
+SELECT NULLIF(btrim(:'betacrew_app_password'), '') IS NOT NULL AS betacrew_password_ok \gset
+\if :betacrew_password_ok
+\else
+  \echo 'empty required psql variable: betacrew_app_password'
+  \quit 1
+\endif
+SELECT NULLIF(btrim(:'keycloak_betacrew_app_password'), '') IS NOT NULL AS keycloak_betacrew_password_ok \gset
+\if :keycloak_betacrew_password_ok
+\else
+  \echo 'empty required psql variable: keycloak_betacrew_app_password'
+  \quit 1
+\endif
+
+SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap'));
+
+DO $$ BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'betacrew_app') THEN
+    CREATE ROLE betacrew_app LOGIN;
+  END IF;
+END $$;
+ALTER ROLE betacrew_app LOGIN PASSWORD :'betacrew_app_password';
+SELECT 'CREATE DATABASE betacrew OWNER betacrew_app'
+WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'betacrew') \gexec
+ALTER DATABASE betacrew OWNER TO betacrew_app;
+GRANT CONNECT ON DATABASE betacrew TO betacrew_app;
+
+DO $$ BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_betacrew_app') THEN
+    CREATE ROLE keycloak_betacrew_app LOGIN;
+  END IF;
+END $$;
+ALTER ROLE keycloak_betacrew_app LOGIN PASSWORD :'keycloak_betacrew_app_password';
+SELECT 'CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app'
+WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_betacrew') \gexec
+ALTER DATABASE keycloak_betacrew OWNER TO keycloak_betacrew_app;
+GRANT CONNECT ON DATABASE keycloak_betacrew TO keycloak_betacrew_app;
+
+SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap'));

compose.betacrew.host.yml

@@ -0,0 +1,42 @@
+services:
+  betacrew_backup:
+    image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8}
+    user: "70:70"
+    network_mode: host
+    command: [/usr/local/bin/run-betacrew-backup-loop.sh]
+    environment:
+      PGHOST: 127.0.0.1
+      PGPORT: "5432"
+      PGUSER: ${POSTGRES_USER:-postgres}
+      PGSSLMODE: verify-full
+      PGSSLROOTCERT: /etc/postgresql/ca.crt
+      POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password
+      BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem
+      BETACREW_BACKUP_ROOT: /backups
+      BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600}
+      BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300}
+      BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35}
+    read_only: true
+    tmpfs:
+      - /tmp:mode=0700,uid=70,gid=70
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]
+    restart: unless-stopped
+    volumes:
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro"
+      - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro"
+    healthcheck:
+      test: [CMD, /usr/local/bin/run-betacrew-backup-loop.sh, healthcheck]
+      interval: 5m
+      timeout: 10s
+      retries: 3
+      start_period: 10m
+    logging:
+      driver: json-file
+      options:
+        max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
+        max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"

compose.host.yml

@@ -129,3 +129,51 @@ services:
       options:
         max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
         max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"
+
+  betacrew_backup:
+    image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8}
+    user: "70:70"
+    network_mode: host
+    command:
+      - /usr/local/bin/run-betacrew-backup-loop.sh
+    environment:
+      PGHOST: 127.0.0.1
+      PGPORT: "5432"
+      PGUSER: ${POSTGRES_USER:-postgres}
+      PGSSLMODE: verify-full
+      PGSSLROOTCERT: /etc/postgresql/ca.crt
+      POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password
+      BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem
+      BETACREW_BACKUP_ROOT: /backups
+      BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600}
+      BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300}
+      BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35}
+    read_only: true
+    tmpfs:
+      - /tmp:mode=0700,uid=70,gid=70
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro"
+      - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro"
+    healthcheck:
+      test: ["CMD", "/usr/local/bin/run-betacrew-backup-loop.sh", "healthcheck"]
+      interval: 5m
+      timeout: 10s
+      retries: 3
+      start_period: 10m
+    logging:
+      driver: json-file
+      options:
+        max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
+        max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"

config/runtrace-pg_hba.conf

@@ -1,5 +1,5 @@
-# Runtrace and Jotwink customer/identity traffic must use TLS. Jotwink is also
-# source-restricted to the application WireGuard peer and Keycloak host.
+# Runtrace, Jotwink, and BetaCrew customer/identity traffic must use TLS.
+# Jotwink and BetaCrew are source-restricted to the app WireGuard peer and Keycloak host.
 # Other shared databases retain their current SCRAM policy during migration.
 local   all                 all                                     trust
 hostnossl runtrace          all                 all                 reject
@@ -14,4 +14,12 @@ hostssl jotwink             postgres            127.0.0.1/32        scram-sha-25
 hostssl keycloak_jotwink    postgres            127.0.0.1/32        scram-sha-256
 hostssl jotwink             all                 all                 reject
 hostssl keycloak_jotwink    all                 all                 reject
+hostnossl betacrew          all                 all                 reject
+hostnossl keycloak_betacrew all                 all                 reject
+hostssl betacrew            betacrew_app         10.80.0.1/32        scram-sha-256
+hostssl keycloak_betacrew   keycloak_betacrew_app 88.99.209.165/32   scram-sha-256
+hostssl betacrew            postgres             127.0.0.1/32        scram-sha-256
+hostssl keycloak_betacrew   postgres             127.0.0.1/32        scram-sha-256
+hostssl betacrew            all                  all                 reject
+hostssl keycloak_betacrew   all                  all                 reject
 host    all                 all                 all                 scram-sha-256

envs/production/.env.db

@@ -1,6 +1,7 @@
 # Production PostgreSQL settings
 POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
 JOTWINK_BACKUP_IMAGE=postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8
+BETACREW_BACKUP_IMAGE=postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8
 POSTGRES_DB=postgres
 POSTGRES_USER=postgres
 MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres
@@ -21,6 +22,12 @@ MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-
 MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS=21600
 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS=300
 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS=35
+MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH=/var/lib/makepad/postgres-backups/betacrew
+MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password
+MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-backup/betacrew-recipient.pem
+MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600
+MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS=300
+MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS=35
 MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT=1.0
 MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT=1G
 MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION=0.1

scripts/apply-betacrew-hba.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+hba_path=${1:?Usage: apply-betacrew-hba.sh <pg_hba.conf>}
+[[ -f "${hba_path}" && ! -L "${hba_path}" ]] || { echo "HBA path must be a regular non-symlink file." >&2; exit 1; }
+
+marker='hostnossl betacrew'
+if grep -Fq "${marker}" "${hba_path}"; then
+  for required in \
+    'hostssl betacrew            betacrew_app         10.80.0.1/32' \
+    'hostssl keycloak_betacrew   keycloak_betacrew_app 88.99.209.165/32' \
+    'hostssl keycloak_betacrew   all                  all                 reject'; do
+    grep -Fq "${required}" "${hba_path}" || { echo "Partial BetaCrew HBA policy found; refusing to modify it." >&2; exit 1; }
+  done
+  exit 0
+fi
+
+fallback='host    all                 all                 all                 scram-sha-256'
+[[ $(grep -Fxc "${fallback}" "${hba_path}") == 1 ]] || { echo "Expected exactly one shared HBA fallback rule." >&2; exit 1; }
+
+temporary=$(mktemp "${hba_path}.betacrew.XXXXXX")
+cleanup() { [[ ! -e "${temporary}" ]] || rm -f "${temporary}"; }
+trap cleanup EXIT HUP INT TERM
+
+awk -v fallback="${fallback}" '
+  $0 == fallback {
+    print "# BetaCrew app and identity traffic require TLS and exact source roles."
+    print "hostnossl betacrew          all                  all                 reject"
+    print "hostnossl keycloak_betacrew all                  all                 reject"
+    print "hostssl betacrew            betacrew_app         10.80.0.1/32        scram-sha-256"
+    print "hostssl keycloak_betacrew   keycloak_betacrew_app 88.99.209.165/32   scram-sha-256"
+    print "hostssl betacrew            postgres             127.0.0.1/32        scram-sha-256"
+    print "hostssl keycloak_betacrew   postgres             127.0.0.1/32        scram-sha-256"
+    print "hostssl betacrew            all                  all                 reject"
+    print "hostssl keycloak_betacrew   all                  all                 reject"
+  }
+  { print }
+' "${hba_path}" > "${temporary}"
+if original_mode=$(stat -c '%a' "${hba_path}" 2>/dev/null); then :; else original_mode=$(stat -f '%Lp' "${hba_path}"); fi
+chmod "${original_mode}" "${temporary}"
+mv "${temporary}" "${hba_path}"

scripts/run-betacrew-backup-loop.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+set -eu
+
+interval_seconds=${BETACREW_BACKUP_INTERVAL_SECONDS:-21600}
+retry_seconds=${BETACREW_BACKUP_RETRY_SECONDS:-300}
+backup_root=${BETACREW_BACKUP_ROOT:-/backups}
+
+case "${interval_seconds}:${retry_seconds}" in
+  *[!0-9:]*) echo "BetaCrew backup timing values must be positive integers." >&2; exit 1 ;;
+esac
+[ "${interval_seconds}" -ge 300 ] && [ "${retry_seconds}" -ge 30 ] || { echo "BetaCrew backup interval or retry delay is too short." >&2; exit 1; }
+
+if [ "${1:-}" = healthcheck ]; then
+  status_file=${backup_root}/last-success.json
+  [ -s "${status_file}" ] || exit 1
+  now=$(date +%s)
+  if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then :; else modified=$(stat -f %m "${status_file}"); fi
+  [ $((now - modified)) -le $((interval_seconds * 2 + retry_seconds)) ]
+  exit
+fi
+
+while :; do
+  if /usr/local/bin/run-betacrew-backup.sh; then
+    sleep "${interval_seconds}"
+  else
+    echo "BetaCrew PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2
+    sleep "${retry_seconds}"
+  fi
+done