From d731a7a5adfaa795fa273d5048abc63e2aca0693 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 13 Aug 2026 23:29:13 +0200 Subject: [PATCH 1/3] Provision BetaCrew databases and encrypted backups --- README.md | 38 +++++++++++++- bootstrap/betacrew-databases.sql | 53 +++++++++++++++++++ compose.host.yml | 48 +++++++++++++++++ config/runtrace-pg_hba.conf | 12 ++++- envs/production/.env.db | 7 +++ scripts/run-betacrew-backup-loop.sh | 29 +++++++++++ scripts/run-betacrew-backup.sh | 81 +++++++++++++++++++++++++++++ scripts/test-betacrew-backup.sh | 48 +++++++++++++++++ scripts/validate-betacrew-config.sh | 19 +++++++ scripts/verify-betacrew-restore.sh | 36 +++++++++++++ 10 files changed, 368 insertions(+), 3 deletions(-) create mode 100644 bootstrap/betacrew-databases.sql create mode 100755 scripts/run-betacrew-backup-loop.sh create mode 100755 scripts/run-betacrew-backup.sh create mode 100755 scripts/test-betacrew-backup.sh create mode 100755 scripts/validate-betacrew-config.sh create mode 100755 scripts/verify-betacrew-restore.sh diff --git a/README.md b/README.md index 26b7d6d..0d75e41 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,14 @@ This repository owns the shared PostgreSQL server. Application repositories conn - `bootstrap/keycloak-runtrace-app.sql`: targeted idempotent bootstrap for the Runtrace Keycloak database - `bootstrap/runtrace-app.sql`: idempotent SQL bootstrap for the Runtrace application database - `bootstrap/jotwink-databases.sql`: idempotent bootstrap for the isolated Jotwink application and identity databases +- `bootstrap/betacrew-databases.sql`: idempotent bootstrap for the isolated BetaCrew application and identity databases - `bootstrap/openpanel-app.sql`: idempotent SQL bootstrap for the OpenPanel application database - `scripts/run-runtrace-backup.sh`: certificate-verified logical backup for Runtrace app and identity data - `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets - `scripts/run-jotwink-backup.sh`: certificate-verified logical backup for Jotwink application and identity data - `scripts/verify-jotwink-restore.sh`: destructive Jotwink restore verification against explicit non-production targets +- `scripts/run-betacrew-backup.sh`: six-hour encrypted BetaCrew application and identity backup +- `scripts/verify-betacrew-restore.sh`: destructive BetaCrew restore verification against explicit non-production targets ## Networks @@ -72,7 +75,8 @@ Keycloak while requiring TLS and SCRAM for `runtrace` and `keycloak_runtrace`. Jotwink is stricter: `jotwink_app` can reach `jotwink` only from the application WireGuard peer `10.80.0.1`, and `keycloak_jotwink_app` can reach `keycloak_jotwink` only from the dedicated -Keycloak host. Other databases keep their existing SCRAM transport policy. +Keycloak host. BetaCrew uses the same source isolation with independent roles +and databases. Other databases keep their existing SCRAM transport policy. Required environment secrets: @@ -172,6 +176,29 @@ scripts/verify-jotwink-restore.sh /var/lib/makepad/postgres-backups/jotwink/ +``` + ## Application Databases Create one database and one dedicated user per application. @@ -197,6 +224,8 @@ Jotwink application and identity persistence use separate roles: | --- | --- | --- | | Jotwink app | `jotwink` | `jotwink_app` | | Jotwink Keycloak | `keycloak_jotwink` | `keycloak_jotwink_app` | +| BetaCrew app | `betacrew` | `betacrew_app` | +| BetaCrew Keycloak | `keycloak_betacrew` | `keycloak_betacrew_app` | OpenPanel application persistence uses: @@ -216,6 +245,8 @@ Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must : "${OPENPANEL_DB_PASSWORD:?set OPENPANEL_DB_PASSWORD to a generated password}" : "${JOTWINK_DB_PASSWORD:?set JOTWINK_DB_PASSWORD to a generated password}" : "${KEYCLOAK_JOTWINK_DB_PASSWORD:?set KEYCLOAK_JOTWINK_DB_PASSWORD to a generated password}" +: "${BETACREW_DB_PASSWORD:?set BETACREW_DB_PASSWORD to a generated password}" +: "${KEYCLOAK_BETACREW_DB_PASSWORD:?set KEYCLOAK_BETACREW_DB_PASSWORD to a generated password}" psql "$POSTGRES_ADMIN_URL" \ -v keycloak_vif_app_password="$KEYCLOAK_VIF_DB_PASSWORD" \ @@ -241,6 +272,11 @@ psql "$POSTGRES_ADMIN_URL" \ -v jotwink_app_password="$JOTWINK_DB_PASSWORD" \ -v keycloak_jotwink_app_password="$KEYCLOAK_JOTWINK_DB_PASSWORD" \ -f bootstrap/jotwink-databases.sql + +psql "$POSTGRES_ADMIN_URL" \ + -v betacrew_app_password="$BETACREW_DB_PASSWORD" \ + -v keycloak_betacrew_app_password="$KEYCLOAK_BETACREW_DB_PASSWORD" \ + -f bootstrap/betacrew-databases.sql ``` Jotwink application traffic follows the existing WireGuard path from diff --git a/bootstrap/betacrew-databases.sql b/bootstrap/betacrew-databases.sql new file mode 100644 index 0000000..e8b63bb --- /dev/null +++ b/bootstrap/betacrew-databases.sql @@ -0,0 +1,53 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. Passwords are supplied as psql +-- variables so credentials never enter source control. +\if :{?betacrew_app_password} +\else + \echo 'missing required psql variable: betacrew_app_password' + \quit 1 +\endif +\if :{?keycloak_betacrew_app_password} +\else + \echo 'missing required psql variable: keycloak_betacrew_app_password' + \quit 1 +\endif + +SELECT NULLIF(btrim(:'betacrew_app_password'), '') IS NOT NULL AS betacrew_password_ok \gset +\if :betacrew_password_ok +\else + \echo 'empty required psql variable: betacrew_app_password' + \quit 1 +\endif +SELECT NULLIF(btrim(:'keycloak_betacrew_app_password'), '') IS NOT NULL AS keycloak_betacrew_password_ok \gset +\if :keycloak_betacrew_password_ok +\else + \echo 'empty required psql variable: keycloak_betacrew_app_password' + \quit 1 +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap')); + +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'betacrew_app') THEN + CREATE ROLE betacrew_app LOGIN; + END IF; +END $$; +ALTER ROLE betacrew_app LOGIN PASSWORD :'betacrew_app_password'; +SELECT 'CREATE DATABASE betacrew OWNER betacrew_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'betacrew') \gexec +ALTER DATABASE betacrew OWNER TO betacrew_app; +GRANT CONNECT ON DATABASE betacrew TO betacrew_app; + +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_betacrew_app') THEN + CREATE ROLE keycloak_betacrew_app LOGIN; + END IF; +END $$; +ALTER ROLE keycloak_betacrew_app LOGIN PASSWORD :'keycloak_betacrew_app_password'; +SELECT 'CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_betacrew') \gexec +ALTER DATABASE keycloak_betacrew OWNER TO keycloak_betacrew_app; +GRANT CONNECT ON DATABASE keycloak_betacrew TO keycloak_betacrew_app; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap')); diff --git a/compose.host.yml b/compose.host.yml index 79f1d25..9f3c8f2 100644 --- a/compose.host.yml +++ b/compose.host.yml @@ -129,3 +129,51 @@ services: options: max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + + betacrew_backup: + image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8} + user: "70:70" + network_mode: host + command: + - /usr/local/bin/run-betacrew-backup-loop.sh + environment: + PGHOST: 127.0.0.1 + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:-postgres} + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem + BETACREW_BACKUP_ROOT: /backups + BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600} + BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300} + BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=70,gid=70 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + volumes: + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro" + healthcheck: + test: ["CMD", "/usr/local/bin/run-betacrew-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index 82597e0..0f3b8d1 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -1,5 +1,5 @@ -# Runtrace and Jotwink customer/identity traffic must use TLS. Jotwink is also -# source-restricted to the application WireGuard peer and Keycloak host. +# Runtrace, Jotwink, and BetaCrew customer/identity traffic must use TLS. +# Jotwink and BetaCrew are source-restricted to the app WireGuard peer and Keycloak host. # Other shared databases retain their current SCRAM policy during migration. local all all trust hostnossl runtrace all all reject @@ -14,4 +14,12 @@ hostssl jotwink postgres 127.0.0.1/32 scram-sha-25 hostssl keycloak_jotwink postgres 127.0.0.1/32 scram-sha-256 hostssl jotwink all all reject hostssl keycloak_jotwink all all reject +hostnossl betacrew all all reject +hostnossl keycloak_betacrew all all reject +hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256 +hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256 +hostssl betacrew postgres 127.0.0.1/32 scram-sha-256 +hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256 +hostssl betacrew all all reject +hostssl keycloak_betacrew all all reject host all all all scram-sha-256 diff --git a/envs/production/.env.db b/envs/production/.env.db index da9638b..7b963c9 100644 --- a/envs/production/.env.db +++ b/envs/production/.env.db @@ -1,6 +1,7 @@ # Production PostgreSQL settings POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 JOTWINK_BACKUP_IMAGE=postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8 +BETACREW_BACKUP_IMAGE=postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8 POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres @@ -21,6 +22,12 @@ MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres- MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS=21600 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS=300 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS=35 +MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH=/var/lib/makepad/postgres-backups/betacrew +MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password +MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-backup/betacrew-recipient.pem +MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600 +MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS=300 +MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS=35 MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT=1.0 MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT=1G MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION=0.1 diff --git a/scripts/run-betacrew-backup-loop.sh b/scripts/run-betacrew-backup-loop.sh new file mode 100755 index 0000000..fc351dd --- /dev/null +++ b/scripts/run-betacrew-backup-loop.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +interval_seconds=${BETACREW_BACKUP_INTERVAL_SECONDS:-21600} +retry_seconds=${BETACREW_BACKUP_RETRY_SECONDS:-300} +backup_root=${BETACREW_BACKUP_ROOT:-/backups} + +case "${interval_seconds}:${retry_seconds}" in + *[!0-9:]*) echo "BetaCrew backup timing values must be positive integers." >&2; exit 1 ;; +esac +[ "${interval_seconds}" -ge 300 ] && [ "${retry_seconds}" -ge 30 ] || { echo "BetaCrew backup interval or retry delay is too short." >&2; exit 1; } + +if [ "${1:-}" = healthcheck ]; then + status_file=${backup_root}/last-success.json + [ -s "${status_file}" ] || exit 1 + now=$(date +%s) + if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then :; else modified=$(stat -f %m "${status_file}"); fi + [ $((now - modified)) -le $((interval_seconds * 2 + retry_seconds)) ] + exit +fi + +while :; do + if /usr/local/bin/run-betacrew-backup.sh; then + sleep "${interval_seconds}" + else + echo "BetaCrew PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2 + sleep "${retry_seconds}" + fi +done diff --git a/scripts/run-betacrew-backup.sh b/scripts/run-betacrew-backup.sh new file mode 100755 index 0000000..e4f2bf9 --- /dev/null +++ b/scripts/run-betacrew-backup.sh @@ -0,0 +1,81 @@ +#!/bin/sh +set -eu + +umask 077 + +backup_root=${BETACREW_BACKUP_ROOT:-/backups} +password_file=${POSTGRES_SUPERUSER_PASSWORD_FILE:-/run/secrets/postgres_superuser_password} +encryption_cert=${BETACREW_BACKUP_ENCRYPTION_CERT:-/run/config/betacrew_backup_recipient.pem} +retention_days=${BETACREW_BACKUP_RETENTION_DAYS:-35} +pg_host=${PGHOST:-127.0.0.1} +pg_port=${PGPORT:-5432} +pg_user=${PGUSER:-postgres} + +case "${retention_days}" in + ''|*[!0-9]*) echo "BETACREW_BACKUP_RETENTION_DAYS must be a positive integer." >&2; exit 1 ;; +esac +[ "${retention_days}" -ge 1 ] || { echo "BETACREW_BACKUP_RETENTION_DAYS must be at least 1." >&2; exit 1; } +[ -s "${password_file}" ] || { echo "PostgreSQL backup credential file is missing or empty." >&2; exit 1; } +[ -s "${encryption_cert}" ] || { echo "BetaCrew backup encryption certificate is missing or empty." >&2; exit 1; } +command -v openssl >/dev/null 2>&1 || { echo "openssl is required to encrypt BetaCrew backups." >&2; exit 1; } +openssl x509 -in "${encryption_cert}" -noout >/dev/null 2>&1 || { echo "BetaCrew backup certificate is invalid." >&2; exit 1; } + +mkdir -p "${backup_root}" +chmod 700 "${backup_root}" +lock_dir=${backup_root}/.betacrew-backup.lock +mkdir "${lock_dir}" 2>/dev/null || { echo "Another BetaCrew backup is running." >&2; exit 1; } + +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +partial_dir=${backup_root}/.${timestamp}.partial +final_dir=${backup_root}/${timestamp} +work_dir=$(mktemp -d "${TMPDIR:-/tmp}/betacrew-backup.${timestamp}.XXXXXX") +pgpass=${work_dir}/.pgpass + +cleanup() { + if [ -d "${partial_dir}" ]; then find "${partial_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${partial_dir}" 2>/dev/null || true; fi + if [ -d "${work_dir}" ]; then find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; fi + rmdir "${lock_dir}" 2>/dev/null || true +} +trap cleanup EXIT HUP INT TERM + +[ ! -e "${final_dir}" ] && [ ! -L "${final_dir}" ] || { echo "Backup destination already exists." >&2; exit 1; } +password=$(tr -d '\r\n' < "${password_file}") +[ -n "${password}" ] || { echo "PostgreSQL backup credential contains no usable value." >&2; exit 1; } +escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') +printf '%s:%s:*:%s:%s\n' "${pg_host}" "${pg_port}" "${pg_user}" "${escaped_password}" > "${pgpass}" +unset password escaped_password +chmod 600 "${pgpass}" +export PGPASSFILE="${pgpass}" +export PGSSLMODE="${PGSSLMODE:-verify-full}" +export PGSSLROOTCERT="${PGSSLROOTCERT:-/etc/postgresql/ca.crt}" + +mkdir "${partial_dir}" +for database in betacrew keycloak_betacrew; do + dump_path=${work_dir}/${database}.dump + encrypted_path=${partial_dir}/${database}.dump.cms + pg_dump --host="${pg_host}" --port="${pg_port}" --username="${pg_user}" --dbname="${database}" \ + --format=custom --compress=9 --no-owner --no-acl --file="${dump_path}" + pg_restore --list "${dump_path}" >/dev/null + openssl cms -encrypt -binary -outform DER -aes-256-gcm -in "${dump_path}" -out "${encrypted_path}" "${encryption_cert}" + openssl cms -cmsout -inform DER -in "${encrypted_path}" -noout >/dev/null + find "${dump_path}" -delete +done + +(cd "${partial_dir}" && sha256sum betacrew.dump.cms keycloak_betacrew.dump.cms > SHA256SUMS) +cat > "${partial_dir}/metadata.json" <&2; exit 1; } +ln -sfn "${timestamp}" "${backup_root}/latest" +status_tmp=${backup_root}/.last-success.json.tmp +printf '{"createdAt":"%s","backup":"%s","databases":["betacrew","keycloak_betacrew"]}\n' "${timestamp}" "${timestamp}" > "${status_tmp}" +chmod 600 "${status_tmp}" +mv "${status_tmp}" "${backup_root}/last-success.json" + +find "${backup_root}" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' -mtime "+${retention_days}" -exec sh -c ' + for directory do find "$directory" -mindepth 1 -delete; rmdir "$directory"; done +' sh {} + + +echo "BetaCrew PostgreSQL backup completed: ${final_dir}" diff --git a/scripts/test-betacrew-backup.sh b/scripts/test-betacrew-backup.sh new file mode 100755 index 0000000..3f0c406 --- /dev/null +++ b/scripts/test-betacrew-backup.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) +cleanup() { find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; } +trap cleanup EXIT + +mkdir -p "${work_dir}/bin" "${work_dir}/backups" +printf 'validation-password\n' > "${work_dir}/password" +chmod 600 "${work_dir}/password" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=betacrew-backup-test' \ + -keyout "${work_dir}/recipient.key" -out "${work_dir}/recipient.pem" >/dev/null 2>&1 + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +for argument in "$@"; do + case "${argument}" in --file=*) output=${argument#--file=} ;; --dbname=*) database=${argument#--dbname=} ;; esac +done +[ -n "${output:-}" ] && [ -n "${database:-}" ] +printf 'validated dump for %s\n' "${database}" > "${output}" +EOF +cat > "${work_dir}/bin/pg_restore" <<'EOF' +#!/bin/sh +set -eu +[ "$1" = "--list" ] && [ -s "$2" ] +EOF +chmod 700 "${work_dir}/bin/pg_dump" "${work_dir}/bin/pg_restore" + +PATH="${work_dir}/bin:${PATH}" BETACREW_BACKUP_ROOT="${work_dir}/backups" \ +POSTGRES_SUPERUSER_PASSWORD_FILE="${work_dir}/password" \ +BETACREW_BACKUP_ENCRYPTION_CERT="${work_dir}/recipient.pem" \ +PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" sh "${repo_root}/scripts/run-betacrew-backup.sh" + +latest=$(readlink "${work_dir}/backups/latest") +backup_dir=${work_dir}/backups/${latest} +for expected in betacrew.dump.cms keycloak_betacrew.dump.cms SHA256SUMS metadata.json; do test -s "${backup_dir}/${expected}"; done +(cd "${backup_dir}" && sha256sum --check SHA256SUMS >/dev/null) +for database in betacrew keycloak_betacrew; do + ! grep -a -q 'validated dump' "${backup_dir}/${database}.dump.cms" + openssl cms -decrypt -binary -inform DER -in "${backup_dir}/${database}.dump.cms" \ + -recip "${work_dir}/recipient.pem" -inkey "${work_dir}/recipient.key" -out "${work_dir}/${database}.decrypted" + grep -q "validated dump for ${database}" "${work_dir}/${database}.decrypted" +done +BETACREW_BACKUP_ROOT="${work_dir}/backups" BETACREW_BACKUP_INTERVAL_SECONDS=300 \ +BETACREW_BACKUP_RETRY_SECONDS=30 sh "${repo_root}/scripts/run-betacrew-backup-loop.sh" healthcheck +echo "BetaCrew PostgreSQL encrypted backup contract passed." diff --git a/scripts/validate-betacrew-config.sh b/scripts/validate-betacrew-config.sh new file mode 100755 index 0000000..0bfb53f --- /dev/null +++ b/scripts/validate-betacrew-config.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +require_literal() { grep -Fq -- "$2" "${repo_root}/$1" || { echo "Missing BetaCrew contract in $1: $2" >&2; exit 1; }; } + +require_literal bootstrap/betacrew-databases.sql "pg_advisory_lock" +require_literal bootstrap/betacrew-databases.sql "CREATE DATABASE betacrew OWNER betacrew_app" +require_literal bootstrap/betacrew-databases.sql "CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app" +require_literal config/runtrace-pg_hba.conf "hostssl betacrew betacrew_app 10.80.0.1/32" +require_literal config/runtrace-pg_hba.conf "hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32" +require_literal compose.host.yml "betacrew_backup:" +require_literal compose.host.yml "BETACREW_BACKUP_INTERVAL_SECONDS" +require_literal compose.host.yml "BETACREW_BACKUP_ENCRYPTION_CERT" +require_literal envs/production/.env.db "MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600" +require_literal scripts/run-betacrew-backup.sh "-aes-256-gcm" +require_literal scripts/verify-betacrew-restore.sh "replace-nonproduction-restore-targets" +require_literal scripts/verify-betacrew-restore.sh "--single-transaction" +echo "BetaCrew PostgreSQL configuration is valid." diff --git a/scripts/verify-betacrew-restore.sh b/scripts/verify-betacrew-restore.sh new file mode 100755 index 0000000..15dda1b --- /dev/null +++ b/scripts/verify-betacrew-restore.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +backup_dir=${1:?Usage: verify-betacrew-restore.sh } +: "${PGSERVICEFILE:?PGSERVICEFILE must identify a root-owned libpq service file}" +: "${BETACREW_RESTORE_SERVICE:?BETACREW_RESTORE_SERVICE must name an empty non-production database}" +: "${KEYCLOAK_BETACREW_RESTORE_SERVICE:?KEYCLOAK_BETACREW_RESTORE_SERVICE must name an empty non-production database}" +: "${BETACREW_RESTORE_CONFIRM:?set BETACREW_RESTORE_CONFIRM=replace-nonproduction-restore-targets}" +: "${BETACREW_BACKUP_DECRYPTION_CERT:?BETACREW_BACKUP_DECRYPTION_CERT must identify the recipient certificate}" +: "${BETACREW_BACKUP_DECRYPTION_KEY:?BETACREW_BACKUP_DECRYPTION_KEY must identify the offline private key}" + +[[ "${BETACREW_RESTORE_CONFIRM}" == replace-nonproduction-restore-targets ]] || { echo "Restore requires explicit non-production confirmation." >&2; exit 1; } +for path in "${PGSERVICEFILE}" "${BETACREW_BACKUP_DECRYPTION_CERT}" "${BETACREW_BACKUP_DECRYPTION_KEY}"; do + [[ -f "${path}" && ! -L "${path}" ]] || { echo "Restore input must be a regular non-symlink file: ${path}" >&2; exit 1; } +done +[[ -d "${backup_dir}" && ! -L "${backup_dir}" ]] || { echo "Backup directory is invalid." >&2; exit 1; } +for required in betacrew.dump.cms keycloak_betacrew.dump.cms SHA256SUMS metadata.json; do [[ -s "${backup_dir}/${required}" ]] || exit 1; done +(cd "${backup_dir}" && sha256sum --check SHA256SUMS) + +work_dir=$(mktemp -d) +cleanup() { find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; } +trap cleanup EXIT HUP INT TERM + +for database in betacrew keycloak_betacrew; do + openssl cms -decrypt -binary -inform DER -in "${backup_dir}/${database}.dump.cms" \ + -recip "${BETACREW_BACKUP_DECRYPTION_CERT}" -inkey "${BETACREW_BACKUP_DECRYPTION_KEY}" \ + -out "${work_dir}/${database}.dump" + pg_restore --list "${work_dir}/${database}.dump" >/dev/null +done + +pg_restore --dbname="service=${BETACREW_RESTORE_SERVICE}" --clean --if-exists --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/betacrew.dump" +pg_restore --dbname="service=${KEYCLOAK_BETACREW_RESTORE_SERVICE}" --clean --if-exists --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/keycloak_betacrew.dump" + +[[ $(psql "service=${BETACREW_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.schema_migrations') IS NOT NULL;") == t ]] +[[ $(psql "service=${KEYCLOAK_BETACREW_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.realm') IS NOT NULL;") == t ]] +echo "BetaCrew and Keycloak restore verification completed against non-production targets." From f686305e5d7d97d1d1823b6490100b026ffd58cb Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 13 Aug 2026 23:37:15 +0200 Subject: [PATCH 2/3] Preserve deployed PostgreSQL host additions --- compose.betacrew.host.yml | 41 +++++++++++++++++++++++++++++ scripts/apply-betacrew-hba.sh | 41 +++++++++++++++++++++++++++++ scripts/validate-betacrew-config.sh | 2 ++ 3 files changed, 84 insertions(+) create mode 100644 compose.betacrew.host.yml create mode 100755 scripts/apply-betacrew-hba.sh diff --git a/compose.betacrew.host.yml b/compose.betacrew.host.yml new file mode 100644 index 0000000..cf6c163 --- /dev/null +++ b/compose.betacrew.host.yml @@ -0,0 +1,41 @@ +services: + betacrew_backup: + image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8} + user: "70:70" + network_mode: host + command: [/usr/local/bin/run-betacrew-backup-loop.sh] + environment: + PGHOST: 127.0.0.1 + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:-postgres} + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem + BETACREW_BACKUP_ROOT: /backups + BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600} + BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300} + BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: [/tmp:mode=0700,uid=70,gid=70] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + restart: unless-stopped + volumes: + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro" + healthcheck: + test: [CMD, /usr/local/bin/run-betacrew-backup-loop.sh, healthcheck] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/scripts/apply-betacrew-hba.sh b/scripts/apply-betacrew-hba.sh new file mode 100755 index 0000000..01341f7 --- /dev/null +++ b/scripts/apply-betacrew-hba.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +hba_path=${1:?Usage: apply-betacrew-hba.sh } +[[ -f "${hba_path}" && ! -L "${hba_path}" ]] || { echo "HBA path must be a regular non-symlink file." >&2; exit 1; } + +marker='hostnossl betacrew' +if grep -Fq "${marker}" "${hba_path}"; then + for required in \ + 'hostssl betacrew betacrew_app 10.80.0.1/32' \ + 'hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32' \ + 'hostssl keycloak_betacrew all all reject'; do + grep -Fq "${required}" "${hba_path}" || { echo "Partial BetaCrew HBA policy found; refusing to modify it." >&2; exit 1; } + done + exit 0 +fi + +fallback='host all all all scram-sha-256' +[[ $(grep -Fxc "${fallback}" "${hba_path}") == 1 ]] || { echo "Expected exactly one shared HBA fallback rule." >&2; exit 1; } + +temporary=$(mktemp "${hba_path}.betacrew.XXXXXX") +cleanup() { [[ ! -e "${temporary}" ]] || rm -f "${temporary}"; } +trap cleanup EXIT HUP INT TERM + +awk -v fallback="${fallback}" ' + $0 == fallback { + print "# BetaCrew app and identity traffic require TLS and exact source roles." + print "hostnossl betacrew all all reject" + print "hostnossl keycloak_betacrew all all reject" + print "hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256" + print "hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256" + print "hostssl betacrew postgres 127.0.0.1/32 scram-sha-256" + print "hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256" + print "hostssl betacrew all all reject" + print "hostssl keycloak_betacrew all all reject" + } + { print } +' "${hba_path}" > "${temporary}" +if original_mode=$(stat -c '%a' "${hba_path}" 2>/dev/null); then :; else original_mode=$(stat -f '%Lp' "${hba_path}"); fi +chmod "${original_mode}" "${temporary}" +mv "${temporary}" "${hba_path}" diff --git a/scripts/validate-betacrew-config.sh b/scripts/validate-betacrew-config.sh index 0bfb53f..e4ed5fa 100755 --- a/scripts/validate-betacrew-config.sh +++ b/scripts/validate-betacrew-config.sh @@ -10,10 +10,12 @@ require_literal bootstrap/betacrew-databases.sql "CREATE DATABASE keycloak_betac require_literal config/runtrace-pg_hba.conf "hostssl betacrew betacrew_app 10.80.0.1/32" require_literal config/runtrace-pg_hba.conf "hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32" require_literal compose.host.yml "betacrew_backup:" +require_literal compose.betacrew.host.yml "betacrew_backup:" require_literal compose.host.yml "BETACREW_BACKUP_INTERVAL_SECONDS" require_literal compose.host.yml "BETACREW_BACKUP_ENCRYPTION_CERT" require_literal envs/production/.env.db "MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600" require_literal scripts/run-betacrew-backup.sh "-aes-256-gcm" require_literal scripts/verify-betacrew-restore.sh "replace-nonproduction-restore-targets" require_literal scripts/verify-betacrew-restore.sh "--single-transaction" +bash -n "${repo_root}/scripts/apply-betacrew-hba.sh" echo "BetaCrew PostgreSQL configuration is valid." From c33a72d0c890dd50c999a804235f5a4a95f1d1b3 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 13 Aug 2026 23:38:27 +0200 Subject: [PATCH 3/3] Fix host backup tmpfs mount --- compose.betacrew.host.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose.betacrew.host.yml b/compose.betacrew.host.yml index cf6c163..414ab99 100644 --- a/compose.betacrew.host.yml +++ b/compose.betacrew.host.yml @@ -17,7 +17,8 @@ services: BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300} BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35} read_only: true - tmpfs: [/tmp:mode=0700,uid=70,gid=70] + tmpfs: + - /tmp:mode=0700,uid=70,gid=70 cap_drop: [ALL] security_opt: [no-new-privileges:true] restart: unless-stopped