From b67c32c2e9949bb44e7d70921ccadb14b6ea405e Mon Sep 17 00:00:00 2001 From: Brian Pooe Date: Wed, 17 Jun 2026 22:15:54 +0200 Subject: [PATCH] Harden strict substitution workflow --- Makefile | 60 +++++++++++------------ README.md | 6 ++- docker-compose-files/gramps-web/README.md | 27 ++++++---- docker-compose-files/immich/README.md | 13 +++++ docker-compose-files/immich/template.yaml | 18 +++++-- docs/RECYCLARR.md | 6 +-- docs/TROUBLESHOOTING.md | 8 +-- substitute_env.sh | 45 +++++++++-------- 8 files changed, 109 insertions(+), 74 deletions(-) diff --git a/Makefile b/Makefile index ff8c510..8bdf08a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Homelab Blueprint Makefile -.PHONY: help check-env status logs-caddy deploy-caddy deploy-ha deploy-arr deploy-immich deploy-dns deploy-vault deploy-postgres deploy-gramps deploy-beszel deploy-proxy +.PHONY: help check-env status deploy-caddy deploy-ha deploy-arr deploy-immich deploy-dns deploy-vault deploy-postgres deploy-gramps deploy-beszel deploy-proxy # Default shell SHELL := /bin/bash @@ -39,7 +39,9 @@ check-env: if [ -d "$$dir" ]; then \ stack=$$(basename $$dir); \ local_env="$$dir/.env"; \ - placeholders=$$(grep -roE '\{\{[A-Za-z_][A-Za-z0-9_]*\}\}' "$$dir" | cut -d: -f2 | sort -u | sed 's/{{//;s/}}//' | grep -v ':-' || true); \ + placeholders=$$(find "$$dir" -type f \( -name '*template*' -o -name '*.yaml' -o -name '*.yml' \) -print0 | \ + xargs -0 awk '!/^[[:space:]]*#/ { while (match($$0, /\{\{[A-Za-z_][A-Za-z0-9_]*\}\}/)) { print substr($$0, RSTART + 2, RLENGTH - 4); $$0 = substr($$0, RSTART + RLENGTH) } }' | \ + sort -u || true); \ if [ -n "$$placeholders" ]; then \ echo "Validating environment for: $$stack"; \ ERR=0; \ @@ -65,6 +67,7 @@ status: done ip-check: + @test -x ./ipfconfig || { echo "Error: ipfconfig script not found or not executable." >&2; exit 1; } @./ipfconfig clean: @@ -72,7 +75,17 @@ clean: @rm -f docker-compose.*.yml # Individual Stack Deployments +# +# deploy_stack(stack-dir, compose-suffix) - shared recipe for the common case: +# check-env -> substitute template -> docker compose up -d +define deploy_stack +@$(MAKE) check-env STACK=$(1) +@echo "Deploying $(1)..." +@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/$(1)/template.yaml docker-compose.$(2).yml $(ENV_FILE) $(STACKS_DIR)/$(1)/.env +@docker compose -f docker-compose.$(2).yml up -d +endef +# Caddy is special: also renders the Caddyfile and forces an image rebuild. deploy-caddy: @$(MAKE) check-env STACK=caddy @echo "Deploying Caddy..." @@ -81,61 +94,46 @@ deploy-caddy: @chmod 644 $(STACKS_DIR)/caddy/Caddyfile @docker compose -f docker-compose.caddy.yml up -d --build +# HA uses an external setup script instead of substitute_env.sh. deploy-ha: + @test -x ./setup-homeassistant.sh || { echo "Error: setup-homeassistant.sh not found or not executable." >&2; exit 1; } @$(MAKE) check-env STACK=homeassistant @echo "Deploying Home Assistant..." @./setup-homeassistant.sh @docker compose -f docker-compose.homeassistant.yml up -d deploy-arr: - @$(MAKE) check-env STACK=arr-stack - @echo "Deploying Arr Stack..." - @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/arr-stack/template.yaml docker-compose.arr-stack.yml $(ENV_FILE) $(STACKS_DIR)/arr-stack/.env - @docker compose -f docker-compose.arr-stack.yml up -d + $(call deploy_stack,arr-stack,arr-stack) deploy-immich: - @$(MAKE) check-env STACK=immich - @echo "Deploying Immich..." - @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/immich/template.yaml docker-compose.immich.yml $(ENV_FILE) $(STACKS_DIR)/immich/.env - @docker compose -f docker-compose.immich.yml up -d + $(call deploy_stack,immich,immich) deploy-dns: - @$(MAKE) check-env STACK=technitium - @echo "Deploying Technitium..." - @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/technitium/template.yaml docker-compose.technitium.yml $(ENV_FILE) $(STACKS_DIR)/technitium/.env - @docker compose -f docker-compose.technitium.yml up -d + $(call deploy_stack,technitium,technitium) deploy-vault: - @$(MAKE) check-env STACK=vault - @echo "Deploying Vault..." - @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/vault/template.yaml docker-compose.vault.yml $(ENV_FILE) $(STACKS_DIR)/vault/.env - @docker compose -f docker-compose.vault.yml up -d + $(call deploy_stack,vault,vault) deploy-postgres: - @$(MAKE) check-env STACK=postgres - @echo "Deploying PostgreSQL..." - @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/postgres/template.yaml docker-compose.postgres.yml $(ENV_FILE) $(STACKS_DIR)/postgres/.env - @docker compose -f docker-compose.postgres.yml up -d + $(call deploy_stack,postgres,postgres) deploy-gramps: @$(MAKE) check-env STACK=gramps-web - @echo "Deploying Gramps Web..." + @echo "Deploying gramps-web..." + @mkdir -p gramps_config + @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/gramps-web/config/config_template.cfg gramps_config/config.cfg $(ENV_FILE) $(STACKS_DIR)/gramps-web/.env @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/gramps-web/template.yaml docker-compose.gramps.yml $(ENV_FILE) $(STACKS_DIR)/gramps-web/.env @docker compose -f docker-compose.gramps.yml up -d deploy-beszel: - @$(MAKE) check-env STACK=beszel-agent - @echo "Deploying Beszel Agent..." - @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/beszel-agent/template.yaml docker-compose.beszel.yml $(ENV_FILE) $(STACKS_DIR)/beszel-agent/.env - @docker compose -f docker-compose.beszel.yml up -d + $(call deploy_stack,beszel-agent,beszel) deploy-proxy: - @$(MAKE) check-env STACK=socket-proxy - @echo "Deploying Socket Proxy..." - @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/socket-proxy/template.yaml docker-compose.proxy.yml $(ENV_FILE) $(STACKS_DIR)/socket-proxy/.env - @docker compose -f docker-compose.proxy.yml up -d + $(call deploy_stack,socket-proxy,proxy) reload-caddy: + @docker compose -f docker-compose.caddy.yml ps --status running --services 2>/dev/null | grep -q '^caddy$$' \ + || { echo "Error: Caddy container is not running. Run 'make deploy-caddy' first." >&2; exit 1; } @$(MAKE) check-env STACK=caddy @echo "Performing zero-downtime Caddy reload..." @$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/caddy/Caddyfile_template $(STACKS_DIR)/caddy/Caddyfile $(ENV_FILE) $(STACKS_DIR)/caddy/.env diff --git a/README.md b/README.md index 9f4c404..9c5220d 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The repository includes a `Makefile` to unify management of all stacks. It suppo | Command | Description | |---|---| -| `make check-env` | Validates all `.env` files against placeholders in templates | +| `make check-env` | Validates root and stack `.env` files against required placeholders in renderable templates | | `make status` | Shows `docker compose ps` for all generated compose files | | `make reload-caddy` | Zero-downtime reload of the Caddy configuration | | `make ip-check` | Verifies VPN connectivity for Arr Stack containers | @@ -65,6 +65,9 @@ docker compose -f docker-compose..yml up -d Home Assistant uses its setup script instead. Arr Stack, Caddy, and Gramps Web render additional configuration templates. +`substitute_env.sh` only reads env files passed on the command line. It does +not fall back to OS environment variables or an implicit `.env`. + ## Requirements - Linux host with Docker Engine and Docker Compose @@ -83,4 +86,3 @@ Git. - [Recyclarr](docs/RECYCLARR.md) - [Troubleshooting](docs/TROUBLESHOOTING.md) - [UDM Pro SE zones and VLAN migration](docs/network/udm-pro-zones-migration.md) - zones and VLAN migration](docs/network/udm-pro-zones-migration.md) diff --git a/docker-compose-files/gramps-web/README.md b/docker-compose-files/gramps-web/README.md index ddefe98..e60c8ed 100644 --- a/docker-compose-files/gramps-web/README.md +++ b/docker-compose-files/gramps-web/README.md @@ -24,27 +24,36 @@ stack. 2. Set the Gramps, OIDC, and email values in `.env`. At minimum configure `TREE`, `BASE_URL`, `SECRET_KEY`, and `USER_DB_URI`. -3. Create the config directory and render both templates: +3. Deploy with `make`: + + ```bash + make deploy-gramps + ``` + + This creates `gramps_config/config.cfg`, renders + `docker-compose.gramps.yml`, and starts the stack. + + Manual equivalent: ```bash mkdir -p gramps_config ./substitute_env.sh docker-compose-files/gramps-web/config/config_template.cfg gramps_config/config.cfg .env chmod 644 gramps_config/config.cfg - ./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps-web.yml .env + ./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps.yml .env ``` 4. Validate and start Gramps Web: ```bash - docker compose -f docker-compose.gramps-web.yml config --quiet - docker compose -f docker-compose.gramps-web.yml up -d + docker compose -f docker-compose.gramps.yml config --quiet + docker compose -f docker-compose.gramps.yml up -d ``` 5. Verify the containers and open `http://`: ```bash - docker compose -f docker-compose.gramps-web.yml ps - docker compose -f docker-compose.gramps-web.yml logs --tail=100 + docker compose -f docker-compose.gramps.yml ps + docker compose -f docker-compose.gramps.yml logs --tail=100 ``` ## Apply Configuration Changes @@ -52,8 +61,8 @@ stack. ```bash ./substitute_env.sh docker-compose-files/gramps-web/config/config_template.cfg gramps_config/config.cfg .env chmod 644 gramps_config/config.cfg -./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps-web.yml .env -docker compose -f docker-compose.gramps-web.yml up -d +./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps.yml .env +docker compose -f docker-compose.gramps.yml up -d ``` ## Initial Setup @@ -74,7 +83,7 @@ writable by the container before starting the stack. ## Useful Commands ```bash -docker compose -f docker-compose.gramps-web.yml restart grampsweb +docker compose -f docker-compose.gramps.yml restart grampsweb ``` ## Backup diff --git a/docker-compose-files/immich/README.md b/docker-compose-files/immich/README.md index ea80103..65d1259 100644 --- a/docker-compose-files/immich/README.md +++ b/docker-compose-files/immich/README.md @@ -22,6 +22,19 @@ Self-hosted photo and video management solution with high performance and visual IMMICH_REDIS_HOSTNAME=immich-redis ``` + Optional OIDC settings can also be supplied. Set `OIDC_ENABLED=true` + explicitly when enabling OAuth; leaving it unset keeps Immich local-login + only. + ```dotenv + OIDC_ENABLED=true + OIDC_ISSUER=https://auth.example.com/application/o/immich/ + OIDC_CLIENT_ID= + OIDC_CLIENT_SECRET= + OIDC_NAME=Single Sign-On + OIDC_AUTO_REDIRECT=false + OIDC_MOBILE_OVERRIDE_ENABLED=false + ``` + 2. **Create Directories**: Create the local appdata directories and the shared media directory: ```bash diff --git a/docker-compose-files/immich/template.yaml b/docker-compose-files/immich/template.yaml index 4969721..704c727 100644 --- a/docker-compose-files/immich/template.yaml +++ b/docker-compose-files/immich/template.yaml @@ -8,8 +8,6 @@ services: volumes: - {{IMMICH_UPLOAD_LOCATION}}:/usr/src/app/upload - /etc/localtime:/etc/localtime:ro - env_file: - - .env environment: - DB_PASSWORD={{IMMICH_DB_PASSWORD}} - DB_USERNAME={{IMMICH_DB_USERNAME}} @@ -17,6 +15,20 @@ services: - DB_HOSTNAME={{IMMICH_DB_HOSTNAME}} - REDIS_HOSTNAME={{IMMICH_REDIS_HOSTNAME}} - TZ={{TZ}} + # OIDC (optional - leave disabled unless explicitly configured) + - OAUTH_ENABLED={{OIDC_ENABLED:-false}} + - OAUTH_ISSUER_URL={{OIDC_ISSUER:-}} + - OAUTH_CLIENT_ID={{OIDC_CLIENT_ID:-}} + - OAUTH_CLIENT_SECRET={{OIDC_CLIENT_SECRET:-}} + - OAUTH_BUTTON_TEXT={{OIDC_NAME:-}} + - OAUTH_AUTO_LAUNCH={{OIDC_AUTO_REDIRECT:-}} + - OAUTH_MOBILE_OVERRIDE_ENABLED={{OIDC_MOBILE_OVERRIDE_ENABLED:-false}} + # Email (optional - leave blank to disable) + - SMTP_HOST={{EMAIL_HOST:-}} + - SMTP_PORT={{EMAIL_PORT:-}} + - SMTP_USERNAME={{EMAIL_HOST_USER:-}} + - SMTP_PASSWORD={{EMAIL_HOST_PASSWORD:-}} + - SMTP_FROM={{DEFAULT_FROM_EMAIL:-}} ports: - "2283:2283" depends_on: @@ -41,8 +53,6 @@ services: runtime: nvidia volumes: - {{DOCKERCONFDIR}}/immich/model-cache:/cache - env_file: - - .env environment: - NVIDIA_VISIBLE_DEVICES=all - NVIDIA_DRIVER_CAPABILITIES=compute,utility diff --git a/docs/RECYCLARR.md b/docs/RECYCLARR.md index 9a82d9f..119cee9 100644 --- a/docs/RECYCLARR.md +++ b/docs/RECYCLARR.md @@ -8,7 +8,7 @@ This repository now uses Recyclarr **v8** with guide-backed quality profiles (no ### 1. Generate `recyclarr.yml` ```bash -./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml +./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env nano /opt/arr-stack/appdata/recyclarr/recyclarr.yml ``` @@ -150,10 +150,10 @@ Fix: ```bash # Regenerate v8 config from current template -./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml +./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env # Recreate recyclarr with v8 image from compose template -./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml +./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env docker-compose -f docker-compose.arr-stack.yml pull recyclarr docker-compose -f docker-compose.arr-stack.yml up -d recyclarr ``` diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 805fcc6..a146529 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -23,7 +23,7 @@ Symptoms: Fix: ```bash -./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml +./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env ``` The script prints any unresolved placeholders. Add missing keys to `.env` (or provide defaults in templates). @@ -45,7 +45,7 @@ radarr: Generate your file from this repo template: ```bash -./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml +./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env ``` ### 2b. Recyclarr errors (`Unable to find include template ...`) @@ -64,10 +64,10 @@ Fix (v8 path used by this repository): ```bash # 1) Regenerate recyclarr config from the v8 template -./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml +./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env # 2) Regenerate compose and recreate recyclarr (v8.6.0 pin) -./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml +./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env docker-compose -f docker-compose.arr-stack.yml pull recyclarr docker-compose -f docker-compose.arr-stack.yml up -d recyclarr ``` diff --git a/substitute_env.sh b/substitute_env.sh index 26574b2..17efd31 100755 --- a/substitute_env.sh +++ b/substitute_env.sh @@ -8,7 +8,7 @@ set -euo pipefail # Ensure we have Bash 4+ for associative arrays if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then - echo "Error: This script requires Bash 4.0 or higher." + echo "Error: This script requires Bash 4.0 or higher." >&2 exit 1 fi @@ -23,8 +23,8 @@ trim() { # 1. Validation: Check arguments if [[ $# -lt 2 ]]; then - echo "Usage: $0 [env_file1] [env_file2] ..." - echo "Example: $0 config_template.cfg config.cfg .env global.env" + echo "Usage: $0 [env_file1] [env_file2] ..." >&2 + echo "Example: $0 config_template.cfg config.cfg .env global.env" >&2 exit 1 fi @@ -32,19 +32,19 @@ SOURCE_FILE="$1" OUTPUT_FILE="$2" if [[ ! -f "$SOURCE_FILE" ]]; then - echo "Error: Source file '$SOURCE_FILE' not found." + echo "Error: Source file '$SOURCE_FILE' not found." >&2 exit 1 fi if [[ "$SOURCE_FILE" == "$OUTPUT_FILE" ]]; then - echo "Error: Source and output files must be different." + echo "Error: Source and output files must be different." >&2 exit 1 fi load_env() { local env_file="$1" if [[ -f "$env_file" ]]; then - echo "Info: Loading environment from $env_file" + echo "Info: Loading environment from $env_file" >&2 while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do line="${raw_line%$'\r'}" line="$(trim "$line")" @@ -53,7 +53,7 @@ load_env() { [[ -z "$line" || "$line" == \#* ]] && continue if [[ "$line" != *"="* ]]; then - echo "Warning: Skipping invalid line in $env_file: $line" + echo "Warning: Skipping invalid line in $env_file: $line" >&2 continue fi @@ -63,7 +63,7 @@ load_env() { key="$(trim "$key")" if [[ ! "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then - echo "Warning: Skipping invalid variable name '$key' in $env_file" + echo "Warning: Skipping invalid variable name '$key' in $env_file" >&2 continue fi @@ -85,7 +85,7 @@ load_env() { LOADED_VARS["$key"]="$value" done <"$env_file" else - echo "Notice: $env_file not found; skipping." + echo "Notice: $env_file not found; skipping." >&2 fi } @@ -94,8 +94,6 @@ if [[ $# -ge 3 ]]; then for (( i=3; i<=$#; i++ )); do load_env "${!i}" done -elif [[ -f ".env" ]]; then - load_env ".env" fi # 3. Process the template (handles {{VAR}} and {{VAR:-default}}) @@ -128,12 +126,17 @@ render_template() { elif [[ -n "$has_default" ]]; then replacement="$default" else + # Leave placeholder intact so post-render validation can report it. replacement="$full" fi + # %% finds the first occurrence of $full by removing the longest + # suffix that starts with it, then #* advances past it. before="${rest%%"$full"*}" rendered+="$before$replacement" rest="${rest#*"$full"}" + # Replacements are appended to `rendered`, which is never re-scanned. + # Values containing {{...}} are passed through verbatim (no nested expansion). done line="$rendered$rest" printf '%s\n' "$line" >>"$target" @@ -143,23 +146,23 @@ render_template() { TMP_OUTPUT="$(mktemp)" trap 'rm -f "$TMP_OUTPUT"' EXIT render_template "$SOURCE_FILE" "$TMP_OUTPUT" -mv "$TMP_OUTPUT" "$OUTPUT_FILE" -trap - EXIT # 4. Final validation: check for unreplaced placeholders if [[ "$SOURCE_FILE" =~ \.ya?ml$ ]]; then - MISSING_VARS="$(awk '!/^[[:space:]]*#/' "$OUTPUT_FILE" | grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' | sort -u || true)" + MISSING_VARS="$(awk '!/^[[:space:]]*#/' "$TMP_OUTPUT" | grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' | sort -u || true)" else - MISSING_VARS="$(grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' "$OUTPUT_FILE" | sort -u || true)" + MISSING_VARS="$(grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' "$TMP_OUTPUT" | sort -u || true)" fi if [[ -n "$MISSING_VARS" ]]; then - echo "--------------------------------------------------------" - echo "ERROR: The following placeholders were not replaced:" - echo "$MISSING_VARS" - echo "These MUST be defined in your .env files (OS environment ignored)." - echo "--------------------------------------------------------" + echo "--------------------------------------------------------" >&2 + echo "ERROR: The following placeholders were not replaced:" >&2 + echo "$MISSING_VARS" >&2 + echo "These MUST be defined in your .env files (OS environment ignored)." >&2 + echo "--------------------------------------------------------" >&2 exit 1 else - echo "Success: All variables substituted in $OUTPUT_FILE" + mv "$TMP_OUTPUT" "$OUTPUT_FILE" + trap - EXIT + echo "Success: All variables substituted in $OUTPUT_FILE" >&2 fi