Harden strict substitution workflow

2026-06-17 22:15:54 +02:00 · 2026-06-17 22:15:54 +02:00 · b67c32c2e9
commit b67c32c2e9
parent 7bbee7bf60
8 changed files with 109 additions and 74 deletions
--- a/60
+++ b/60
@ -1,6 +1,6 @@
 # Homelab Blueprint Makefile

-.PHONY: help check-env status logs-caddy deploy-caddy deploy-ha deploy-arr deploy-immich deploy-dns deploy-vault deploy-postgres deploy-gramps deploy-beszel deploy-proxy
+.PHONY: help check-env status deploy-caddy deploy-ha deploy-arr deploy-immich deploy-dns deploy-vault deploy-postgres deploy-gramps deploy-beszel deploy-proxy

 # Default shell
 SHELL := /bin/bash
@ -39,7 +39,9 @@ check-env:
 		if [ -d "$$dir" ]; then \
 			stack=$$(basename $$dir); \
 			local_env="$$dir/.env"; \
-			placeholders=$$(grep -roE '\{\{[A-Za-z_][A-Za-z0-9_]*\}\}' "$$dir" | cut -d: -f2 | sort -u | sed 's/{{//;s/}}//' | grep -v ':-' || true); \
+			placeholders=$$(find "$$dir" -type f \( -name '*template*' -o -name '*.yaml' -o -name '*.yml' \) -print0 | \
+				xargs -0 awk '!/^[[:space:]]*#/ { while (match($$0, /\{\{[A-Za-z_][A-Za-z0-9_]*\}\}/)) { print substr($$0, RSTART + 2, RLENGTH - 4); $$0 = substr($$0, RSTART + RLENGTH) } }' | \
+				sort -u || true); \
 			if [ -n "$$placeholders" ]; then \
 				echo "Validating environment for: $$stack"; \
 				ERR=0; \
@ -65,6 +67,7 @@ status:
 	done

 ip-check:
+	@test -x ./ipfconfig || { echo "Error: ipfconfig script not found or not executable." >&2; exit 1; }
 	@./ipfconfig

 clean:
@ -72,7 +75,17 @@ clean:
 	@rm -f docker-compose.*.yml

 # Individual Stack Deployments
+#
+# deploy_stack(stack-dir, compose-suffix) - shared recipe for the common case:
+#   check-env -> substitute template -> docker compose up -d
+define deploy_stack
+@$(MAKE) check-env STACK=$(1)
+@echo "Deploying $(1)..."
+@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/$(1)/template.yaml docker-compose.$(2).yml $(ENV_FILE) $(STACKS_DIR)/$(1)/.env
+@docker compose -f docker-compose.$(2).yml up -d
+endef

+# Caddy is special: also renders the Caddyfile and forces an image rebuild.
 deploy-caddy:
 	@$(MAKE) check-env STACK=caddy
 	@echo "Deploying Caddy..."
@ -81,61 +94,46 @@ deploy-caddy:
 	@chmod 644 $(STACKS_DIR)/caddy/Caddyfile
 	@docker compose -f docker-compose.caddy.yml up -d --build

+# HA uses an external setup script instead of substitute_env.sh.
 deploy-ha:
+	@test -x ./setup-homeassistant.sh || { echo "Error: setup-homeassistant.sh not found or not executable." >&2; exit 1; }
 	@$(MAKE) check-env STACK=homeassistant
 	@echo "Deploying Home Assistant..."
 	@./setup-homeassistant.sh
 	@docker compose -f docker-compose.homeassistant.yml up -d

 deploy-arr:
-	@$(MAKE) check-env STACK=arr-stack
-	@echo "Deploying Arr Stack..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/arr-stack/template.yaml docker-compose.arr-stack.yml $(ENV_FILE) $(STACKS_DIR)/arr-stack/.env
-	@docker compose -f docker-compose.arr-stack.yml up -d
+	$(call deploy_stack,arr-stack,arr-stack)

 deploy-immich:
-	@$(MAKE) check-env STACK=immich
-	@echo "Deploying Immich..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/immich/template.yaml docker-compose.immich.yml $(ENV_FILE) $(STACKS_DIR)/immich/.env
-	@docker compose -f docker-compose.immich.yml up -d
+	$(call deploy_stack,immich,immich)

 deploy-dns:
-	@$(MAKE) check-env STACK=technitium
-	@echo "Deploying Technitium..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/technitium/template.yaml docker-compose.technitium.yml $(ENV_FILE) $(STACKS_DIR)/technitium/.env
-	@docker compose -f docker-compose.technitium.yml up -d
+	$(call deploy_stack,technitium,technitium)

 deploy-vault:
-	@$(MAKE) check-env STACK=vault
-	@echo "Deploying Vault..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/vault/template.yaml docker-compose.vault.yml $(ENV_FILE) $(STACKS_DIR)/vault/.env
-	@docker compose -f docker-compose.vault.yml up -d
+	$(call deploy_stack,vault,vault)

 deploy-postgres:
-	@$(MAKE) check-env STACK=postgres
-	@echo "Deploying PostgreSQL..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/postgres/template.yaml docker-compose.postgres.yml $(ENV_FILE) $(STACKS_DIR)/postgres/.env
-	@docker compose -f docker-compose.postgres.yml up -d
+	$(call deploy_stack,postgres,postgres)

 deploy-gramps:
 	@$(MAKE) check-env STACK=gramps-web
-	@echo "Deploying Gramps Web..."
+	@echo "Deploying gramps-web..."
+	@mkdir -p gramps_config
+	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/gramps-web/config/config_template.cfg gramps_config/config.cfg $(ENV_FILE) $(STACKS_DIR)/gramps-web/.env
 	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/gramps-web/template.yaml docker-compose.gramps.yml $(ENV_FILE) $(STACKS_DIR)/gramps-web/.env
 	@docker compose -f docker-compose.gramps.yml up -d

 deploy-beszel:
-	@$(MAKE) check-env STACK=beszel-agent
-	@echo "Deploying Beszel Agent..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/beszel-agent/template.yaml docker-compose.beszel.yml $(ENV_FILE) $(STACKS_DIR)/beszel-agent/.env
-	@docker compose -f docker-compose.beszel.yml up -d
+	$(call deploy_stack,beszel-agent,beszel)

 deploy-proxy:
-	@$(MAKE) check-env STACK=socket-proxy
-	@echo "Deploying Socket Proxy..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/socket-proxy/template.yaml docker-compose.proxy.yml $(ENV_FILE) $(STACKS_DIR)/socket-proxy/.env
-	@docker compose -f docker-compose.proxy.yml up -d
+	$(call deploy_stack,socket-proxy,proxy)

 reload-caddy:
+	@docker compose -f docker-compose.caddy.yml ps --status running --services 2>/dev/null | grep -q '^caddy$$' \
+		|| { echo "Error: Caddy container is not running. Run 'make deploy-caddy' first." >&2; exit 1; }
 	@$(MAKE) check-env STACK=caddy
 	@echo "Performing zero-downtime Caddy reload..."
 	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/caddy/Caddyfile_template $(STACKS_DIR)/caddy/Caddyfile $(ENV_FILE) $(STACKS_DIR)/caddy/.env
--- a/README.md
+++ b/README.md
@ -39,7 +39,7 @@ The repository includes a `Makefile` to unify management of all stacks. It suppo

 | Command | Description |
 |---|---|
-| `make check-env` | Validates all `.env` files against placeholders in templates |
+| `make check-env` | Validates root and stack `.env` files against required placeholders in renderable templates |
 | `make status` | Shows `docker compose ps` for all generated compose files |
 | `make reload-caddy` | Zero-downtime reload of the Caddy configuration |
 | `make ip-check` | Verifies VPN connectivity for Arr Stack containers |
@ -65,6 +65,9 @@ docker compose -f docker-compose.<stack>.yml up -d
 Home Assistant uses its setup script instead. Arr Stack, Caddy, and Gramps Web
 render additional configuration templates.

+`substitute_env.sh` only reads env files passed on the command line. It does
+not fall back to OS environment variables or an implicit `.env`.
+
 ## Requirements

 - Linux host with Docker Engine and Docker Compose
@ -83,4 +86,3 @@ Git.
 - [Recyclarr](docs/RECYCLARR.md)
 - [Troubleshooting](docs/TROUBLESHOOTING.md)
 - [UDM Pro SE zones and VLAN migration](docs/network/udm-pro-zones-migration.md)
- zones and VLAN migration](docs/network/udm-pro-zones-migration.md)
--- a/docker-compose-files/gramps-web/README.md
+++ b/docker-compose-files/gramps-web/README.md
@ -24,27 +24,36 @@ stack.
 2. Set the Gramps, OIDC, and email values in `.env`. At minimum configure
   `TREE`, `BASE_URL`, `SECRET_KEY`, and `USER_DB_URI`.

-3. Create the config directory and render both templates:
+3. Deploy with `make`:
+
+   ```bash
+   make deploy-gramps
+   ```
+
+   This creates `gramps_config/config.cfg`, renders
+   `docker-compose.gramps.yml`, and starts the stack.
+
+   Manual equivalent:

   ```bash
   mkdir -p gramps_config
   ./substitute_env.sh docker-compose-files/gramps-web/config/config_template.cfg gramps_config/config.cfg .env
   chmod 644 gramps_config/config.cfg
-   ./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps-web.yml .env
+   ./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps.yml .env
   ```

 4. Validate and start Gramps Web:

   ```bash
-   docker compose -f docker-compose.gramps-web.yml config --quiet
-   docker compose -f docker-compose.gramps-web.yml up -d
+   docker compose -f docker-compose.gramps.yml config --quiet
+   docker compose -f docker-compose.gramps.yml up -d
   ```

 5. Verify the containers and open `http://<HOST_IP>`:

   ```bash
-   docker compose -f docker-compose.gramps-web.yml ps
-   docker compose -f docker-compose.gramps-web.yml logs --tail=100
+   docker compose -f docker-compose.gramps.yml ps
+   docker compose -f docker-compose.gramps.yml logs --tail=100
   ```

 ## Apply Configuration Changes
@ -52,8 +61,8 @@ stack.
 ```bash
 ./substitute_env.sh docker-compose-files/gramps-web/config/config_template.cfg gramps_config/config.cfg .env
 chmod 644 gramps_config/config.cfg
-./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps-web.yml .env
-docker compose -f docker-compose.gramps-web.yml up -d
+./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps.yml .env
+docker compose -f docker-compose.gramps.yml up -d
 ```

 ## Initial Setup
@ -74,7 +83,7 @@ writable by the container before starting the stack.
 ## Useful Commands

 ```bash
-docker compose -f docker-compose.gramps-web.yml restart grampsweb
+docker compose -f docker-compose.gramps.yml restart grampsweb
 ```

 ## Backup
--- a/docker-compose-files/immich/README.md
+++ b/docker-compose-files/immich/README.md
@ -22,6 +22,19 @@ Self-hosted photo and video management solution with high performance and visual
    IMMICH_REDIS_HOSTNAME=immich-redis
    ```

+    Optional OIDC settings can also be supplied. Set `OIDC_ENABLED=true`
+    explicitly when enabling OAuth; leaving it unset keeps Immich local-login
+    only.
+    ```dotenv
+    OIDC_ENABLED=true
+    OIDC_ISSUER=https://auth.example.com/application/o/immich/
+    OIDC_CLIENT_ID=<immich-client-id>
+    OIDC_CLIENT_SECRET=<immich-client-secret>
+    OIDC_NAME=Single Sign-On
+    OIDC_AUTO_REDIRECT=false
+    OIDC_MOBILE_OVERRIDE_ENABLED=false
+    ```
+
 2.  **Create Directories**:
    Create the local appdata directories and the shared media directory:
    ```bash
--- a/docker-compose-files/immich/template.yaml
+++ b/docker-compose-files/immich/template.yaml
@ -8,8 +8,6 @@ services:
    volumes:
      - {{IMMICH_UPLOAD_LOCATION}}:/usr/src/app/upload
      - /etc/localtime:/etc/localtime:ro
-    env_file:
-      - .env
    environment:
      - DB_PASSWORD={{IMMICH_DB_PASSWORD}}
      - DB_USERNAME={{IMMICH_DB_USERNAME}}
@ -17,6 +15,20 @@ services:
      - DB_HOSTNAME={{IMMICH_DB_HOSTNAME}}
      - REDIS_HOSTNAME={{IMMICH_REDIS_HOSTNAME}}
      - TZ={{TZ}}
+      # OIDC (optional - leave disabled unless explicitly configured)
+      - OAUTH_ENABLED={{OIDC_ENABLED:-false}}
+      - OAUTH_ISSUER_URL={{OIDC_ISSUER:-}}
+      - OAUTH_CLIENT_ID={{OIDC_CLIENT_ID:-}}
+      - OAUTH_CLIENT_SECRET={{OIDC_CLIENT_SECRET:-}}
+      - OAUTH_BUTTON_TEXT={{OIDC_NAME:-}}
+      - OAUTH_AUTO_LAUNCH={{OIDC_AUTO_REDIRECT:-}}
+      - OAUTH_MOBILE_OVERRIDE_ENABLED={{OIDC_MOBILE_OVERRIDE_ENABLED:-false}}
+      # Email (optional - leave blank to disable)
+      - SMTP_HOST={{EMAIL_HOST:-}}
+      - SMTP_PORT={{EMAIL_PORT:-}}
+      - SMTP_USERNAME={{EMAIL_HOST_USER:-}}
+      - SMTP_PASSWORD={{EMAIL_HOST_PASSWORD:-}}
+      - SMTP_FROM={{DEFAULT_FROM_EMAIL:-}}
    ports:
      - "2283:2283"
    depends_on:
@ -41,8 +53,6 @@ services:
    runtime: nvidia
    volumes:
      - {{DOCKERCONFDIR}}/immich/model-cache:/cache
-    env_file:
-      - .env
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
--- a/docs/RECYCLARR.md
+++ b/docs/RECYCLARR.md
@ -8,7 +8,7 @@ This repository now uses Recyclarr **v8** with guide-backed quality profiles (no
 ### 1. Generate `recyclarr.yml`

 ```bash
-./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml
+./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env
 nano /opt/arr-stack/appdata/recyclarr/recyclarr.yml
 ```

@ -150,10 +150,10 @@ Fix:

 ```bash
 # Regenerate v8 config from current template
-./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml
+./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env

 # Recreate recyclarr with v8 image from compose template
-./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml
+./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env
 docker-compose -f docker-compose.arr-stack.yml pull recyclarr
 docker-compose -f docker-compose.arr-stack.yml up -d recyclarr
 ```
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@ -23,7 +23,7 @@ Symptoms:
 Fix:

 ```bash
-./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml
+./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env
 ```

 The script prints any unresolved placeholders. Add missing keys to `.env` (or provide defaults in templates).
@ -45,7 +45,7 @@ radarr:
 Generate your file from this repo template:

 ```bash
-./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml
+./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env
 ```

 ### 2b. Recyclarr errors (`Unable to find include template ...`)
@ -64,10 +64,10 @@ Fix (v8 path used by this repository):

 ```bash
 # 1) Regenerate recyclarr config from the v8 template
-./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml
+./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env

 # 2) Regenerate compose and recreate recyclarr (v8.6.0 pin)
-./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml
+./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env
 docker-compose -f docker-compose.arr-stack.yml pull recyclarr
 docker-compose -f docker-compose.arr-stack.yml up -d recyclarr
 ```
--- a/substitute_env.sh
+++ b/substitute_env.sh
@ -8,7 +8,7 @@ set -euo pipefail

 # Ensure we have Bash 4+ for associative arrays
 if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then
-    echo "Error: This script requires Bash 4.0 or higher."
+    echo "Error: This script requires Bash 4.0 or higher." >&2
    exit 1
 fi

@ -23,8 +23,8 @@ trim() {

 # 1. Validation: Check arguments
 if [[ $# -lt 2 ]]; then
-    echo "Usage: $0 <source_template_file> <output_destination_file> [env_file1] [env_file2] ..."
-    echo "Example: $0 config_template.cfg config.cfg .env global.env"
+    echo "Usage: $0 <source_template_file> <output_destination_file> [env_file1] [env_file2] ..." >&2
+    echo "Example: $0 config_template.cfg config.cfg .env global.env" >&2
    exit 1
 fi

@ -32,19 +32,19 @@ SOURCE_FILE="$1"
 OUTPUT_FILE="$2"

 if [[ ! -f "$SOURCE_FILE" ]]; then
-    echo "Error: Source file '$SOURCE_FILE' not found."
+    echo "Error: Source file '$SOURCE_FILE' not found." >&2
    exit 1
 fi

 if [[ "$SOURCE_FILE" == "$OUTPUT_FILE" ]]; then
-    echo "Error: Source and output files must be different."
+    echo "Error: Source and output files must be different." >&2
    exit 1
 fi

 load_env() {
    local env_file="$1"
    if [[ -f "$env_file" ]]; then
-        echo "Info: Loading environment from $env_file"
+        echo "Info: Loading environment from $env_file" >&2
        while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do
            line="${raw_line%$'\r'}"
            line="$(trim "$line")"
@ -53,7 +53,7 @@ load_env() {
            [[ -z "$line" || "$line" == \#* ]] && continue

            if [[ "$line" != *"="* ]]; then
-                echo "Warning: Skipping invalid line in $env_file: $line"
+                echo "Warning: Skipping invalid line in $env_file: $line" >&2
                continue
            fi

@ -63,7 +63,7 @@ load_env() {
            key="$(trim "$key")"

            if [[ ! "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
-                echo "Warning: Skipping invalid variable name '$key' in $env_file"
+                echo "Warning: Skipping invalid variable name '$key' in $env_file" >&2
                continue
            fi

@ -85,7 +85,7 @@ load_env() {
            LOADED_VARS["$key"]="$value"
        done <"$env_file"
    else
-        echo "Notice: $env_file not found; skipping."
+        echo "Notice: $env_file not found; skipping." >&2
    fi
 }

@ -94,8 +94,6 @@ if [[ $# -ge 3 ]]; then
    for (( i=3; i<=$#; i++ )); do
        load_env "${!i}"
    done
-elif [[ -f ".env" ]]; then
-    load_env ".env"
 fi

 # 3. Process the template (handles {{VAR}} and {{VAR:-default}})
@ -128,12 +126,17 @@ render_template() {
            elif [[ -n "$has_default" ]]; then
                replacement="$default"
            else
+                # Leave placeholder intact so post-render validation can report it.
                replacement="$full"
            fi

+            # %% finds the first occurrence of $full by removing the longest
+            # suffix that starts with it, then #* advances past it.
            before="${rest%%"$full"*}"
            rendered+="$before$replacement"
            rest="${rest#*"$full"}"
+            # Replacements are appended to `rendered`, which is never re-scanned.
+            # Values containing {{...}} are passed through verbatim (no nested expansion).
        done
        line="$rendered$rest"
        printf '%s\n' "$line" >>"$target"
@ -143,23 +146,23 @@ render_template() {
 TMP_OUTPUT="$(mktemp)"
 trap 'rm -f "$TMP_OUTPUT"' EXIT
 render_template "$SOURCE_FILE" "$TMP_OUTPUT"
-mv "$TMP_OUTPUT" "$OUTPUT_FILE"
-trap - EXIT

 # 4. Final validation: check for unreplaced placeholders
 if [[ "$SOURCE_FILE" =~ \.ya?ml$ ]]; then
-    MISSING_VARS="$(awk '!/^[[:space:]]*#/' "$OUTPUT_FILE" | grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' | sort -u || true)"
+    MISSING_VARS="$(awk '!/^[[:space:]]*#/' "$TMP_OUTPUT" | grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' | sort -u || true)"
 else
-    MISSING_VARS="$(grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' "$OUTPUT_FILE" | sort -u || true)"
+    MISSING_VARS="$(grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' "$TMP_OUTPUT" | sort -u || true)"
 fi

 if [[ -n "$MISSING_VARS" ]]; then
-    echo "--------------------------------------------------------"
-    echo "ERROR: The following placeholders were not replaced:"
-    echo "$MISSING_VARS"
-    echo "These MUST be defined in your .env files (OS environment ignored)."
-    echo "--------------------------------------------------------"
+    echo "--------------------------------------------------------" >&2
+    echo "ERROR: The following placeholders were not replaced:" >&2
+    echo "$MISSING_VARS" >&2
+    echo "These MUST be defined in your .env files (OS environment ignored)." >&2
+    echo "--------------------------------------------------------" >&2
    exit 1
 else
-    echo "Success: All variables substituted in $OUTPUT_FILE"
+    mv "$TMP_OUTPUT" "$OUTPUT_FILE"
+    trap - EXIT
+    echo "Success: All variables substituted in $OUTPUT_FILE" >&2
 fi