Harden strict substitution workflow

Makefile

@@ -1,6 +1,6 @@
 # Homelab Blueprint Makefile
 
-.PHONY: help check-env status logs-caddy deploy-caddy deploy-ha deploy-arr deploy-immich deploy-dns deploy-vault deploy-postgres deploy-gramps deploy-beszel deploy-proxy
+.PHONY: help check-env status deploy-caddy deploy-ha deploy-arr deploy-immich deploy-dns deploy-vault deploy-postgres deploy-gramps deploy-beszel deploy-proxy
 
 # Default shell
 SHELL := /bin/bash
@@ -39,7 +39,9 @@ check-env:
 		if [ -d "$$dir" ]; then \
 			stack=$$(basename $$dir); \
 			local_env="$$dir/.env"; \
-			placeholders=$$(grep -roE '\{\{[A-Za-z_][A-Za-z0-9_]*\}\}' "$$dir" | cut -d: -f2 | sort -u | sed 's/{{//;s/}}//' | grep -v ':-' || true); \
+			placeholders=$$(find "$$dir" -type f \( -name '*template*' -o -name '*.yaml' -o -name '*.yml' \) -print0 | \
+				xargs -0 awk '!/^[[:space:]]*#/ { while (match($$0, /\{\{[A-Za-z_][A-Za-z0-9_]*\}\}/)) { print substr($$0, RSTART + 2, RLENGTH - 4); $$0 = substr($$0, RSTART + RLENGTH) } }' | \
+				sort -u || true); \
 			if [ -n "$$placeholders" ]; then \
 				echo "Validating environment for: $$stack"; \
 				ERR=0; \
@@ -65,6 +67,7 @@ status:
 	done
 
 ip-check:
+	@test -x ./ipfconfig || { echo "Error: ipfconfig script not found or not executable." >&2; exit 1; }
 	@./ipfconfig
 
 clean:
@@ -72,7 +75,17 @@ clean:
 	@rm -f docker-compose.*.yml
 
 # Individual Stack Deployments
+#
+# deploy_stack(stack-dir, compose-suffix) - shared recipe for the common case:
+#   check-env -> substitute template -> docker compose up -d
+define deploy_stack
+@$(MAKE) check-env STACK=$(1)
+@echo "Deploying $(1)..."
+@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/$(1)/template.yaml docker-compose.$(2).yml $(ENV_FILE) $(STACKS_DIR)/$(1)/.env
+@docker compose -f docker-compose.$(2).yml up -d
+endef
 
+# Caddy is special: also renders the Caddyfile and forces an image rebuild.
 deploy-caddy:
 	@$(MAKE) check-env STACK=caddy
 	@echo "Deploying Caddy..."
@@ -81,61 +94,46 @@ deploy-caddy:
 	@chmod 644 $(STACKS_DIR)/caddy/Caddyfile
 	@docker compose -f docker-compose.caddy.yml up -d --build
 
+# HA uses an external setup script instead of substitute_env.sh.
 deploy-ha:
+	@test -x ./setup-homeassistant.sh || { echo "Error: setup-homeassistant.sh not found or not executable." >&2; exit 1; }
 	@$(MAKE) check-env STACK=homeassistant
 	@echo "Deploying Home Assistant..."
 	@./setup-homeassistant.sh
 	@docker compose -f docker-compose.homeassistant.yml up -d
 
 deploy-arr:
-	@$(MAKE) check-env STACK=arr-stack
+	$(call deploy_stack,arr-stack,arr-stack)
-	@echo "Deploying Arr Stack..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/arr-stack/template.yaml docker-compose.arr-stack.yml $(ENV_FILE) $(STACKS_DIR)/arr-stack/.env
-	@docker compose -f docker-compose.arr-stack.yml up -d
 
 deploy-immich:
-	@$(MAKE) check-env STACK=immich
+	$(call deploy_stack,immich,immich)
-	@echo "Deploying Immich..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/immich/template.yaml docker-compose.immich.yml $(ENV_FILE) $(STACKS_DIR)/immich/.env
-	@docker compose -f docker-compose.immich.yml up -d
 
 deploy-dns:
-	@$(MAKE) check-env STACK=technitium
+	$(call deploy_stack,technitium,technitium)
-	@echo "Deploying Technitium..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/technitium/template.yaml docker-compose.technitium.yml $(ENV_FILE) $(STACKS_DIR)/technitium/.env
-	@docker compose -f docker-compose.technitium.yml up -d
 
 deploy-vault:
-	@$(MAKE) check-env STACK=vault
+	$(call deploy_stack,vault,vault)
-	@echo "Deploying Vault..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/vault/template.yaml docker-compose.vault.yml $(ENV_FILE) $(STACKS_DIR)/vault/.env
-	@docker compose -f docker-compose.vault.yml up -d
 
 deploy-postgres:
-	@$(MAKE) check-env STACK=postgres
+	$(call deploy_stack,postgres,postgres)
-	@echo "Deploying PostgreSQL..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/postgres/template.yaml docker-compose.postgres.yml $(ENV_FILE) $(STACKS_DIR)/postgres/.env
-	@docker compose -f docker-compose.postgres.yml up -d
 
 deploy-gramps:
 	@$(MAKE) check-env STACK=gramps-web
-	@echo "Deploying Gramps Web..."
+	@echo "Deploying gramps-web..."
+	@mkdir -p gramps_config
+	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/gramps-web/config/config_template.cfg gramps_config/config.cfg $(ENV_FILE) $(STACKS_DIR)/gramps-web/.env
 	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/gramps-web/template.yaml docker-compose.gramps.yml $(ENV_FILE) $(STACKS_DIR)/gramps-web/.env
 	@docker compose -f docker-compose.gramps.yml up -d
 
 deploy-beszel:
-	@$(MAKE) check-env STACK=beszel-agent
+	$(call deploy_stack,beszel-agent,beszel)
-	@echo "Deploying Beszel Agent..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/beszel-agent/template.yaml docker-compose.beszel.yml $(ENV_FILE) $(STACKS_DIR)/beszel-agent/.env
-	@docker compose -f docker-compose.beszel.yml up -d
 
 deploy-proxy:
-	@$(MAKE) check-env STACK=socket-proxy
+	$(call deploy_stack,socket-proxy,proxy)
-	@echo "Deploying Socket Proxy..."
-	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/socket-proxy/template.yaml docker-compose.proxy.yml $(ENV_FILE) $(STACKS_DIR)/socket-proxy/.env
-	@docker compose -f docker-compose.proxy.yml up -d
 
 reload-caddy:
+	@docker compose -f docker-compose.caddy.yml ps --status running --services 2>/dev/null | grep -q '^caddy$$' \
+		|| { echo "Error: Caddy container is not running. Run 'make deploy-caddy' first." >&2; exit 1; }
 	@$(MAKE) check-env STACK=caddy
 	@echo "Performing zero-downtime Caddy reload..."
 	@$(SUBSTITUTE_SCRIPT) $(STACKS_DIR)/caddy/Caddyfile_template $(STACKS_DIR)/caddy/Caddyfile $(ENV_FILE) $(STACKS_DIR)/caddy/.env

README.md

@@ -39,7 +39,7 @@ The repository includes a `Makefile` to unify management of all stacks. It suppo
 
 | Command | Description |
 |---|---|
-| `make check-env` | Validates all `.env` files against placeholders in templates |
+| `make check-env` | Validates root and stack `.env` files against required placeholders in renderable templates |
 | `make status` | Shows `docker compose ps` for all generated compose files |
 | `make reload-caddy` | Zero-downtime reload of the Caddy configuration |
 | `make ip-check` | Verifies VPN connectivity for Arr Stack containers |
@@ -65,6 +65,9 @@ docker compose -f docker-compose.<stack>.yml up -d
 Home Assistant uses its setup script instead. Arr Stack, Caddy, and Gramps Web
 render additional configuration templates.
 
+`substitute_env.sh` only reads env files passed on the command line. It does
+not fall back to OS environment variables or an implicit `.env`.
+
 ## Requirements
 
 - Linux host with Docker Engine and Docker Compose
@@ -83,4 +86,3 @@ Git.
 - [Recyclarr](docs/RECYCLARR.md)
 - [Troubleshooting](docs/TROUBLESHOOTING.md)
 - [UDM Pro SE zones and VLAN migration](docs/network/udm-pro-zones-migration.md)
- zones and VLAN migration](docs/network/udm-pro-zones-migration.md)

docker-compose-files/gramps-web/README.md

@@ -24,27 +24,36 @@ stack.
 2. Set the Gramps, OIDC, and email values in `.env`. At minimum configure
    `TREE`, `BASE_URL`, `SECRET_KEY`, and `USER_DB_URI`.
 
-3. Create the config directory and render both templates:
+3. Deploy with `make`:
+
+   ```bash
+   make deploy-gramps
+   ```
+
+   This creates `gramps_config/config.cfg`, renders
+   `docker-compose.gramps.yml`, and starts the stack.
+
+   Manual equivalent:
 
    ```bash
    mkdir -p gramps_config
    ./substitute_env.sh docker-compose-files/gramps-web/config/config_template.cfg gramps_config/config.cfg .env
    chmod 644 gramps_config/config.cfg
-   ./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps-web.yml .env
+   ./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps.yml .env
    ```
 
 4. Validate and start Gramps Web:
 
    ```bash
-   docker compose -f docker-compose.gramps-web.yml config --quiet
+   docker compose -f docker-compose.gramps.yml config --quiet
-   docker compose -f docker-compose.gramps-web.yml up -d
+   docker compose -f docker-compose.gramps.yml up -d
    ```
 
 5. Verify the containers and open `http://<HOST_IP>`:
 
    ```bash
-   docker compose -f docker-compose.gramps-web.yml ps
+   docker compose -f docker-compose.gramps.yml ps
-   docker compose -f docker-compose.gramps-web.yml logs --tail=100
+   docker compose -f docker-compose.gramps.yml logs --tail=100
    ```
 
 ## Apply Configuration Changes
@@ -52,8 +61,8 @@ stack.
 ```bash
 ./substitute_env.sh docker-compose-files/gramps-web/config/config_template.cfg gramps_config/config.cfg .env
 chmod 644 gramps_config/config.cfg
-./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps-web.yml .env
+./substitute_env.sh docker-compose-files/gramps-web/template.yaml docker-compose.gramps.yml .env
-docker compose -f docker-compose.gramps-web.yml up -d
+docker compose -f docker-compose.gramps.yml up -d
 ```
 
 ## Initial Setup
@@ -74,7 +83,7 @@ writable by the container before starting the stack.
 ## Useful Commands
 
 ```bash
-docker compose -f docker-compose.gramps-web.yml restart grampsweb
+docker compose -f docker-compose.gramps.yml restart grampsweb
 ```
 
 ## Backup

docker-compose-files/immich/README.md

@@ -22,6 +22,19 @@ Self-hosted photo and video management solution with high performance and visual
     IMMICH_REDIS_HOSTNAME=immich-redis
     ```
 
+    Optional OIDC settings can also be supplied. Set `OIDC_ENABLED=true`
+    explicitly when enabling OAuth; leaving it unset keeps Immich local-login
+    only.
+    ```dotenv
+    OIDC_ENABLED=true
+    OIDC_ISSUER=https://auth.example.com/application/o/immich/
+    OIDC_CLIENT_ID=<immich-client-id>
+    OIDC_CLIENT_SECRET=<immich-client-secret>
+    OIDC_NAME=Single Sign-On
+    OIDC_AUTO_REDIRECT=false
+    OIDC_MOBILE_OVERRIDE_ENABLED=false
+    ```
+
 2.  **Create Directories**:
     Create the local appdata directories and the shared media directory:
     ```bash

docker-compose-files/immich/template.yaml

@@ -8,8 +8,6 @@ services:
     volumes:
       - {{IMMICH_UPLOAD_LOCATION}}:/usr/src/app/upload
       - /etc/localtime:/etc/localtime:ro
-    env_file:
-      - .env
     environment:
       - DB_PASSWORD={{IMMICH_DB_PASSWORD}}
       - DB_USERNAME={{IMMICH_DB_USERNAME}}
@@ -17,6 +15,20 @@ services:
       - DB_HOSTNAME={{IMMICH_DB_HOSTNAME}}
       - REDIS_HOSTNAME={{IMMICH_REDIS_HOSTNAME}}
       - TZ={{TZ}}
+      # OIDC (optional - leave disabled unless explicitly configured)
+      - OAUTH_ENABLED={{OIDC_ENABLED:-false}}
+      - OAUTH_ISSUER_URL={{OIDC_ISSUER:-}}
+      - OAUTH_CLIENT_ID={{OIDC_CLIENT_ID:-}}
+      - OAUTH_CLIENT_SECRET={{OIDC_CLIENT_SECRET:-}}
+      - OAUTH_BUTTON_TEXT={{OIDC_NAME:-}}
+      - OAUTH_AUTO_LAUNCH={{OIDC_AUTO_REDIRECT:-}}
+      - OAUTH_MOBILE_OVERRIDE_ENABLED={{OIDC_MOBILE_OVERRIDE_ENABLED:-false}}
+      # Email (optional - leave blank to disable)
+      - SMTP_HOST={{EMAIL_HOST:-}}
+      - SMTP_PORT={{EMAIL_PORT:-}}
+      - SMTP_USERNAME={{EMAIL_HOST_USER:-}}
+      - SMTP_PASSWORD={{EMAIL_HOST_PASSWORD:-}}
+      - SMTP_FROM={{DEFAULT_FROM_EMAIL:-}}
     ports:
       - "2283:2283"
     depends_on:
@@ -41,8 +53,6 @@ services:
     runtime: nvidia
     volumes:
       - {{DOCKERCONFDIR}}/immich/model-cache:/cache
-    env_file:
-      - .env
     environment:
       - NVIDIA_VISIBLE_DEVICES=all
       - NVIDIA_DRIVER_CAPABILITIES=compute,utility

docs/RECYCLARR.md

@@ -8,7 +8,7 @@ This repository now uses Recyclarr **v8** with guide-backed quality profiles (no
 ### 1. Generate `recyclarr.yml`
 
 ```bash
-./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml
+./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env
 nano /opt/arr-stack/appdata/recyclarr/recyclarr.yml
 ```
 
@@ -150,10 +150,10 @@ Fix:
 
 ```bash
 # Regenerate v8 config from current template
-./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml
+./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env
 
 # Recreate recyclarr with v8 image from compose template
-./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml
+./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env
 docker-compose -f docker-compose.arr-stack.yml pull recyclarr
 docker-compose -f docker-compose.arr-stack.yml up -d recyclarr
 ```

docs/TROUBLESHOOTING.md

@@ -23,7 +23,7 @@ Symptoms:
 Fix:
 
 ```bash
-./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml
+./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env
 ```
 
 The script prints any unresolved placeholders. Add missing keys to `.env` (or provide defaults in templates).
@@ -45,7 +45,7 @@ radarr:
 Generate your file from this repo template:
 
 ```bash
-./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml
+./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env
 ```
 
 ### 2b. Recyclarr errors (`Unable to find include template ...`)
@@ -64,10 +64,10 @@ Fix (v8 path used by this repository):
 
 ```bash
 # 1) Regenerate recyclarr config from the v8 template
-./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml
+./substitute_env.sh docker-compose-files/arr-stack/config/recyclarr_template.yml /opt/arr-stack/appdata/recyclarr/recyclarr.yml .env
 
 # 2) Regenerate compose and recreate recyclarr (v8.6.0 pin)
-./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml
+./substitute_env.sh docker-compose-files/arr-stack/template.yaml docker-compose.arr-stack.yml .env
 docker-compose -f docker-compose.arr-stack.yml pull recyclarr
 docker-compose -f docker-compose.arr-stack.yml up -d recyclarr
 ```

substitute_env.sh

@@ -8,7 +8,7 @@ set -euo pipefail
 
 # Ensure we have Bash 4+ for associative arrays
 if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then
-    echo "Error: This script requires Bash 4.0 or higher."
+    echo "Error: This script requires Bash 4.0 or higher." >&2
     exit 1
 fi
 
@@ -23,8 +23,8 @@ trim() {
 
 # 1. Validation: Check arguments
 if [[ $# -lt 2 ]]; then
-    echo "Usage: $0 <source_template_file> <output_destination_file> [env_file1] [env_file2] ..."
+    echo "Usage: $0 <source_template_file> <output_destination_file> [env_file1] [env_file2] ..." >&2
-    echo "Example: $0 config_template.cfg config.cfg .env global.env"
+    echo "Example: $0 config_template.cfg config.cfg .env global.env" >&2
     exit 1
 fi
 
@@ -32,19 +32,19 @@ SOURCE_FILE="$1"
 OUTPUT_FILE="$2"
 
 if [[ ! -f "$SOURCE_FILE" ]]; then
-    echo "Error: Source file '$SOURCE_FILE' not found."
+    echo "Error: Source file '$SOURCE_FILE' not found." >&2
     exit 1
 fi
 
 if [[ "$SOURCE_FILE" == "$OUTPUT_FILE" ]]; then
-    echo "Error: Source and output files must be different."
+    echo "Error: Source and output files must be different." >&2
     exit 1
 fi
 
 load_env() {
     local env_file="$1"
     if [[ -f "$env_file" ]]; then
-        echo "Info: Loading environment from $env_file"
+        echo "Info: Loading environment from $env_file" >&2
         while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do
             line="${raw_line%$'\r'}"
             line="$(trim "$line")"
@@ -53,7 +53,7 @@ load_env() {
             [[ -z "$line" || "$line" == \#* ]] && continue
 
             if [[ "$line" != *"="* ]]; then
-                echo "Warning: Skipping invalid line in $env_file: $line"
+                echo "Warning: Skipping invalid line in $env_file: $line" >&2
                 continue
             fi
 
@@ -63,7 +63,7 @@ load_env() {
             key="$(trim "$key")"
 
             if [[ ! "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
-                echo "Warning: Skipping invalid variable name '$key' in $env_file"
+                echo "Warning: Skipping invalid variable name '$key' in $env_file" >&2
                 continue
             fi
 
@@ -85,7 +85,7 @@ load_env() {
             LOADED_VARS["$key"]="$value"
         done <"$env_file"
     else
-        echo "Notice: $env_file not found; skipping."
+        echo "Notice: $env_file not found; skipping." >&2
     fi
 }
 
@@ -94,8 +94,6 @@ if [[ $# -ge 3 ]]; then
     for (( i=3; i<=$#; i++ )); do
         load_env "${!i}"
     done
-elif [[ -f ".env" ]]; then
-    load_env ".env"
 fi
 
 # 3. Process the template (handles {{VAR}} and {{VAR:-default}})
@@ -128,12 +126,17 @@ render_template() {
             elif [[ -n "$has_default" ]]; then
                 replacement="$default"
             else
+                # Leave placeholder intact so post-render validation can report it.
                 replacement="$full"
             fi
 
+            # %% finds the first occurrence of $full by removing the longest
+            # suffix that starts with it, then #* advances past it.
             before="${rest%%"$full"*}"
             rendered+="$before$replacement"
             rest="${rest#*"$full"}"
+            # Replacements are appended to `rendered`, which is never re-scanned.
+            # Values containing {{...}} are passed through verbatim (no nested expansion).
         done
         line="$rendered$rest"
         printf '%s\n' "$line" >>"$target"
@@ -143,23 +146,23 @@ render_template() {
 TMP_OUTPUT="$(mktemp)"
 trap 'rm -f "$TMP_OUTPUT"' EXIT
 render_template "$SOURCE_FILE" "$TMP_OUTPUT"
-mv "$TMP_OUTPUT" "$OUTPUT_FILE"
-trap - EXIT
 
 # 4. Final validation: check for unreplaced placeholders
 if [[ "$SOURCE_FILE" =~ \.ya?ml$ ]]; then
-    MISSING_VARS="$(awk '!/^[[:space:]]*#/' "$OUTPUT_FILE" | grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' | sort -u || true)"
+    MISSING_VARS="$(awk '!/^[[:space:]]*#/' "$TMP_OUTPUT" | grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' | sort -u || true)"
 else
-    MISSING_VARS="$(grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' "$OUTPUT_FILE" | sort -u || true)"
+    MISSING_VARS="$(grep -oE '\{\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}\}' "$TMP_OUTPUT" | sort -u || true)"
 fi
 
 if [[ -n "$MISSING_VARS" ]]; then
-    echo "--------------------------------------------------------"
+    echo "--------------------------------------------------------" >&2
-    echo "ERROR: The following placeholders were not replaced:"
+    echo "ERROR: The following placeholders were not replaced:" >&2
-    echo "$MISSING_VARS"
+    echo "$MISSING_VARS" >&2
-    echo "These MUST be defined in your .env files (OS environment ignored)."
+    echo "These MUST be defined in your .env files (OS environment ignored)." >&2
-    echo "--------------------------------------------------------"
+    echo "--------------------------------------------------------" >&2
     exit 1
 else
-    echo "Success: All variables substituted in $OUTPUT_FILE"
+    mv "$TMP_OUTPUT" "$OUTPUT_FILE"
+    trap - EXIT
+    echo "Success: All variables substituted in $OUTPUT_FILE" >&2
 fi