From 0768f6d783181ad1789538e57cb25ca185103b46 Mon Sep 17 00:00:00 2001 From: Chris Amow Date: Wed, 26 Aug 2026 00:36:13 -0500 Subject: [PATCH] add restricted production diagnostics --- docs/install_debug_charts.md | 200 ++++++++++++++++++++++++++ docs/plan_diagnostics_improvements.md | 61 ++++++++ ops/chart-debug-command | 101 +++++++++++++ ops/install-chart-debug | 69 +++++++++ 4 files changed, 431 insertions(+) create mode 100644 docs/install_debug_charts.md create mode 100755 ops/chart-debug-command create mode 100755 ops/install-chart-debug diff --git a/docs/install_debug_charts.md b/docs/install_debug_charts.md new file mode 100644 index 0000000..85a160c --- /dev/null +++ b/docs/install_debug_charts.md @@ -0,0 +1,200 @@ +# Install Restricted Production Chart Diagnostics + +This handoff is for the agent administering the Coolify Docker host for +`chart.amow.com`. + +The implementation files are: + +```text +ops/chart-debug-command +ops/install-chart-debug +``` + +Run the installer on the Coolify Docker **host** as root, not inside the chart +application container. + +## Dedicated key + +Install this public key: + +```text +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAdYzYs9RJ9Az9ypyYhrKsqdfbdv0EbjOGQVox+UyZBz chart-debug restricted production diagnostics +``` + +Expected fingerprint: + +```text +SHA256:i2VPhUqgN9gHcta27aH6HV8d5YMvKABQ1EXworhnjGw +``` + +The private key must never be copied to production or pasted into chat. It stays +on the diagnostics client at: + +```text +/home/chris/.ssh/chart_debug_ed25519 +``` + +## Security requirements + +- Do not add `chart-debug` to the Docker group. +- Do not enable password authentication for the account. +- Do not add an unrestricted SSH key. +- Do not grant a normal production shell. +- Keep the wrapper, configuration and sudoers file root-owned. +- Keep the account password locked. +- Preserve the `restrict` and forced-command options in `authorized_keys`. +- Verify arbitrary commands and malformed arguments are denied. +- Do not print container environment variables or application secrets. + +The account may run only: + +```text +logs --since DURATION +status +container-state +recent-deploy +capture-read CAPTURE_ID +capture-delete CAPTURE_ID +``` + +## Installation + +### 1. Identify the chart container + +```bash +docker ps --format 'table {{.ID}}\t{{.Names}}\t{{.Image}}' +``` + +The Coolify resource UUID is: + +```text +dgvch0xqv8uvjfor7dl8bwl9 +``` + +A likely stable selector is: + +```text +^dgvch0xqv8uvjfor7dl8bwl9 +``` + +Do not assume it. Verify the regex matches exactly one running chart container +and will continue matching after a Coolify redeploy. + +### 2. Run the installer + +From a checkout containing `ops/`: + +```bash +sudo ./ops/install-chart-debug \ + --public-key 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAdYzYs9RJ9Az9ypyYhrKsqdfbdv0EbjOGQVox+UyZBz chart-debug restricted production diagnostics' \ + --container-pattern '^VERIFIED-STABLE-PREFIX' \ + --url https://chart.amow.com +``` + +### 3. Verify account and file security + +```bash +passwd -S chart-debug + +stat -c '%U:%G %a %n' \ + /usr/local/sbin/chart-debug-command \ + /etc/chart-debug.conf \ + /etc/sudoers.d/chart-debug \ + /home/chart-debug/.ssh/authorized_keys + +visudo -cf /etc/sudoers.d/chart-debug +``` + +Expected permissions: + +| Path | Owner | Mode | +|---|---|---:| +| `/usr/local/sbin/chart-debug-command` | `root:root` | `755` | +| `/etc/chart-debug.conf` | `root:root` | `600` | +| `/etc/sudoers.d/chart-debug` | `root:root` | `440` | +| `/home/chart-debug/.ssh/authorized_keys` | `chart-debug:chart-debug` | `600` | + +The account status must show a locked password. + +### 4. Test allowed commands locally + +```bash +sudo -u chart-debug sudo -n \ + /usr/local/sbin/chart-debug-command 'container-state' + +sudo -u chart-debug sudo -n \ + /usr/local/sbin/chart-debug-command 'logs --since 5m' + +sudo -u chart-debug sudo -n \ + /usr/local/sbin/chart-debug-command 'status' + +sudo -u chart-debug sudo -n \ + /usr/local/sbin/chart-debug-command 'recent-deploy' +``` + +### 5. Verify denial behavior + +```bash +if sudo -u chart-debug sudo -n \ + /usr/local/sbin/chart-debug-command 'shell' +then + echo 'ERROR: arbitrary command was allowed' + exit 1 +else + echo 'arbitrary command correctly denied' +fi +``` + +Also verify these fail closed: + +- Invalid durations. +- Extra arguments. +- Invalid capture IDs. +- A container selector matching zero containers. +- A container selector matching multiple containers. + +### 6. Verify SSH policy + +If `sshd` uses `AllowUsers`, add `chart-debug`. Keep password authentication +disabled. Ensure the firewall permits SSH from the diagnostics client's network. + +Do not modify the installed forced-command `authorized_keys` entry. + +### 7. Verify auditing + +```bash +journalctl -t chart-debug +``` + +## Report back + +Return only: + +- Production SSH hostname or IP. +- SSH port. +- Exact stable container regex. +- Whether each allowed command succeeded. +- Confirmation arbitrary commands were denied. +- Confirmation the account password is locked. +- Confirmation the account is not in the Docker group. +- Errors with secrets redacted. + +Do not return `CHART_AUTH_TOKEN`, Schwab credentials, container environment, or +private key material. + +## Client verification + +After receiving the host and port: + +```bash +ssh -i ~/.ssh/chart_debug_ed25519 \ + -p PORT chart-debug@HOST 'logs --since 20m' +``` + +Other examples: + +```bash +ssh -i ~/.ssh/chart_debug_ed25519 -p PORT chart-debug@HOST status +ssh -i ~/.ssh/chart_debug_ed25519 -p PORT chart-debug@HOST container-state +ssh -i ~/.ssh/chart_debug_ed25519 -p PORT chart-debug@HOST recent-deploy +``` diff --git a/docs/plan_diagnostics_improvements.md b/docs/plan_diagnostics_improvements.md index ca2c917..f1d7c13 100644 --- a/docs/plan_diagnostics_improvements.md +++ b/docs/plan_diagnostics_improvements.md @@ -49,6 +49,67 @@ Expected agent usage: ssh chart-debug@production logs --since 20m ``` +### Installation on the Coolify host + +The implementation lives in `ops/chart-debug-command` and +`ops/install-chart-debug`. The Coolify-side agent must run as root on the Docker +host, not inside the application container. + +1. Identify the current chart container and a stable name prefix that survives + deploys: + + ```bash + docker ps --format 'table {{.ID}}\t{{.Names}}\t{{.Image}}' + ``` + + The Coolify resource UUID is `dgvch0xqv8uvjfor7dl8bwl9`; a likely anchored + pattern is `^dgvch0xqv8uvjfor7dl8bwl9`, but the agent must verify it matches + exactly one running container before installation. + +2. Run the installer from a checkout containing `ops/`, using the dedicated + public key supplied out-of-band: + + ```bash + sudo ./ops/install-chart-debug \ + --public-key 'ssh-ed25519 AAAA... chart-debug restricted production diagnostics' \ + --container-pattern '^VERIFIED-STABLE-PREFIX' \ + --url https://chart.amow.com + ``` + +3. Verify the account is locked, files are root-owned, sudo policy is valid, + the selector matches exactly one container, allowed commands work, and an + arbitrary command is denied: + + ```bash + passwd -S chart-debug + stat -c '%U:%G %a %n' /usr/local/sbin/chart-debug-command \ + /etc/chart-debug.conf /etc/sudoers.d/chart-debug + visudo -cf /etc/sudoers.d/chart-debug + sudo -u chart-debug sudo -n /usr/local/sbin/chart-debug-command 'container-state' + sudo -u chart-debug sudo -n /usr/local/sbin/chart-debug-command 'logs --since 5m' + if sudo -u chart-debug sudo -n /usr/local/sbin/chart-debug-command 'shell'; then + echo 'ERROR: arbitrary command was allowed'; exit 1 + else + echo 'arbitrary command correctly denied' + fi + ``` + +4. Confirm SSH permits the `chart-debug` user. If `AllowUsers` is configured, + add `chart-debug`; do not enable password authentication. Report the host or + IP and SSH port so the client alias can be configured. + +Security invariants: + +- Do not add `chart-debug` to the Docker group. +- Do not install the private key on production or paste it into chat. +- Keep `/etc/chart-debug.conf`, the wrapper and sudoers entry root-owned. +- Keep the account password locked and the `authorized_keys` `restrict` forced + command intact; no additional unrestricted keys. +- The wrapper must match exactly one running container. Zero or multiple matches + fail closed. +- Verify denials as well as successful commands. Audit records are available via + `journalctl -t chart-debug`. + ## Observability Keep browser performance telemetry separate from production access. A future diff --git a/ops/chart-debug-command b/ops/chart-debug-command new file mode 100755 index 0000000..09b963f --- /dev/null +++ b/ops/chart-debug-command @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONFIG=/etc/chart-debug.conf +MAX_LOG_LINES=4000 + +if [[ ! -r "$CONFIG" ]]; then + echo "chart-debug is not configured" >&2 + exit 1 +fi + +# Root-owned configuration written by install-chart-debug. +# shellcheck source=/dev/null +source "$CONFIG" + +original=${1:-${SSH_ORIGINAL_COMMAND:-}} +if [[ -z "$original" || "$original" == *$'\n'* || "$original" == *$'\r'* ]]; then + echo "usage: logs --since 20m | status | container-state | recent-deploy | capture-read ID | capture-delete ID" >&2 + exit 2 +fi +read -r -a args <<< "$original" + +logger -t chart-debug -- "user=${SUDO_USER:-${USER:-unknown}} from=${SSH_CONNECTION:-local} command=$original" + +resolve_container() { + local id name + local -a matches=() + while read -r id name; do + [[ -n "$id" && "$name" =~ $CHART_CONTAINER_PATTERN ]] && matches+=("$id") + done < <(docker ps --format '{{.ID}} {{.Names}}') + if [[ ${#matches[@]} -ne 1 ]]; then + echo "container selector matched ${#matches[@]} running containers" >&2 + exit 3 + fi + printf '%s' "${matches[0]}" +} + +redact() { + sed -E \ + -e 's/(Authorization:[[:space:]]*Bearer[[:space:]]+)[A-Za-z0-9._-]+/\1[REDACTED]/Ig' \ + -e 's/((token|secret|password|api[_-]?key)["=:[:space:]]+)[^,[:space:]"}]+/\1[REDACTED]/Ig' +} + +valid_capture_id() { + [[ "$1" =~ ^c-[A-Za-z0-9_-]{12}$ ]] +} + +case "${args[0]}" in + logs) + [[ ${#args[@]} -eq 3 && "${args[1]}" == "--since" ]] || { + echo "usage: logs --since 20m" >&2; exit 2; + } + [[ "${args[2]}" =~ ^[1-9][0-9]*(s|m|h|d)$ ]] || { + echo "invalid duration" >&2; exit 2; + } + docker logs --timestamps --tail "$MAX_LOG_LINES" --since "${args[2]}" "$(resolve_container)" 2>&1 | redact + ;; + status) + [[ ${#args[@]} -eq 1 ]] || { echo "usage: status" >&2; exit 2; } + container=$(resolve_container) + token="" + while IFS= read -r entry; do + [[ "$entry" == CHART_AUTH_TOKEN=* ]] && token=${entry#CHART_AUTH_TOKEN=} + done < <(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$container") + if [[ -n "$token" ]]; then + curl -fsS -H "X-Chart-Token: $token" "$CHART_PUBLIC_URL/api/status" | redact + else + echo "CHART_AUTH_TOKEN is unavailable" >&2 + exit 4 + fi + ;; + container-state) + [[ ${#args[@]} -eq 1 ]] || { echo "usage: container-state" >&2; exit 2; } + docker inspect --format \ + 'name={{.Name}} image={{.Config.Image}} status={{.State.Status}} started={{.State.StartedAt}} restarts={{.RestartCount}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}}' \ + "$(resolve_container)" + ;; + recent-deploy) + [[ ${#args[@]} -eq 1 ]] || { echo "usage: recent-deploy" >&2; exit 2; } + curl -fsS "$CHART_PUBLIC_URL/api/version" + printf '\n' + docker inspect --format 'container_started={{.State.StartedAt}} image={{.Config.Image}}' "$(resolve_container)" + ;; + capture-read) + [[ ${#args[@]} -eq 2 ]] && valid_capture_id "${args[1]}" || { + echo "usage: capture-read c-XXXXXXXXXXXX" >&2; exit 2; + } + docker exec "$(resolve_container)" base64 "/tmp/chart-captures/${args[1]}.png" + ;; + capture-delete) + [[ ${#args[@]} -eq 2 ]] && valid_capture_id "${args[1]}" || { + echo "usage: capture-delete c-XXXXXXXXXXXX" >&2; exit 2; + } + curl -fsS -X DELETE "$CHART_PUBLIC_URL/api/debug/captures/${args[1]}" >/dev/null + echo "deleted ${args[1]}" + ;; + *) + echo "command not allowed" >&2 + exit 2 + ;; +esac diff --git a/ops/install-chart-debug b/ops/install-chart-debug new file mode 100755 index 0000000..a3bd08e --- /dev/null +++ b/ops/install-chart-debug @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: sudo ./ops/install-chart-debug \ + --public-key 'ssh-ed25519 AAAA...' \ + --container-pattern '^chart-app-' \ + [--url https://chart.amow.com] +EOF +} + +public_key="" +container_pattern="" +public_url="https://chart.amow.com" +while [[ $# -gt 0 ]]; do + case "$1" in + --public-key) public_key=${2:-}; shift 2 ;; + --container-pattern) container_pattern=${2:-}; shift 2 ;; + --url) public_url=${2:-}; shift 2 ;; + *) usage >&2; exit 2 ;; + esac +done + +[[ $EUID -eq 0 ]] || { echo "run as root" >&2; exit 1; } +[[ "$public_key" =~ ^(ssh-ed25519|sk-ssh-ed25519@openssh.com)[[:space:]]+[A-Za-z0-9+/=]+([[:space:]].*)?$ ]] || { + echo "an Ed25519 public key is required" >&2; exit 2; +} +[[ -n "$container_pattern" && "$container_pattern" != *$'\n'* ]] || { + echo "a container-name regex is required" >&2; exit 2; +} +[[ "$public_url" =~ ^https://[^/]+$ ]] || { echo "--url must be an HTTPS origin" >&2; exit 2; } + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +install -o root -g root -m 0755 "$script_dir/chart-debug-command" /usr/local/sbin/chart-debug-command + +if ! id chart-debug >/dev/null 2>&1; then + useradd --create-home --shell /bin/bash chart-debug +fi +passwd --lock chart-debug >/dev/null +install -d -o chart-debug -g chart-debug -m 0700 /home/chart-debug/.ssh + +forced='restrict,command="/usr/bin/sudo -n /usr/local/sbin/chart-debug-command \"$SSH_ORIGINAL_COMMAND\""' +printf '%s %s\n' "$forced" "$public_key" \ + > /home/chart-debug/.ssh/authorized_keys +chown chart-debug:chart-debug /home/chart-debug/.ssh/authorized_keys +chmod 0600 /home/chart-debug/.ssh/authorized_keys + +printf 'CHART_CONTAINER_PATTERN=%q\nCHART_PUBLIC_URL=%q\n' \ + "$container_pattern" "$public_url" > /etc/chart-debug.conf +chown root:root /etc/chart-debug.conf +chmod 0600 /etc/chart-debug.conf + +cat > /etc/sudoers.d/chart-debug <<'EOF' +Defaults:chart-debug !requiretty +chart-debug ALL=(root) NOPASSWD: /usr/local/sbin/chart-debug-command * +EOF +chmod 0440 /etc/sudoers.d/chart-debug +visudo -cf /etc/sudoers.d/chart-debug >/dev/null + +matches=0 +while read -r _ name; do + [[ "$name" =~ $container_pattern ]] && matches=$((matches + 1)) +done < <(docker ps --format '{{.ID}} {{.Names}}') +[[ $matches -eq 1 ]] || { + echo "warning: container pattern currently matches $matches running containers" >&2 +} + +echo "installed restricted chart-debug access"