diff --git a/CHANGELOG.md b/CHANGELOG.md index d78e1ac..d327a45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,10 @@ the linux-only ufw-docker patching with something that behaves the same on macOS and needs no privilege. The scope is read from the compose file rather than declared beside it, so it cannot disagree with what is published +- `myos cert` derives the certificates a server needs from the route tags its + stacks publish, and asks dehydrated for them: a wildcard where a tag uses one, + a certificate per name otherwise. The `host/dehydrated` stack answers http-01 + itself and delegates dns-01 to a provider hook - commands chain: `myos build up logs host/fabio`, as make targets did - the stack catalogue no longer needs make at all: its settings are hooks, and only six stacks keep a .mk, for targets diff --git a/bin/myos b/bin/myos index 17a6e83..24ac2b2 100755 --- a/bin/myos +++ b/bin/myos @@ -19,7 +19,7 @@ done MYOS_ROOT=$(cd "$(dirname "$_self")/.." && pwd -P) export MYOS_ROOT -for _m in core str var tags naming stack config compose hooks context expose; do +for _m in core str var tags naming stack config compose hooks context expose cert; do # shellcheck source=/dev/null . "$MYOS_ROOT/lib/$_m.sh" done @@ -70,6 +70,7 @@ Commands: export every setting of the stacks, as KEY=value env-update fill .env from the .env.dist templates expose [--strict] what the stacks publish, and to whom + cert list|issue|renew|show certificates, derived from the routes doctor check the installation version print the myos version @@ -148,6 +149,16 @@ for _c in $MYOS_CMDS; do done MYOS_CMDS=$_cmds +# cert takes a subcommand where the others take only stacks +case $MYOS_CMDS in + *cert*) + case ${MYOS_REFS%% *} in + list|domains|issue|renew|show) + MYOS_VARS=${MYOS_REFS%% *} + case $MYOS_REFS in *' '*) MYOS_REFS=${MYOS_REFS#* } ;; *) MYOS_REFS= ;; esac ;; + esac ;; +esac + # env, ls and doctor take variable names where the others take stacks case $MYOS_CMDS in env|ls|doctor) diff --git a/lib/cert.sh b/lib/cert.sh new file mode 100644 index 0000000..d746cb2 --- /dev/null +++ b/lib/cert.sh @@ -0,0 +1,98 @@ +#shellcheck shell=sh +# cert: which certificates a server needs, derived from what its stacks route. +# +# The hostnames are already declared, once, in the fabio route tags a stack +# publishes: urlprefix-/. Asking for them a second time in a +# domains.txt would be a second source of truth, free to disagree with what is +# actually served. They are read from the resolved compose configuration +# instead. +# +# A name written *.example.org needs a wildcard, which ACME only issues over +# dns-01; a concrete name can be had over http-01. That is the whole of +# "per-site or wildcard according to need": the tags say which. + +# myos_cert_names the hostnames the requested stacks route, one per line +myos_cert_names() { + for _ref in $MYOS_STACKS; do + _files=$(myos_stack_compose_files "$_ref" 2>/dev/null) || continue + [ -n "$_files" ] || continue + _fw=$(myos_framework_compose_files) + [ -n "$_fw" ] && _files="$_files +$_fw" + _app=$(myos_stack_name "$_ref") + _project=$(myos_project_name "$(myos_scope "$_ref")" "$USER" "$ENV" "$_app") + DRYRUN=false myos_compose "$_project" "$_files" -- config 2>/dev/null + done | myos_cert_parse +} + +# myos_cert_parse (compose config on stdin) -> hostnames +# A tag is urlprefix-[:]/ with options after a space; the +# bare "*" is fabio's catch-all and names nothing. +myos_cert_parse() { + grep -oE 'urlprefix-[^",[:space:]]*' 2>/dev/null | + sed -e 's/^urlprefix-//' -e 's|/.*||' -e 's/:[0-9]*$//' | + grep -vE '^\*?$' | + sort -u +} + +# myos_cert_covers WILDCARD_PARENT NAME does *.parent cover this name? +# A wildcard matches one label, so *.example.org covers a.example.org but +# neither example.org nor a.b.example.org. +myos_cert_covers() { + case $2 in + *".$1") + _head=${2%".$1"} + case $_head in *.*|'') return 1 ;; *) return 0 ;; esac ;; + *) return 1 ;; + esac +} + +# myos_cert_groups the certificates to ask for, one per line, in the shape +# dehydrated reads: the common name first, then its subject alternative names. +# +# MYOS_CERT_MODE: +# auto a wildcard where the tags use one, a certificate per name otherwise +# wildcard one wildcard per domain, whether or not a tag asked for it +# per-site never a wildcard: one certificate per name, dns-01 not required +myos_cert_groups() { + _names=$(myos_cert_names) + [ -n "$_names" ] || return 0 + _mode=${MYOS_CERT_MODE:-auto} + + # the parents a wildcard is wanted for + _wild= + for _n in $_names; do + case $_n in + \*.*) [ "$_mode" = per-site ] || _wild="$_wild ${_n#\*.}" ;; + esac + done + if [ "$_mode" = wildcard ]; then + for _n in $_names; do + case $_n in + \*.*) ;; + *.*.*) _wild="$_wild ${_n#*.}" ;; + esac + done + fi + _wild=$(printf '%s' "$_wild" | tr ' ' '\n' | sed '/^$/d' | sort -u) + + # one line per wildcard, the parent first so it is the common name + for _p in $_wild; do + printf '%s *.%s\n' "$_p" "$_p" + done + + # the concrete names a wildcard does not already cover + for _n in $_names; do + case $_n in \*.*) continue ;; esac + _covered=no + for _p in $_wild; do + [ "$_n" = "$_p" ] && { _covered=yes; break; } + myos_cert_covers "$_p" "$_n" && { _covered=yes; break; } + done + [ "$_covered" = no ] && printf '%s\n' "$_n" + done + return 0 +} + +# myos_cert_needs_dns true when any certificate asked for is a wildcard +myos_cert_needs_dns() { myos_cert_groups | grep -q '\*\.'; } diff --git a/lib/cmd/cert.sh b/lib/cmd/cert.sh new file mode 100644 index 0000000..f8ad03c --- /dev/null +++ b/lib/cmd/cert.sh @@ -0,0 +1,91 @@ +#shellcheck shell=sh +# shellcheck disable=SC1091 # lib/cmd files are sourced by path at run time +# shellcheck disable=SC3028 # HOSTNAME is a myos variable, set by bin/myos +# myos cert the certificates a server needs +# +# The hostnames come from the route tags of the stacks, so a site gets a +# certificate by being routed, not by being written down a second time. +myos_cmd_cert() { + _sub=$(myos_firstword "${MYOS_VARS:-}${MYOS_ARGS:+ $MYOS_ARGS}") + [ -n "$_sub" ] || _sub=list + case $_sub in + list) myos_cert_list ;; + domains) myos_cert_write_domains ;; + issue) myos_cert_run "" ;; + renew) myos_cert_run "--cron" ;; + show) myos_cert_show ;; + *) myos_die "$MYOS_E_USAGE" "myos cert " ;; + esac +} + +# myos_cert_list the certificates that would be asked for, and how +myos_cert_list() { + _groups=$(myos_cert_groups) + [ -n "$_groups" ] || { + printf 'no routed hostname: nothing to certify\n' + return 0 + } + printf '%s%-46s %-9s %s%s\n' "$MYOS_C_HIGHLIGHT" CERTIFICATE CHALLENGE NAMES "$MYOS_C_RESET" + printf '%s\n' "$_groups" | while IFS= read -r _line; do + _cn=$(myos_firstword "$_line") + case $_line in + *'*.'*) _ch=dns-01 ;; + *) _ch=http-01 ;; + esac + printf '%-46s %-9s %s\n' "$_cn" "$_ch" "$_line" + done + myos_cert_needs_dns && + myos_info "a wildcard is asked for: dns-01 needs MYOS_CERT_HOOK to talk to your dns provider" + return 0 +} + +# myos_cert_write_domains the domains.txt dehydrated reads +myos_cert_write_domains() { + _dir=${MYOS_CERT_DIR:-$WORKDIR/.myos/dehydrated} + _file=$_dir/domains.txt + _groups=$(myos_cert_groups) + [ -n "$_groups" ] || { myos_warning "no routed hostname: not writing $_file"; return 0; } + myos_run mkdir -p "$_dir" + if [ "${DRYRUN:-false}" = true ]; then + printf 'would write %s:\n%s\n' "$_file" "$_groups" + else + printf '%s\n' "$_groups" > "$_file" + printf '%s\n' "$_file" + fi +} + +# myos_cert_run ARGS run dehydrated in the host stack, on the domains derived +myos_cert_run() { + myos_cert_write_domains >/dev/null || return $? + _args=$1 + [ -n "${MYOS_CERT_STAGING:-}" ] && _args="$_args --staging" + case ${MYOS_ARGS:-} in + *--staging*) _args="$_args --staging" ;; + esac + case ${MYOS_ARGS:-} in + *--force*) _args="$_args --force" ;; + esac + MYOS_ARGS="$_args" SERVICE=${SERVICE:-dehydrated} \ + MYOS_STACKS="host/dehydrated" myos_cert_exec +} + +myos_cert_exec() { + # shellcheck source=lib/cmd/exec.sh + . "$MYOS_ROOT/lib/cmd/exec.sh" + myos_cmd_exec +} + +# myos_cert_show the certificates that exist, and when they expire +myos_cert_show() { + _vol=${HOST_DOCKER_VOLUME:-${HOSTNAME:-localhost}} + # shellcheck disable=SC2016 # the script runs in the container, not here + myos_run docker run --rm -v "$_vol:/host" alpine:3.20 sh -c ' + apk add -q openssl 2>/dev/null + for c in /host/certs/*-cert.pem; do + [ -f "$c" ] || continue + n=$(basename "$c" -cert.pem) + e=$(openssl x509 -in "$c" -noout -enddate 2>/dev/null | sed "s/notAfter=//") + i=$(openssl x509 -in "$c" -noout -issuer 2>/dev/null | sed "s/.*CN *= *//;s/,.*//") + printf "%-46s %-28s %s\n" "$n" "$e" "$i" + done' +} diff --git a/lib/expose.sh b/lib/expose.sh index 6d97205..4cd1187 100644 --- a/lib/expose.sh +++ b/lib/expose.sh @@ -87,26 +87,33 @@ myos_stack_prefix() { # 0.0.0.0 exactly like a deliberate public one. myos_expose_declared() { awk ' - function emit(entry, e, scope, target) { + function indent(line, n) { match(line, /^ */); return RLENGTH } + function emit(entry, e, scope, target, n, parts) { e = entry - gsub(/^[ \t"'"'"'-]+/, "", e); gsub(/["'"'"']+$/, "", e) + sub(/^ *- */, "", e) + gsub(/^["'"'"']|["'"'"']$/, "", e) if (e ~ /\$\{MYOS_BIND_PUBLIC[^}]*\}/) scope = "public" else if (e ~ /\$\{MYOS_BIND_MESH[^}]*\}/) scope = "mesh" else if (e ~ /\$\{MYOS_BIND_PRIVATE[^}]*\}/) scope = "private" else if (e ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:/) scope = "pinned" else if (e ~ /^\[/) scope = "pinned" else scope = "unbound" - # the container port is the last field, minus any /protocol target = e sub(/\/[a-z]+$/, "", target) n = split(target, parts, ":") target = parts[n] if (target ~ /^[0-9]+(-[0-9]+)?$/) printf "%s|%s|%s\n", svc, target, scope } - /^services:[ \t]*$/ { insvc = 1; next } - insvc && /^ [a-zA-Z0-9_.-]+:[ \t]*$/ { svc = $1; sub(/:$/, "", svc); inports = 0 } - insvc && /^ ports:/ { inports = 1; next } - inports && /^ [a-zA-Z]/ { inports = 0 } - inports && /^ *-/ { emit($0) } + /^services:[ \t]*$/ { insvc = 1; svcind = -1; next } + !insvc { next } + # a service is the first level of keys under services: + /^ *[a-zA-Z0-9_.-]+:[ \t]*$/ && (svcind == -1 || indent($0) == svcind) { + if (svcind == -1) svcind = indent($0) + svc = $1; sub(/:$/, "", svc); inports = 0; next + } + /^ *ports:/ { inports = 1; portind = indent($0); next } + # the list items of a ports: block, whatever indent they use + inports && /^ *- / && indent($0) >= portind { emit($0); next } + inports && /^ *[a-zA-Z0-9_.-]+:/ { inports = 0 } ' "$@" } diff --git a/skills/myos/references/commands.md b/skills/myos/references/commands.md index 901faf4..042972e 100644 --- a/skills/myos/references/commands.md +++ b/skills/myos/references/commands.md @@ -28,6 +28,7 @@ myos [options] [stack...] [VAR=value...] [-- args...] | `env [VAR...]` | resolved variables | | `env-update` | fill the `.env` from the `.env.dist` templates | | `expose [--strict]` | what the stacks publish, and to whom | +| `cert list\|issue\|renew\|show` | certificates, derived from the route tags | | `export [--make]` | every setting of the stacks, as `KEY=value` | | `doctor` | check the installation | | `version` | the myos version | diff --git a/skills/myos/references/conventions.md b/skills/myos/references/conventions.md index dc5300d..69bd4c2 100644 --- a/skills/myos/references/conventions.md +++ b/skills/myos/references/conventions.md @@ -216,6 +216,43 @@ and an unbound port becomes `0.0.0.0` exactly like a deliberate public one. The split of responsibility: the **scope** belongs to the stack, in its compose file; the **address** of a scope belongs to the host, in its configuration. +## Certificates + +A site gets a certificate by being routed, not by being written down a second +time. `myos cert` reads the same `urlprefix-` tags fabio routes on, and decides +what to ask for: + +```sh +myos cert list # what would be asked for, and over which challenge +myos cert issue # ask for it +myos cert renew # what is close to expiry, for a cron +myos cert show # what exists, and when it expires +``` + +| a tag routes | myos asks for | challenge | +|---|---|---| +| `app.example.org` | a certificate for that name | http-01 | +| `*.ipns.example.org` | `ipns.example.org` **and** `*.ipns.example.org` | dns-01 | + +That is the whole of "per site or wildcard as needed": a wildcard is asked for +where a tag uses one, and it absorbs the concrete names it covers. A wildcard +covers one label, so `*.example.org` absorbs `a.example.org` but not +`a.b.example.org`, which keeps its own certificate. + +`MYOS_CERT_MODE=per-site` never asks for a wildcard, which keeps everything on +http-01 and needs no DNS credentials. `wildcard` asks for one per domain. + +The issuer is [dehydrated](https://github.com/dehydrated-io/dehydrated), a +shell script, in the `host/dehydrated` stack. It answers http-01 itself on a +port bound to the loopback, which fabio routes +`/.well-known/acme-challenge/` to. A wildcard needs dns-01, so point +`HOST_DEHYDRATED_DNS_HOOK` at your provider's script; it receives dehydrated's +own hook arguments. + +Certificates land where fabio looks for them, `-cert.pem` and +`-key.pem` under `/host/certs`, written to a temporary name and moved, so +fabio never reads half a file. + ## Groups A group is a lowercase name whose value lists stacks. It can live in a `.env`, diff --git a/spec/golden/cases.txt b/spec/golden/cases.txt index a44a36b..92d5179 100644 --- a/spec/golden/cases.txt +++ b/spec/golden/cases.txt @@ -91,3 +91,4 @@ cmd-recreate | host-project | recreate host/consul cmd-status | host-project | status host/consul expose-none | host-project | expose host/consul expose-default | app-git | expose +cert-none | host-project | cert list host/consul diff --git a/spec/golden/expected.cli/cert-none.txt b/spec/golden/expected.cli/cert-none.txt new file mode 100644 index 0000000..5788d2f --- /dev/null +++ b/spec/golden/expected.cli/cert-none.txt @@ -0,0 +1,2 @@ +no routed hostname: nothing to certify +[exit 0] diff --git a/spec/golden/expected/cert-none.txt b/spec/golden/expected/cert-none.txt new file mode 100644 index 0000000..dc40928 --- /dev/null +++ b/spec/golden/expected/cert-none.txt @@ -0,0 +1,4 @@ +WARNING: myos[0] cert-rule-exists: target cert unavailable in app myos +WARNING: myos[0] list-rule-exists: target list unavailable in app myos +WARNING: myos[0] host/consul-rule-exists: target host/consul unavailable in app myos +[exit 0] diff --git a/spec/unit/cert_dns_helper.sh b/spec/unit/cert_dns_helper.sh new file mode 100644 index 0000000..0bcf719 --- /dev/null +++ b/spec/unit/cert_dns_helper.sh @@ -0,0 +1,4 @@ +#shellcheck shell=sh +MYOS_CERT_MODE=$1 +myos_cert_names() { printf 'urlprefix-*.ipns.ex.org/*\nurlprefix-a.ex.org/*\n' | myos_cert_parse; } +myos_cert_needs_dns && echo yes || echo no diff --git a/spec/unit/cert_groups_helper.sh b/spec/unit/cert_groups_helper.sh new file mode 100644 index 0000000..59cfda3 --- /dev/null +++ b/spec/unit/cert_groups_helper.sh @@ -0,0 +1,10 @@ +#shellcheck shell=sh +# myos_cert_names normally reads the compose configuration; here it is replaced +# by a fixed list so the grouping can be checked on its own. +MYOS_CERT_MODE=$1 +_fixture=${2-'urlprefix-ipfs.ex.org/* +urlprefix-*.ipns.ex.org/* +urlprefix-a.ipns.ex.org/* +urlprefix-ex.org/*'} +myos_cert_names() { printf '%s' "$_fixture" | myos_cert_parse; } +myos_cert_groups diff --git a/spec/unit/cert_parse_helper.sh b/spec/unit/cert_parse_helper.sh new file mode 100644 index 0000000..0473afc --- /dev/null +++ b/spec/unit/cert_parse_helper.sh @@ -0,0 +1,2 @@ +#shellcheck shell=sh +printf 'urlprefix-ipfs.ex.org/api/*\nurlprefix-*.ipns.ex.org/*\nurlprefix-ex.org:443/* proto=https\nurlprefix-*/*\n' | myos_cert_parse diff --git a/spec/unit/cert_spec.sh b/spec/unit/cert_spec.sh new file mode 100644 index 0000000..850ed96 --- /dev/null +++ b/spec/unit/cert_spec.sh @@ -0,0 +1,74 @@ +#shellcheck shell=sh +Include lib/core.sh +Include lib/str.sh +Include lib/var.sh +Include lib/tags.sh +Include lib/naming.sh +Include lib/stack.sh +Include lib/cert.sh + +# The hostnames a server must certify are already declared in the fabio route +# tags. Reading them there rather than in a domains.txt keeps one source of +# truth, and decides on its own which name needs a wildcard. +Describe 'lib/cert.sh' + Describe 'myos_cert_parse' + one_tag() { printf 'urlprefix-a.ex.org/x/*\n' | myos_cert_parse; } + It 'reads a hostname out of a route tag' + When call one_tag + The output should equal "a.ex.org" + End + It 'drops the path, the port and the options' + When run source spec/unit/cert_parse_helper.sh + The line 1 should equal "*.ipns.ex.org" + The line 2 should equal "ex.org" + The line 3 should equal "ipfs.ex.org" + The lines of output should equal 3 + End + End + + Describe 'myos_cert_covers' + Parameters + "ex.org" "a.ex.org" success + "ex.org" "ex.org" failure + "ex.org" "a.b.ex.org" failure + "ex.org" "other.org" failure + End + It "*.$1 against $2" + When call myos_cert_covers "$1" "$2" + The status should be "$3" + End + End + + Describe 'myos_cert_groups' + It 'asks for a wildcard where a tag uses one, and absorbs what it covers' + When run source spec/unit/cert_groups_helper.sh auto + The line 1 should equal "ipns.ex.org *.ipns.ex.org" + The output should include "ipfs.ex.org" + The output should not include "a.ipns.ex.org *" + End + It 'never asks for a wildcard in per-site mode' + When run source spec/unit/cert_groups_helper.sh per-site + The output should not include "*" + The output should include "a.ipns.ex.org" + End + It 'asks for one per domain in wildcard mode' + When run source spec/unit/cert_groups_helper.sh wildcard + The output should include "ex.org *.ex.org" + End + It 'says nothing when nothing is routed' + When run source spec/unit/cert_groups_helper.sh auto "" + The output should equal "" + End + End + + Describe 'myos_cert_needs_dns' + It 'is true when a wildcard is asked for' + When run source spec/unit/cert_dns_helper.sh auto + The output should equal "yes" + End + It 'is false without one' + When run source spec/unit/cert_dns_helper.sh per-site + The output should equal "no" + End + End +End