From 55fae625d601d2c1bc1c444311a7a64293a2b17c Mon Sep 17 00:00:00 2001 From: Yann Autissier Date: Thu, 3 Sep 2026 20:46:28 +0200 Subject: [PATCH] keep the dynamism of make in pure shell Two mechanisms, matching what the make engine actually did: Lazy defaults. A stack setting is a function myos_default_, called only when the variable has no value, and called again at every reference. That is exactly a recursive ?=: an explicit value wins, and the default follows a DOMAIN that a .env changes later. The prefix is what makes it safe; the first version used a bare function named after the variable, and the test suite caught it running /usr/bin/host for a stack group called host. Templates. myos env-update fills a .env from the .env.dist files, expanding ${VAR} against the current values and running $(command), forward references included. Also fixed: the project .env now wins over /etc/conf.d/myos, which is what the documentation claimed and the code did not. share/make/shim.mk lets a project keep make as a front end: every myos command becomes a target that shells out to bin/myos, and the project keeps its own targets and its stack .mk files. It sits outside make/ because the legacy engine globs every .mk in there. --- CHANGELOG.md | 8 ++ README.md | 13 +++ bin/myos | 34 +++++-- lib/cmd/env-update.sh | 26 ++++++ lib/config.sh | 91 ++++++++++++++++++ lib/tags.sh | 9 +- lib/var.sh | 56 +++++++++++ share/make/shim.mk | 57 ++++++++++++ skills/myos/references/authoring.md | 6 +- skills/myos/references/commands.md | 1 + skills/myos/references/conventions.md | 44 +++++++-- spec/fixtures/shim-project/Makefile | 2 + .../shim-project/stack/host/consul.yml | 12 +++ .../shim-project/stack/host/fabio.yml | 13 +++ spec/fixtures/shim-project/stack/host/host.mk | 7 ++ .../shim-project/stack/host/registrator.yml | 9 ++ spec/golden/cases.txt | 4 + .../expected.cli/cat-unknown-target.txt | 1 + spec/golden/expected/shim-config.txt | 2 + spec/golden/expected/shim-env.txt | 2 + spec/golden/expected/shim-project-target.txt | 2 + spec/golden/expected/shim-up-group.txt | 4 + spec/support/run.sh | 5 +- spec/unit/compose_spec.sh | 1 + spec/unit/config_spec.sh | 93 +++++++++++++++++++ spec/unit/config_update_helper.sh | 5 + spec/unit/hooks_spec.sh | 1 + spec/unit/stack_spec.sh | 1 + spec/unit/tags_spec.sh | 1 + spec/unit/var_default_helper.sh | 3 + spec/unit/var_lazy_helper.sh | 7 ++ spec/unit/var_loop_helper.sh | 3 + spec/unit/var_spec.sh | 73 +++++++++++++++ 33 files changed, 571 insertions(+), 25 deletions(-) create mode 100644 lib/cmd/env-update.sh create mode 100644 lib/var.sh create mode 100644 share/make/shim.mk create mode 100644 spec/fixtures/shim-project/Makefile create mode 100644 spec/fixtures/shim-project/stack/host/consul.yml create mode 100644 spec/fixtures/shim-project/stack/host/fabio.yml create mode 100644 spec/fixtures/shim-project/stack/host/host.mk create mode 100644 spec/fixtures/shim-project/stack/host/registrator.yml create mode 100644 spec/golden/expected/shim-config.txt create mode 100644 spec/golden/expected/shim-env.txt create mode 100644 spec/golden/expected/shim-project-target.txt create mode 100644 spec/golden/expected/shim-up-group.txt create mode 100644 spec/unit/config_update_helper.sh create mode 100644 spec/unit/var_default_helper.sh create mode 100644 spec/unit/var_lazy_helper.sh create mode 100644 spec/unit/var_loop_helper.sh create mode 100644 spec/unit/var_spec.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b2f6c7..89a3f79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ - agent skill in `skills/myos/`, contributor notes in `AGENTS.md` - stacks carry their settings in `.env` and `.sh` hooks, so the catalogue no longer needs make to be installed +- lazy defaults (`myos_default_` functions) give the recursive `?=` of + make in pure shell: an explicit value wins, and the default is recomputed + at each reference +- `myos env-update` generates a `.env` from the `.env.dist` templates, + expanding `${VAR}` and `$(command)`, including forward references +- the project `.env` now wins over `/etc/conf.d/myos`, as documented; + `MYOS_CONF_PRIORITY=system` restores the previous order +- `share/make/shim.mk`: make as an optional front end over the same shell code - `--color always|never|auto`, and no colour when the output is piped - verified under the /bin/sh of Alpine (busybox) and Debian (dash) - the make engine still works and is still covered by the golden tests diff --git a/README.md b/README.md index 3e9848f..917feef 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,19 @@ The make targets keep working: `print-VAR`, `stack--`, `SETUP_UFW=true` enables the ufw/ufw-docker integration (`myos setup-ufw`). +## With make + +A project that would rather drive make can include the shim, which turns every +myos command into a make target while leaving its own targets alone: + +```make +MYOS ?= /usr/local/lib/myos +include $(MYOS)/share/make/shim.mk +``` + +`make up STACK=host` then runs exactly what `myos up host` runs: the shim only +forwards. make is not needed otherwise, and the CLI never calls it. + ## For agents `skills/myos/SKILL.md` is a skill describing how to drive myos, with diff --git a/bin/myos b/bin/myos index c2b8c35..a442e47 100755 --- a/bin/myos +++ b/bin/myos @@ -19,7 +19,7 @@ done MYOS_ROOT=$(cd "$(dirname "$_self")/.." && pwd -P) export MYOS_ROOT -for _m in core str tags naming stack config compose hooks; do +for _m in core str var tags naming stack config compose hooks; do # shellcheck source=/dev/null . "$MYOS_ROOT/lib/$_m.sh" done @@ -58,6 +58,7 @@ Commands: ps logs config exec run inspect and enter them ls [--groups] list the stacks myos can see env [VAR...] show resolved variables + env-update fill .env from the .env.dist templates doctor check the installation version print the myos version @@ -121,11 +122,30 @@ esac WORKDIR=${WORKDIR:-$PWD} WORKDIR=$(cd "$WORKDIR" 2>/dev/null && pwd -P) || myos_die "$MYOS_E_USAGE" "no such directory: $WORKDIR" -for _f in $(myos_conf_files); do myos_dotenv_load "$_f"; done -myos_dotenv_load "${HOME:-}/.config/myos/config" +# ENV decides which .env. to read, so it is resolved first, from the most +# specific source that names it. +if [ -z "${ENV:-}" ]; then + for _f in "$WORKDIR/.env" "${HOME:-}/.config/myos/config" $(myos_conf_files); do + ENV=$(myos_dotenv_parse "$_f" | sed -n 's/^ENV=//p' | tail -1) + [ -n "$ENV" ] && break + done +fi ENV=${ENV:-local} -myos_dotenv_load "$WORKDIR/.env.$ENV" -myos_dotenv_load "$WORKDIR/.env" + +# The layers, most specific first: the loader never overwrites a value, so the +# order below is the order of precedence. The environment and the VAR=value +# arguments are already set, and therefore win over every file. +# MYOS_CONF_PRIORITY=system puts the machine files first, as the make engine did. +myos_config_layers() { + if [ "${MYOS_CONF_PRIORITY:-}" = system ]; then + myos_conf_files + printf '%s\n' "${HOME:-}/.config/myos/config" "$WORKDIR/.env.$ENV" "$WORKDIR/.env" + else + printf '%s\n' "$WORKDIR/.env.$ENV" "$WORKDIR/.env" "${HOME:-}/.config/myos/config" + myos_conf_files + fi +} +for _f in $(myos_config_layers); do myos_dotenv_load "$_f"; done # Overlay switches. Their names drive which ..yml files load, # so these defaults decide that e.g. supabase.labels.yml is picked up. @@ -159,7 +179,7 @@ if [ -z "$MYOS_REFS" ]; then fi if [ -z "$MYOS_REFS" ]; then case $MYOS_CMD in - env|ls|doctor|version|help) ;; + env|env-update|ls|doctor|version|help) ;; *) myos_die "$MYOS_E_USAGE" "no stack given, and no compose file in $WORKDIR" ;; esac fi @@ -211,7 +231,7 @@ esac # shellcheck source=/dev/null if [ -f "$MYOS_ROOT/lib/cmd/$MYOS_CMD.sh" ]; then . "$MYOS_ROOT/lib/cmd/$MYOS_CMD.sh" - "myos_cmd_$MYOS_CMD" + "myos_cmd_$(printf '%s' "$MYOS_CMD" | tr '-' '_')" exit $? fi diff --git a/lib/cmd/env-update.sh b/lib/cmd/env-update.sh new file mode 100644 index 0000000..1629fbd --- /dev/null +++ b/lib/cmd/env-update.sh @@ -0,0 +1,26 @@ +#shellcheck shell=sh +# myos env-update fill the .env of the workdir from the .env.dist it finds +# +# Templates are read from the workdir and from every requested stack, so a +# stack can ship the variables it expects and their defaults. +myos_cmd_env_update() { + _target=${ENV_FILE:-$WORKDIR/.env} + _dists=$MYOS_ARGS + if [ -z "$_dists" ]; then + for _ref in $MYOS_STACKS; do + _d=$(myos_stack_resolve "$_ref" 2>/dev/null) || continue + for _c in "$_d/.env.dist" "$_d/$(myos_stack_name "$_ref").env.dist"; do + [ -f "$_c" ] && _dists="${_dists:+$_dists }$_c" + done + done + for _c in "$WORKDIR/.env.dist" "$WORKDIR/.env.example" "$WORKDIR/.env.sample"; do + [ -f "$_c" ] && _dists="${_dists:+$_dists }$_c" + done + fi + [ -n "$_dists" ] || { myos_warning "no .env.dist found for $MYOS_STACKS"; return 0; } + for _dist in $_dists; do + myos_info "env-update $_target from $_dist" + myos_env_update "$_target" "$_dist" "$WORKDIR/.env.$ENV" + done + printf '%s\n' "$_target" +} diff --git a/lib/config.sh b/lib/config.sh index 16865dc..078ecef 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -80,3 +80,94 @@ myos_env_export() { done return 0 } + +# myos_expand STRING substitute ${VAR} and $(command) in STRING. +# shellcheck disable=SC2016 # the single quotes are deliberate: these patterns +# match the literal characters ${ and $( in the input, they are not expansions +# This is what the make engine did when it generated a .env out of a .env.dist: +# ${VAR} takes the current value, $(cmd) runs the command. Nothing else is +# interpreted, so the rest of the line can hold anything. +myos_expand() { + _in=$1 + _guard=0 + while [ "$_guard" -lt 16 ]; do + _guard=$((_guard + 1)) + case $_in in + *'${'*'}'*) + _pre=${_in%%'${'*} + _rest=${_in#*'${'} + _name=${_rest%%\}*} + _post=${_rest#*\}} + case $_name in + ''|*[!A-Za-z0-9_]*) _in="$_pre\${$_name}$_post"; break ;; + esac + _in="$_pre$(myos_var "$_name")$_post" ;; + *) break ;; + esac + done + _guard=0 + while [ "$_guard" -lt 16 ]; do + _guard=$((_guard + 1)) + case $_in in + *'$('*')'*) + _pre=${_in%%'$('*} + _rest=${_in#*'$('} + _cmd=${_rest%%)*} + _post=${_rest#*)} + _in="$_pre$(eval "$_cmd" 2>/dev/null)$_post" ;; + *) break ;; + esac + done + printf '%s' "$_in" +} + +# myos_env_update FILE DIST [OVER...] +# Add to FILE every variable of DIST that is missing from it, expanded. +# A variable that already has a value keeps it, whether it comes from the +# environment, from FILE, or from one of the OVER files: the .env is a record +# of the choices already made, never something that overwrites them. +myos_env_update() { + _file=$1; _dist=$2; shift 2 + [ -f "$_dist" ] || return 0 + [ -e "$_file" ] || : > "$_file" + + # what the overrides pin, read before anything else + for _over in "$@"; do + [ -f "$_over" ] || continue + myos_dotenv_load "$_over" + done + + # Which keys already hold a choice, made in the environment, in the .env or + # in an override. Those are kept verbatim; everything else is a template. + _preset=" " + while IFS= read -r _kv; do + [ -n "$_kv" ] || continue + _k=${_kv%%=*} + [ -n "$_k" ] || continue + [ -n "$(myos_var "$_k")" ] && _preset="$_preset$_k " + done <> "$_file" + _added=$((_added + 1)) + done <_TAGS label # to consul, fabio routes on the urlprefix- tags it finds there. - -# myos_var NAME value of the variable named NAME, empty when unset -myos_var() { - [ -n "${1:-}" ] || return 0 - eval "printf '%s' \"\${$1:-}\"" -} +# +# Every lookup goes through myos_var (lib/var.sh), so a stack setting may be a +# plain value or a lazy default, and the two behave the same here. # myos_uri SERVICE PORT [BASE_URI] # ., unless _SERVICE[_]_NAME overrides the prefix diff --git a/lib/var.sh b/lib/var.sh new file mode 100644 index 0000000..c9ddc9a --- /dev/null +++ b/lib/var.sh @@ -0,0 +1,56 @@ +#shellcheck shell=sh +# var: variable resolution, with the lazy defaults the make engine had. +# +# make gives every `VAR ?= $(call ...)` two properties at once: an explicit +# value wins, and the default is re-evaluated at each reference, so it sees +# whatever a .env loaded later has changed. +# +# The shell gets both by keeping defaults in functions: `myos_var NAME` reads +# the variable when it has a value, and otherwise calls the function +# `myos_default_NAME`. The prefix matters: a bare function named after the +# variable would collide with commands on PATH, and a stack setting called +# `host` or `test` would then run a program instead of returning a value. +# +# myos_default_HOST_FABIO_SERVICE_9998_TAGS() { myos_tagprefix HOST_FABIO 9998; } +# +# is the exact equivalent of +# +# HOST_FABIO_SERVICE_9998_TAGS ?= $(call tagprefix,HOST_FABIO,9998) + +MYOS_VAR_MAX_DEPTH=${MYOS_VAR_MAX_DEPTH:-32} + +# myos_var NAME the value of NAME: the variable if it has one, else the lazy +# default, else empty. +myos_var() { + [ -n "${1:-}" ] || return 0 + eval "_myos_set=\${$1+yes}" + if [ "${_myos_set:-}" = yes ]; then + eval "printf '%s' \"\$$1\"" + return 0 + fi + myos_var_is_lazy "$1" || return 0 + + # a default written in terms of itself would loop for ever + _myos_depth=$((${MYOS_VAR_DEPTH:-0} + 1)) + if [ "$_myos_depth" -gt "$MYOS_VAR_MAX_DEPTH" ]; then + myos_error "variable $1 is defined in terms of itself" + return 1 + fi + MYOS_VAR_DEPTH=$_myos_depth "myos_default_$1" +} + +# myos_default NAME BODY declare a lazy default from a string, for callers +# that build the variable name at run time +myos_default() { + eval "myos_default_$1() { $2; }" +} + +# myos_var_is_lazy NAME true when NAME has no value but has a lazy default +myos_var_is_lazy() { + eval "_myos_set=\${$1+yes}" + [ "${_myos_set:-}" = yes ] && return 1 + # a shell function, never a command on PATH: command -v prints the name back + # for a function and a path for a program + _myos_fn=$(command -v "myos_default_$1" 2>/dev/null) || return 1 + [ "$_myos_fn" = "myos_default_$1" ] +} diff --git a/share/make/shim.mk b/share/make/shim.mk new file mode 100644 index 0000000..f67e18c --- /dev/null +++ b/share/make/shim.mk @@ -0,0 +1,57 @@ +## +# myos, from make. +# +# This file gives a project the myos commands as make targets, without the make +# engine: every target shells out to bin/myos, which is pure POSIX sh. What the +# project keeps from make is what make is actually good at, and the CLI is not: +# its own targets, its own dependencies, and the .mk files of its stacks. +# +# MYOS ?= /usr/local/lib/myos +# include $(MYOS)/share/make/shim.mk +# +# It lives outside make/ on purpose: the legacy engine includes every .mk of +# that directory, and would pull this one in too. +# +# Then `make up STACK=host` and `myos up host` do the same thing, through the +# same code. Variables given on the command line are forwarded, so +# `make up STACK=host DOMAIN=example.org` behaves as expected. + +MYOS ?= $(patsubst %/share/make/shim.mk,%,$(lastword $(MAKEFILE_LIST))) +MYOS_BIN ?= $(MYOS)/bin/myos +STACK_DIR_NAME ?= stack +STACK_DIR ?= $(wildcard $(CURDIR)/$(STACK_DIR_NAME)) + +# variable MYOS_ARGS: variables set on the make command line, forwarded to myos +MYOS_ARGS ?= $(foreach v,$(MAKEOVERRIDES),$(v)) + +.DEFAULT_GOAL := help + +## the stack files of the project may add their own targets and variables +include $(wildcard $(STACK_DIR)/*.mk $(STACK_DIR)/*/*.mk) + +# function make: run a myos command, for the stack .mk files that call it +define make + $(MYOS_BIN) $(MYOS_ARGS) $(1) +endef + +# target help: List the myos commands +.PHONY: help +help: + @$(MYOS_BIN) help + +# target myos: Run an arbitrary myos command, as in `make myos ARGS="up host"` +.PHONY: myos +myos: + @$(MYOS_BIN) $(MYOS_ARGS) $(ARGS) + +# make tries to remake every makefile it read, and the catch-all below would +# hand each of them to myos as a command. An empty rule stops that. +$(MAKEFILE_LIST): ; + +# target %: Hand anything else to myos +## a target the project defines itself keeps precedence over this rule +%: FORCE + @$(MYOS_BIN) $(MYOS_ARGS) $@ $(ARGS) + +.PHONY: FORCE +FORCE: ; diff --git a/skills/myos/references/authoring.md b/skills/myos/references/authoring.md index b9bdfc8..c7959e4 100644 --- a/skills/myos/references/authoring.md +++ b/skills/myos/references/authoring.md @@ -7,7 +7,7 @@ stack//.yml the services stack//.local.yml what only makes sense on a workstation (published ports…) stack//.labels.yml the registrator labels, so routing stays optional stack//.env plain settings: versions, defaults -stack//.sh computed settings (fabio tags), no make needed +stack//.sh lazy defaults (fabio tags), no make needed stack//.env.dist the variables it expects, with defaults stack//README.md what it is and what it needs ``` @@ -54,7 +54,9 @@ lib/core.sh logging, exit codes, dry run lib/str.sh strings and version comparison lib/naming.sh project names, networks, user identity lib/stack.sh stack path, references, overlays, groups -lib/config.sh dotenv, variables of the compose files +lib/var.sh variable resolution and lazy defaults +lib/config.sh dotenv, templates, variables of the compose files +lib/hooks.sh the per-stack .env and .sh lib/compose.sh finding and calling docker compose lib/tags.sh fabio tags lib/cmd/.sh one file per command diff --git a/skills/myos/references/commands.md b/skills/myos/references/commands.md index 58657ca..6916d8a 100644 --- a/skills/myos/references/commands.md +++ b/skills/myos/references/commands.md @@ -26,6 +26,7 @@ myos [options] [stack...] [VAR=value...] [-- args...] | `build` / `pull` | images | | `ls [--groups]` | the stacks and groups myos can see | | `env [VAR...]` | resolved variables | +| `env-update` | fill the `.env` from the `.env.dist` templates | | `doctor` | check the installation | | `version` | the myos version | diff --git a/skills/myos/references/conventions.md b/skills/myos/references/conventions.md index bcca7e4..bc8f619 100644 --- a/skills/myos/references/conventions.md +++ b/skills/myos/references/conventions.md @@ -110,21 +110,49 @@ A stack keeps its own settings next to its compose files: | `.sh` | values that have to be computed | | `.mk` | the legacy make snippet; still read for its groups | -A `.sh` hook is sourced with the myos helpers available, and sets variables -directly. This is what lets a stack work on a machine that has no make: +A `.sh` hook declares **lazy defaults**: a function named +`myos_default_`, called only when the variable has no value, and +called again at each reference. That is the make `?=` on a recursive variable, +in shell: ```sh # stack/host/fabio.sh -HOST_FABIO_VERSION=${HOST_FABIO_VERSION:-1.6.3} -HOST_FABIO_SERVICE_9998_NAME=${HOST_FABIO_SERVICE_9998_NAME:-fabio} -HOST_FABIO_SERVICE_9998_AUTH=${HOST_FABIO_SERVICE_9998_AUTH:-default} -HOST_FABIO_SERVICE_9998_TAGS=${HOST_FABIO_SERVICE_9998_TAGS:-$(myos_tagprefix HOST_FABIO 9998)} +myos_default_HOST_FABIO_VERSION() { printf '1.6.3'; } +myos_default_HOST_FABIO_SERVICE_9998_NAME() { printf 'fabio'; } +myos_default_HOST_FABIO_SERVICE_9998_AUTH() { printf 'default'; } +myos_default_HOST_FABIO_SERVICE_9998_TAGS() { myos_tagprefix HOST_FABIO 9998; } ``` -Always write `${VAR:-default}` so the environment and the `.env` still win. +Two things follow, and they are the point: + +- a value given anywhere (environment, `.env`, command line) wins over the + default, without the hook having to say so; +- the default is computed against the values current **at the moment it is + read**, so a `DOMAIN` set in a `.env` loaded later is taken into account. + +The prefix is not decoration: a bare function named `host` or `test` would be +indistinguishable from the program of that name, and myos would run it. + Helpers available in a hook: `myos_tagprefix`, `myos_urlprefix`, `myos_uri`, `myos_url`, `myos_envprefix`, `myos_servicenvs`, `myos_var`, `myos_lower`, -`myos_upper`. +`myos_upper`, and `myos_default NAME 'body'` when the name is built at run time. + +## Templates: .env.dist + +A stack may ship a `.env.dist` listing the variables it expects, with their +defaults. `myos env-update` writes the missing ones into the `.env`, expanding +`${VAR}` against the current values and running `$(command)`: + +```sh +# stack/demo/.env.dist +DEMO_IMAGE=alpine:${DEMO_VERSION} +DEMO_VERSION=3.20 +DEMO_SECRET=$(openssl rand -hex 16) +``` + +A line may refer to a variable defined further down. A variable that already +has a value keeps it: the `.env` records choices, it never overwrites them, and +running the command twice changes nothing. ## Groups diff --git a/spec/fixtures/shim-project/Makefile b/spec/fixtures/shim-project/Makefile new file mode 100644 index 0000000..10ab102 --- /dev/null +++ b/spec/fixtures/shim-project/Makefile @@ -0,0 +1,2 @@ +MYOS ?= /usr/local/lib/myos +include $(MYOS)/share/make/shim.mk diff --git a/spec/fixtures/shim-project/stack/host/consul.yml b/spec/fixtures/shim-project/stack/host/consul.yml new file mode 100644 index 0000000..31a5028 --- /dev/null +++ b/spec/fixtures/shim-project/stack/host/consul.yml @@ -0,0 +1,12 @@ +services: + consul: + image: hashicorp/consul:1.15 + container_name: ${HOST_COMPOSE_PROJECT_NAME:-localhost}-consul + network_mode: host + restart: always + environment: + CONSUL_HTTP_TOKEN: ${HOST_CONSUL_HTTP_TOKEN} + volumes: + - consul:/consul/data +volumes: + consul: diff --git a/spec/fixtures/shim-project/stack/host/fabio.yml b/spec/fixtures/shim-project/stack/host/fabio.yml new file mode 100644 index 0000000..eb82e46 --- /dev/null +++ b/spec/fixtures/shim-project/stack/host/fabio.yml @@ -0,0 +1,13 @@ +services: + fabio: + image: fabiolb/fabio:1.6.3 + container_name: ${HOST_COMPOSE_PROJECT_NAME:-localhost}-fabio + depends_on: [consul] + ports: + - "80:80" + - "443:443" + environment: + FABIO_REGISTRY_CONSUL_ADDR: ${DOCKER_HOST_INET4:-127.0.0.1}:8500 + FABIO_LOG_ACCESS_TARGET: ${HOST_FABIO_LOG_ACCESS:-} + networks: + - public diff --git a/spec/fixtures/shim-project/stack/host/host.mk b/spec/fixtures/shim-project/stack/host/host.mk new file mode 100644 index 0000000..03c3613 --- /dev/null +++ b/spec/fixtures/shim-project/stack/host/host.mk @@ -0,0 +1,7 @@ +host ?= host/consul host/fabio + +# a target of the project, on top of the myos commands: this is what make is +# kept for, and what the shim leaves alone +.PHONY: host-certs +host-certs: + @echo "would renew the certificates of $(host)" diff --git a/spec/fixtures/shim-project/stack/host/registrator.yml b/spec/fixtures/shim-project/stack/host/registrator.yml new file mode 100644 index 0000000..0af2821 --- /dev/null +++ b/spec/fixtures/shim-project/stack/host/registrator.yml @@ -0,0 +1,9 @@ +services: + registrator: + image: gliderlabs/registrator:master + container_name: ${HOST_COMPOSE_PROJECT_NAME:-localhost}-registrator + network_mode: host + depends_on: [consul] + command: -internal=false -useIpFromLabel SERVICE_ADDRESS consul://127.0.0.1:8500 + volumes: + - ${DOCKER_SOCKET_LOCATION:-/var/run/docker.sock}:/tmp/docker.sock diff --git a/spec/golden/cases.txt b/spec/golden/cases.txt index daba4bc..e5acb99 100644 --- a/spec/golden/cases.txt +++ b/spec/golden/cases.txt @@ -77,3 +77,7 @@ cmd-scale | app-nogit | scale postgres SERVICE=postgre cmd-ls | host-project | ls cmd-ls-groups | host-project | ls --groups cmd-env-domain | host-project | env DOMAIN +shim-up-group | shim-project | @make up STACK=host DRYRUN=true +shim-config | shim-project | @make config STACK=host/consul DRYRUN=true +shim-project-target | shim-project | @make host-certs +shim-env | shim-project | @make env ARGS=COMPOSE_PROJECT_NAME STACK=host diff --git a/spec/golden/expected.cli/cat-unknown-target.txt b/spec/golden/expected.cli/cat-unknown-target.txt index 08636f8..dcc23ae 100644 --- a/spec/golden/expected.cli/cat-unknown-target.txt +++ b/spec/golden/expected.cli/cat-unknown-target.txt @@ -16,6 +16,7 @@ Commands: ps logs config exec run inspect and enter them ls [--groups] list the stacks myos can see env [VAR...] show resolved variables + env-update fill .env from the .env.dist templates doctor check the installation version print the myos version diff --git a/spec/golden/expected/shim-config.txt b/spec/golden/expected/shim-config.txt new file mode 100644 index 0000000..d03e10d --- /dev/null +++ b/spec/golden/expected/shim-config.txt @@ -0,0 +1,2 @@ +docker compose -f @WD@/stack/host/consul.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host config +[exit 0] diff --git a/spec/golden/expected/shim-env.txt b/spec/golden/expected/shim-env.txt new file mode 100644 index 0000000..53723d0 --- /dev/null +++ b/spec/golden/expected/shim-env.txt @@ -0,0 +1,2 @@ +COMPOSE_PROJECT_NAME testhost +[exit 0] diff --git a/spec/golden/expected/shim-project-target.txt b/spec/golden/expected/shim-project-target.txt new file mode 100644 index 0000000..e353772 --- /dev/null +++ b/spec/golden/expected/shim-project-target.txt @@ -0,0 +1,2 @@ +would renew the certificates of host/consul host/fabio +[exit 0] diff --git a/spec/golden/expected/shim-up-group.txt b/spec/golden/expected/shim-up-group.txt new file mode 100644 index 0000000..a95d1b3 --- /dev/null +++ b/spec/golden/expected/shim-up-group.txt @@ -0,0 +1,4 @@ +docker network create tester-local +docker network create testhost +docker compose -f @WD@/stack/host/consul.yml -f @WD@/stack/host/fabio.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host up -d +[exit 0] diff --git a/spec/support/run.sh b/spec/support/run.sh index 9692b89..2689ad1 100644 --- a/spec/support/run.sh +++ b/spec/support/run.sh @@ -47,8 +47,9 @@ myos_normalize() { # shellcheck disable=SC2046 # myos_hermetic_env output is meant to be word-split myos_run_engine() { _engine=$1; _sb=$2; shift 2 - # "@make" as first arg = include mode: the project Makefile includes make/include.mk - # and make runs from the project dir (CURDIR = project). Same for both engines. + # "@make" as first arg = the project drives make itself: its Makefile includes + # either the legacy engine (make/include.mk) or the shim (make/shim.mk), and + # make runs from the project directory. if [ "${1:-}" = "@make" ]; then shift _out=$(cd "$_sb/wd" && env -i $(myos_hermetic_env "$_sb") MYOS_CONF=/dev/null \ diff --git a/spec/unit/compose_spec.sh b/spec/unit/compose_spec.sh index d13c3c6..9fb3aac 100644 --- a/spec/unit/compose_spec.sh +++ b/spec/unit/compose_spec.sh @@ -1,6 +1,7 @@ #shellcheck shell=sh Include lib/str.sh Include lib/core.sh +Include lib/var.sh Include lib/tags.sh Include lib/config.sh Include lib/compose.sh diff --git a/spec/unit/config_spec.sh b/spec/unit/config_spec.sh index a001bd7..4def3fd 100644 --- a/spec/unit/config_spec.sh +++ b/spec/unit/config_spec.sh @@ -1,6 +1,7 @@ #shellcheck shell=sh Include lib/str.sh Include lib/core.sh +Include lib/var.sh Include lib/tags.sh Include lib/config.sh @@ -104,3 +105,95 @@ Describe 'lib/config.sh robustness' The status should be success End End + +# The make engine generated a .env out of a .env.dist, expanding ${VAR} against +# the current values and running $(cmd). These check the shell equivalent. +Describe 'lib/config.sh templates' + setup() { MYOS_TMP=$(mktemp -d "${TMPDIR:-/tmp}/myos-tpl.XXXXXX"); } + cleanup() { rm -rf "$MYOS_TMP"; } + BeforeEach setup + AfterEach cleanup + + Describe 'myos_expand' + It 'substitutes a variable' + DOMAIN=example.org + When call myos_expand 'https://app.${DOMAIN}/' + The output should equal "https://app.example.org/" + End + It 'substitutes several, including twice the same' + DOMAIN=example.org + When call myos_expand '${DOMAIN}:${DOMAIN}' + The output should equal "example.org:example.org" + End + It 'substitutes a lazy default like any other value' + # shellcheck disable=SC2317 + myos_default_LAZY_DOMAIN() { printf 'lazy.example.org'; } + When call myos_expand 'https://${LAZY_DOMAIN}/' + The output should equal "https://lazy.example.org/" + End + It 'empties an unknown variable, as make does' + When call myos_expand 'a${NOT_SET_ANYWHERE}b' + The output should equal "ab" + End + It 'runs a command substitution' + When call myos_expand 'pre-$(echo mid)-post' + The output should equal "pre-mid-post" + End + It 'leaves a malformed reference alone' + When call myos_expand 'a${not-a-name}b' + The output should equal 'a${not-a-name}b' + End + End + + Describe 'myos_env_update' + It 'adds the missing variables, expanded' + printf 'DOMAIN=example.org\nAPP_URL=https://app.${DOMAIN}/\n' > "$MYOS_TMP/.env.dist" + DOMAIN=chosen.org + When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist" + The status should be success + The contents of file "$MYOS_TMP/.env" should include "APP_URL=https://app.chosen.org/" + The contents of file "$MYOS_TMP/.env" should include "DOMAIN=chosen.org" + End + It 'never touches a value already recorded' + printf 'KEEP=default\n' > "$MYOS_TMP/.env.dist" + printf 'KEEP=already-chosen\n' > "$MYOS_TMP/.env" + When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist" + The contents of file "$MYOS_TMP/.env" should equal "KEEP=already-chosen" + End + It 'is idempotent' + printf 'A=1\nB=${A}2\n' > "$MYOS_TMP/.env.dist" + When run source spec/unit/config_update_helper.sh "$MYOS_TMP" + The output should equal "2" + End + It 'does nothing without a template' + When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/nope.dist" + The status should be success + The path "$MYOS_TMP/.env" should not be exist + End + End +End + +Describe 'lib/config.sh forward references' + setup() { MYOS_TMP=$(mktemp -d "${TMPDIR:-/tmp}/myos-fwd.XXXXXX"); } + cleanup() { rm -rf "$MYOS_TMP"; } + BeforeEach setup + AfterEach cleanup + + It 'resolves a reference to a variable defined further down the template' + printf 'IMAGE=alpine:${VERSION}\nVERSION=3.20\n' > "$MYOS_TMP/.env.dist" + When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist" + The contents of file "$MYOS_TMP/.env" should include "IMAGE=alpine:3.20" + End + It 'resolves a chain of references' + printf 'A=${B}\nB=${C}\nC=deep\n' > "$MYOS_TMP/.env.dist" + When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist" + The contents of file "$MYOS_TMP/.env" should include "A=deep" + End + It 'still lets an explicit choice win over the template' + printf 'IMAGE=alpine:${VERSION}\nVERSION=3.20\n' > "$MYOS_TMP/.env.dist" + VERSION=3.19 + When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist" + The contents of file "$MYOS_TMP/.env" should include "IMAGE=alpine:3.19" + The contents of file "$MYOS_TMP/.env" should include "VERSION=3.19" + End +End diff --git a/spec/unit/config_update_helper.sh b/spec/unit/config_update_helper.sh new file mode 100644 index 0000000..a2a793c --- /dev/null +++ b/spec/unit/config_update_helper.sh @@ -0,0 +1,5 @@ +#shellcheck shell=sh +# Running the update twice must leave the same file: print how many lines it has. +myos_env_update "$1/.env" "$1/.env.dist" >/dev/null 2>&1 +myos_env_update "$1/.env" "$1/.env.dist" >/dev/null 2>&1 +wc -l < "$1/.env" | tr -d ' ' diff --git a/spec/unit/hooks_spec.sh b/spec/unit/hooks_spec.sh index 078ff65..6b72eae 100644 --- a/spec/unit/hooks_spec.sh +++ b/spec/unit/hooks_spec.sh @@ -1,6 +1,7 @@ #shellcheck shell=sh Include lib/str.sh Include lib/core.sh +Include lib/var.sh Include lib/tags.sh Include lib/naming.sh Include lib/config.sh diff --git a/spec/unit/stack_spec.sh b/spec/unit/stack_spec.sh index deb8a27..73635ab 100644 --- a/spec/unit/stack_spec.sh +++ b/spec/unit/stack_spec.sh @@ -1,6 +1,7 @@ #shellcheck shell=sh Include lib/str.sh Include lib/core.sh +Include lib/var.sh Include lib/tags.sh Include lib/stack.sh diff --git a/spec/unit/tags_spec.sh b/spec/unit/tags_spec.sh index af2d552..d5cd891 100644 --- a/spec/unit/tags_spec.sh +++ b/spec/unit/tags_spec.sh @@ -1,6 +1,7 @@ #shellcheck shell=sh Include lib/str.sh Include lib/core.sh +Include lib/var.sh Include lib/tags.sh # The expectations below are the very examples left as comments in diff --git a/spec/unit/var_default_helper.sh b/spec/unit/var_default_helper.sh new file mode 100644 index 0000000..8d1396b --- /dev/null +++ b/spec/unit/var_default_helper.sh @@ -0,0 +1,3 @@ +#shellcheck shell=sh +myos_default BUILT 'printf "built-%s" here' +myos_var BUILT diff --git a/spec/unit/var_lazy_helper.sh b/spec/unit/var_lazy_helper.sh new file mode 100644 index 0000000..cff4407 --- /dev/null +++ b/spec/unit/var_lazy_helper.sh @@ -0,0 +1,7 @@ +#shellcheck shell=sh +# A lazy default is re-evaluated at each reference, so it follows a value that +# changes later; an explicit value still wins over it. +myos_default_LAZY_ONE() { printf 'from-%s' "$(myos_var LAZY_BASE)"; } +LAZY_BASE=first; myos_var LAZY_ONE; echo +LAZY_BASE=second; myos_var LAZY_ONE; echo +LAZY_ONE=pinned; myos_var LAZY_ONE; echo diff --git a/spec/unit/var_loop_helper.sh b/spec/unit/var_loop_helper.sh new file mode 100644 index 0000000..3b612dd --- /dev/null +++ b/spec/unit/var_loop_helper.sh @@ -0,0 +1,3 @@ +#shellcheck shell=sh +myos_default_LOOPY() { myos_var LOOPY; } +myos_var LOOPY diff --git a/spec/unit/var_spec.sh b/spec/unit/var_spec.sh new file mode 100644 index 0000000..c07b6a8 --- /dev/null +++ b/spec/unit/var_spec.sh @@ -0,0 +1,73 @@ +#shellcheck shell=sh +Include lib/core.sh +Include lib/var.sh + +Describe 'lib/var.sh' + Describe 'myos_var' + It 'returns the value of a plain variable' + SOME_VAR=plain + When call myos_var SOME_VAR + The output should equal "plain" + End + It 'returns empty for an unset variable' + When call myos_var NEVER_SET + The output should equal "" + End + It 'returns empty for an empty name' + When call myos_var "" + The output should equal "" + End + It 'never runs a command that happens to share the name' + # `host` is a real program: a lazy default must never be confused with it + When call myos_var host + The output should equal "" + The status should be success + The stderr should equal "" + End + End + + Describe 'lazy defaults' + lazy() { + myos_default_LAZY_ONE() { printf 'from-%s' "$(myos_var LAZY_BASE)"; } + myos_var LAZY_ONE + } + It 'computes the default when the variable has no value' + LAZY_BASE=a + When call lazy + The output should equal "from-a" + End + It 'recomputes it against the current values' + When run source spec/unit/var_lazy_helper.sh + The line 1 should equal "from-first" + The line 2 should equal "from-second" + The line 3 should equal "pinned" + End + It 'declares a default from a string too' + When run source spec/unit/var_default_helper.sh + The output should equal "built-here" + End + It 'refuses a default written in terms of itself' + When run source spec/unit/var_loop_helper.sh + The stderr should include "defined in terms of itself" + The status should equal 1 + End + End + + Describe 'myos_var_is_lazy' + It 'is true for a variable that only has a default' + myos_default_ONLY_LAZY() { echo x; } + When call myos_var_is_lazy ONLY_LAZY + The status should be success + End + It 'is false once the variable has a value' + myos_default_BOTH() { echo x; } + BOTH=explicit + When call myos_var_is_lazy BOTH + The status should be failure + End + It 'is false for a program on PATH' + When call myos_var_is_lazy host + The status should be failure + End + End +End