keep the dynamism of make in pure shell

Two mechanisms, matching what the make engine actually did:

Lazy defaults. A stack setting is a function myos_default_<VAR>, called only
when the variable has no value, and called again at every reference. That is
exactly a recursive ?=: an explicit value wins, and the default follows a
DOMAIN that a .env changes later. The prefix is what makes it safe; the first
version used a bare function named after the variable, and the test suite
caught it running /usr/bin/host for a stack group called host.

Templates. myos env-update fills a .env from the .env.dist files, expanding
${VAR} against the current values and running $(command), forward references
included.

Also fixed: the project .env now wins over /etc/conf.d/myos, which is what the
documentation claimed and the code did not.

share/make/shim.mk lets a project keep make as a front end: every myos command
becomes a target that shells out to bin/myos, and the project keeps its own
targets and its stack .mk files. It sits outside make/ because the legacy
engine globs every .mk in there.

spec/fixtures/shim-project/Makefile

@@ -0,0 +1,2 @@
+MYOS ?= /usr/local/lib/myos
+include $(MYOS)/share/make/shim.mk

spec/fixtures/shim-project/stack/host/consul.yml

@@ -0,0 +1,12 @@
+services:
+  consul:
+    image: hashicorp/consul:1.15
+    container_name: ${HOST_COMPOSE_PROJECT_NAME:-localhost}-consul
+    network_mode: host
+    restart: always
+    environment:
+      CONSUL_HTTP_TOKEN: ${HOST_CONSUL_HTTP_TOKEN}
+    volumes:
+      - consul:/consul/data
+volumes:
+  consul:

spec/fixtures/shim-project/stack/host/fabio.yml

@@ -0,0 +1,13 @@
+services:
+  fabio:
+    image: fabiolb/fabio:1.6.3
+    container_name: ${HOST_COMPOSE_PROJECT_NAME:-localhost}-fabio
+    depends_on: [consul]
+    ports:
+      - "80:80"
+      - "443:443"
+    environment:
+      FABIO_REGISTRY_CONSUL_ADDR: ${DOCKER_HOST_INET4:-127.0.0.1}:8500
+      FABIO_LOG_ACCESS_TARGET: ${HOST_FABIO_LOG_ACCESS:-}
+    networks:
+      - public

spec/fixtures/shim-project/stack/host/host.mk

@@ -0,0 +1,7 @@
+host ?= host/consul host/fabio
+
+# a target of the project, on top of the myos commands: this is what make is
+# kept for, and what the shim leaves alone
+.PHONY: host-certs
+host-certs:
+	@echo "would renew the certificates of $(host)"

spec/fixtures/shim-project/stack/host/registrator.yml

@@ -0,0 +1,9 @@
+services:
+  registrator:
+    image: gliderlabs/registrator:master
+    container_name: ${HOST_COMPOSE_PROJECT_NAME:-localhost}-registrator
+    network_mode: host
+    depends_on: [consul]
+    command: -internal=false -useIpFromLabel SERVICE_ADDRESS consul://127.0.0.1:8500
+    volumes:
+      - ${DOCKER_SOCKET_LOCATION:-/var/run/docker.sock}:/tmp/docker.sock

spec/golden/cases.txt

@@ -77,3 +77,7 @@ cmd-scale                       | app-nogit     | scale postgres SERVICE=postgre
 cmd-ls                          | host-project  | ls
 cmd-ls-groups                   | host-project  | ls --groups
 cmd-env-domain                  | host-project  | env DOMAIN
+shim-up-group                   | shim-project  | @make up STACK=host DRYRUN=true
+shim-config                     | shim-project  | @make config STACK=host/consul DRYRUN=true
+shim-project-target             | shim-project  | @make host-certs
+shim-env                        | shim-project  | @make env ARGS=COMPOSE_PROJECT_NAME STACK=host

spec/golden/expected.cli/cat-unknown-target.txt

@@ -16,6 +16,7 @@ Commands:
  ps logs config exec run inspect and enter them
  ls [--groups] list the stacks myos can see
  env [VAR...] show resolved variables
+ env-update fill .env from the .env.dist templates
  doctor check the installation
  version print the myos version
 

spec/golden/expected/shim-config.txt

@@ -0,0 +1,2 @@
+docker compose -f @WD@/stack/host/consul.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host config
+[exit 0]

spec/golden/expected/shim-env.txt

@@ -0,0 +1,2 @@
+COMPOSE_PROJECT_NAME testhost
+[exit 0]

spec/golden/expected/shim-project-target.txt

@@ -0,0 +1,2 @@
+would renew the certificates of host/consul host/fabio
+[exit 0]

spec/golden/expected/shim-up-group.txt

@@ -0,0 +1,4 @@
+docker network create tester-local
+docker network create testhost
+docker compose -f @WD@/stack/host/consul.yml -f @WD@/stack/host/fabio.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host up -d
+[exit 0]

spec/support/run.sh

@@ -47,8 +47,9 @@ myos_normalize() {
 # shellcheck disable=SC2046  # myos_hermetic_env output is meant to be word-split
 myos_run_engine() {
   _engine=$1; _sb=$2; shift 2
-  # "@make" as first arg = include mode: the project Makefile includes make/include.mk
-  # and make runs from the project dir (CURDIR = project). Same for both engines.
+  # "@make" as first arg = the project drives make itself: its Makefile includes
+  # either the legacy engine (make/include.mk) or the shim (make/shim.mk), and
+  # make runs from the project directory.
   if [ "${1:-}" = "@make" ]; then
     shift
     _out=$(cd "$_sb/wd" && env -i $(myos_hermetic_env "$_sb") MYOS_CONF=/dev/null \

spec/unit/compose_spec.sh

@@ -1,6 +1,7 @@
 #shellcheck shell=sh
 Include lib/str.sh
 Include lib/core.sh
+Include lib/var.sh
 Include lib/tags.sh
 Include lib/config.sh
 Include lib/compose.sh

spec/unit/config_spec.sh

@@ -1,6 +1,7 @@
 #shellcheck shell=sh
 Include lib/str.sh
 Include lib/core.sh
+Include lib/var.sh
 Include lib/tags.sh
 Include lib/config.sh
 
@@ -104,3 +105,95 @@ Describe 'lib/config.sh robustness'
     The status should be success
   End
 End
+
+# The make engine generated a .env out of a .env.dist, expanding ${VAR} against
+# the current values and running $(cmd). These check the shell equivalent.
+Describe 'lib/config.sh templates'
+  setup() { MYOS_TMP=$(mktemp -d "${TMPDIR:-/tmp}/myos-tpl.XXXXXX"); }
+  cleanup() { rm -rf "$MYOS_TMP"; }
+  BeforeEach setup
+  AfterEach cleanup
+
+  Describe 'myos_expand'
+    It 'substitutes a variable'
+      DOMAIN=example.org
+      When call myos_expand 'https://app.${DOMAIN}/'
+      The output should equal "https://app.example.org/"
+    End
+    It 'substitutes several, including twice the same'
+      DOMAIN=example.org
+      When call myos_expand '${DOMAIN}:${DOMAIN}'
+      The output should equal "example.org:example.org"
+    End
+    It 'substitutes a lazy default like any other value'
+      # shellcheck disable=SC2317
+      myos_default_LAZY_DOMAIN() { printf 'lazy.example.org'; }
+      When call myos_expand 'https://${LAZY_DOMAIN}/'
+      The output should equal "https://lazy.example.org/"
+    End
+    It 'empties an unknown variable, as make does'
+      When call myos_expand 'a${NOT_SET_ANYWHERE}b'
+      The output should equal "ab"
+    End
+    It 'runs a command substitution'
+      When call myos_expand 'pre-$(echo mid)-post'
+      The output should equal "pre-mid-post"
+    End
+    It 'leaves a malformed reference alone'
+      When call myos_expand 'a${not-a-name}b'
+      The output should equal 'a${not-a-name}b'
+    End
+  End
+
+  Describe 'myos_env_update'
+    It 'adds the missing variables, expanded'
+      printf 'DOMAIN=example.org\nAPP_URL=https://app.${DOMAIN}/\n' > "$MYOS_TMP/.env.dist"
+      DOMAIN=chosen.org
+      When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist"
+      The status should be success
+      The contents of file "$MYOS_TMP/.env" should include "APP_URL=https://app.chosen.org/"
+      The contents of file "$MYOS_TMP/.env" should include "DOMAIN=chosen.org"
+    End
+    It 'never touches a value already recorded'
+      printf 'KEEP=default\n' > "$MYOS_TMP/.env.dist"
+      printf 'KEEP=already-chosen\n' > "$MYOS_TMP/.env"
+      When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist"
+      The contents of file "$MYOS_TMP/.env" should equal "KEEP=already-chosen"
+    End
+    It 'is idempotent'
+      printf 'A=1\nB=${A}2\n' > "$MYOS_TMP/.env.dist"
+      When run source spec/unit/config_update_helper.sh "$MYOS_TMP"
+      The output should equal "2"
+    End
+    It 'does nothing without a template'
+      When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/nope.dist"
+      The status should be success
+      The path "$MYOS_TMP/.env" should not be exist
+    End
+  End
+End
+
+Describe 'lib/config.sh forward references'
+  setup() { MYOS_TMP=$(mktemp -d "${TMPDIR:-/tmp}/myos-fwd.XXXXXX"); }
+  cleanup() { rm -rf "$MYOS_TMP"; }
+  BeforeEach setup
+  AfterEach cleanup
+
+  It 'resolves a reference to a variable defined further down the template'
+    printf 'IMAGE=alpine:${VERSION}\nVERSION=3.20\n' > "$MYOS_TMP/.env.dist"
+    When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist"
+    The contents of file "$MYOS_TMP/.env" should include "IMAGE=alpine:3.20"
+  End
+  It 'resolves a chain of references'
+    printf 'A=${B}\nB=${C}\nC=deep\n' > "$MYOS_TMP/.env.dist"
+    When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist"
+    The contents of file "$MYOS_TMP/.env" should include "A=deep"
+  End
+  It 'still lets an explicit choice win over the template'
+    printf 'IMAGE=alpine:${VERSION}\nVERSION=3.20\n' > "$MYOS_TMP/.env.dist"
+    VERSION=3.19
+    When call myos_env_update "$MYOS_TMP/.env" "$MYOS_TMP/.env.dist"
+    The contents of file "$MYOS_TMP/.env" should include "IMAGE=alpine:3.19"
+    The contents of file "$MYOS_TMP/.env" should include "VERSION=3.19"
+  End
+End

spec/unit/config_update_helper.sh

@@ -0,0 +1,5 @@
+#shellcheck shell=sh
+# Running the update twice must leave the same file: print how many lines it has.
+myos_env_update "$1/.env" "$1/.env.dist" >/dev/null 2>&1
+myos_env_update "$1/.env" "$1/.env.dist" >/dev/null 2>&1
+wc -l < "$1/.env" | tr -d ' '

spec/unit/hooks_spec.sh

@@ -1,6 +1,7 @@
 #shellcheck shell=sh
 Include lib/str.sh
 Include lib/core.sh
+Include lib/var.sh
 Include lib/tags.sh
 Include lib/naming.sh
 Include lib/config.sh

spec/unit/stack_spec.sh

@@ -1,6 +1,7 @@
 #shellcheck shell=sh
 Include lib/str.sh
 Include lib/core.sh
+Include lib/var.sh
 Include lib/tags.sh
 Include lib/stack.sh
 

spec/unit/tags_spec.sh

@@ -1,6 +1,7 @@
 #shellcheck shell=sh
 Include lib/str.sh
 Include lib/core.sh
+Include lib/var.sh
 Include lib/tags.sh
 
 # The expectations below are the very examples left as comments in

spec/unit/var_default_helper.sh

@@ -0,0 +1,3 @@
+#shellcheck shell=sh
+myos_default BUILT 'printf "built-%s" here'
+myos_var BUILT

spec/unit/var_lazy_helper.sh

@@ -0,0 +1,7 @@
+#shellcheck shell=sh
+# A lazy default is re-evaluated at each reference, so it follows a value that
+# changes later; an explicit value still wins over it.
+myos_default_LAZY_ONE() { printf 'from-%s' "$(myos_var LAZY_BASE)"; }
+LAZY_BASE=first;  myos_var LAZY_ONE; echo
+LAZY_BASE=second; myos_var LAZY_ONE; echo
+LAZY_ONE=pinned;  myos_var LAZY_ONE; echo

spec/unit/var_loop_helper.sh

@@ -0,0 +1,3 @@
+#shellcheck shell=sh
+myos_default_LOOPY() { myos_var LOOPY; }
+myos_var LOOPY

spec/unit/var_spec.sh

@@ -0,0 +1,73 @@
+#shellcheck shell=sh
+Include lib/core.sh
+Include lib/var.sh
+
+Describe 'lib/var.sh'
+  Describe 'myos_var'
+    It 'returns the value of a plain variable'
+      SOME_VAR=plain
+      When call myos_var SOME_VAR
+      The output should equal "plain"
+    End
+    It 'returns empty for an unset variable'
+      When call myos_var NEVER_SET
+      The output should equal ""
+    End
+    It 'returns empty for an empty name'
+      When call myos_var ""
+      The output should equal ""
+    End
+    It 'never runs a command that happens to share the name'
+      # `host` is a real program: a lazy default must never be confused with it
+      When call myos_var host
+      The output should equal ""
+      The status should be success
+      The stderr should equal ""
+    End
+  End
+
+  Describe 'lazy defaults'
+    lazy() {
+      myos_default_LAZY_ONE() { printf 'from-%s' "$(myos_var LAZY_BASE)"; }
+      myos_var LAZY_ONE
+    }
+    It 'computes the default when the variable has no value'
+      LAZY_BASE=a
+      When call lazy
+      The output should equal "from-a"
+    End
+    It 'recomputes it against the current values'
+      When run source spec/unit/var_lazy_helper.sh
+      The line 1 should equal "from-first"
+      The line 2 should equal "from-second"
+      The line 3 should equal "pinned"
+    End
+    It 'declares a default from a string too'
+      When run source spec/unit/var_default_helper.sh
+      The output should equal "built-here"
+    End
+    It 'refuses a default written in terms of itself'
+      When run source spec/unit/var_loop_helper.sh
+      The stderr should include "defined in terms of itself"
+      The status should equal 1
+    End
+  End
+
+  Describe 'myos_var_is_lazy'
+    It 'is true for a variable that only has a default'
+      myos_default_ONLY_LAZY() { echo x; }
+      When call myos_var_is_lazy ONLY_LAZY
+      The status should be success
+    End
+    It 'is false once the variable has a value'
+      myos_default_BOTH() { echo x; }
+      BOTH=explicit
+      When call myos_var_is_lazy BOTH
+      The status should be failure
+    End
+    It 'is false for a program on PATH'
+      When call myos_var_is_lazy host
+      The status should be failure
+    End
+  End
+End