make the hooks usable by the whole catalogue

- hooks load the _stack files of every directory between the stack path root
  and the stack, outermost first: make included both $(dir)/*.mk and
  $(dir)/*/*.mk, so a stack in a subdirectory saw its parent's settings
- a group may be declared in <group>/<group>.env, where the stack lives
- MYOS_STACK_DIR lets a hook read a file it ships next to itself
- myos_filter no longer confuses a literal * in a make pattern with a wildcard,
  and the list helpers no longer let the shell expand a * into filenames
- MACHINE, SYSTEM, HOST and DOMAINNAME join the framework variables a hook sees
- the make shim gains $(call myos-var,NAME), so a .mk target can read a
  setting that now lives in a hook, and it picks up the .mk of every stack
  directory rather than only the project's

lib/context.sh

@@ -55,6 +55,11 @@ myos_context_defaults() {
   myos_default_DOCKER_NETWORK()            { myos_network_private "$USER" "$ENV"; }
   myos_default_DOCKER_IMAGE_TAG()          { printf 'latest'; }
   myos_default_GIT_USER()                  { printf '%s' "$USER"; }
+  myos_default_HOST()                      { myos_addprefix "${HOSTNAME:-}." "$(myos_var DOMAIN)"; }
+  myos_default_HOSTNAME()                  { printf '%s' "${HOSTNAME:-}"; }
+  myos_default_DOMAINNAME()                { myos_firstword "$(myos_var DOMAIN)"; }
+  myos_default_MACHINE()                   { uname -m 2>/dev/null; }
+  myos_default_SYSTEM()                    { uname -s 2>/dev/null; }
   myos_default_HOST_COMPOSE_PROJECT_NAME() { printf '%s' "${HOSTNAME:-}"; }
   myos_default_HOST_DOCKER_VOLUME()        { printf '%s' "${HOSTNAME:-}"; }
   myos_default_HOST_DOCKER_REPOSITORY()    { printf '%s' "${HOSTNAME:-}" | tr '_-' '//'; }

lib/hooks.sh

@@ -1,30 +1,69 @@
 #shellcheck shell=sh
+# shellcheck disable=SC1090  # hooks are sourced by a path built at run time
 # hooks: the per-stack settings that used to live in a .mk file.
 #
-# A stack may ship, next to its compose files:
+# A stack directory may ship, next to its compose files:
 #   _stack.env       settings shared by every stack of the directory
 #   _stack.sh        the same, computed
-#   <name>.env       dotenv, for plain values
+#   <name>.env       settings of one stack
 #   <name>.env.<env> the same, for one environment
-#   <name>.sh        shell, for values that have to be computed (fabio tags, JWTs)
+#   <name>.sh        computed settings of one stack
 #   <name>.mk        the legacy make snippet, still read for its groups
 #
-# A .sh hook runs with the myos helpers available (myos_tagprefix, myos_uri,
+# A .sh hook declares lazy defaults (see lib/var.sh): functions named
-# myos_var) and sets variables directly. This is what lets a stack of the
+# myos_default_<VARIABLE>, called only when the variable has no value and
-# catalogue work on a machine that has no make.
+# called again at each reference. That is a make `?=` on a recursive variable,
+# and it is what lets a stack of the catalogue work without make installed.
+#
+# The _stack hooks of every directory between the stack path root and the stack
+# itself are loaded, outermost first: make included both $(dir)/*.mk and
+# $(dir)/*/*.mk, so a stack in a subdirectory saw its parent's settings.
 
-# myos_stack_hooks DIR NAME  load the hooks of one stack, most specific last
+# myos_stack_hooks DIR NAME  load the hooks that apply to one stack
 myos_stack_hooks() {
   _hdir=$1; _hname=$2
-  for _h in "$_hdir/_stack.env" "$_hdir/$_hname.env" "$_hdir/$_hname.env.$ENV"; do
+
-    [ -f "$_h" ] && myos_dotenv_load "$_h"
+  # the stack path entry this directory belongs to
+  _root=
+  _IFS=$IFS; IFS=:
+  for _r in $(myos_path); do
+    IFS=$_IFS
+    case $_hdir in "$_r"|"$_r"/*) _root=$_r; break ;; esac
+    IFS=:
+  done
+  IFS=$_IFS
+
+  # every directory from the root down to the stack, outermost first
+  _chain=$_hdir
+  if [ -n "$_root" ]; then
+    _d=$_hdir
+    while [ "$_d" != "$_root" ] && [ "$_d" != "/" ] && [ -n "$_d" ]; do
+      _d=$(dirname "$_d")
+      _chain="$_d
+$_chain"
     done
-  for _h in "$_hdir/_stack.sh" "$_hdir/$_hname.sh" "$_hdir/$_hname.$ENV.sh"; do
-    if [ -f "$_h" ]; then
-      myos_debug "hook $_h"
-      # shellcheck source=/dev/null
-      . "$_h"
   fi
+
+  for _d in $_chain; do
+    myos_hook_load "$_d/_stack.env" dotenv
+    myos_hook_load "$_d/_stack.sh" shell
   done
+  myos_hook_load "$_hdir/$_hname.env" dotenv
+  myos_hook_load "$_hdir/$_hname.env.$ENV" dotenv
+  myos_hook_load "$_hdir/$_hname.sh" shell
+  myos_hook_load "$_hdir/$_hname.$ENV.sh" shell
   return 0
 }
+
+# myos_hook_load FILE KIND
+myos_hook_load() {
+  [ -f "$1" ] || return 0
+  myos_debug "hook $1"
+  # a hook may need to read a file it ships next to itself
+  # shellcheck disable=SC2034  # read by the hooks sourced below
+  MYOS_STACK_DIR=$(dirname "$1")
+  case $2 in
+    dotenv) myos_dotenv_load "$1" ;;
+    shell)  . "$1" ;;
+  esac
+}

lib/stack.sh

@@ -171,9 +171,12 @@ myos_group_value() {
   _IFS=$IFS; IFS=:
   for _d in $(myos_path); do
     IFS=$_IFS
-    [ -f "$_d/$1.env" ] && { _v=$(sed -n "s/^$1=//p" "$_d/$1.env" | tail -1 | tr -d '"'); }
+    for _f in "$_d/$1.env" "$_d/$1/$1.env" "$_d/$1/_stack.env"; do
-    [ -z "$_v" ] && [ -f "$_d/$1.mk" ] && { _v=$(myos_mk_group "$_d/$1.mk" "$1"); }
+      [ -z "$_v" ] && [ -f "$_f" ] && _v=$(sed -n "s/^$1=//p" "$_f" | tail -1 | tr -d '"')
-    [ -z "$_v" ] && [ -f "$_d/$1/$1.mk" ] && { _v=$(myos_mk_group "$_d/$1/$1.mk" "$1"); }
+    done
+    for _f in "$_d/$1.mk" "$_d/$1/$1.mk"; do
+      [ -z "$_v" ] && [ -f "$_f" ] && _v=$(myos_mk_group "$_f" "$1")
+    done
     [ -n "$_v" ] && { printf '%s' "$_v"; return 0; }
     IFS=:
   done

lib/str.sh

@@ -14,8 +14,10 @@ myos_slugify() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9
 
 # myos_reverse WORDS...  reverse the order of space separated words
 myos_reverse() {
+  set -f  # a word may be a pattern such as *.example.org, do not glob it
   _out=
   for _w in $1; do _out="$_w${_out:+ }$_out"; done
+  set +f
   printf '%s' "$_out"
 }
 
@@ -35,7 +37,7 @@ myos_verlt() {
 # The make list functions the catalogue uses, on space separated words.
 
 # myos_firstword LIST / myos_lastword LIST
-myos_firstword() { for _w in $1; do printf '%s' "$_w"; return 0; done; }
+myos_firstword() { set -f; for _w in $1; do set +f; printf '%s' "$_w"; return 0; done; set +f; }
 myos_lastword()  { _l=; for _w in $1; do _l=$_w; done; printf '%s' "$_l"; }
 
 # myos_or A B...  the first argument that is not empty
@@ -44,6 +46,7 @@ myos_or() { for _a in "$@"; do [ -n "$_a" ] && { printf '%s' "$_a"; return 0; };
 # myos_patsubst PATTERN REPLACEMENT LIST
 # The pattern holds one %, standing for any text; the replacement puts it back.
 myos_patsubst() {
+  set -f  # a word may be a pattern such as *.example.org, do not glob it
   _pre=${1%%%*}; _suf=${1#*%}
   _rpre=${2%%%*}; _rsuf=${2#*%}
   _out=
@@ -55,39 +58,51 @@ myos_patsubst() {
       *) _out="${_out:+$_out }$_w" ;;
     esac
   done
+  set +f
   printf '%s' "$_out"
 }
 
+# myos_pattern MAKE_PATTERN  the shell pattern matching a make pattern.
+# In make only % is a wildcard, so a literal *, ? or [ has to be protected
+# before % becomes *: "*.%" means "starts with a star and a dot", not
+# "anything".
+myos_pattern() {
+  printf '%s' "$1" | sed -e 's/[][*?]/\\&/g' -e 's/%/*/g'
+}
+
 # myos_filter PATTERNS LIST / myos_filter_out PATTERNS LIST
-# A make pattern uses % where a shell pattern uses *.
 myos_filter() {
-  _pats=$(printf '%s' "$1" | tr '%' '*')
+  set -f  # a word may itself be a pattern, do not glob it
   _out=
   for _w in $2; do
-    for _p in $_pats; do
+    for _raw in $1; do
-      # shellcheck disable=SC2254  # the pattern is meant to glob
+      _p=$(myos_pattern "$_raw")
+      # shellcheck disable=SC2254  # the pattern is meant to match, not to glob
       case $_w in $_p) _out="${_out:+$_out }$_w"; break ;; esac
     done
   done
+  set +f
   printf '%s' "$_out"
 }
 myos_filter_out() {
-  _pats=$(printf '%s' "$1" | tr '%' '*')
+  set -f
   _out=
   for _w in $2; do
     _keep=yes
-    for _p in $_pats; do
+    for _raw in $1; do
-      # shellcheck disable=SC2254  # the pattern is meant to glob
+      _p=$(myos_pattern "$_raw")
+      # shellcheck disable=SC2254  # the pattern is meant to match, not to glob
       case $_w in $_p) _keep=no; break ;; esac
     done
     [ "$_keep" = yes ] && _out="${_out:+$_out }$_w"
   done
+  set +f
   printf '%s' "$_out"
 }
 
 # myos_addprefix PREFIX LIST / myos_addsuffix SUFFIX LIST
-myos_addprefix() { _out=; for _w in $2; do _out="${_out:+$_out }$1$_w"; done; printf '%s' "$_out"; }
+myos_addprefix() { set -f; _out=; for _w in $2; do _out="${_out:+$_out }$1$_w"; done; set +f; printf '%s' "$_out"; }
-myos_addsuffix() { _out=; for _w in $2; do _out="${_out:+$_out }$_w$1"; done; printf '%s' "$_out"; }
+myos_addsuffix() { set -f; _out=; for _w in $2; do _out="${_out:+$_out }$_w$1"; done; set +f; printf '%s' "$_out"; }
 
 # myos_b64url  read stdin, write url-safe base64 without padding
 myos_b64url() { openssl enc -A -base64 | tr '+/' '-_' | tr -d '='; }
@@ -105,3 +120,16 @@ myos_jwt() {
   _sig=$(printf '%s' "$_hb.$_pb" | openssl dgst -sha256 -binary -hmac "$_s" | myos_b64url)
   printf '%s.%s.%s' "$_hb" "$_pb" "$_sig"
 }
+
+# myos_patsublist PATTERN REPLACEMENT LIST
+# patsubst over a list, joined by commas. The fabio tags are built this way:
+# one route per uri, in a single label.
+myos_patsublist() {
+  set -f  # a word may be a pattern such as *.example.org, do not glob it
+  _out=
+  for _w in $3; do
+    _out="${_out:+$_out,}$(myos_patsubst "$1" "$2" "$_w")"
+  done
+  set +f
+  printf '%s' "$_out"
+}

lib/tags.sh

@@ -8,35 +8,45 @@
 # Every lookup goes through myos_var (lib/var.sh), so a stack setting may be a
 # plain value or a lazy default, and the two behave the same here.
 
-# myos_uri SERVICE PORT [BASE_URI]
+# myos_uri SERVICES PORT [BASE_URI]
-# <service>.<base uri>, unless <SERVICE>_SERVICE[_<port>]_NAME overrides the prefix
+# <service>.<base uri> for each service and each base uri, unless
+# <SERVICE>_SERVICE[_<port>]_NAME overrides the prefix. The first argument is a
+# list: one stack may publish several services on one port.
 myos_uri() {
-  _svc=$1; _port=${2:-}; _base=${3:-${APP_URI:-}}
+  set -f  # a uri may be a pattern such as *.ipns.example.org
+  _svcs=$1; _port=${2:-}; _base=${3:-${APP_URI:-}}
+  _out=
+  for _svc in $_svcs; do
     _u=$(myos_upper "$_svc")
     _name=$(myos_var "${_u}_SERVICE_${_port}_NAME")
     [ -n "$_name" ] || _name=$(myos_var "${_u}_SERVICE_NAME")
     [ -n "$_name" ] || _name=$_svc
-  _out=
     for _b in $_base; do _out="${_out:+$_out }${_name}.${_b}"; done
+  done
+  set +f
   printf '%s' "$_out"
 }
 
 # myos_url SERVICE PORT [BASE_URI]
 myos_url() {
+  set -f
   _out=
   for _u in $(myos_uri "$@"); do _out="${_out:+$_out }${APP_SCHEME:-http}://$_u"; done
+  set +f
   printf '%s' "$_out"
 }
 
 # myos_urlprefix [PATH] [OPTS] [URIS]
 # one comma separated "urlprefix-<uri><path>* [opts]" per uri
 myos_urlprefix() {
+  set -f
   _path=${1:-}; _opts=${2:-}; _uris=${3:-${APP_URI:-}}
   _out=
   for _u in $_uris; do
     _tag="urlprefix-${_u}${_path}${MYOS_URL_SUFFIX:-*}${_opts:+ $_opts}"
     _out="${_out:+$_out,}$_tag"
   done
+  set +f
   printf '%s' "$_out"
 }
 
@@ -62,7 +72,8 @@ myos_tagprefix() {
   [ -n "$_opts" ] || _opts=$(myos_envprefix "$_stack" "$_port" allow auth deny prepend proto register strip)
   _uris=
   for _k in "$@"; do
-    _v=$(myos_var "${_u}_SERVICE_${_port}_${_k}"); [ -n "$_v" ] && _uris="${_uris:+$_uris }$_v"
+    _v=$(myos_var "${_u}_SERVICE_${_port}_$(myos_upper "$_k")")
+    [ -n "$_v" ] && _uris="${_uris:+$_uris }$_v"
   done
   [ -n "$_uris" ] || _uris=$(myos_var "${_u}_SERVICE_${_port}_URIS")
   [ -n "$_uris" ] || _uris=$(myos_uri "$_stack" "$_port")

share/make/shim.mk

@@ -19,21 +19,29 @@
 MYOS                            ?= $(patsubst %/share/make/shim.mk,%,$(lastword $(MAKEFILE_LIST)))
 MYOS_BIN                        ?= $(MYOS)/bin/myos
 STACK_DIR_NAME                  ?= stack
-STACK_DIR                       ?= $(wildcard $(CURDIR)/$(STACK_DIR_NAME))
+## every directory myos looks for stacks in, so the .mk of a stack installed
+## system wide brings its targets along too
+STACK_DIR                       ?= $(subst :, ,$(shell $(MYOS_BIN) env MYOS_PATH --color=never 2>/dev/null | awk '{print $$2}'))
 
 # variable MYOS_ARGS: variables set on the make command line, forwarded to myos
 MYOS_ARGS                       ?= $(foreach v,$(MAKEOVERRIDES),$(v))
 
 .DEFAULT_GOAL                   := help
 
-## the stack files of the project may add their own targets and variables
+## the stack files may add their own targets: that is what make is kept for
-include $(wildcard $(STACK_DIR)/*.mk $(STACK_DIR)/*/*.mk)
+include $(foreach dir,$(STACK_DIR),$(wildcard $(dir)/*.mk $(dir)/*/*.mk))
 
 # function make: run a myos command, for the stack .mk files that call it
 define make
 	$(MYOS_BIN) $(MYOS_ARGS) $(1)
 endef
 
+# function myos-var: the value myos resolves for a variable.
+# A stack keeps its settings in hooks that only myos reads, so a .mk target
+# asks for them rather than defining them itself:
+#   $(call myos-var,HOST_DOCKER_VOLUME)
+myos-var = $(shell $(MYOS_BIN) --color=never $(MYOS_ARGS) env $(1) | awk '{print $$2}')
+
 # target help: List the myos commands
 .PHONY: help
 help:

spec/unit/hooks_spec.sh

@@ -5,6 +5,7 @@ Include lib/var.sh
 Include lib/tags.sh
 Include lib/naming.sh
 Include lib/config.sh
+Include lib/stack.sh
 Include lib/hooks.sh
 
 # A stack ships its computed settings as a shell hook, so the catalogue works
@@ -14,6 +15,7 @@ Describe 'lib/hooks.sh'
     MYOS_TMP=$(mktemp -d "${TMPDIR:-/tmp}/myos-hook.XXXXXX")
     MYOS_TMP=$(cd "$MYOS_TMP" && pwd -P)
     ENV=local; DOMAIN=example.org; USER=tester; HOSTNAME=testhost
+    MYOS_PATH=$MYOS_TMP
     APP_HOST=demo.example.org; APP_URI=demo.example.org/
   }
   cleanup() { rm -rf "$MYOS_TMP"; }