chain commands, and let a project refine a catalogue stack

myos build up logs host/fabio runs the three in order and stops at the first
failure, the way make build up logs STACK=host/fabio did. Leading words that
name commands are commands; the first word that is not one starts the stacks.

A stack found in several directories of the stack path is now merged rather
than shadowed, least specific first, so a project drops
stack/postgres/postgres.local.yml next to the catalogue's postgres.yml and
refines it. Settings hooks follow the same order, so a project can redefine a
default the catalogue ships. Neither engine did this before: the project
directory simply hid the catalogue one.

An unknown command now says so and suggests the command to type, instead of
printing the whole usage.

CHANGELOG.md

@@ -20,6 +20,9 @@
 - the project `.env` now wins over `/etc/conf.d/myos`, as documented;
   `MYOS_CONF_PRIORITY=system` restores the previous order
 - `share/make/shim.mk`: make as an optional front end over the same shell code
+- commands chain: `myos build up logs host/fabio`, as make targets did
+- a stack found in several directories of the stack path is merged, project
+  last, so a project refines a catalogue stack instead of replacing it
 - `--color always|never|auto`, and no colour when the output is piped
 - verified under the /bin/sh of Alpine (busybox) and Debian (dash)
 - the make engine still works and is still covered by the golden tests

bin/myos

@@ -24,11 +24,22 @@ for _m in core str var tags naming stack config compose hooks; do
   . "$MYOS_ROOT/lib/$_m.sh"
 done
 
+# myos_is_command WORD  true when WORD names a command rather than a stack
+myos_is_command() {
+  case $1 in
+    up|down|start|stop|restart|ps|logs|config|build|pull|create|kill|top|images) return 0 ;;
+    version|help) return 0 ;;
+    print-*|stack-*-*) return 0 ;;
+    *@*) myos_is_command "${1%@*}"; return $? ;;
+  esac
+  [ -f "$MYOS_ROOT/lib/cmd/$1.sh" ]
+}
+
 # --- command line ---------------------------------------------------------
 # These are read by the lib/cmd/* files sourced further down.
 # shellcheck disable=SC2034
 {
-  MYOS_CMD=
+  MYOS_CMDS=
   MYOS_REFS=
   MYOS_REFS_RAW=
   MYOS_VARS=
@@ -83,39 +94,63 @@ while [ $# -gt 0 ]; do
     -h|--help) usage; exit 0 ;;
     --) shift; MYOS_ARGS="$*"; break ;;
     -*)
-      if [ -n "$MYOS_CMD" ]; then
+      if [ -n "$MYOS_CMDS" ]; then
         MYOS_ARGS="${MYOS_ARGS:+$MYOS_ARGS }$1"; shift
       else
         myos_error "unknown option: $1"; usage >&2; exit "$MYOS_E_USAGE"
       fi ;;
     *=*) eval "${1%%=*}=\${1#*=}"; export "${1%%=*}"; shift ;;
     *)
-      if [ -z "$MYOS_CMD" ]; then MYOS_CMD=$1; else MYOS_REFS="${MYOS_REFS:+$MYOS_REFS }$1"; fi
+      # Leading words that name commands are commands, the rest are stacks:
+      # `myos build up logs host/fabio` runs three commands on one stack, the
+      # way `make build up logs STACK=host/fabio` did.
+      if [ -z "$MYOS_REFS" ] && myos_is_command "$1"; then
+        MYOS_CMDS="${MYOS_CMDS:+$MYOS_CMDS }$1"
+      else
+        MYOS_REFS="${MYOS_REFS:+$MYOS_REFS }$1"
+      fi
       shift ;;
   esac
 done
 
-[ -n "$MYOS_CMD" ] || { usage; exit "$MYOS_E_USAGE"; }
+if [ -z "$MYOS_CMDS" ]; then
+  if [ -n "$MYOS_REFS" ]; then
+    # the first word was meant as a command; name it rather than dump the usage
+    myos_error "unknown command: ${MYOS_REFS%% *}"
+    myos_error "to act on a stack of that name, say what to do: myos up ${MYOS_REFS%% *}"
+    exit "$MYOS_E_USAGE"
+  fi
+  usage
+  exit "$MYOS_E_USAGE"
+fi
 MYOS_REFS_RAW=$MYOS_REFS
-case $MYOS_CMD in
-  env|ls|doctor) MYOS_VARS=$MYOS_REFS; MYOS_REFS= ;;
-esac
 
-# Targets of the make engine keep working: print-VAR, stack-<stack>-<command>,
+# Targets of the make engine keep working: print-VAR, stack-<stack>-<command>
-# <command>@<env> and a bare group name.
+# and <command>@<env> each translate to a command of the CLI.
-case $MYOS_CMD in
+_cmds=
+for _c in $MYOS_CMDS; do
+  case $_c in
+    *@*) ENV=${_c#*@}; _c=${_c%@*} ;;
+  esac
+  case $_c in
     print-*)
-    MYOS_VARS=${MYOS_CMD#print-}
+      MYOS_VARS="${MYOS_VARS:+$MYOS_VARS }${_c#print-}"
       # shellcheck disable=SC2209  # the literal string "env", not the command
-    MYOS_CMD=env ;;
+      _c=env ;;
-  *@*)     ENV=${MYOS_CMD#*@}; MYOS_CMD=${MYOS_CMD%@*} ;;
-esac
-case $MYOS_CMD in
     stack-*-*)
-    _rest=${MYOS_CMD#stack-}
+      _rest=${_c#stack-}
-    MYOS_CMD=${_rest##*-}
+      _c=${_rest##*-}
-    MYOS_REFS="${_rest%-*} $MYOS_REFS"
+      MYOS_REFS="${_rest%-*}${MYOS_REFS:+ $MYOS_REFS}" ;;
-    MYOS_REFS=${MYOS_REFS% } ;;
+  esac
+  _cmds="${_cmds:+$_cmds }$_c"
+done
+MYOS_CMDS=$_cmds
+
+# env, ls and doctor take variable names where the others take stacks
+case $MYOS_CMDS in
+  env|ls|doctor)
+    [ -n "$MYOS_VARS" ] || MYOS_VARS=$MYOS_REFS
+    MYOS_REFS= ;;
 esac
 
 # --- configuration --------------------------------------------------------
@@ -178,10 +213,13 @@ if [ -z "$MYOS_REFS" ]; then
   fi
 fi
 if [ -z "$MYOS_REFS" ]; then
-  case $MYOS_CMD in
+  # these commands describe the installation rather than act on a stack
+  for _c in $MYOS_CMDS; do
+    case $_c in
       env|env-update|ls|doctor|version|help) ;;
       *) myos_die "$MYOS_E_USAGE" "no stack given, and no compose file in $WORKDIR" ;;
     esac
+  done
 fi
 
 # shellcheck disable=SC2086  # a list of references
@@ -198,21 +236,28 @@ APP_SCHEME=${APP_SCHEME:-http}
 # Per-stack hooks must run in this shell: everything downstream reads the
 # variables they set, and a command substitution would throw them away.
 for _ref in $MYOS_STACKS; do
-  _hdir=$(myos_stack_resolve "$_ref" 2>/dev/null) || continue
+  for _hdir in $(myos_stack_dirs "$_ref"); do
     myos_stack_hooks "$_hdir" "$(myos_stack_name "$_ref")"
+  done
 done
 
+# The two functions below are called from the lib/cmd/* files sourced later,
+# which shellcheck cannot see.
+# shellcheck disable=SC2329
 # myos_framework_compose_files  the networks and volumes overlays myos itself
 # provides; they always come last so a stack can rely on them being there.
 myos_framework_compose_files() {
   myos_compose_files "$MYOS_ROOT/share/compose" "networks volumes" "$(myos_compose_suffixes)" "$ENV"
 }
 
+# shellcheck disable=SC2329
 # myos_stack_compose_files REF  the ordered compose files of one reference
 myos_stack_compose_files() {
-  _dir=$(myos_stack_resolve "$1") || return $?
+  _dirs=$(myos_stack_dirs "$1")
+  [ -n "$_dirs" ] || { myos_stack_resolve "$1" >/dev/null; return $?; }
   _name=$(myos_stack_name "$1")
   _suffixes="$(myos_compose_suffixes) $(myos_stack_version "$1")"
+  for _dir in $_dirs; do
     case $1 in
       .|./*|/*|../*)
         myos_compose_files "$_dir" "docker-compose compose" "$_suffixes" "$ENV"
@@ -220,26 +265,35 @@ myos_stack_compose_files() {
       *)
         myos_compose_files "$_dir" "docker-compose $_name" "$_suffixes" "$ENV" ;;
     esac
+  done
 }
 
 # --- dispatch -------------------------------------------------------------
-case $MYOS_CMD in
+# myos_dispatch COMMAND  run one command
-  version) printf 'myos %s\n' "$MYOS_VERSION"; exit 0 ;;
+myos_dispatch() {
-  help) usage; exit 0 ;;
+  case $1 in
-esac
+    version) printf 'myos %s\n' "$MYOS_VERSION"; return 0 ;;
-
+    help) usage; return 0 ;;
-# shellcheck source=/dev/null
+  esac
-if [ -f "$MYOS_ROOT/lib/cmd/$MYOS_CMD.sh" ]; then
+  if [ -f "$MYOS_ROOT/lib/cmd/$1.sh" ]; then
-  . "$MYOS_ROOT/lib/cmd/$MYOS_CMD.sh"
+    # shellcheck source=/dev/null
-  "myos_cmd_$(printf '%s' "$MYOS_CMD" | tr '-' '_')"
+    . "$MYOS_ROOT/lib/cmd/$1.sh"
-  exit $?
+    "myos_cmd_$(printf '%s' "$1" | tr '-' '_')"
-fi
+    return $?
-
+  fi
-# commands that map straight onto docker compose
+  # commands that map straight onto docker compose
-case $MYOS_CMD in
+  case $1 in
     up|down|start|stop|restart|ps|logs|config|build|pull|create|kill|top|images) ;;
-  *) myos_error "unknown command: $MYOS_CMD"; usage >&2; exit "$MYOS_E_USAGE" ;;
+    *) myos_error "unknown command: $1"; usage >&2; return "$MYOS_E_USAGE" ;;
-esac
+  esac
+  # shellcheck source=/dev/null
+  . "$MYOS_ROOT/lib/cmd/_compose.sh"
+  myos_cmd_compose "$1"
+}
 
-. "$MYOS_ROOT/lib/cmd/_compose.sh"
+# Several commands run in order and stop at the first failure, as make did.
-myos_cmd_compose "$MYOS_CMD"
+_rc=0
+for MYOS_CMD in $MYOS_CMDS; do
+  myos_dispatch "$MYOS_CMD" || { _rc=$?; break; }
+done
+exit "$_rc"

lib/stack.sh

@@ -44,6 +44,37 @@ myos_stack_version() {
   case $_r in *:*) printf '%s' "${_r##*:}" ;; *) printf 'latest' ;; esac
 }
 
+# myos_stack_dirs REF  every directory of MYOS_PATH holding this stack, least
+# specific first. A compose overlay wins over the ones before it, so a project
+# that ships stack/postgres/postgres.local.yml refines the postgres stack of
+# the catalogue instead of replacing it.
+myos_stack_dirs() {
+  _ref=${1%/}
+  case $_ref in *:*) _ref=${_ref%:*} ;; esac
+  _name=$(myos_stack_name "$1")
+  case $_ref in
+    .|./*|/*|../*)
+      [ -d "$_ref" ] && (cd "$_ref" && pwd -P)
+      return 0 ;;
+  esac
+  _found=
+  _IFS=$IFS; IFS=:
+  for _d in $(myos_path); do
+    IFS=$_IFS
+    _hit=
+    if [ -d "$_d/$_ref" ]; then _hit=$_d/$_ref
+    elif [ -f "$_d/$_ref.yml" ] || [ -f "$_d/$_ref.yaml" ]; then _hit=$(dirname "$_d/$_ref")
+    elif [ -d "$_d/$_name" ]; then _hit=$_d/$_name
+    fi
+    # least specific last here, then reversed below
+    [ -n "$_hit" ] && _found="$_hit
+$_found"
+    IFS=:
+  done
+  IFS=$_IFS
+  printf '%s' "$_found" | sed '/^$/d'
+}
+
 # myos_stack_resolve REF  print the directory holding the stack,
 # or fail with MYOS_E_NOSTACK
 myos_stack_resolve() {

skills/myos/SKILL.md

@@ -48,6 +48,9 @@ myos down <stack>             # remove the containers
 
 A reference is `[<group>/]<name>[:<version>]`, or a path.
 
+Commands chain: `myos build up logs host/fabio` runs the three in order and
+stops at the first failure.
+
 ```sh
 myos up                    # the current directory, when it holds a compose file
 myos up host               # a group: expands to host/consul host/fabio ...
@@ -57,9 +60,13 @@ myos -C /opt/app up        # somewhere else
 myos -e master up          # in another environment
 ```
 
-Stacks are looked up along the stack path, project first:
+Stacks are looked up along the stack path: `./stack`, `../stack`,
-`./stack`, `../stack`, `~/.local/share/myos/stack`, `/usr/local/share/myos/stack`,
+`~/.local/share/myos/stack`, `/usr/local/share/myos/stack`, `/usr/share/myos/stack`.
-`/usr/share/myos/stack`. `myos doctor` prints the resolved path.
+`myos doctor` prints the resolved path.
+
+A stack found in several of them is **merged**, the project last, so a project
+adds `stack/postgres/postgres.local.yml` next to the catalogue's `postgres.yml`
+and refines it instead of replacing it. The same goes for the settings hooks.
 
 ## Three kinds of stack
 

skills/myos/references/commands.md

@@ -30,6 +30,20 @@ myos [options] <command> [stack...] [VAR=value...] [-- args...]
 | `doctor` | check the installation |
 | `version` | the myos version |
 
+## Several commands at once
+
+Commands chain, the way make targets did. Leading words that name commands are
+commands; the first word that is not one starts the list of stacks.
+
+```sh
+myos build up logs host/fabio        # like: make build up logs STACK=host/fabio
+myos up ps host
+```
+
+They run in order and stop at the first failure. A stack whose name is also a
+command name has to be given as `STACK=<name>`, otherwise it is read as a
+command.
+
 Anything after `--` goes to docker compose:
 
 ```sh
@@ -65,7 +79,8 @@ does not: a typo is an error.
 |---|---|
 | `make up STACK=host` | `myos up host` |
 | `make print-COMPOSE_FILE` | `myos env COMPOSE_FILE` |
-| `make host` | `myos up host` |
+| `make host` | `myos up host` (a bare stack name is not a command) |
+| `make build up logs STACK=host/fabio` | `myos build up logs host/fabio` |
 | `make stack-host-config` | `myos config host` |
 | `make up@master` | `myos -e master up` |
 | `make exec SERVICE=php ARGS='ls'` | `myos exec <stack> -- php ls` |

skills/myos/references/conventions.md

@@ -99,6 +99,20 @@ parsed, never sourced, so a value may contain a `#` or a `$(...)` without
 breaking anything or being executed. The make engine included `.env` as a
 makefile, where both broke.
 
+## Where a stack lives
+
+The same stack may exist in several directories of the stack path. They are all
+loaded, least specific first, so the project has the last word:
+
+```
+/usr/local/share/myos/stack/postgres/postgres.yml     the catalogue
+./stack/postgres/postgres.local.yml                   the project refines it
+./stack/postgres/postgres.sh                          and may redefine a default
+```
+
+Nothing has to be copied to change one setting: a value in the project `.env`
+already wins over any default a stack ships.
+
 ## Per-stack settings
 
 A stack keeps its own settings next to its compose files:

spec/golden/cases.txt

@@ -81,3 +81,6 @@ shim-up-group                   | shim-project  | @make up STACK=host DRYRUN=tru
 shim-config                     | shim-project  | @make config STACK=host/consul DRYRUN=true
 shim-project-target             | shim-project  | @make host-certs
 shim-env                        | shim-project  | @make env ARGS=COMPOSE_PROJECT_NAME STACK=host
+chain-build-up-logs             | host-project  | build up logs host/fabio
+chain-up-group                  | host-project  | up ps host
+chain-print-two                 | host-project  | print-COMPOSE_PROJECT_NAME print-APP STACK=host/consul

spec/golden/expected.cli/cat-unknown-target.txt

@@ -1,28 +1,3 @@
 ERROR: unknown command: doesnotexist
-Usage: myos [options] <command> [stack...] [VAR=value...] [-- args...]
+ERROR: to act on a stack of that name, say what to do: myos up doesnotexist
-
-Options:
- -C DIR work in DIR instead of the current directory
- -e ENV environment (default: local, or ENV from the config)
- -H HOSTS run on remote hosts instead (comma separated, or "all")
- -n, --dry-run print the commands instead of running them
- --color WHEN always, never or auto (default: colour when on a terminal)
- -v, --verbose show what myos does
- -d, --debug show every command
- -h, --help this help
-
-Commands:
- up down start stop restart recreate manage the containers of a stack
- ps logs config exec run inspect and enter them
- ls [--groups] list the stacks myos can see
- env [VAR...] show resolved variables
- env-update fill .env from the .env.dist templates
- doctor check the installation
- version print the myos version
-
-Stacks:
- myos up the stack of the current directory
- myos up host a group, expanded from host=... in a .env or .mk
- myos up host/fabio a single stack
- myos up postgres:9.6 a versioned stack
 [exit 2]

spec/golden/expected.cli/chain-build-up-logs.txt

@@ -0,0 +1,6 @@
+docker compose -f @WD@/stack/host/fabio.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host build
+docker network create tester-local
+docker network create testhost
+docker compose -f @WD@/stack/host/fabio.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host up -d
+docker compose -f @WD@/stack/host/fabio.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host logs --follow --tail=100
+[exit 0]

spec/golden/expected.cli/chain-print-two.txt

@@ -0,0 +1,5 @@
+COMPOSE_PROJECT_NAME testhost
+APP consul
+COMPOSE_PROJECT_NAME testhost
+APP consul
+[exit 0]

spec/golden/expected.cli/chain-up-group.txt

@@ -0,0 +1,5 @@
+docker network create tester-local
+docker network create testhost
+docker compose -f @WD@/stack/host/consul.yml -f @WD@/stack/host/fabio.yml -f @WD@/stack/host/registrator.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host up -d
+docker compose -f @WD@/stack/host/consul.yml -f @WD@/stack/host/fabio.yml -f @WD@/stack/host/registrator.yml -f @MYOS@/share/compose/networks.yml -p testhost --project-directory @WD@/stack/host ps
+[exit 0]

spec/golden/expected.cli/host-target.txt

@@ -1,2 +1,3 @@
-ERROR: no stack given, and no compose file in @WD@
+ERROR: unknown command: host
+ERROR: to act on a stack of that name, say what to do: myos up host
 [exit 2]

spec/golden/expected/chain-build-up-logs.txt

@@ -0,0 +1,10 @@
+make -o docker-stack-build MAKE_OLDFILE=docker-stack-build ENV=local DOCKER_COMPOSE=docker --log-level=error compose docker-compose-build STACK=myos APP_NAME=myos
+docker --log-level=error compose --ansi=auto -f @MYOS@/share/compose/networks.yml -p tester-myos-local build --build-arg APP=myos --build-arg BRANCH=lightning --build-arg COMPOSE_VERSION=2.24.4 --build-arg DOCKER_MACHINE=x86_64 --build-arg DOCKER_REPOSITORY=tester/myos/local --build-arg DOCKER_SYSTEM=Linux --build-arg GIT_AUTHOR_EMAIL=tester@example.test --build-arg GIT_AUTHOR_NAME=tester --build-arg SSH_REMOTE_HOSTS=github.com gitlab.com --build-arg USER=tester --build-arg VERSION=legacy-1.0-beta-17-g55fae62 --build-arg SSH_AUTHORIZED_KEYS=https://github.com/tester.keys --build-arg SSH_BASTION_USERNAME=tester --build-arg SSH_PORT=22 --build-arg SSH_PUBLIC_HOSTS= github.com gitlab.com --build-arg SSH_USER=tester --build-arg GID=20 --build-arg UID=502
+make -o docker-stack-up MAKE_OLDFILE=docker-stack-up ENV=local DOCKER_COMPOSE=docker --log-level=error compose docker-compose-up STACK=myos APP_NAME=myos
+sh -c docker network create tester-local >/dev/null 2>&1
+sh -c docker network create testhost >/dev/null 2>&1
+docker --log-level=error compose --ansi=auto -f @MYOS@/share/compose/networks.yml -p tester-myos-local up -d
+make -o docker-stack-logs MAKE_OLDFILE=docker-stack-logs ENV=local DOCKER_COMPOSE=docker --log-level=error compose docker-compose-logs STACK=myos APP_NAME=myos
+docker --log-level=error compose --ansi=auto -f @MYOS@/share/compose/networks.yml -p tester-myos-local logs --follow --tail=100
+WARNING: myos[0] host/fabio-rule-exists: target host/fabio unavailable in app myos
+[exit 0]

spec/golden/expected/chain-print-two.txt

@@ -0,0 +1,3 @@
+COMPOSE_PROJECT_NAME testhost
+APP myos
+[exit 0]

spec/golden/expected/chain-up-group.txt

@@ -0,0 +1,20 @@
+make -o docker-stack-up MAKE_OLDFILE=docker-stack-up ENV=local DOCKER_COMPOSE=docker --log-level=error compose docker-compose-up STACK=myos APP_NAME=myos
+sh -c docker network create tester-local >/dev/null 2>&1
+sh -c docker network create testhost >/dev/null 2>&1
+docker --log-level=error compose --ansi=auto -f @MYOS@/share/compose/networks.yml -p tester-myos-local up -d
+make -o docker-stack-ps MAKE_OLDFILE=docker-stack-ps ENV=local DOCKER_COMPOSE=docker --log-level=error compose docker-compose-ps STACK=myos APP_NAME=myos
+docker --log-level=error compose --ansi=auto -f @MYOS@/share/compose/networks.yml -p tester-myos-local ps
+make -o host MAKE_OLDFILE=host ENV=local DOCKER_COMPOSE=docker --log-level=error compose up STACK=host/consul host/fabio host/registrator
+make -o host MAKE_OLDFILE=host ENV=local DOCKER_COMPOSE=docker --log-level=error compose docker-compose-up STACK=host/consul APP_NAME=host
+sh -c docker network create tester-local >/dev/null 2>&1
+sh -c docker network create testhost >/dev/null 2>&1
+docker --log-level=error compose --ansi=auto -f @WD@/stack/host/consul.yml -f @MYOS@/share/compose/networks.yml -p testhost up -d
+make -o host MAKE_OLDFILE=host ENV=local DOCKER_COMPOSE=docker --log-level=error compose docker-compose-up STACK=host/fabio APP_NAME=host
+sh -c docker network create tester-local >/dev/null 2>&1
+sh -c docker network create testhost >/dev/null 2>&1
+docker --log-level=error compose --ansi=auto -f @WD@/stack/host/fabio.yml -f @MYOS@/share/compose/networks.yml -p testhost up -d
+make -o host MAKE_OLDFILE=host ENV=local DOCKER_COMPOSE=docker --log-level=error compose docker-compose-up STACK=host/registrator APP_NAME=host
+sh -c docker network create tester-local >/dev/null 2>&1
+sh -c docker network create testhost >/dev/null 2>&1
+docker --log-level=error compose --ansi=auto -f @WD@/stack/host/registrator.yml -f @MYOS@/share/compose/networks.yml -p testhost up -d
+[exit 0]