diff --git a/spec/bench/README.md b/spec/bench/README.md new file mode 100644 index 0000000..8a73738 --- /dev/null +++ b/spec/bench/README.md @@ -0,0 +1,40 @@ +# Benchmark of the engines + +Same work, four engines, five runs, median. Docker is the mock of +`spec/support/bin`, the catalogue is `myos-stacks` reached through +`$HOME/.local/share/myos/stack`, the environment is `env.sh`. + + sh spec/bench/run.sh + +`justfile` is a prototype of just as the engine: shebang recipes that source +`lib/*.sh` once. `go/main.go` is a prototype of the core in Go: stack path, +groups, compose files, project name, dry-run command; `export` runs one `sh` +per stack directory to evaluate the shell hooks. + +## Results, 2026-09-05, Mac Studio M2 Ultra + +| work | make | sh (bin/myos) | just | go | +|---|---:|---:|---:|---:| +| fixed cost, empty target | 312 ms | 50 ms | 170 ms | 23 ms | +| `up` 1 / 3 stacks, no hooks in the stack | 792 / 2201 | 177 / 332 | 204 / 268 | 23 / 23 | +| `up` 1 / 3 stacks, real catalogue with hooks | — | 353 / 855 | (prototype does not load hooks) | (idem) | +| `export`, 80 settings of the `host` group | — | 1477 | 1314 | 720 (1 sh) | +| same, `MYOS_VAR_MEMO=1` | — | 1503 | 1318 | 774 | +| one computed setting (`HOST_FABIO_SERVICE_9998_TAGS`) | — | ~43 ms net (69 − 26) | | | + +Reference points: `sh -c :` 24 ms, `just --version` 27 ms, sourcing `lib/*.sh` +2 ms. + +## What it says + +- The engine's own cost: go flat at 23 ms; sh 177 ms + ~63 ms per stack; just + 204 ms + ~32 ms per stack; make 792 ms + ~700 ms per stack (it re-reads + itself for every stack). +- The shell hooks cost ~40 ms per computed setting, on every engine: 80 + settings ≈ 0.7 s even from Go, which runs the very same `sh`. Memoisation + changes nothing, because the cost is not repeated lookups: each `tagprefix` + spawns 15-20 command substitutions for distinct, mostly empty, variables. +- `bin/myos` doubles that to 1.5 s by loading the hooks of a directory once + per stack reference instead of once per directory: `host/consul`, + `host/fabio` and `host/registrator` share `stack/host/_stack.sh`. +- just's fixed cost (170 ms for a shebang recipe, against 27 ms for `just + --version`) is its own overhead of writing and running the recipe script. diff --git a/spec/bench/bench.sh b/spec/bench/bench.sh new file mode 100755 index 0000000..183b40f --- /dev/null +++ b/spec/bench/bench.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# bench.sh LABEL N -- CMD... run CMD N times, print the median wall time in ms +# Hermetic: docker is the mock of spec/support/bin, config comes from the +# environment only, HOME points at the fixture catalogue. +set -u +label=$1; n=$2; shift 2; [ "$1" = "--" ] && shift +i=0; times="" +while [ "$i" -lt "$n" ]; do + s=$(python3 -c 'import time;print(int(time.time()*1e6))') + "$@" >/dev/null 2>&1 + e=$(python3 -c 'import time;print(int(time.time()*1e6))') + times="$times $(( (e - s) / 1000 ))" + i=$((i + 1)) +done +median=$(printf '%s\n' $times | sort -n | awk '{a[NR]=$1} END {print a[int((NR+1)/2)]}') +printf '%-44s %6s ms (runs:%s)\n' "$label" "$median" "$times" diff --git a/spec/bench/env.sh b/spec/bench/env.sh new file mode 100644 index 0000000..893517d --- /dev/null +++ b/spec/bench/env.sh @@ -0,0 +1,5 @@ +# the hermetic environment every engine runs in +export PATH=/Users/aya/dev/myos/spec/support/bin:/Users/aya/.local/bin:/usr/bin:/bin +export HOME=/tmp/myos-bench/home WORKDIR=/tmp/myos-bench/wd MYOS_ROOT=/Users/aya/dev/myos +export USER=tester HOSTNAME=testhost DOMAIN=example.test ENV=local DRYRUN=true +export MYOS_CONF=/dev/null MYOS_PROJECT_FORMAT=user-app-env DOCKER_MACHINE=x86_64 DOCKER_SYSTEM=Linux diff --git a/spec/bench/go/main.go b/spec/bench/go/main.go new file mode 100644 index 0000000..64113a7 --- /dev/null +++ b/spec/bench/go/main.go @@ -0,0 +1,238 @@ +// A prototype of the myos core in Go, just large enough to be benchmarked +// fairly against the other engines: stack path, group expansion, compose file +// resolution across every directory of the path, project name, and the +// dry-run compose command. Same rules as lib/stack.sh and lib/naming.sh. +// +// export delegates the shell hooks to ONE sh per stack directory, which is +// what a Go engine would do to keep the developer contract in shell. +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" +) + +func env(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +// stackPath: the directories stacks are looked up in, project first +func stackPath(workdir string) []string { + home := env("HOME", "/nonexistent") + root := env("MYOS_ROOT", ".") + prefix := filepath.Dir(filepath.Dir(root)) + var out []string + seen := map[string]bool{} + for _, d := range []string{workdir, filepath.Join(workdir, ".."), filepath.Join(home, ".local/share"), filepath.Join(prefix, "share"), "/usr/local/share", "/usr/share"} { + for _, c := range []string{filepath.Join(d, "stack"), filepath.Join(d, "myos/stack")} { + if st, err := os.Stat(c); err == nil && st.IsDir() { + if r, err := filepath.EvalSymlinks(c); err == nil { + c = r + } + if !seen[c] { + seen[c] = true + out = append(out, c) + } + } + } + } + return out +} + +// groupValue: the list a lowercase group name expands to, from .env, +// /.env or /_stack.env along the path +func groupValue(path []string, name string) string { + if strings.ContainsAny(name, "/:.") || strings.ToLower(name) != name { + return "" + } + if v := os.Getenv(name); v != "" { + return v + } + for _, d := range path { + for _, f := range []string{filepath.Join(d, name+".env"), filepath.Join(d, name, name+".env"), filepath.Join(d, name, "_stack.env")} { + b, err := os.ReadFile(f) + if err != nil { + continue + } + for _, line := range strings.Split(string(b), "\n") { + if strings.HasPrefix(line, name+"=") { + return strings.Trim(strings.TrimPrefix(line, name+"="), "\"") + } + } + } + } + return "" +} + +func expand(path []string, refs []string, depth int) []string { + var out []string + for _, r := range refs { + if v := groupValue(path, r); v != "" && depth < 16 { + out = append(out, expand(path, strings.Fields(v), depth+1)...) + } else { + out = append(out, r) + } + } + return out +} + +func stackName(ref string) string { + r := strings.TrimSuffix(ref, "/") + if i := strings.LastIndex(r, ":"); i >= 0 { + r = r[:i] + } + return strings.TrimSuffix(filepath.Base(r), ".yml") +} + +// stackDirs: every directory of the path holding the stack, least specific first +func stackDirs(path []string, ref string) []string { + r := strings.TrimSuffix(ref, "/") + if i := strings.LastIndex(r, ":"); i >= 0 { + r = r[:i] + } + name := stackName(ref) + var found []string + for _, d := range path { + var hit string + if st, err := os.Stat(filepath.Join(d, r)); err == nil && st.IsDir() { + hit = filepath.Join(d, r) + } else if _, err := os.Stat(filepath.Join(d, r+".yml")); err == nil { + hit = filepath.Dir(filepath.Join(d, r)) + } else if st, err := os.Stat(filepath.Join(d, name)); err == nil && st.IsDir() { + hit = filepath.Join(d, name) + } + if hit != "" { + found = append([]string{hit}, found...) + } + } + return found +} + +func exists(p string) bool { _, err := os.Stat(p); return err == nil } + +// composeFiles: the files that exist, in the order the framework loads them +func composeFiles(dir string, names, suffixes []string, envName string) []string { + var out []string + for _, e := range []string{"yml", "yaml"} { + for _, n := range names { + for _, f := range []string{ + filepath.Join(dir, n+"."+e), filepath.Join(dir, n+"."+envName+"."+e), + filepath.Join(dir, envName, n+"."+e), filepath.Join(dir, envName, n+"."+envName+"."+e)} { + if exists(f) { + out = append(out, f) + } + } + for _, s := range suffixes { + for _, f := range []string{filepath.Join(dir, n+"."+s+"."+e), filepath.Join(dir, n+"."+s+"."+envName+"."+e)} { + if exists(f) { + out = append(out, f) + } + } + } + } + } + return out +} + +func scope(ref string) string { + switch strings.SplitN(ref, "/", 2)[0] { + case "host": + return "host" + case "User", "user": + return "user" + case "cluster": + return "cluster" + } + return "app" +} + +func projectName(sc, user, envName, app string) string { + switch sc { + case "host": + return env("HOST_COMPOSE_PROJECT_NAME", env("HOSTNAME", "localhost")) + case "user": + return user + case "cluster": + return strings.ToLower(app) + } + n := strings.NewReplacer(".", "", "-", "", "_", "").Replace(strings.ToLower(app)) + if env("MYOS_PROJECT_FORMAT", "user-env-app") == "user-app-env" { + return user + "-" + n + "-" + envName + } + return user + "-" + envName + "-" + n +} + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: myos-go [stack...]") + os.Exit(2) + } + workdir := env("WORKDIR", ".") + envName := env("ENV", "local") + user := env("USER", "tester") + path := stackPath(workdir) + suffixes := []string{"app", "labels", "networks", "ssh", "volumes", "latest"} + + switch os.Args[1] { + case "noop": + return + case "up": + refs := expand(path, os.Args[2:], 0) + byProject := map[string][]string{} + var order []string + for _, ref := range refs { + app := stackName(ref) + var files []string + for _, d := range stackDirs(path, ref) { + files = append(files, composeFiles(d, []string{"docker-compose", app}, suffixes, envName)...) + } + p := projectName(scope(ref), user, envName, app) + if _, ok := byProject[p]; !ok { + order = append(order, p) + } + byProject[p] = append(byProject[p], files...) + } + for _, p := range order { + files := append(byProject[p], filepath.Join(env("MYOS_ROOT", "."), "share/compose/networks.yml")) + var b strings.Builder + b.WriteString("docker compose") + for _, f := range files { + b.WriteString(" -f " + f) + } + fmt.Printf("%s -p %s up -d\n", b.String(), p) + } + case "export": + // one sh per stack directory evaluates its hooks and prints every value + refs := expand(path, os.Args[2:], 0) + seen := map[string]bool{} + var dirs []string + for _, ref := range refs { + for _, d := range stackDirs(path, ref) { + if !seen[d] { + seen[d] = true + dirs = append(dirs, d) + } + } + } + sort.Strings(dirs) + root := env("MYOS_ROOT", ".") + for _, d := range dirs { + script := fmt.Sprintf(`for m in core str var tags naming stack config compose hooks; do . %s/lib/$m.sh; done +[ -f %s/_stack.sh ] || exit 0 +myos_stack_hooks %s _ +for v in $(sed -n 's/^myos_default_\([A-Za-z_][A-Za-z0-9_]*\)().*/\1/p' %s/_stack.sh | sort -u); do printf '%%s=%%s\n' "$v" "$(myos_var "$v")"; done`, root, d, d, d) + cmd := exec.Command("sh", "-c", script) + cmd.Env = os.Environ() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + _ = cmd.Run() + } + } +} diff --git a/spec/bench/justfile b/spec/bench/justfile new file mode 100644 index 0000000..4131fb2 --- /dev/null +++ b/spec/bench/justfile @@ -0,0 +1,52 @@ +# just as the engine: the interface is a justfile, the logic stays in lib/*.sh. +# Recipes are shebang recipes, so a whole body runs in ONE sh that sources +# lib/ once; what is measured is just's own overhead on top of the shell. + +set export + +MYOS_ROOT := env_var_or_default("MYOS_ROOT", "/Users/aya/dev/myos") +WORKDIR := env_var_or_default("WORKDIR", justfile_directory()) +ENV := env_var_or_default("ENV", "local") +USER := env_var_or_default("USER", "tester") +HOSTNAME := env_var_or_default("HOSTNAME", "testhost") +DOMAIN := env_var_or_default("DOMAIN", "example.test") +DRYRUN := env_var_or_default("DRYRUN", "true") + +# a recipe that does nothing: the fixed cost of just + one sh + sourcing lib/ +noop: + #!/bin/sh + for m in core str var tags naming stack config compose hooks; do . $MYOS_ROOT/lib/$m.sh; done + +# up STACKS: resolve every stack of the groups, group by compose project, +# print one compose command per project (what bin/myos does) +up +stacks: + #!/bin/sh + for m in core str var tags naming stack config compose hooks; do . $MYOS_ROOT/lib/$m.sh; done + rows="" + for ref in $(myos_group_expand {{stacks}}); do + files="" + for d in $(myos_stack_dirs "$ref"); do + files="$files $(myos_compose_files "$d" "docker-compose $(myos_stack_name "$ref")" "$(myos_compose_suffixes)" "$ENV" | tr '\n' ' ')" + done + app=$(myos_stack_name "$ref") + project=$(myos_project_name "$(myos_scope "$ref")" "$USER" "$ENV" "$app") + rows="$rows + $project|$files" + done + for project in $(printf '%s\n' "$rows" | sed '/^$/d' | cut -d'|' -f1 | awk '!s[$0]++'); do + files=$(printf '%s\n' "$rows" | awk -F'|' -v p="$project" '$1==p {print $2}' | tr ' ' '\n' | sed '/^$/d' | awk '!s[$0]++') + fargs=""; for f in $files $MYOS_ROOT/share/compose/networks.yml; do fargs="$fargs -f $f"; done + echo "docker compose$fargs -p $project up -d" + done + +# export STACKS: every setting the hooks of the stacks declare +export +stacks: + #!/bin/sh + for m in core str var tags naming stack config compose hooks; do . $MYOS_ROOT/lib/$m.sh; done + refs=$(myos_group_expand {{stacks}}) + for ref in $refs; do + for d in $(myos_stack_dirs "$ref"); do myos_stack_hooks "$d" "$(myos_stack_name "$ref")"; done + done + names=$(for ref in $refs; do for d in $(myos_stack_dirs "$ref"); do + [ -f "$d/_stack.sh" ] && sed -n 's/^myos_default_\([A-Za-z_][A-Za-z0-9_]*\)().*/\1/p' "$d/_stack.sh"; done; done | sort -u) + for v in $names; do printf '%s=%s\n' "$v" "$(myos_var "$v")"; done diff --git a/spec/bench/run.sh b/spec/bench/run.sh new file mode 100755 index 0000000..04cfb96 --- /dev/null +++ b/spec/bench/run.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# the full matrix: 5 runs each, median, every engine on the same work +. /tmp/myos-bench/env.sh; cd "$WORKDIR" +B=/tmp/myos-bench/bench.sh; N=5 +MK="make -esC $MYOS_ROOT MYOS=. WORKDIR=$WORKDIR" +SH="$MYOS_ROOT/bin/myos" +JU="just --justfile /tmp/myos-bench/justfile" +GO=/tmp/myos-bench/myos-go +S1="host/consul"; S2="host/consul host/fabio"; S3="host/consul host/fabio host/registrator" + +echo "== cout fixe : demarrage + cible vide" +$B "make noop" $N -- $MK FORCE +$B "sh noop (myos version)" $N -- $SH version +$B "just noop (parse + 1 sh + source lib/)" $N -- $JU noop +$B "go noop" $N -- $GO noop +echo +echo "== up : 1 / 2 / 3 stacks, dry-run" +$B "make up 1" $N -- $MK up STACK="$S1" +$B "make up 2" $N -- $MK up STACK="$S2" +$B "make up 3" $N -- $MK up STACK="$S3" +$B "sh up 1" $N -- $SH up host/consul +$B "sh up 2" $N -- $SH up host/consul host/fabio +$B "sh up 3" $N -- $SH up host/consul host/fabio host/registrator +$B "just up 1" $N -- $JU up host/consul +$B "just up 2" $N -- $JU up host/consul host/fabio +$B "just up 3" $N -- $JU up host/consul host/fabio host/registrator +$B "go up 1" $N -- $GO up host/consul +$B "go up 2" $N -- $GO up host/consul host/fabio +$B "go up 3" $N -- $GO up host/consul host/fabio host/registrator +echo +echo "== export : les 80 reglages du groupe host (evaluation des hooks shell)" +$B "sh export, hooks tels quels" $N -- $SH export STACK=host +$B "just export, hooks tels quels" $N -- $JU export host +$B "go export, hooks tels quels (1 sh/repertoire)" $N -- $GO export host +echo +echo "== export : memes hooks, evalues en une passe (MYOS_VAR_MEMO=1)" +MYOS_VAR_MEMO=1 $B "sh export, memoise" $N -- $SH export STACK=host +MYOS_VAR_MEMO=1 $B "just export, memoise" $N -- $JU export host +MYOS_VAR_MEMO=1 $B "go export, memoise" $N -- $GO export host