Publish packages from the host that can reach Arty.

deploy/capy-bearer-receive

@@ -20,7 +20,8 @@ HEALTH_HOST=${HEALTH_HOST:-}
 BACKUP=${BACKUP:-/var/backups/capy-bearer}
 FILES=(bin/bearer_fastcgi.linux.bin bin/capyc bin/wasm/core.wasm)
 
-log() { printf '%s %s\n' "$(date -Is)" "$*" >&2; }
+# The distributor's updater log adds the timestamp.
+log() { printf '%s\n' "$*" >&2; }
 fail() { log "FAILED: $*"; exit 1; }
 
 # The distributor runs `ssh target capy-bearer-receive REVISION`; sshd hands

deploy/capy-package-release.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
-"""Publishes the Debian and RPM packages for the revision that CT100 runs.
+"""Publishes the Debian and RPM packages for the revision that the builder runs.
 
-Runs from capy-package-release.timer on the packaging host. It publishes a
-revision once, and only after the updater on capy-bearer.openfu.com has built,
-gated, and deployed it. The code gates therefore run once, on CT100. This job
-builds the binaries for the packages, builds both packages, checks the package
-layout and the release contract, publishes to Arty, and fetches the published
-digest back to compare bytes.
+Two hosts share this script. The publisher runs it from
+capy-package-release.timer with no arguments. It publishes a revision once,
+and only after the updater on the builder has deployed it, so the code gates
+run once, on the builder. The publisher asks the packaging host to build the
+binaries and both packages with `capy-package-release stage REVISION`, copies
+the stage back, publishes it to Arty, and fetches the published digest to
+compare bytes.
 """
 import hashlib
 import json
@@ -23,10 +24,15 @@ STATE = pathlib.Path(os.environ.get("STATE", "/var/lib/capy-package-release"))
 SRC = STATE / "src"
 REMOTE = os.environ.get("REMOTE", "ssh://git@git.openfu.com:11622/udo/capy-bearer.git")
 DEPLOYED_URL = os.environ.get("DEPLOYED_URL", "https://capy-bearer.openfu.com/deployed-revision.txt")
+PACKAGING_HOST = os.environ.get("PACKAGING_HOST", "capy-bearer-dev")
 EVIDENCE = pathlib.Path(os.environ.get("EVIDENCE", "/root/capy-package-releases"))
 HELPER = os.environ.get("RELEASE_HELPER", "/root/scripts/arty/release.py")
 TOKEN_FILE = os.environ.get("TOKEN_FILE", "/root/.config/arty/worker-publish.token")
 CONTRACT = "deploy/arty-packages.json"
+# A revision that leaves these paths unchanged gets no new package.
+PACKAGE_PATHS = ["src", "etc", "version.txt", "LICENSE", "README.md", "deploy/arty-packages.json", "scripts/make_deb.sh", "scripts/make_rpm.sh",
+                 "scripts/deb", "scripts/rpm", "scripts/bearer-cli", "scripts/systemd", "scripts/build_capy.sh", "scripts/build_linux.sh",
+                 "scripts/build_core_wasm.sh", "bin/assets"]
 GATES = [
     "bash scripts/build_capy.sh",
     "bash scripts/build_core_wasm.sh && bash scripts/build_linux.sh",
@@ -38,45 +44,33 @@ GATES = [
 ]
 
 
-def run(command, cwd, timeout, **kw):
+def run(command, cwd=None, timeout=300, **kw):
     return subprocess.run(command, cwd=cwd, timeout=timeout, check=True, text=True, capture_output=True, **kw).stdout.strip()
 
 
 def git(*args):
-    return run(["git", "-C", str(SRC), *args], None, 300)
+    return run(["git", "-C", str(SRC), *args])
 
 
-def main():
+def checkout(head):
     STATE.mkdir(parents=True, exist_ok=True)
-    head = run(["git", "ls-remote", REMOTE, "refs/heads/main"], None, 120).split()[0]
-    with urllib.request.urlopen(DEPLOYED_URL, timeout=30) as response:
-        deployed = response.read().decode().strip()
-    if deployed != head:
-        print(f"remote head {head[:7]} is not deployed yet (deployed {deployed[:7]}), nothing to do")
-        return 0
-    marker = STATE / f"published-{head}"
-    if marker.exists():
-        print(f"{head[:7]} is already published, nothing to do")
-        return 0
-    failed = STATE / f"failed-{head}"
-    if failed.exists():
-        print(f"{head[:7]} failed before, see {failed}; remove the file to retry")
-        return 0
-
     if not (SRC / ".git").is_dir():
-        run(["git", "clone", "-q", REMOTE, str(SRC)], None, 600)
+        run(["git", "clone", "-q", REMOTE, str(SRC)], timeout=600)
     git("fetch", "-q", "origin", "main")
     git("checkout", "-q", "--detach", head)
     git("clean", "-qfd", "-e", "bin")
+
+
+def stage(head):
+    """Builds and packages HEAD on the packaging host. Prints the stage path."""
+    checkout(head)
     for old in (SRC / "dist").glob("*"):
         old.unlink()
-
-    out = EVIDENCE / head[:7]
-    stage = out / "stage"
-    if stage.exists():
-        shutil.rmtree(stage)
-    stage.mkdir(parents=True, mode=0o700)
-    log = open(stage / "build-and-test.log", "w")
+    out = EVIDENCE / head[:7] / "stage"
+    if out.exists():
+        shutil.rmtree(out)
+    out.mkdir(parents=True, mode=0o700)
+    log = open(out / "build-and-test.log", "w")
     tests = []
     for command in GATES:
         log.write(f"\nRUN {command}\n")
@@ -84,51 +78,93 @@ def main():
         started = time.monotonic()
         result = subprocess.run(["bash", "-o", "pipefail", "-c", command], cwd=SRC, stdout=log, stderr=subprocess.STDOUT, timeout=1800)
         seconds = round(time.monotonic() - started, 3)
-        print(f"{command}: exit {result.returncode} in {seconds}s", flush=True)
+        print(f"{command}: exit {result.returncode} in {seconds}s", file=sys.stderr, flush=True)
         if result.returncode != 0:
             log.write(f"FAILED exit {result.returncode}\n")
             log.close()
-            failed.write_text(f"gate failed: {command}\n")
-            return 1
+            sys.exit(f"gate failed: {command}")
         tests.append({"command": command, "result": "passed", "seconds": seconds})
     log.close()
     if git("status", "--porcelain", "--untracked-files=all"):
-        failed.write_text("builds changed the source tree\n")
-        return 1
-
+        sys.exit("builds changed the source tree")
     debs = list((SRC / "dist").glob("*.deb"))
     rpms = list((SRC / "dist").glob("*.rpm"))
     if len(debs) != 1 or len(rpms) != 1:
-        failed.write_text("expected exactly one deb and one rpm\n")
-        return 1
+        sys.exit("expected exactly one deb and one rpm")
     artifacts = []
     for name, source, info in (("bearer.deb", debs[0], ["dpkg-deb", "-f"]), ("bearer.rpm", rpms[0], ["rpm", "-qip"])):
-        shutil.copyfile(source, stage / name)
-        data = (stage / name).read_bytes()
-        meta = run([*info, str(stage / name)], None, 60)
-        artifacts.append({"path": name, "original_filename": source.name, "sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data), "metadata": meta})
+        shutil.copyfile(source, out / name)
+        data = (out / name).read_bytes()
+        artifacts.append({"path": name, "original_filename": source.name, "sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data),
+                          "metadata": run([*info, str(out / name)], timeout=60)})
     receipt = {
         "schema_version": 1, "source_revision": head, "build_platform": platform.platform(),
         "tests": tests, "artifacts": artifacts,
         "limitations": [
-            "Compiler, runtime, and artifact gates ran on capy-bearer.openfu.com through capy-bearer-update before this revision was deployed there. This job repeats only the package gates.",
+            "Compiler, runtime, and artifact gates ran on the builder through capy-bearer-update before this revision was deployed there. This stage repeats only the package gates.",
             "No package installation test on a disposable target-distribution host.",
             "No accepted-reference promotion or server installation.",
         ],
     }
-    (stage / "test-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")
+    (out / "test-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")
+    print(out)
+
+
+def publish():
+    """Runs on the publisher. Stages on the packaging host, then publishes."""
+    STATE.mkdir(parents=True, exist_ok=True)
+    head = run(["git", "ls-remote", REMOTE, "refs/heads/main"], timeout=120).split()[0]
+    with urllib.request.urlopen(DEPLOYED_URL, timeout=30) as response:
+        deployed = response.read().decode().strip()
+    if deployed != head:
+        print(f"remote head {head[:7]} is not deployed yet (deployed {deployed[:7]}), nothing to do")
+        return 0
+    marker = STATE / f"published-{head}"
+    failed = STATE / f"failed-{head}"
+    if marker.exists():
+        print(f"{head[:7]} is already published, nothing to do")
+        return 0
+    if failed.exists():
+        print(f"{head[:7]} failed before, see {failed}; remove the file to retry")
+        return 0
+    checkout(head)
+    published = sorted(STATE.glob("published-*"), key=lambda path: path.stat().st_mtime)
+    if published:
+        previous = published[-1].name[len("published-"):]
+        if not git("diff", "--name-only", previous, head, "--", *PACKAGE_PATHS):
+            marker.write_text(f"unchanged package inputs since {previous}\n")
+            print(f"{head[:7]} changes no package input since {previous[:7]}, nothing to publish")
+            return 0
+
+    out = EVIDENCE / head[:7]
+    local_stage = out / "stage"
+    if local_stage.exists():
+        shutil.rmtree(local_stage)
+    out.mkdir(parents=True, exist_ok=True)
+    try:
+        remote_stage = run(["ssh", "-o", "BatchMode=yes", PACKAGING_HOST, f"capy-package-release stage {head}"], timeout=3600)
+    except subprocess.CalledProcessError as error:
+        failed.write_text(f"stage failed on {PACKAGING_HOST}:\n{error.stderr}\n")
+        print(error.stderr, file=sys.stderr)
+        return 1
+    run(["scp", "-q", "-r", f"{PACKAGING_HOST}:{remote_stage}", str(local_stage)], timeout=900)
 
     env = dict(os.environ, ARTY_TOKEN=pathlib.Path(TOKEN_FILE).read_text().strip())
-    publish = out / f"capy-package-{head[:7]}-publish.json"
-    print(run(["python3", HELPER, "publish", CONTRACT, "--stage", str(stage), "--receipt", str(publish)], SRC, 900, env=env))
-    digest = json.loads(publish.read_text())["manifest"]["digest"]
-    fetched = out / "fetched"
-    if fetched.exists():
-        shutil.rmtree(fetched)
-    run(["python3", HELPER, "fetch", CONTRACT, digest, str(fetched), "--expected-revision", head,
-         "--receipt", str(out / f"capy-package-{head[:7]}-fetch.json")], SRC, 900, env=env)
+    receipt = out / f"capy-package-{head[:7]}-publish.json"
+    try:
+        print(run(["python3", HELPER, "publish", CONTRACT, "--stage", str(local_stage), "--receipt", str(receipt)], SRC, 1800, env=env))
+        digest = json.loads(receipt.read_text())["manifest"]["digest"]
+        fetched = out / "fetched"
+        if fetched.exists():
+            shutil.rmtree(fetched)
+        run(["python3", HELPER, "fetch", CONTRACT, digest, str(fetched), "--expected-revision", head,
+             "--receipt", str(out / f"capy-package-{head[:7]}-fetch.json")], SRC, 1800, env=env)
+    except subprocess.CalledProcessError as error:
+        failed.write_text(f"publish failed:\n{error.stderr}\n")
+        print(error.stderr, file=sys.stderr)
+        return 1
     for name in ("bearer.deb", "bearer.rpm", "test-receipt.json", "build-and-test.log"):
-        if (fetched / name).read_bytes() != (stage / name).read_bytes():
+        if (fetched / name).read_bytes() != (local_stage / name).read_bytes():
             failed.write_text(f"fetched {name} differs from the stage\n")
             return 1
     shutil.rmtree(fetched)
@@ -138,4 +174,9 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    if len(sys.argv) == 3 and sys.argv[1] == "stage":
+        stage(sys.argv[2])
+    elif len(sys.argv) == 1:
+        sys.exit(publish())
+    else:
+        sys.exit("usage: capy-package-release [stage REVISION]")

deploy/capy-package-release.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Publish the Capy-Bearer packages for the deployed revision
+Description=Publish the Capy-Bearer packages for the deployed revision (runs on the publisher)
 After=network-online.target
 Wants=network-online.target
 

docs/deploy.md

@@ -6,7 +6,7 @@ A push to `main` on Gitea reaches every runtime and the package repository witho
 
 1. `capy-bearer-update.timer` on the builder (CT100, capy-bearer.openfu.com) runs `deploy/capy-bearer-update` every 5 minutes. When `origin/main` has a new commit, the updater builds it in a staging clone, runs the gates, installs the binaries and the site, restarts Bearer, and checks health. It then writes the revision to `DEPLOYED_FILE`, which nginx serves as `/deployed-revision.txt`.
 2. The updater runs `deploy/capy-bearer-distribute`. For each host in `DISTRIBUTE_TARGETS` it streams the three runtime artifacts over SSH with `DISTRIBUTE_KEY`. On the target, `deploy/capy-bearer-receive` is the forced command of that key. It checks the application sources with the new `capyc`, backs up the current artifacts, installs, restarts, and checks health. A failed check or a failed health probe restores the backup. The target records the revision in `bin/REVISION`.
-3. `capy-package-release.timer` on the packaging host (capy-bearer-dev) runs `deploy/capy-package-release.py` every 10 minutes. When the Gitea head equals `/deployed-revision.txt` and no package exists for it, the job builds the binaries and both packages in `/var/lib/capy-package-release/src`, runs the package gates, publishes to Arty with the worker token, fetches the published digest, and compares bytes. Evidence lands in `/root/capy-package-releases/<rev>/`.
+3. `capy-package-release.timer` on the publisher (udo-aiworker) runs `deploy/capy-package-release.py` every 10 minutes. When the Gitea head equals `/deployed-revision.txt`, no package exists for it, and the revision changes a package input, the publisher asks the packaging host (capy-bearer-dev) to run `capy-package-release stage REVISION`. The packaging host builds the binaries and both packages in `/var/lib/capy-package-release/src` and runs the package gates. The publisher copies the stage back, publishes to Arty with the worker token, fetches the published digest, and compares bytes. The publisher holds the token and has a working upload path to Arty. Evidence lands in `/root/capy-package-releases/<rev>/` on both hosts.
 
 ## Files
 
@@ -18,8 +18,8 @@ A push to `main` on Gitea reaches every runtime and the package repository witho
 | `deploy/capy-bearer-distribute` | builder `/usr/local/sbin/capy-bearer-distribute` |
 | `deploy/capy-bearer-receive` | each target `/usr/local/sbin/capy-bearer-receive` |
 | `deploy/capy-bearer-receive.default` | each target `/etc/default/capy-bearer-receive` |
-| `deploy/capy-package-release.py` | packaging host `/usr/local/sbin/capy-package-release` |
-| `deploy/capy-package-release.service`, `.timer` | packaging host `/etc/systemd/system/` |
+| `deploy/capy-package-release.py` | publisher and packaging host `/usr/local/sbin/capy-package-release` |
+| `deploy/capy-package-release.service`, `.timer` | publisher `/etc/systemd/system/` |
 
 These files do not update themselves. Install them by hand after a change to `deploy/`.
 
@@ -31,4 +31,4 @@ These files do not update themselves. Install them by hand after a change to `de
 
 ## Timing
 
-Builder: about 2.5 minutes from fetch to health. Distribution: about 30 seconds per target. Package job: about 5 minutes after the builder has finished, plus the timer interval.
+Builder: about 2.5 minutes from fetch to health. Distribution: about 30 seconds per target. Package job: about 6 minutes after the builder has finished, plus the timer interval. A revision that changes no package input gets no new package.