Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa9306912d | ||
|
|
cebe9e2737 | ||
|
|
b91e2d53ff | ||
|
|
fdfa4bac08 | ||
|
|
bcac92669a | ||
|
|
a43148ee13 | ||
|
|
71f3fa09c4 | ||
|
|
153333d055 | ||
|
|
b0f19eb0c5 | ||
|
|
36ec6d0f93 | ||
|
|
3f37a6cf34 | ||
|
|
98f8825701 | ||
|
|
4ab34d3be2 | ||
|
|
8a7bff71aa | ||
|
|
3b10b29ef5 | ||
|
|
883f15045b | ||
|
|
cd2b417962 | ||
|
|
405d29d933 | ||
|
|
bb16f915f3 | ||
|
|
25bc952dff | ||
|
|
5243854129 | ||
|
|
486146ab34 | ||
|
|
fba9b09d3e | ||
|
|
8d2ad6424b | ||
|
|
75c4a8de33 | ||
|
|
fd83ff2f53 | ||
|
|
38bdfa2ab4 | ||
|
|
5b340e7b20 | ||
|
|
2bd41d980c | ||
|
|
80efb7fcfc | ||
|
|
b9af302baf | ||
|
|
c1523e8acf | ||
|
|
82865956d2 | ||
|
|
ecbed641bb | ||
|
|
db27d0dbf3 | ||
|
|
2335fc0bbb | ||
|
|
ba1617934a | ||
|
|
c91ebb810c | ||
|
|
b9d130bf64 | ||
|
|
9db8a5006a | ||
|
|
d4dde5d436 | ||
|
|
a2b7b62960 | ||
|
|
d77399aced | ||
|
|
6174828e18 | ||
|
|
e991c6ac63 | ||
|
|
bb5d7879ff | ||
|
|
3ed3fdd6b0 | ||
|
|
c717ae7041 | ||
|
|
0f4b0c9536 | ||
|
|
5d672f5bf6 | ||
|
|
043c77f077 | ||
|
|
b241661941 | ||
|
|
dac0278218 | ||
|
|
ea8e6edc94 | ||
|
|
eda14976d0 | ||
|
|
0d26ac5a2a | ||
|
|
39dd6e0151 | ||
|
|
1d71877af6 | ||
|
|
ae47dd29e8 | ||
|
|
faa7825799 | ||
|
|
97ed97885a | ||
|
|
de7d22cb9f | ||
|
|
0e684fb06f | ||
|
|
e0ed35adf1 | ||
|
|
aebe75cd04 | ||
|
|
0738005863 | ||
|
|
efd0097f76 | ||
|
|
a4e2d4bc25 | ||
|
|
0eacffa08d | ||
|
|
e5df6232cf | ||
|
|
61acf98909 | ||
|
|
77e0394195 | ||
|
|
1e0e581bd7 | ||
|
|
0ac68afa4c | ||
|
|
489c415fd9 | ||
|
|
18b065f77a | ||
|
|
6f8a8ffd87 | ||
|
|
6a1ee8377d | ||
|
|
a464febdcb | ||
|
|
8ae2eaaad3 | ||
|
|
23f2934e1f | ||
|
|
69df921856 | ||
|
|
f73689cfd8 | ||
|
|
1a15347b02 | ||
|
|
a4bf11dbc1 | ||
|
|
182771192b | ||
|
|
0a53c1ff0a | ||
|
|
ca05bd7224 | ||
|
|
aa977482db | ||
|
|
e67eff17ad | ||
|
|
6cceec9c26 | ||
|
|
be0c88c801 | ||
|
|
ba0a38d403 | ||
|
|
01b24ff2ae | ||
|
|
f39c4b7c4b | ||
|
|
83f941b77e | ||
|
|
e8253a06b7 | ||
|
|
18ba57a7b0 | ||
|
|
07b9e1605d | ||
|
|
4d22171d16 | ||
|
|
a1ae532c6e | ||
|
|
2a7db16a77 | ||
|
|
549e0349d7 | ||
|
|
16d4294f08 | ||
|
|
c733fb2bf9 | ||
|
|
4c965c73b2 | ||
|
|
3342d129c2 | ||
|
|
32646c6256 | ||
|
|
0a0ab65908 | ||
|
|
feb35ddd10 | ||
|
|
75f41088b1 | ||
|
|
b662a7da95 | ||
|
|
146205c83b | ||
|
|
705f5bcc89 | ||
|
|
8285a98f80 | ||
|
|
2fb935e726 | ||
|
|
7be3bf1dbf | ||
|
|
2a5359051e | ||
|
|
a3ac09f8a3 | ||
|
|
846b3e1b4c | ||
|
|
0c207981e9 | ||
|
|
e7456d3391 | ||
|
|
5bca1f6cec | ||
|
|
86b9d069b1 | ||
|
|
f6ee90e169 | ||
|
|
9a286315a3 | ||
|
|
5d0801854c | ||
|
|
343473bc01 | ||
|
|
2593b36072 | ||
|
|
34af2f1820 | ||
|
|
be7bdf8304 | ||
|
|
918f18c200 | ||
|
|
1db5473f50 | ||
|
|
de638603cd | ||
|
|
20ec690e22 | ||
|
|
9a5b6f4d92 | ||
|
|
398726a14a | ||
|
|
c8be24f833 | ||
|
|
f8ff63af0e | ||
|
|
34e78bdbd5 | ||
|
|
fb4e52dd09 | ||
|
|
e58ff4465a | ||
|
|
b1dec36e1c | ||
|
|
4aaf7dbdc6 | ||
|
|
95c0aafb68 | ||
|
|
5062e008c6 | ||
|
|
55badc1e08 | ||
|
|
bdc621fb36 | ||
|
|
0ec5c05de8 | ||
|
|
d10634c7d6 | ||
|
|
a7bad217eb | ||
|
|
f2c13c7d29 | ||
|
|
3444ff4e29 | ||
|
|
26f4da7ba5 | ||
|
|
4c6b4f5d4d | ||
|
|
70797753df | ||
|
|
ec3216072d | ||
|
|
810c8d4d72 | ||
|
|
ce688fc0bf | ||
|
|
c131896432 | ||
|
|
42e6eec9e9 | ||
|
|
fe00fe6a25 | ||
|
|
c97b7c841f | ||
|
|
2d0387fe63 | ||
|
|
71d3047ef0 | ||
|
|
d86cc38b2a | ||
|
|
21b2efd268 | ||
|
|
badd522d60 | ||
|
|
ecd3f600d9 | ||
|
|
099df17e77 | ||
|
|
c88e42eba2 | ||
|
|
89058ebd49 | ||
|
|
d8204ab7ed | ||
|
|
e2ea1af4c8 | ||
|
|
08780475d0 | ||
|
|
6eb2742e7d | ||
|
|
c1b7e12b0b | ||
|
|
53d44ff42a | ||
|
|
6331dea8b0 | ||
|
|
240beec7de | ||
|
|
7de167b21b | ||
|
|
49af014a84 | ||
|
|
73cf1c6ff9 | ||
|
|
f8b1e14b74 | ||
|
|
265e6f9a15 | ||
|
|
40e995da88 | ||
|
|
6e4fb7fd4b | ||
|
|
0695ad7ae6 | ||
|
|
eb6b07531a | ||
|
|
2d6846fe03 | ||
|
|
a5bfd40233 | ||
|
|
a40500eea9 | ||
|
|
f8212f102f | ||
|
|
59302b465d | ||
|
|
efafe44db1 | ||
|
|
6a2f81e873 | ||
|
|
3a43337735 | ||
|
|
b6df89d24c | ||
|
|
10d992a7e4 | ||
|
|
5c63618b30 | ||
|
|
7db0b78e88 | ||
|
|
979492449e | ||
|
|
6fbe239313 | ||
|
|
26527e7dae | ||
|
|
04a57e92c2 | ||
|
|
d59efa0b5c | ||
|
|
4216ced493 | ||
|
|
9f4f493486 | ||
|
|
63d86f1263 | ||
|
|
398a5806e7 | ||
|
|
1adc734801 | ||
|
|
0ae6dfd565 | ||
|
|
8531bac6cd | ||
|
|
ce13c00ebd | ||
|
|
2857c3b46b | ||
|
|
d944885ce9 | ||
|
|
62d1535e76 | ||
|
|
46556d308a | ||
|
|
fc5481dbe4 | ||
|
|
01e60a670c | ||
|
|
c4010854a5 | ||
|
|
fb1311cdae | ||
|
|
4aa76ce673 | ||
|
|
fb82bd7bca | ||
|
|
b7df5cbe4e | ||
|
|
fe9804b32c | ||
|
|
8446ab1c71 | ||
|
|
b5054b08d3 | ||
|
|
df32a1542b | ||
|
|
e82602db77 | ||
|
|
2d6105c778 | ||
|
|
d89940e85b | ||
|
|
60fceee204 | ||
|
|
13f7dab5f0 | ||
|
|
f7413cc711 | ||
|
|
b053f14d58 | ||
|
|
56179d712e | ||
|
|
a06bf6778b | ||
|
|
1506141155 | ||
|
|
ae69cd50b1 | ||
|
|
0c23d209a0 | ||
|
|
eac5763e03 | ||
|
|
6530eeab6d | ||
|
|
5eb8f2f8a7 | ||
|
|
f9b140add9 | ||
|
|
d4db96f709 | ||
|
|
5f08573db8 | ||
|
|
da83fcb78d | ||
|
|
f04bdd1f14 | ||
|
|
bb6a6063e8 | ||
|
|
bf813d910b | ||
|
|
374aa77f27 |
Executable
+23
@@ -0,0 +1,23 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
#
|
||||||
|
# Pre-commit hook installed by huskies.
|
||||||
|
# Runs script/check (fmt-check, clippy, cargo check, source-map-check)
|
||||||
|
# before every commit. Aborts if any gate fails.
|
||||||
|
#
|
||||||
|
# Emergency bypass: git commit --no-verify (see AGENT.md — avoid this)
|
||||||
|
|
||||||
|
REPO_ROOT="$(git rev-parse --show-toplevel)"
|
||||||
|
|
||||||
|
printf '[pre-commit] Running script/check ...\n'
|
||||||
|
OUTPUT=$("$REPO_ROOT/script/check" 2>&1)
|
||||||
|
STATUS=$?
|
||||||
|
|
||||||
|
if [ "$STATUS" -ne 0 ]; then
|
||||||
|
printf '\n=== PRE-COMMIT HOOK FAILED ===\n\n'
|
||||||
|
printf '%s\n' "$OUTPUT"
|
||||||
|
printf '\nFix the issues above, then re-validate with:\n'
|
||||||
|
printf ' script/check\n'
|
||||||
|
printf '\nEmergency bypass (see AGENT.md -- avoid this):\n'
|
||||||
|
printf ' git commit --no-verify\n\n'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# Gitea Actions workflows
|
||||||
|
|
||||||
|
## `release-artifact.yml`
|
||||||
|
|
||||||
|
Triggers on every push to `master`. Builds the `linux-arm64` sled binary and
|
||||||
|
publishes it to the "dev" release channel via `script/ci-publish-artifact`.
|
||||||
|
|
||||||
|
### Runner registration
|
||||||
|
|
||||||
|
The job targets the `arm64-mac` runner label. Register an `act_runner` on an
|
||||||
|
Apple Silicon macOS host that has `cargo`/`rustc` and `curl` on `PATH`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
act_runner register \
|
||||||
|
--instance https://code.crashlabs.io \
|
||||||
|
--token <runner-registration-token> \
|
||||||
|
--labels arm64-mac
|
||||||
|
act_runner daemon
|
||||||
|
```
|
||||||
|
|
||||||
|
The registration token comes from the repo's **Settings → Actions →
|
||||||
|
Runners → Create new Runner** page in Gitea. Without a runner carrying the
|
||||||
|
`arm64-mac` label, jobs from this workflow queue indefinitely.
|
||||||
|
|
||||||
|
### Secrets
|
||||||
|
|
||||||
|
Configure these under the repo's **Settings → Actions → Secrets**. Never
|
||||||
|
commit credentials — the workflow only ever references them via
|
||||||
|
`${{ secrets.* }}`.
|
||||||
|
|
||||||
|
| Secret | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `HUSKIES_CHANNEL_URL` | Base URL of the dev release channel host. |
|
||||||
|
| `HUSKIES_CHANNEL_TOKEN` | Bearer token authorised to publish artifacts to that channel. |
|
||||||
|
|
||||||
|
### Channel host contract
|
||||||
|
|
||||||
|
`script/ci-publish-artifact` expects the channel host at
|
||||||
|
`HUSKIES_CHANNEL_URL` to implement:
|
||||||
|
|
||||||
|
- `POST {HUSKIES_CHANNEL_URL}/<artifact-name>` — accepts the raw artifact
|
||||||
|
bytes as the request body. Requires `Authorization: Bearer <token>` and
|
||||||
|
`X-Git-Hash: <short-git-hash>` headers. Non-2xx responses in the 4xx range
|
||||||
|
(including 401/403) are treated as permanent failures; 5xx responses and
|
||||||
|
network errors are retried with backoff.
|
||||||
|
- `GET {HUSKIES_CHANNEL_URL}/manifest.json` — returns a JSON object with a
|
||||||
|
`git_hash` field reflecting the most recently published artifact.
|
||||||
|
Requires `Authorization: Bearer <token>`.
|
||||||
|
|
||||||
|
This is a separate, unsigned channel distinct from the Ed25519-signed
|
||||||
|
release channels the `pull <channel>` gateway command consumes (see
|
||||||
|
`server/src/service/gateway/release_manifest.rs`) — CI has no safe place to
|
||||||
|
hold a channel signing key, so the dev channel trusts the bearer token alone.
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
name: Publish sled artifact
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-dev-artifact:
|
||||||
|
# Registered on an arm64 macOS act_runner host — see
|
||||||
|
# .gitea/workflows/README.md for registration instructions.
|
||||||
|
runs-on: [arm64-mac]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Build linux-arm64 sled binary
|
||||||
|
env:
|
||||||
|
# Dedicated target dir so this CI build never clobbers a developer's
|
||||||
|
# incremental target/release build on the shared runner host, mirroring
|
||||||
|
# SLED_TARGET_DIR in server/src/chat/transport/matrix/release.rs.
|
||||||
|
CARGO_TARGET_DIR: target/ci-release
|
||||||
|
run: cargo build --release -p huskies
|
||||||
|
|
||||||
|
- name: Stage artifact
|
||||||
|
run: cp target/ci-release/release/huskies target/ci-release/release/huskies-linux-arm64
|
||||||
|
|
||||||
|
- name: Publish to dev channel
|
||||||
|
env:
|
||||||
|
HUSKIES_CHANNEL_URL: ${{ secrets.HUSKIES_CHANNEL_URL }}
|
||||||
|
HUSKIES_CHANNEL_TOKEN: ${{ secrets.HUSKIES_CHANNEL_TOKEN }}
|
||||||
|
run: script/ci-publish-artifact target/ci-release/release/huskies-linux-arm64 "$(git rev-parse --short HEAD)"
|
||||||
+2
-3
@@ -6,15 +6,14 @@
|
|||||||
# Local environment (secrets)
|
# Local environment (secrets)
|
||||||
.env
|
.env
|
||||||
|
|
||||||
# Local-only scripts
|
|
||||||
script/local-release
|
|
||||||
|
|
||||||
# App specific (root-level; huskies subdirectory patterns live in .huskies/.gitignore)
|
# App specific (root-level; huskies subdirectory patterns live in .huskies/.gitignore)
|
||||||
store.json
|
store.json
|
||||||
_merge_parsed.json
|
_merge_parsed.json
|
||||||
.huskies_port
|
.huskies_port
|
||||||
.huskies/bot.toml.bak
|
.huskies/bot.toml.bak
|
||||||
.huskies/build_hash
|
.huskies/build_hash
|
||||||
|
# Phantom 0-byte pipeline.db sometimes appears at repo root from old code; canonical DB lives at .huskies/pipeline.db
|
||||||
|
/pipeline.db
|
||||||
|
|
||||||
# Per-worktree planning file (written by coder agents, must never reach squash commits)
|
# Per-worktree planning file (written by coder agents, must never reach squash commits)
|
||||||
PLAN.md
|
PLAN.md
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ timers.json
|
|||||||
|
|
||||||
# Misc
|
# Misc
|
||||||
wishlist.md
|
wishlist.md
|
||||||
|
double_timmy_log.md
|
||||||
|
|
||||||
# Database
|
# Database
|
||||||
pipeline.db
|
pipeline.db
|
||||||
|
|||||||
+1
-1
@@ -56,7 +56,7 @@ There are no exceptions. The merge gate runs `source-map-check` and rejects the
|
|||||||
Before committing, run `cargo run -p source-map-gen --bin source-map-check -- --worktree . --base master` and address every missing-docs direction it prints. If you added a new module file (e.g. `foo.rs` or `foo/mod.rs`), the FIRST line of that file MUST be a `//! What this module is for` doc comment.
|
Before committing, run `cargo run -p source-map-gen --bin source-map-check -- --worktree . --base master` and address every missing-docs direction it prints. If you added a new module file (e.g. `foo.rs` or `foo/mod.rs`), the FIRST line of that file MUST be a `//! What this module is for` doc comment.
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
Docs live in `website/docs/*.html` (static HTML), **not** Markdown files. When a story asks you to document something, edit the relevant `.html` file in `website/docs/`.
|
Docs live in `website/app/docs/*.tsx` (Next.js pages), **not** Markdown files. When a story asks you to document something, edit the relevant `.tsx` file under `website/app/docs/`. Run `npm run build` in `website/` to verify your changes render correctly.
|
||||||
|
|
||||||
## Configuration files
|
## Configuration files
|
||||||
- Agent config: `.huskies/agents.toml` (preferred) or `[[agent]]` blocks in `.huskies/project.toml`
|
- Agent config: `.huskies/agents.toml` (preferred) or `[[agent]]` blocks in `.huskies/project.toml`
|
||||||
|
|||||||
+409
-58
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,306 @@
|
|||||||
|
# Chat-Driven Project Bootstrap
|
||||||
|
|
||||||
|
Design overview for going from "I want a new project" to a running,
|
||||||
|
container-isolated, editor-accessible huskies project in one chat command.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
A user can say to Timmy in chat:
|
||||||
|
|
||||||
|
```
|
||||||
|
new project myapp --stack rust
|
||||||
|
new project legacy-rails --git git@github.com:me/legacy-rails.git
|
||||||
|
```
|
||||||
|
|
||||||
|
and end up with:
|
||||||
|
|
||||||
|
1. A fresh docker container running the project's huskies node.
|
||||||
|
2. The project's source code bind-mounted from the host so the user can
|
||||||
|
edit it in any editor.
|
||||||
|
3. SSH into the container so editors can run LSPs, builds, and tests
|
||||||
|
inside the container — never on the host.
|
||||||
|
4. Optional git remote configured for push to GitHub or Gitea.
|
||||||
|
5. The new sled registered with the gateway, so Timmy can drive coders /
|
||||||
|
mergemaster / etc. on the project via existing chat commands.
|
||||||
|
|
||||||
|
Manual repo creation on GitHub/Gitea remains the user's job. Everything
|
||||||
|
downstream of that is orchestrated.
|
||||||
|
|
||||||
|
## Architecture at a Glance
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────────────┐
|
||||||
|
│ Browser / Matrix │───┐
|
||||||
|
└──────────────────────┘ │
|
||||||
|
▼
|
||||||
|
┌───────────────────────┐
|
||||||
|
│ Gateway (huskies-gw) │
|
||||||
|
│ • chat dispatcher │
|
||||||
|
│ • new-project │
|
||||||
|
│ • routing │
|
||||||
|
└─────────┬─────────────┘
|
||||||
|
│
|
||||||
|
┌─────────┴───────────────────────────────────┐
|
||||||
|
│ docker engine (host) │
|
||||||
|
│ ┌────────────┐ ┌────────────┐ ┌─────────┐ │
|
||||||
|
│ │ project-A │ │ project-B │ │ ... │ │
|
||||||
|
│ │ sled + │ │ sled + │ │ │ │
|
||||||
|
│ │ sshd + │ │ sshd + │ │ │ │
|
||||||
|
│ │ LSPs │ │ LSPs │ │ │ │
|
||||||
|
│ └─────┬──────┘ └─────┬──────┘ └─────────┘ │
|
||||||
|
└────────┼──────────────┼─────────────────────┘
|
||||||
|
│ │
|
||||||
|
bind mount │ │ bind mount
|
||||||
|
┌────────┴───┐ ┌─────┴──────┐
|
||||||
|
│ ~/code/A │ │ ~/code/B │ ◄── host
|
||||||
|
└────────────┘ └────────────┘ editor opens
|
||||||
|
these paths
|
||||||
|
```
|
||||||
|
|
||||||
|
- One container per project. The container runs the project's huskies
|
||||||
|
binary (sled), an SSH server, and the stack-appropriate LSP(s).
|
||||||
|
- Source lives on the host (e.g. `~/code/<project>`), bind-mounted into
|
||||||
|
the container at a known path. Host can git-diff, back up, or edit.
|
||||||
|
- The gateway is editor-agnostic and project-agnostic — it talks to each
|
||||||
|
sled via the existing rendezvous / CRDT-sync protocol.
|
||||||
|
|
||||||
|
## Three Personas
|
||||||
|
|
||||||
|
| Persona | What they do | What they need |
|
||||||
|
|---------|--------------|----------------|
|
||||||
|
| Chat-only user | Drives everything via Matrix/web chat | Installed huskies binary; chat client |
|
||||||
|
| Editor-using technical user | Same + edits source in their editor | SSH config to the container + editor-specific remote-dev setup |
|
||||||
|
| Multi-project user | Several projects running in parallel | Gateway-listed projects, all routable from one chat |
|
||||||
|
|
||||||
|
Chat-only users never touch SSH. Editor users go through a one-time
|
||||||
|
"copy this SSH command into your editor's remote settings" handoff at
|
||||||
|
project creation time.
|
||||||
|
|
||||||
|
## The Bootstrap Chat Command
|
||||||
|
|
||||||
|
```
|
||||||
|
new project <name> [--stack <stack>] [--git <url>] [--path <host-path>]
|
||||||
|
```
|
||||||
|
|
||||||
|
Flow:
|
||||||
|
|
||||||
|
1. **Validate**: name unique among existing projects; host path doesn't already
|
||||||
|
exist; stack (if declared) is one of the supported overlays.
|
||||||
|
2. **Allocate** a fresh per-project port range (gateway picks).
|
||||||
|
3. **Create host directory** at `--path` (default `~/huskies/<name>/`).
|
||||||
|
4. If `--git` provided, `git clone` into that directory; else `git init`.
|
||||||
|
5. **Detect stack** from cloned content if not declared:
|
||||||
|
- `Cargo.toml` → `rust`
|
||||||
|
- `package.json` → `node`
|
||||||
|
- `go.mod` → `go`
|
||||||
|
- `pyproject.toml` / `requirements.txt` / `setup.py` → `python`
|
||||||
|
- `Gemfile` → `ruby`
|
||||||
|
- `pom.xml` / `build.gradle` → `jvm`
|
||||||
|
- Multiple → pick the dominant, warn.
|
||||||
|
- None → minimal base image, user can install tooling later.
|
||||||
|
6. **Compose the container** from `huskies-project-base` + the stack
|
||||||
|
overlay (Dockerfile fragments under `docker/stacks/<stack>/`).
|
||||||
|
7. **Launch** the container with bind mount + port forwards + an
|
||||||
|
auto-generated SSH key.
|
||||||
|
8. **Seed `.huskies/project.toml`** with sensible defaults.
|
||||||
|
9. **Register** the project with the gateway (`gateway_projects` LWW-map).
|
||||||
|
10. **Reply in chat** with: project name, host path, SSH command, and
|
||||||
|
a `huskies status <name>` invocation to verify.
|
||||||
|
|
||||||
|
## Container Template
|
||||||
|
|
||||||
|
Layered:
|
||||||
|
|
||||||
|
- **`huskies-project-base`**: debian-slim + git + huskies binary + sshd
|
||||||
|
+ sudo + a `huskies` user with the SSH pubkey installed.
|
||||||
|
- **`huskies-project-<stack>`**: per-stack additions, pre-built by
|
||||||
|
`script/build-project-images`. E.g. rust gets `rustup` +
|
||||||
|
`rust-analyzer` + `cargo-nextest`; node gets `node@22` +
|
||||||
|
`typescript-language-server`; etc. Stack fragments live in
|
||||||
|
`docker/stacks/<stack>/Dockerfile.fragment`.
|
||||||
|
- **`huskies-project-local-<name>`** *(optional)*: built on the fly at
|
||||||
|
container launch time when the project contains
|
||||||
|
`.huskies/Dockerfile.fragment`. This file is appended after the
|
||||||
|
stack overlay (`FROM huskies-project-<stack>`) so agents can extend
|
||||||
|
their own image without editing shared stack files. Because the
|
||||||
|
fragment lives inside the bind-mounted `/workspace/.huskies/`, changes
|
||||||
|
survive container recreation and are committed alongside the project
|
||||||
|
source. The `project-rebuild` command picks up the fragment
|
||||||
|
automatically when rebuilding.
|
||||||
|
|
||||||
|
Example `.huskies/Dockerfile.fragment` that adds `jq`:
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
RUN apt-get update && apt-get install -y jq
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Project layer**: the bind-mounted `/workspace` is the project source,
|
||||||
|
written by the host's editor, read by the in-container tooling.
|
||||||
|
|
||||||
|
The container's SSH server is bound to a host-local port (not exposed
|
||||||
|
externally). Auth is the per-project keypair generated at bootstrap;
|
||||||
|
the public key sits inside the container, the private key on host.
|
||||||
|
|
||||||
|
## Build Sandbox Model
|
||||||
|
|
||||||
|
The threat: editing code in a host-side editor causes the editor (or its
|
||||||
|
LSP plugin) to run `cargo check` / `npm install` / `pip install` /
|
||||||
|
similar, which executes arbitrary code from project dependencies —
|
||||||
|
`build.rs`, proc-macros, npm `postinstall`, Python `setup.py`, Ruby
|
||||||
|
native-extension build scripts, etc. A malicious dependency compromises
|
||||||
|
the host.
|
||||||
|
|
||||||
|
The mitigation: all build / type-check / dependency-install commands
|
||||||
|
execute **inside the project container**. The host's editor connects to
|
||||||
|
the container over SSH; rust-analyzer (or equivalent) runs inside the
|
||||||
|
container; the host process never `exec`s untrusted build scripts.
|
||||||
|
|
||||||
|
Container isolation is the docker default plus:
|
||||||
|
- No `--privileged`.
|
||||||
|
- No host bind mounts beyond the project source and the SSH key.
|
||||||
|
- No host network beyond the gateway's CRDT sync port.
|
||||||
|
- `--cap-drop=ALL` plus the minimum caps needed (probably none).
|
||||||
|
|
||||||
|
This isn't a hardened sandbox in the gvisor / Firecracker sense — a
|
||||||
|
docker-escape exploit on a compromised container still escalates to
|
||||||
|
host. For most consumer threat models (malicious crate from
|
||||||
|
crates.io / npm), docker's default isolation is sufficient. Tighter
|
||||||
|
sandboxing (gvisor) is a separate future spike if needed.
|
||||||
|
|
||||||
|
## Editor Connection — Editor-Agnostic SSH
|
||||||
|
|
||||||
|
| Editor | Connection mechanism |
|
||||||
|
|--------|----------------------|
|
||||||
|
| VSCode | Remote-SSH extension |
|
||||||
|
| JetBrains (IntelliJ/Rover) | JetBrains Gateway (SSH) |
|
||||||
|
| Zed | Built-in SSH remoting (mac/linux only today) |
|
||||||
|
| Vim/Neovim | SSH terminal session, or local nvim + LSP-over-SSH |
|
||||||
|
| Emacs | TRAMP + remote LSP via lsp-mode |
|
||||||
|
|
||||||
|
All converge on: `ssh huskies@127.0.0.1 -p <project-port> -i ~/.huskies/<name>/id_ed25519`.
|
||||||
|
That string is emitted in the bootstrap chat reply.
|
||||||
|
|
||||||
|
## Git Integration
|
||||||
|
|
||||||
|
- Initial setup is `git init` or `git clone` inside the container.
|
||||||
|
- For push: user's existing GitHub / Gitea SSH key is bind-mounted
|
||||||
|
read-only into the container at `~/.ssh/id_*`, OR the user supplies a
|
||||||
|
push token via `huskies secrets set GIT_TOKEN=...` (stored as a Fly
|
||||||
|
secret equivalent — for now, a chmod 600 file in the container).
|
||||||
|
- The container's `git` config gets `user.name` / `user.email` from the
|
||||||
|
gateway-level user identity.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
| Decision | Choice | Alternative |
|
||||||
|
|----------|--------|-------------|
|
||||||
|
| Container per project | One container per project | One container many projects: simpler but breaks isolation, breaks per-project deps |
|
||||||
|
| Editor model | SSH-remote (any editor) | VSCode Dev Containers only: simpler config but locks out everyone else |
|
||||||
|
| Source location | Bind mount from host | Inside container only: breaks "I can also edit on my laptop" requirement |
|
||||||
|
| Stack detection | Auto from project files, override with `--stack` | Always declared: more friction at bootstrap |
|
||||||
|
| Push secrets | Bind-mounted host SSH key OR per-project token | Gateway holds tokens: bigger blast radius |
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
1. **Per-project resource limits.** Should each container have a hard
|
||||||
|
CPU / RAM cap so a runaway agent doesn't starve the host?
|
||||||
|
2. **Lifecycle / cleanup.** If the user deletes a project from chat,
|
||||||
|
what gets removed? Container yes; host source no (data loss); git
|
||||||
|
remotes yes? Need a confirm step.
|
||||||
|
3. **Multi-tenant.** Out of scope for this design (that's huskies.dev
|
||||||
|
territory). This doc assumes single-user local-only.
|
||||||
|
4. **Windows specifics.** Bind mounts work but line-ending /
|
||||||
|
permission edge cases. Probably document "use WSL2 for best
|
||||||
|
experience" rather than fight Windows native paths.
|
||||||
|
5. **Gateway-on-host vs gateway-in-container.** The gateway today runs
|
||||||
|
in its own container. New per-project containers connect via docker
|
||||||
|
network. Need to confirm the network plumbing works for arbitrary
|
||||||
|
per-project containers, not just the manually-configured ones.
|
||||||
|
|
||||||
|
## Phasing
|
||||||
|
|
||||||
|
The work breaks naturally into:
|
||||||
|
|
||||||
|
- **Phase 0 (now):** this design doc.
|
||||||
|
- **Phase 1:** chat command exists and provisions a bare project
|
||||||
|
container (no stack overlay, no SSH, no git clone — just
|
||||||
|
"start a container, register with gateway"). Validates the
|
||||||
|
orchestration shell.
|
||||||
|
- **Phase 2:** stack-aware container template — base image + overlays;
|
||||||
|
detection from project files.
|
||||||
|
- **Phase 3:** SSH-remote editor access — sshd in the container,
|
||||||
|
per-project keypair, chat-reply emits the connection string.
|
||||||
|
- **Phase 4:** git integration — `--git <url>` clones, host SSH key
|
||||||
|
mount, push verification.
|
||||||
|
- **Phase 5:** per-project resource limits + cleanup chat commands.
|
||||||
|
- **Phase 6:** `--adopt <dir>` wraps a container around an existing
|
||||||
|
checkout. No clone or init — bind-mount only.
|
||||||
|
- **Phase 7 (story 1137):** First-run init flow — config summary and
|
||||||
|
chat-driven overrides (see below).
|
||||||
|
|
||||||
|
Each phase ships independently and is usable on its own. Phase 1 alone
|
||||||
|
gives chat-only users a working project; later phases add the editor
|
||||||
|
and git polish.
|
||||||
|
|
||||||
|
## First-Run Init Flow (Story 1137)
|
||||||
|
|
||||||
|
After a successful `new project ... --adopt` (or any new-project
|
||||||
|
bootstrap), the bot appends a **Default configuration** block to the
|
||||||
|
adoption success reply. This block lists every scaffolded agent with
|
||||||
|
its model, budget cap, and turn limit, and provides ready-to-send
|
||||||
|
override commands.
|
||||||
|
|
||||||
|
### Example reply tail
|
||||||
|
|
||||||
|
```
|
||||||
|
**Default configuration** (3 agents):
|
||||||
|
- coder-1 (coder): model=`sonnet`, budget=$5.00, max_turns=50
|
||||||
|
- qa (qa): model=`sonnet`, budget=$4.00, max_turns=40
|
||||||
|
- mergemaster (mergemaster): model=`sonnet`, budget=$5.00, max_turns=30
|
||||||
|
|
||||||
|
Override via chat: `huskies config myapp coder.model=opus`
|
||||||
|
Project settings: `huskies config myapp default_qa=human`
|
||||||
|
Accept all defaults silently: add `--skip-config` to the bootstrap command.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Config override command
|
||||||
|
|
||||||
|
```
|
||||||
|
huskies config <project> <key>=<value>
|
||||||
|
```
|
||||||
|
|
||||||
|
The gateway resolves the project's `host_path` from `projects.toml`,
|
||||||
|
then writes the setting to `.huskies/agents.toml` or
|
||||||
|
`.huskies/project.toml` on the host.
|
||||||
|
|
||||||
|
**Agent fields** (`<stage_or_name>.<field>=<value>`):
|
||||||
|
|
||||||
|
| Key | Target | Supported values |
|
||||||
|
|-----|--------|-----------------|
|
||||||
|
| `coder.model` | agents.toml, coder stage | `sonnet`, `opus`, any model string |
|
||||||
|
| `qa.model` | agents.toml, qa stage | same |
|
||||||
|
| `mergemaster.model` | agents.toml, mergemaster stage | same |
|
||||||
|
| `coder.max_turns` | agents.toml, coder stage | integer |
|
||||||
|
| `coder.max_budget` | agents.toml, coder stage | decimal (USD) |
|
||||||
|
|
||||||
|
**Project keys** (bare `<key>=<value>`):
|
||||||
|
|
||||||
|
| Key | Notes |
|
||||||
|
|-----|-------|
|
||||||
|
| `default_qa` | `"server"`, `"agent"`, or `"human"` |
|
||||||
|
| `max_retries` | integer |
|
||||||
|
| `max_coders` | integer |
|
||||||
|
| `base_branch` | branch name string |
|
||||||
|
| `timezone` | IANA timezone (e.g. `"Europe/London"`) |
|
||||||
|
| `default_coder_model` | model string |
|
||||||
|
|
||||||
|
### Skip path
|
||||||
|
|
||||||
|
Pass `--skip-config` to suppress the config block entirely:
|
||||||
|
|
||||||
|
```
|
||||||
|
new project myapp --adopt /path/to/checkout --skip-config
|
||||||
|
```
|
||||||
|
|
||||||
|
The success reply is identical to pre-1137 output — only the SSH
|
||||||
|
command and registration summary, no agent listing.
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# CRDT Snapshot Compaction
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The huskies project CRDT has grown to 55K ops / 276MB in `pipeline.db`.
|
||||||
|
Every container restart replays all operations in a tight synchronous loop
|
||||||
|
on the tokio runtime (`crdt_state/state/init.rs:68-78`), taking 27+ minutes
|
||||||
|
and freezing the runtime so the HTTP server never becomes ready.
|
||||||
|
|
||||||
|
### Op bloat
|
||||||
|
|
||||||
|
55K ops across only 4,154 sequence numbers (~13 ops per seq on average).
|
||||||
|
Many zeroed-out `MergeJobCrdt` entries appear to be tombstones never cleaned
|
||||||
|
up. Some individual ops are up to 523KB. Average op size is 4.6KB.
|
||||||
|
|
||||||
|
## Proposed Fix
|
||||||
|
|
||||||
|
### 1. Snapshot (checkpoint)
|
||||||
|
|
||||||
|
After replaying all ops, serialize the materialized CRDT state to a
|
||||||
|
checkpoint blob (e.g. a `crdt_snapshot` table or a separate file). On next
|
||||||
|
startup, load the snapshot and only replay ops with `rowid > snapshot_rowid`.
|
||||||
|
|
||||||
|
At snapshot time, back up the database file so corruption is recoverable.
|
||||||
|
|
||||||
|
### 2. Op pruning / compaction
|
||||||
|
|
||||||
|
Delete ops that are superseded by the snapshot. Tombstoned/deleted items with
|
||||||
|
all-zero fields contribute nothing to materialized state and can be dropped
|
||||||
|
from the log once snapshotted.
|
||||||
|
|
||||||
|
### 3. Immediate fix: spawn_blocking
|
||||||
|
|
||||||
|
Move the replay loop to `tokio::task::spawn_blocking` so the HTTP server and
|
||||||
|
liveness ticks are not starved during replay. This does not reduce replay
|
||||||
|
time but prevents the runtime freeze.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- Startup loads a snapshot when available and only replays ops newer than the snapshot sequence
|
||||||
|
- A snapshot is written after full CRDT replay completes (or periodically in background)
|
||||||
|
- DB backup is created at each snapshot time
|
||||||
|
- Startup time for 55K ops drops from 27+ minutes to under 30 seconds
|
||||||
|
- Dead/zeroed CRDT ops are pruned during compaction
|
||||||
|
- CRDT sync protocol continues to work correctly across nodes after compaction
|
||||||
|
- The replay loop runs in spawn_blocking so it does not freeze the tokio runtime
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
# Story 1208: Cross-Project MCP for Ops/LLM Sessions
|
||||||
|
|
||||||
|
## 1. Problem Statement
|
||||||
|
|
||||||
|
An ops/LLM session connects to a `huskies --gateway` instance's `/mcp`
|
||||||
|
endpoint. Before this story, the *only* way to act on a specific registered
|
||||||
|
project was:
|
||||||
|
|
||||||
|
1. Call `switch_project` (mutates the gateway's shared, global
|
||||||
|
`GatewayState.active_project`), then
|
||||||
|
2. Call the ordinary project-level tool (`create_story`, `get_story_todos`,
|
||||||
|
`show`, …), which the gateway silently proxies to whichever project is
|
||||||
|
currently active.
|
||||||
|
|
||||||
|
This has two problems:
|
||||||
|
- **Race condition**: `active_project` is one value shared by every
|
||||||
|
connected client. Two concurrent ops sessions targeting different projects
|
||||||
|
will step on each other's `switch_project` calls.
|
||||||
|
- **No true "read a named project once" path**: for a single lookup against
|
||||||
|
a project that isn't the current default, a caller had to mutate shared
|
||||||
|
state just to read something, then (optionally) switch back.
|
||||||
|
|
||||||
|
The practical consequence (and the reason this story exists) is that
|
||||||
|
operators and LLM agents fall back to hand-crafting raw JSON-RPC requests
|
||||||
|
directly against a project's own container port, bypassing the gateway
|
||||||
|
entirely — the "shell-fallback" this story is named for.
|
||||||
|
|
||||||
|
## 2. Chosen Mechanism: Per-Call `project` Argument
|
||||||
|
|
||||||
|
Any `tools/call` request for a non-gateway (proxied) tool may now include an
|
||||||
|
optional top-level `project` field inside `arguments`:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"jsonrpc": "2.0",
|
||||||
|
"id": 1,
|
||||||
|
"method": "tools/call",
|
||||||
|
"params": {
|
||||||
|
"name": "create_story",
|
||||||
|
"arguments": {
|
||||||
|
"name": "Fix login bug",
|
||||||
|
"acceptance_criteria": ["..."],
|
||||||
|
"origin": "...",
|
||||||
|
"project": "robot-studio"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- If `project` is present and non-empty, the gateway looks it up in
|
||||||
|
`projects.toml` (`GatewayState.projects`) and proxies the call directly to
|
||||||
|
that project's live sled-uplink WebSocket connection —
|
||||||
|
`GatewayState::proxy_mcp_for_project` in
|
||||||
|
`server/src/service/gateway/mod.rs`. `GatewayState.active_project` is
|
||||||
|
**not read or mutated** by this path.
|
||||||
|
- If `project` is absent (the common case, and all pre-existing behavior),
|
||||||
|
the call proxies to whichever project is currently active, exactly as
|
||||||
|
before — full backward compatibility with existing sessions and
|
||||||
|
`switch_project`-based workflows.
|
||||||
|
- An unknown project name returns a JSON-RPC `-32602` (invalid params)
|
||||||
|
error listing the registered project names. A known project with no live
|
||||||
|
WS-uplink connection returns `-32603` naming the sled, matching the
|
||||||
|
existing `active_project` proxy error shape.
|
||||||
|
|
||||||
|
Implementation: `server/src/http/gateway/mcp.rs`
|
||||||
|
(`gateway_mcp_post_handler`'s `tools/call` branch,
|
||||||
|
`proxy_and_respond_for_project`) and
|
||||||
|
`server/src/service/gateway/mod.rs` (`GatewayState::sled_connection_for`,
|
||||||
|
`GatewayState::proxy_mcp_for_project`, generalized from the existing
|
||||||
|
`active_sled_connection` / `proxy_active_mcp`).
|
||||||
|
|
||||||
|
### Schema discoverability
|
||||||
|
|
||||||
|
`tools/list` merges gateway tools with the active project's own tool list.
|
||||||
|
Every merged (proxied) tool's `inputSchema.properties` gets a `project`
|
||||||
|
property injected (`inject_project_arg_schema` in `http/gateway/mcp.rs`) so
|
||||||
|
MCP clients that validate call arguments against the declared schema before
|
||||||
|
sending don't strip or reject the extra field. This is additive only — no
|
||||||
|
existing property, and no `required` list, is touched.
|
||||||
|
|
||||||
|
### Why not mirror every tool at the gateway level?
|
||||||
|
|
||||||
|
Rejected alternative: define a `project_create_story`, `project_show`, etc.
|
||||||
|
for every project-level tool at the gateway. This was rejected because it
|
||||||
|
duplicates ~15+ tool schemas and dispatch arms and drifts out of sync every
|
||||||
|
time a project-level tool's schema changes. A single per-call argument that
|
||||||
|
every proxied tool call can carry scales to new project-level tools for
|
||||||
|
free.
|
||||||
|
|
||||||
|
## 3. Fleet-Wide Reads (AC 2)
|
||||||
|
|
||||||
|
These already existed as gateway-level tools before this story and needed
|
||||||
|
no code change — listed here for completeness of the "how an ops session
|
||||||
|
connects" picture:
|
||||||
|
|
||||||
|
| Tool | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `list_projects` | Every registered project: name, url, ssh_port, host_path, adopted/built-in marker, active marker. No liveness check. |
|
||||||
|
| `gateway_health` | Per-project health (WS heartbeat or HTTP poll) plus CRDT event-relay staleness. |
|
||||||
|
| `aggregate_pipeline_status` | Pipeline stage counts and blocked/failing items across every registered project, fetched in parallel. |
|
||||||
|
| `fleet_identity` | (Story 1206) Per-sled identity pin vs. live signed identity, and TOFU re-pin. |
|
||||||
|
|
||||||
|
## 4. How an Ops Session Should Connect
|
||||||
|
|
||||||
|
1. Point the MCP client at the gateway's `/mcp` endpoint
|
||||||
|
(`http://<gateway-host>:<port>/mcp`), the same endpoint local agents use
|
||||||
|
— there is no separate "ops" endpoint.
|
||||||
|
2. Call `tools/list` to see the merged tool surface (gateway tools + the
|
||||||
|
active project's tools, each carrying the optional `project` schema
|
||||||
|
property).
|
||||||
|
3. For a one-off call against a specific project, pass `project: "<name>"`
|
||||||
|
inside `arguments` on that call — no `switch_project` required, and no
|
||||||
|
risk of racing another session's active-project selection.
|
||||||
|
4. For fleet-wide questions (is anything down, what's blocked everywhere),
|
||||||
|
use `list_projects`, `gateway_health`, or `aggregate_pipeline_status`
|
||||||
|
directly; they already scan every registered project.
|
||||||
|
5. `switch_project` remains available for sessions that want a persistent
|
||||||
|
default (e.g. an interactive chat session working one project at a
|
||||||
|
time) — it is unaffected by this change.
|
||||||
|
|
||||||
|
## 5. Design Review Note (AC 4)
|
||||||
|
|
||||||
|
This document captures the chosen approach (per-call `project` argument,
|
||||||
|
generalized proxy functions, additive schema injection) as required by AC 4.
|
||||||
|
No new gateway-level tool surface was added for AC 1 — the existing proxy
|
||||||
|
path was extended instead, minimizing new schema/dispatch surface area and
|
||||||
|
keeping every future project-level tool automatically cross-project-capable.
|
||||||
@@ -0,0 +1,245 @@
|
|||||||
|
# LLM Context From Events
|
||||||
|
|
||||||
|
Design overview for making any LLM-driven chat persona (Timmy at the
|
||||||
|
gateway, Sally at a single sled, future personas) aware of huskies
|
||||||
|
events without the user having to re-narrate them.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
**Update the LLM's context non-intrusively when a state transition
|
||||||
|
happens.** No new LLM turn is fired; events are simply visible to the
|
||||||
|
LLM the next time the user (or anything else) does cause it to run.
|
||||||
|
The LLM should never need to be told what already happened inside
|
||||||
|
huskies.
|
||||||
|
|
||||||
|
## Guiding Principle
|
||||||
|
|
||||||
|
**Transports have nothing to do with LLMs.** A transport (Matrix bot,
|
||||||
|
web UI, CLI, future TUIs) is a pure courier — it relays user text in,
|
||||||
|
LLM text out, and never owns LLM-facing state. Anything the LLM needs
|
||||||
|
to know lives in huskies, behind a single `assemble_prompt_context`
|
||||||
|
helper that the transport calls. Adding a new transport must require
|
||||||
|
zero changes to the event-awareness path.
|
||||||
|
|
||||||
|
## Three things this doc is NOT
|
||||||
|
|
||||||
|
1. **Triggers** — `on StoryMerged{1122} do Rebuild`. These are
|
||||||
|
deterministic subscribers; they should never invoke the LLM. Covered
|
||||||
|
in a separate design.
|
||||||
|
2. **Proactive wake** — running an LLM turn *because* an event fired,
|
||||||
|
without the user typing. Costs tokens, risks ramble. Explicitly out
|
||||||
|
of scope here; a separate decision to make later.
|
||||||
|
3. **A transport feature** — this design assumes any transport that
|
||||||
|
invokes the LLM uses the same context-assembly helper. Matrix bot,
|
||||||
|
web UI, CLI all funnel through it.
|
||||||
|
|
||||||
|
## Why Past Attempts Have Failed
|
||||||
|
|
||||||
|
- **Buffer lived on the transport**, not on huskies. The current
|
||||||
|
`BotContext.pending_pipeline_events` (`server/src/chat/transport/matrix/bot/context.rs:103-116`)
|
||||||
|
is Matrix-only; web UI users see nothing of the kind, and the buffer
|
||||||
|
dies with the bot process.
|
||||||
|
- **Process-local, RAM-only**. Server rebuild → buffer empty. Any
|
||||||
|
events between the old binary's last user turn and the new binary's
|
||||||
|
first are silently lost.
|
||||||
|
- **Unbounded `mpsc` channels drop under lag.** The server logs
|
||||||
|
routinely show `[xxx-sub] Subscriber lagged, skipped N event(s)`.
|
||||||
|
When the subscriber feeding the buffer falls behind, events vanish
|
||||||
|
without being recorded.
|
||||||
|
- **No end-to-end test.** Nothing asserts "fire event E, send user
|
||||||
|
message M, the LLM's prompt contains E."
|
||||||
|
- **No cross-process aggregation.** Events in a sled have no path to
|
||||||
|
the gateway-side LLM context without bespoke plumbing per event type.
|
||||||
|
|
||||||
|
## Architecture at a Glance
|
||||||
|
|
||||||
|
```
|
||||||
|
┌────────────┐ ┌────────────┐ ┌────────────┐
|
||||||
|
│ Sled A │ │ Sled B │ │ Sled C │
|
||||||
|
│ event_log/ │ │ event_log/ │ │ event_log/ │ ◄── source of truth
|
||||||
|
└─────┬──────┘ └─────┬──────┘ └─────┬──────┘ (CRDT-backed)
|
||||||
|
│ │ │
|
||||||
|
└───────────────┼───────────────┘
|
||||||
|
▼
|
||||||
|
┌────────────────────┐
|
||||||
|
│ Gateway aggregator │ ◄── tail-merges all sled logs
|
||||||
|
│ event_view/ │ into a single ordered stream
|
||||||
|
└─────────┬──────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌────────────────────────┐
|
||||||
|
│ Per-LLM-session state │ ◄── scope filter +
|
||||||
|
│ sessions/<id>/ │ high-water mark per stream
|
||||||
|
└─────────┬──────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌────────────────────────┐
|
||||||
|
│ assemble_prompt_context│ ◄── single helper used by
|
||||||
|
│ (session_id) -> Str │ every transport before
|
||||||
|
└─────────┬──────────────┘ each LLM turn
|
||||||
|
│
|
||||||
|
┌──────────────┼──────────────┐
|
||||||
|
▼ ▼ ▼
|
||||||
|
Matrix bot Web UI CLI / TUI
|
||||||
|
```
|
||||||
|
|
||||||
|
## Event Model — Reuse What Already Exists
|
||||||
|
|
||||||
|
There is no need to invent a parallel event taxonomy. Huskies already
|
||||||
|
has a complete typed enum and a single broadcast bus:
|
||||||
|
|
||||||
|
- `server/src/pipeline_state/transition.rs` defines `PipelineEvent`
|
||||||
|
with **30 variants** covering every state-machine transition
|
||||||
|
(`DepsMet`, `GatesStarted/Passed/Failed`, `QaSkipped`,
|
||||||
|
`MergeSucceeded/Failed/FailedFinal`, `Accepted`, `Block/Unblock`,
|
||||||
|
`Abandon`, `Supersede`, `ReviewHold/Cleared`, `Reject`, `Triage`,
|
||||||
|
`Close`, `Demote`, `Freeze/Unfreeze`, `MergemasterAttempted`,
|
||||||
|
`FixupRequested`, `ReQueuedForQa`, `MergeAborted`,
|
||||||
|
`HotfixRequested`, `MergeRetryStarted`).
|
||||||
|
- The same module defines `ExecutionEvent` with 7 variants for agent
|
||||||
|
lifecycle (`SpawnRequested`, `SpawnedSuccessfully`, `Heartbeat`,
|
||||||
|
`HitRateLimit`, `Exited`, `Stopped`, `Reset`).
|
||||||
|
- Every transition fires a `TransitionFired` event on a single internal
|
||||||
|
bus. Ten subscribers already consume it (audit-log,
|
||||||
|
worktree-create-sub, worktree-cleanup-sub, merge-failure-sub,
|
||||||
|
merge-block-sub, done-archive-sub, content-gc, cost-rollup-sub,
|
||||||
|
stage-notification-sub, event-triggers).
|
||||||
|
|
||||||
|
**The LLM context injector is just the 11th subscriber on the same
|
||||||
|
bus.** It writes typed events into the per-sled CRDT event log
|
||||||
|
described below; everything downstream reuses the existing taxonomy.
|
||||||
|
|
||||||
|
Each persisted entry carries:
|
||||||
|
|
||||||
|
```
|
||||||
|
struct LoggedEvent {
|
||||||
|
id: EventId, // monotonic per sled
|
||||||
|
sled_id: SledId,
|
||||||
|
timestamp: UnixSeconds,
|
||||||
|
transition: TransitionFired, // story_id + from + to + PipelineEvent
|
||||||
|
// (or ExecutionEvent — see open question)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The few events that genuinely don't fit the pipeline state machine
|
||||||
|
(e.g. `ProjectAdopted`, `Rebuilt`, `GatewayHealthChanged`) live in a
|
||||||
|
small, separately-enumerated `InfraEvent` enum, but the same log and
|
||||||
|
the same subscriber pattern still apply.
|
||||||
|
|
||||||
|
## Session Model
|
||||||
|
|
||||||
|
An LLM session is a first-class CRDT entity:
|
||||||
|
|
||||||
|
```
|
||||||
|
struct LlmSession {
|
||||||
|
id: SessionId,
|
||||||
|
persona: Persona, // "Timmy", "Sally", ...
|
||||||
|
scope: ScopeFilter, // { sleds: All } | { sleds: Set<SledId> }
|
||||||
|
high_water: BTreeMap<SledId, EventId>, // per-stream
|
||||||
|
created: UnixSeconds,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The session id is what the transport carries; it's not the Matrix room,
|
||||||
|
not the web socket id. A given Matrix room may map to one session; a
|
||||||
|
web UI tab may map to another. Multiple transports for the same human
|
||||||
|
can share a session if you want — that's a separate UX call.
|
||||||
|
|
||||||
|
## Prompt Assembly Contract
|
||||||
|
|
||||||
|
Every transport calls one helper before invoking the LLM:
|
||||||
|
|
||||||
|
```
|
||||||
|
fn assemble_prompt_context(session_id: SessionId) -> String
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
|
||||||
|
1. Read the session's scope filter and high-water marks.
|
||||||
|
2. Fetch events from the gateway aggregator that match the scope and
|
||||||
|
are newer than the high-water marks.
|
||||||
|
3. Render them as a single `<system-reminder>` block, ordered by sled
|
||||||
|
then timestamp.
|
||||||
|
4. Advance the high-water marks to the latest event seen, atomically
|
||||||
|
with the LLM-turn-start CRDT op (so a crash mid-turn doesn't double-
|
||||||
|
inject).
|
||||||
|
5. Return the rendered block (empty string if no new events).
|
||||||
|
|
||||||
|
The transport prepends the result to the user's prompt and invokes the
|
||||||
|
LLM as usual.
|
||||||
|
|
||||||
|
## Persistence & Reliability Rules
|
||||||
|
|
||||||
|
- **Event log is CRDT-backed.** Survives sled restart.
|
||||||
|
- **High-water marks are CRDT-backed.** Survives gateway restart.
|
||||||
|
- **Aggregator uses bounded queues with drop-oldest semantics**, and
|
||||||
|
every drop logs `[event-agg] dropped N events for session <id>; client
|
||||||
|
must re-fetch from <high-water>`. The aggregator never silently
|
||||||
|
swallows events — if the queue is full, the session gets a sentinel
|
||||||
|
event `EventStreamGap { from, to }` so the LLM can see it missed
|
||||||
|
context.
|
||||||
|
- **End-to-end test required**: `fire(Event::StoryMerged{1122}) → user
|
||||||
|
sends "what's going on?" → assembled prompt contains "1122 merged"`.
|
||||||
|
|
||||||
|
## Multi-Persona Scoping
|
||||||
|
|
||||||
|
The same machinery serves both Timmy and Sally:
|
||||||
|
|
||||||
|
| Persona | Scope filter | Notes |
|
||||||
|
|---------|------------------------------------|--------------------------------|
|
||||||
|
| Timmy | `{ sleds: All }` | Gateway-wide; aware of every sled |
|
||||||
|
| Sally | `{ sleds: { huskies-server } }` | Single-sled; sled-local events only |
|
||||||
|
| Manny | `{ sleds: { huskies, ketflix } }` | Hypothetical; subset |
|
||||||
|
|
||||||
|
Sally never has to know Timmy exists, and vice versa. Their sessions
|
||||||
|
advance their own high-water marks against the same underlying log.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
| Decision | Choice | Alternative |
|
||||||
|
|------------------------|-------------------------------------|----------------------------------------------|
|
||||||
|
| Event publication | Each sled owns its log | Single global log: cross-sled bottleneck |
|
||||||
|
| Aggregation | Gateway tail-merges | Each session pulls from each sled directly: N×M fanout |
|
||||||
|
| Buffer location | CRDT-persisted | In-process: lost on rebuild (current bug) |
|
||||||
|
| Event identity | Typed enum | Strings: structured-log creep, no compile-time safety |
|
||||||
|
| Drop semantics | Drop-oldest + `EventStreamGap` | Silent drop (current bug): LLM lies confidently |
|
||||||
|
| Session ↔ transport | Session is separate from transport | One per transport: web tab + Matrix get different views |
|
||||||
|
| Proactive LLM wake | OUT OF SCOPE | Wake on every event: cost + ramble |
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
1. **Session lifecycle**. How are sessions created and garbage-
|
||||||
|
collected? Created on first transport message? GC'd after N days
|
||||||
|
idle?
|
||||||
|
2. **Event retention**. How long are events kept in the log? Forever
|
||||||
|
feels wrong; "since last terminal session turn" feels right but
|
||||||
|
needs care for multi-session readers.
|
||||||
|
3. **Multi-transport same session**. Should one human's Matrix and web
|
||||||
|
UI share a session by default, or always be separate?
|
||||||
|
4. **Render budget**. If 500 events accumulated between turns, do we
|
||||||
|
render all 500 or summarize? A `summarize_events` fallback path is
|
||||||
|
probably worth designing in from the start.
|
||||||
|
5. **Aggregator placement when there is no gateway**. A standalone
|
||||||
|
single-sled install has no gateway — does the sled itself host the
|
||||||
|
aggregator? (Probably yes; trivially "aggregates" its own log.)
|
||||||
|
|
||||||
|
## Phasing
|
||||||
|
|
||||||
|
- **Phase 0 (now):** this design doc.
|
||||||
|
- **Phase 1:** typed `Event` enum + per-sled CRDT-backed event log;
|
||||||
|
one publisher subscribes to existing pipeline transitions and writes
|
||||||
|
`StoryStaged` / `StoryMerged` / `StoryMergeFailed`.
|
||||||
|
- **Phase 2:** `LlmSession` CRDT entity + `assemble_prompt_context`
|
||||||
|
helper, wired into the Matrix bot's `handle_message` (replaces the
|
||||||
|
existing `pending_pipeline_events` Vec). End-to-end test covering the
|
||||||
|
fire-event → user-turn → prompt-contains-event contract.
|
||||||
|
- **Phase 3:** Gateway aggregator over multiple sleds; Timmy's session
|
||||||
|
scoped to `All`. Sally's session scoped to a single sled.
|
||||||
|
- **Phase 4:** Web UI and any other transports migrated onto
|
||||||
|
`assemble_prompt_context`; the Matrix-specific Vec deleted.
|
||||||
|
- **Phase 5:** Bounded queues + `EventStreamGap` sentinel; observability
|
||||||
|
for `assemble_prompt_context` runs (events injected, gaps observed).
|
||||||
|
|
||||||
|
Each phase ships independently. Phase 2 alone delivers the user-facing
|
||||||
|
fix: Timmy sees what merged when you next say anything, without you
|
||||||
|
needing to re-narrate.
|
||||||
Generated
+543
-948
File diff suppressed because it is too large
Load Diff
+46
-36
@@ -1,58 +1,68 @@
|
|||||||
[workspace]
|
[workspace]
|
||||||
members = ["server", "crates/bft-json-crdt", "crates/source-map-gen"]
|
members = [
|
||||||
|
"server",
|
||||||
|
"crates/bft-json-crdt",
|
||||||
|
"crates/source-map-gen",
|
||||||
|
"crates/release-manifest",
|
||||||
|
"crates/release-tool",
|
||||||
|
]
|
||||||
resolver = "3"
|
resolver = "3"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
async-stream = "0.3"
|
async-stream = "0.3.6"
|
||||||
async-trait = "0.1.89"
|
async-trait = "0.1.89"
|
||||||
bytes = "1"
|
bytes = "1.12.1"
|
||||||
chrono = { version = "0.4.44", features = ["serde"] }
|
chrono = { version = "0.4.45", features = ["serde"] }
|
||||||
chrono-tz = "0.10"
|
chrono-tz = "0.10.4"
|
||||||
eventsource-stream = "0.2.3"
|
eventsource-stream = "0.2.3"
|
||||||
futures = "0.3"
|
futures = "0.3.32"
|
||||||
homedir = "0.3.6"
|
homedir = "0.3.6"
|
||||||
ignore = "0.4.25"
|
ignore = "0.4.29"
|
||||||
mime_guess = "2"
|
mime_guess = "2.0.5"
|
||||||
notify = "8.2.0"
|
notify = "8.2.0"
|
||||||
poem = { version = "3", features = ["websocket", "test"] }
|
poem = { version = "3.1.12", features = ["websocket", "test"] }
|
||||||
portable-pty = "0.9.0"
|
portable-pty = "0.9.0"
|
||||||
reqwest = { version = "0.13.3", features = ["json", "stream"] }
|
reqwest = { version = "0.13.4", features = ["json", "stream"] }
|
||||||
rust-embed = "8"
|
rust-embed = "8.12.0"
|
||||||
ed25519-dalek = { version = "2", default-features = false, features = ["rand_core"] }
|
ed25519-dalek = { version = "3.0.0", default-features = false, features = ["rand_core"] }
|
||||||
indexmap = { version = "2.14.0", features = ["serde"] }
|
indexmap = { version = "2.14.0", features = ["serde"] }
|
||||||
rand = "0.10"
|
rand = "0.10.2"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1.0.228", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1.0.150"
|
||||||
serde_urlencoded = "0.7"
|
serde_urlencoded = "0.7.1"
|
||||||
sha1 = "0.11"
|
sha1 = "0.11.0"
|
||||||
sha2 = "0.11.0"
|
sha2 = "0.11.0"
|
||||||
hmac = "0.13"
|
hmac = "0.13.0"
|
||||||
subtle = "2"
|
subtle = "2.6.1"
|
||||||
base64 = "0.22"
|
base64 = "0.22.1"
|
||||||
serde_yaml = "0.9"
|
strip-ansi-escapes = "0.2.1"
|
||||||
strip-ansi-escapes = "0.2"
|
tempfile = "3.27.0"
|
||||||
tempfile = "3"
|
tokio = { version = "1.52.4", features = ["rt-multi-thread", "macros", "sync"] }
|
||||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] }
|
toml = "1.1.3"
|
||||||
toml = "1.1.2"
|
uuid = { version = "1.24.0", features = ["v4", "serde"] }
|
||||||
uuid = { version = "1.23.1", features = ["v4", "serde"] }
|
tokio-tungstenite = { version = "0.30.0", features = ["connect", "rustls-tls-native-roots"] }
|
||||||
tokio-tungstenite = { version = "0.29.0", features = ["connect", "rustls-tls-native-roots"] }
|
|
||||||
walkdir = "2.5.0"
|
walkdir = "2.5.0"
|
||||||
filetime = "0.2"
|
filetime = "0.2.29"
|
||||||
matrix-sdk = { version = "0.17", default-features = false, features = [
|
# 0.18 is current as of this pin (verified via `cargo search matrix-sdk`).
|
||||||
|
# matrix-sdk-sqlite 0.18 pulls in rusqlite 0.37, which requires libsqlite3-sys
|
||||||
|
# ==0.35.x — see the pin rationale in server/Cargo.toml. Bumping matrix-sdk
|
||||||
|
# past 0.18 may require re-checking that ceiling.
|
||||||
|
matrix-sdk = { version = "0.18", default-features = false, features = [
|
||||||
"sqlite",
|
"sqlite",
|
||||||
"e2e-encryption",
|
"e2e-encryption",
|
||||||
] }
|
] }
|
||||||
pulldown-cmark = { version = "0.13.3", default-features = false, features = [
|
pulldown-cmark = { version = "0.13.4", default-features = false, features = [
|
||||||
"html",
|
"html",
|
||||||
] }
|
] }
|
||||||
regex = "1"
|
regex = "1.13.1"
|
||||||
libc = "0.2"
|
libc = "0.2.186"
|
||||||
nutype = { version = "0.7", features = ["serde"] }
|
nutype = { version = "0.7.0", features = ["serde"] }
|
||||||
garde = { version = "0.22", features = ["derive"] }
|
garde = { version = "0.23", features = ["derive"] }
|
||||||
ammonia = "4.1"
|
ammonia = "4.1.3"
|
||||||
sqlx = { version = "=0.9.0-alpha.1", default-features = false, features = [
|
sqlx = { version = "0.9.0", default-features = false, features = [
|
||||||
"runtime-tokio",
|
"runtime-tokio",
|
||||||
"sqlite",
|
"sqlite",
|
||||||
"macros",
|
"macros",
|
||||||
"migrate",
|
"migrate",
|
||||||
] }
|
] }
|
||||||
|
serde_yaml = "0.9.34"
|
||||||
|
|||||||
@@ -79,6 +79,10 @@ cd frontend && npm install && npm run dev
|
|||||||
|
|
||||||
Configuration lives in `.huskies/project.toml`. See `.huskies/bot.toml.*.example` for transport setup.
|
Configuration lives in `.huskies/project.toml`. See `.huskies/bot.toml.*.example` for transport setup.
|
||||||
|
|
||||||
|
## Website
|
||||||
|
|
||||||
|
The huskies.dev website source has moved to [crashlabs/huskies-server](https://code.crashlabs.io/crashlabs/huskies-server).
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
Internal architecture documentation lives in [`docs/architecture/`](docs/architecture/):
|
Internal architecture documentation lives in [`docs/architecture/`](docs/architecture/):
|
||||||
@@ -91,10 +95,10 @@ Internal architecture documentation lives in [`docs/architecture/`](docs/archite
|
|||||||
Requires a Gitea API token in `.env` (`GITEA_TOKEN=your_token`).
|
Requires a Gitea API token in `.env` (`GITEA_TOKEN=your_token`).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
script/release 0.7.1
|
script/release 0.14.0
|
||||||
```
|
```
|
||||||
|
|
||||||
This bumps version in `Cargo.toml` and `package.json`, builds macOS arm64 and Linux amd64 binaries, tags the repo, and publishes a Gitea release with changelog and binaries attached.
|
This bumps version in `Cargo.toml` and `package.json`, builds macOS arm64, Linux amd64, and Linux arm64 binaries, tags the repo, pushes the branch and tag, and publishes a Gitea release with changelog and binaries attached.
|
||||||
|
|
||||||
## Multi-node CRDT sync (rendezvous)
|
## Multi-node CRDT sync (rendezvous)
|
||||||
|
|
||||||
|
|||||||
@@ -6,8 +6,14 @@ edition = "2021"
|
|||||||
[lib]
|
[lib]
|
||||||
crate-type = ["lib"]
|
crate-type = ["lib"]
|
||||||
|
|
||||||
|
# The logging-* features print multi-KB debug dumps on every CRDT op — and
|
||||||
|
# they execute inside the global CRDT state mutex in the server, so a stalled
|
||||||
|
# stdout write while holding that lock can pin every tokio worker and freeze
|
||||||
|
# the whole process (bug 1170). They are development tools: opt in explicitly
|
||||||
|
# with `--features bft-json-crdt/logging-list` when debugging CRDT internals.
|
||||||
|
# Never enable them in a production build.
|
||||||
[features]
|
[features]
|
||||||
default = ["bft", "logging-list", "logging-json"]
|
default = ["bft"]
|
||||||
logging-list = ["logging-base"]
|
logging-list = ["logging-base"]
|
||||||
logging-json = ["logging-base"]
|
logging-json = ["logging-base"]
|
||||||
logging-base = []
|
logging-base = []
|
||||||
@@ -15,18 +21,18 @@ bft = []
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
bft-crdt-derive = { path = "bft-crdt-derive" }
|
bft-crdt-derive = { path = "bft-crdt-derive" }
|
||||||
colored = "3"
|
colored = "3.1.1"
|
||||||
ed25519-dalek = { workspace = true }
|
ed25519-dalek = { workspace = true }
|
||||||
indexmap = { workspace = true, features = ["serde"] }
|
indexmap = { workspace = true, features = ["serde"] }
|
||||||
rand = { workspace = true }
|
rand = { workspace = true }
|
||||||
random_color = "1"
|
random_color = "1.1.0"
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||||
serde_with = "3"
|
serde_with = "3.21.0"
|
||||||
sha2 = { workspace = true }
|
sha2 = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
criterion = { version = "0.8", features = ["html_reports"] }
|
criterion = { version = "0.8.2", features = ["html_reports"] }
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||||
|
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ publish = false
|
|||||||
proc-macro = true
|
proc-macro = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
indexmap = { version = "2.2.6", features = ["serde"] }
|
indexmap = { version = "2.14.0", features = ["serde"] }
|
||||||
proc-macro2 = "1.0.47"
|
proc-macro2 = "1.0.106"
|
||||||
proc-macro-crate = "3"
|
proc-macro-crate = "3.5.0"
|
||||||
quote = "1.0.21"
|
quote = "1.0.46"
|
||||||
syn = { version = "2", features = ["full"] }
|
syn = { version = "2.0.119", features = ["full"] }
|
||||||
|
|||||||
@@ -20,6 +20,10 @@ use std::{
|
|||||||
|
|
||||||
/// An RGA-like list CRDT that can store a CRDT-like datatype
|
/// An RGA-like list CRDT that can store a CRDT-like datatype
|
||||||
#[derive(Clone, Serialize, Deserialize)]
|
#[derive(Clone, Serialize, Deserialize)]
|
||||||
|
#[serde(bound(
|
||||||
|
serialize = "T: serde::Serialize",
|
||||||
|
deserialize = "T: serde::de::DeserializeOwned"
|
||||||
|
))]
|
||||||
pub struct ListCrdt<T>
|
pub struct ListCrdt<T>
|
||||||
where
|
where
|
||||||
T: CrdtNode,
|
T: CrdtNode,
|
||||||
@@ -32,6 +36,7 @@ where
|
|||||||
pub ops: Vec<Op<T>>,
|
pub ops: Vec<Op<T>>,
|
||||||
/// Queue of messages where K is the ID of the message yet to arrive
|
/// Queue of messages where K is the ID of the message yet to arrive
|
||||||
/// and V is the list of operations depending on it
|
/// and V is the list of operations depending on it
|
||||||
|
#[serde(skip)]
|
||||||
message_q: HashMap<OpId, Vec<Op<T>>>,
|
message_q: HashMap<OpId, Vec<Op<T>>>,
|
||||||
/// The sequence number of this node
|
/// The sequence number of this node
|
||||||
our_seq: SequenceNumber,
|
our_seq: SequenceNumber,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
use crate::debug::DebugView;
|
use crate::debug::DebugView;
|
||||||
use crate::json_crdt::{CrdtNode, JsonValue, OpState};
|
use crate::json_crdt::{CrdtNode, JsonValue, OpState};
|
||||||
use crate::op::{join_path, print_path, Op, PathSegment, SequenceNumber};
|
use crate::op::{join_path, print_path, Op, PathSegment, SequenceNumber};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
use std::cmp::{max, Ordering};
|
use std::cmp::{max, Ordering};
|
||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
|
|
||||||
@@ -14,7 +15,11 @@ use crate::keypair::AuthorId;
|
|||||||
|
|
||||||
/// A simple delete-wins, last-writer-wins (LWW) register CRDT.
|
/// A simple delete-wins, last-writer-wins (LWW) register CRDT.
|
||||||
/// Basically only for adding support for primitives within a more complex CRDT
|
/// Basically only for adding support for primitives within a more complex CRDT
|
||||||
#[derive(Clone)]
|
#[derive(Clone, Serialize, Deserialize)]
|
||||||
|
#[serde(bound(
|
||||||
|
serialize = "T: serde::Serialize",
|
||||||
|
deserialize = "T: serde::de::DeserializeOwned"
|
||||||
|
))]
|
||||||
pub struct LwwRegisterCrdt<T>
|
pub struct LwwRegisterCrdt<T>
|
||||||
where
|
where
|
||||||
T: CrdtNode,
|
T: CrdtNode,
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
[package]
|
||||||
|
name = "release-manifest"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
serde = { workspace = true, features = ["derive"] }
|
||||||
|
serde_json = { workspace = true }
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
//! Shared release-manifest type for the signed release-channel pull pipeline.
|
||||||
|
//!
|
||||||
|
//! The publisher tool (`crates/release-tool`) builds a [`ReleaseManifest`],
|
||||||
|
//! serializes it to canonical bytes, signs those bytes with the channel's
|
||||||
|
//! Ed25519 private key, and publishes the resulting [`SignedManifest`] as
|
||||||
|
//! `manifest.json` on the release channel. The gateway (`huskies-server`)
|
||||||
|
//! fetches that file, re-serializes the embedded manifest with
|
||||||
|
//! [`ReleaseManifest::canonical_bytes`], and verifies the signature against
|
||||||
|
//! its pinned public key before trusting anything in it.
|
||||||
|
//!
|
||||||
|
//! Keeping the type in its own dependency-light crate lets both sides agree
|
||||||
|
//! on the exact byte representation to sign/verify without the server crate
|
||||||
|
//! ever linking signing code, and without the publisher tool depending on
|
||||||
|
//! the full `huskies` server crate.
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// The signed payload describing one published release artifact.
|
||||||
|
///
|
||||||
|
/// Field order is significant: [`ReleaseManifest::canonical_bytes`] relies on
|
||||||
|
/// `serde_json`'s struct serialization preserving declaration order, so the
|
||||||
|
/// signer and verifier always agree on the exact bytes being signed.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct ReleaseManifest {
|
||||||
|
/// Filename of the artifact within the channel (e.g. `huskies-linux-arm64`).
|
||||||
|
pub artifact: String,
|
||||||
|
/// Lowercase hex sha256 digest of the artifact's bytes.
|
||||||
|
pub sha256: String,
|
||||||
|
/// Version identifier — the short git commit hash the artifact was built from.
|
||||||
|
pub version: String,
|
||||||
|
/// Release channel name this manifest was signed for (e.g. `stable`).
|
||||||
|
pub channel: String,
|
||||||
|
/// Unix timestamp (seconds) the manifest was signed at.
|
||||||
|
///
|
||||||
|
/// Used for rollback/replay detection: a pull refuses any manifest whose
|
||||||
|
/// timestamp is not strictly newer than the currently installed one.
|
||||||
|
pub timestamp: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReleaseManifest {
|
||||||
|
/// Serialize this manifest deterministically for signing and verification.
|
||||||
|
///
|
||||||
|
/// Both the publisher and the gateway construct this independently from
|
||||||
|
/// their own in-memory `ReleaseManifest` value — the manifest.json file's
|
||||||
|
/// exact on-disk byte layout is never itself the signed payload.
|
||||||
|
pub fn canonical_bytes(&self) -> Vec<u8> {
|
||||||
|
serde_json::to_vec(self).expect("ReleaseManifest serialization cannot fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A [`ReleaseManifest`] plus its Ed25519 signature (lowercase hex), as
|
||||||
|
/// published to a release channel's `manifest.json`.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SignedManifest {
|
||||||
|
/// The manifest describing the published artifact.
|
||||||
|
pub manifest: ReleaseManifest,
|
||||||
|
/// Hex-encoded Ed25519 signature over `manifest.canonical_bytes()`.
|
||||||
|
pub signature: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn sample() -> ReleaseManifest {
|
||||||
|
ReleaseManifest {
|
||||||
|
artifact: "huskies-linux-arm64".to_string(),
|
||||||
|
sha256: "a".repeat(64),
|
||||||
|
version: "abc1234".to_string(),
|
||||||
|
channel: "stable".to_string(),
|
||||||
|
timestamp: 1_700_000_000,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonical_bytes_is_deterministic() {
|
||||||
|
let m = sample();
|
||||||
|
assert_eq!(m.canonical_bytes(), m.canonical_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonical_bytes_changes_with_any_field() {
|
||||||
|
let m1 = sample();
|
||||||
|
let mut m2 = sample();
|
||||||
|
m2.timestamp += 1;
|
||||||
|
assert_ne!(m1.canonical_bytes(), m2.canonical_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn signed_manifest_roundtrips_through_json() {
|
||||||
|
let signed = SignedManifest {
|
||||||
|
manifest: sample(),
|
||||||
|
signature: "deadbeef".to_string(),
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&signed).unwrap();
|
||||||
|
let parsed: SignedManifest = serde_json::from_str(&json).unwrap();
|
||||||
|
assert_eq!(parsed.manifest, signed.manifest);
|
||||||
|
assert_eq!(parsed.signature, signed.signature);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manifest_missing_field_fails_to_parse() {
|
||||||
|
let bad = serde_json::json!({
|
||||||
|
"artifact": "huskies-linux-arm64",
|
||||||
|
"sha256": "a".repeat(64),
|
||||||
|
"version": "abc1234",
|
||||||
|
"channel": "stable"
|
||||||
|
// timestamp missing
|
||||||
|
});
|
||||||
|
let result: Result<ReleaseManifest, _> = serde_json::from_value(bad);
|
||||||
|
assert!(
|
||||||
|
result.is_err(),
|
||||||
|
"manifest missing a field must fail to parse"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn signed_manifest_missing_signature_fails_to_parse() {
|
||||||
|
let bad = serde_json::json!({ "manifest": sample() });
|
||||||
|
let result: Result<SignedManifest, _> = serde_json::from_value(bad);
|
||||||
|
assert!(
|
||||||
|
result.is_err(),
|
||||||
|
"signed manifest missing signature must fail to parse"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
[package]
|
||||||
|
name = "release-tool"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "release-tool"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
release-manifest = { path = "../release-manifest" }
|
||||||
|
ed25519-dalek = { workspace = true }
|
||||||
|
sha2 = { workspace = true }
|
||||||
|
rand = { workspace = true }
|
||||||
|
serde_json = { workspace = true }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tempfile = { workspace = true }
|
||||||
@@ -0,0 +1,311 @@
|
|||||||
|
//! `release-tool` — offline publisher CLI for signed release channels.
|
||||||
|
//!
|
||||||
|
//! Generates a release-channel Ed25519 keypair and signs release manifests
|
||||||
|
//! for a channel's `manifest.json`. This binary is intentionally its own
|
||||||
|
//! crate, depending only on [`release_manifest`] and `ed25519-dalek` — it
|
||||||
|
//! never links against the `huskies` server crate, so the running gateway
|
||||||
|
//! has no code path that can read a channel's private signing key. Run this
|
||||||
|
//! tool offline (or in a separate publish pipeline) and copy only the
|
||||||
|
//! resulting public key hex into the gateway's `projects.toml`.
|
||||||
|
//!
|
||||||
|
//! Usage:
|
||||||
|
//! ```text
|
||||||
|
//! release-tool keygen <key-out-path>
|
||||||
|
//! release-tool sign --key <path> --artifact <path> --version <str> --channel <str> --out <path> [--timestamp <unix-secs>]
|
||||||
|
//! ```
|
||||||
|
|
||||||
|
use ed25519_dalek::{Signer, SigningKey};
|
||||||
|
use rand::Rng;
|
||||||
|
use release_manifest::{ReleaseManifest, SignedManifest};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let args: Vec<String> = std::env::args().collect();
|
||||||
|
let result = match args.get(1).map(String::as_str) {
|
||||||
|
Some("keygen") => run_keygen(&args[2..]),
|
||||||
|
Some("sign") => run_sign(&args[2..]),
|
||||||
|
_ => Err(
|
||||||
|
"usage: release-tool keygen <key-out-path> | release-tool sign --key <path> \
|
||||||
|
--artifact <path> --version <str> --channel <str> --out <path> [--timestamp <unix-secs>]"
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Err(e) = result {
|
||||||
|
eprintln!("error: {e}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── keygen ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
fn run_keygen(args: &[String]) -> Result<(), String> {
|
||||||
|
let key_path = args.first().ok_or("keygen requires a key-out-path")?;
|
||||||
|
let signing_key = generate_signing_key();
|
||||||
|
write_seed_file(Path::new(key_path), &signing_key)?;
|
||||||
|
|
||||||
|
let pubkey_hex = hex_encode(signing_key.verifying_key().as_bytes());
|
||||||
|
println!("Wrote private key seed to {key_path}");
|
||||||
|
println!("Pinned release public key (paste into projects.toml as `pubkey`):");
|
||||||
|
println!("{pubkey_hex}");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn generate_signing_key() -> SigningKey {
|
||||||
|
let mut seed = [0u8; 32];
|
||||||
|
rand::rng().fill_bytes(&mut seed);
|
||||||
|
SigningKey::from_bytes(&seed)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_seed_file(path: &Path, signing_key: &SigningKey) -> Result<(), String> {
|
||||||
|
if let Some(parent) = path.parent()
|
||||||
|
&& !parent.as_os_str().is_empty()
|
||||||
|
{
|
||||||
|
std::fs::create_dir_all(parent)
|
||||||
|
.map_err(|e| format!("cannot create {}: {e}", parent.display()))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::io::Write;
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
let mut file = std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create(true)
|
||||||
|
.truncate(true)
|
||||||
|
.mode(0o600)
|
||||||
|
.open(path)
|
||||||
|
.map_err(|e| format!("cannot create {}: {e}", path.display()))?;
|
||||||
|
file.write_all(&signing_key.to_bytes())
|
||||||
|
.map_err(|e| format!("cannot write {}: {e}", path.display()))
|
||||||
|
}
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
{
|
||||||
|
std::fs::write(path, signing_key.to_bytes())
|
||||||
|
.map_err(|e| format!("cannot write {}: {e}", path.display()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_seed_file(path: &Path) -> Result<SigningKey, String> {
|
||||||
|
let bytes = std::fs::read(path).map_err(|e| format!("cannot read {}: {e}", path.display()))?;
|
||||||
|
let seed: [u8; 32] = bytes
|
||||||
|
.try_into()
|
||||||
|
.map_err(|_| format!("{} must contain exactly 32 bytes", path.display()))?;
|
||||||
|
Ok(SigningKey::from_bytes(&seed))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── sign ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Parsed `sign` subcommand arguments.
|
||||||
|
struct SignArgs {
|
||||||
|
key: PathBuf,
|
||||||
|
artifact: PathBuf,
|
||||||
|
version: String,
|
||||||
|
channel: String,
|
||||||
|
out: PathBuf,
|
||||||
|
timestamp: Option<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_sign_args(args: &[String]) -> Result<SignArgs, String> {
|
||||||
|
let mut key = None;
|
||||||
|
let mut artifact = None;
|
||||||
|
let mut version = None;
|
||||||
|
let mut channel = None;
|
||||||
|
let mut out = None;
|
||||||
|
let mut timestamp = None;
|
||||||
|
|
||||||
|
let mut i = 0;
|
||||||
|
while i < args.len() {
|
||||||
|
let flag = args[i].as_str();
|
||||||
|
let value = args
|
||||||
|
.get(i + 1)
|
||||||
|
.ok_or_else(|| format!("missing value for {flag}"))?;
|
||||||
|
match flag {
|
||||||
|
"--key" => key = Some(PathBuf::from(value)),
|
||||||
|
"--artifact" => artifact = Some(PathBuf::from(value)),
|
||||||
|
"--version" => version = Some(value.clone()),
|
||||||
|
"--channel" => channel = Some(value.clone()),
|
||||||
|
"--out" => out = Some(PathBuf::from(value)),
|
||||||
|
"--timestamp" => {
|
||||||
|
timestamp = Some(
|
||||||
|
value
|
||||||
|
.parse::<i64>()
|
||||||
|
.map_err(|_| format!("--timestamp must be an integer, got `{value}`"))?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
other => return Err(format!("unknown flag `{other}`")),
|
||||||
|
}
|
||||||
|
i += 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(SignArgs {
|
||||||
|
key: key.ok_or("--key is required")?,
|
||||||
|
artifact: artifact.ok_or("--artifact is required")?,
|
||||||
|
version: version.ok_or("--version is required")?,
|
||||||
|
channel: channel.ok_or("--channel is required")?,
|
||||||
|
out: out.ok_or("--out is required")?,
|
||||||
|
timestamp,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_sign(args: &[String]) -> Result<(), String> {
|
||||||
|
let parsed = parse_sign_args(args)?;
|
||||||
|
let signing_key = load_seed_file(&parsed.key)?;
|
||||||
|
let artifact_bytes = std::fs::read(&parsed.artifact)
|
||||||
|
.map_err(|e| format!("cannot read {}: {e}", parsed.artifact.display()))?;
|
||||||
|
let artifact_name = parsed
|
||||||
|
.artifact
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.ok_or("--artifact path has no filename")?
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let timestamp = match parsed.timestamp {
|
||||||
|
Some(t) => t,
|
||||||
|
None => std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map_err(|e| format!("system clock before epoch: {e}"))?
|
||||||
|
.as_secs() as i64,
|
||||||
|
};
|
||||||
|
|
||||||
|
let signed = sign_manifest(
|
||||||
|
&signing_key,
|
||||||
|
artifact_name,
|
||||||
|
&artifact_bytes,
|
||||||
|
parsed.version,
|
||||||
|
parsed.channel,
|
||||||
|
timestamp,
|
||||||
|
);
|
||||||
|
|
||||||
|
let json =
|
||||||
|
serde_json::to_string_pretty(&signed).map_err(|e| format!("serialise manifest: {e}"))?;
|
||||||
|
std::fs::write(&parsed.out, json)
|
||||||
|
.map_err(|e| format!("cannot write {}: {e}", parsed.out.display()))?;
|
||||||
|
println!("Signed manifest written to {}", parsed.out.display());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build and sign a [`SignedManifest`] for the given artifact bytes.
|
||||||
|
///
|
||||||
|
/// Pure aside from the signature computation — split out from `run_sign` so
|
||||||
|
/// tests can exercise it without touching the filesystem.
|
||||||
|
fn sign_manifest(
|
||||||
|
signing_key: &SigningKey,
|
||||||
|
artifact: String,
|
||||||
|
artifact_bytes: &[u8],
|
||||||
|
version: String,
|
||||||
|
channel: String,
|
||||||
|
timestamp: i64,
|
||||||
|
) -> SignedManifest {
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(artifact_bytes);
|
||||||
|
let sha256 = hex_encode(&hasher.finalize());
|
||||||
|
|
||||||
|
let manifest = ReleaseManifest {
|
||||||
|
artifact,
|
||||||
|
sha256,
|
||||||
|
version,
|
||||||
|
channel,
|
||||||
|
timestamp,
|
||||||
|
};
|
||||||
|
let signature = hex_encode(&signing_key.sign(&manifest.canonical_bytes()).to_bytes());
|
||||||
|
SignedManifest {
|
||||||
|
manifest,
|
||||||
|
signature,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
fn hex_encode(bytes: &[u8]) -> String {
|
||||||
|
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn keygen_then_sign_produces_verifiable_signature() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let key_path = tmp.path().join("channel.key");
|
||||||
|
let signing_key = generate_signing_key();
|
||||||
|
write_seed_file(&key_path, &signing_key).unwrap();
|
||||||
|
|
||||||
|
let loaded = load_seed_file(&key_path).unwrap();
|
||||||
|
assert_eq!(loaded.verifying_key(), signing_key.verifying_key());
|
||||||
|
|
||||||
|
let signed = sign_manifest(
|
||||||
|
&loaded,
|
||||||
|
"huskies-linux-arm64".to_string(),
|
||||||
|
b"fake binary contents",
|
||||||
|
"abc1234".to_string(),
|
||||||
|
"stable".to_string(),
|
||||||
|
1_700_000_000,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Verify with ed25519-dalek directly, mirroring how the gateway verifies.
|
||||||
|
use ed25519_dalek::Verifier;
|
||||||
|
let sig_bytes: [u8; 64] = hex_bytes(&signed.signature).try_into().unwrap();
|
||||||
|
let sig = ed25519_dalek::Signature::from_bytes(&sig_bytes);
|
||||||
|
assert!(
|
||||||
|
signing_key
|
||||||
|
.verifying_key()
|
||||||
|
.verify(&signed.manifest.canonical_bytes(), &sig)
|
||||||
|
.is_ok(),
|
||||||
|
"signature produced by sign_manifest must verify against the signing key's pubkey"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sign_manifest_hashes_artifact_bytes() {
|
||||||
|
let signing_key = generate_signing_key();
|
||||||
|
let signed = sign_manifest(
|
||||||
|
&signing_key,
|
||||||
|
"art".to_string(),
|
||||||
|
b"hello world",
|
||||||
|
"v1".to_string(),
|
||||||
|
"stable".to_string(),
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(b"hello world");
|
||||||
|
let expected = hex_encode(&hasher.finalize());
|
||||||
|
assert_eq!(signed.manifest.sha256, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_sign_args_rejects_missing_required_flag() {
|
||||||
|
let args: Vec<String> = vec!["--key".into(), "k".into()];
|
||||||
|
assert!(parse_sign_args(&args).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_sign_args_accepts_all_flags() {
|
||||||
|
let args: Vec<String> = vec![
|
||||||
|
"--key".into(),
|
||||||
|
"k".into(),
|
||||||
|
"--artifact".into(),
|
||||||
|
"a".into(),
|
||||||
|
"--version".into(),
|
||||||
|
"v1".into(),
|
||||||
|
"--channel".into(),
|
||||||
|
"stable".into(),
|
||||||
|
"--out".into(),
|
||||||
|
"o".into(),
|
||||||
|
"--timestamp".into(),
|
||||||
|
"42".into(),
|
||||||
|
];
|
||||||
|
let parsed = parse_sign_args(&args).unwrap();
|
||||||
|
assert_eq!(parsed.timestamp, Some(42));
|
||||||
|
assert_eq!(parsed.channel, "stable");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex_bytes(s: &str) -> Vec<u8> {
|
||||||
|
(0..s.len())
|
||||||
|
.step_by(2)
|
||||||
|
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -111,10 +111,9 @@ fn parse_pub_item(line: &str) -> Option<(String, String)> {
|
|||||||
let rest = if let Some(r) = trimmed.strip_prefix("pub(") {
|
let rest = if let Some(r) = trimmed.strip_prefix("pub(") {
|
||||||
let end = r.find(')')?;
|
let end = r.find(')')?;
|
||||||
r[end + 1..].trim_start()
|
r[end + 1..].trim_start()
|
||||||
} else if let Some(r) = trimmed.strip_prefix("pub ") {
|
|
||||||
r.trim_start()
|
|
||||||
} else {
|
} else {
|
||||||
return None;
|
let r = trimmed.strip_prefix("pub ")?;
|
||||||
|
r.trim_start()
|
||||||
};
|
};
|
||||||
|
|
||||||
// Handle "async fn"
|
// Handle "async fn"
|
||||||
@@ -139,10 +138,9 @@ fn parse_pub_item(line: &str) -> Option<(String, String)> {
|
|||||||
("const", r.trim_start())
|
("const", r.trim_start())
|
||||||
} else if let Some(r) = rest.strip_prefix("static ") {
|
} else if let Some(r) = rest.strip_prefix("static ") {
|
||||||
("static", r.trim_start())
|
("static", r.trim_start())
|
||||||
} else if let Some(r) = rest.strip_prefix("mod ") {
|
|
||||||
("mod", r.trim_start())
|
|
||||||
} else {
|
} else {
|
||||||
return None;
|
let r = rest.strip_prefix("mod ")?;
|
||||||
|
("mod", r.trim_start())
|
||||||
};
|
};
|
||||||
|
|
||||||
let name: String = name_part
|
let name: String = name_part
|
||||||
|
|||||||
@@ -123,10 +123,9 @@ fn parse_exported_item(line: &str) -> Option<(String, String)> {
|
|||||||
// Strip "export default" or "export"
|
// Strip "export default" or "export"
|
||||||
let rest = if let Some(r) = trimmed.strip_prefix("export default ") {
|
let rest = if let Some(r) = trimmed.strip_prefix("export default ") {
|
||||||
r.trim_start()
|
r.trim_start()
|
||||||
} else if let Some(r) = trimmed.strip_prefix("export ") {
|
|
||||||
r.trim_start()
|
|
||||||
} else {
|
} else {
|
||||||
return None;
|
let r = trimmed.strip_prefix("export ")?;
|
||||||
|
r.trim_start()
|
||||||
};
|
};
|
||||||
|
|
||||||
// Strip optional "async"
|
// Strip optional "async"
|
||||||
@@ -148,10 +147,9 @@ fn parse_exported_item(line: &str) -> Option<(String, String)> {
|
|||||||
("const", r.trim_start())
|
("const", r.trim_start())
|
||||||
} else if let Some(r) = rest.strip_prefix("let ") {
|
} else if let Some(r) = rest.strip_prefix("let ") {
|
||||||
("let", r.trim_start())
|
("let", r.trim_start())
|
||||||
} else if let Some(r) = rest.strip_prefix("enum ") {
|
|
||||||
("enum", r.trim_start())
|
|
||||||
} else {
|
} else {
|
||||||
return None;
|
let r = rest.strip_prefix("enum ")?;
|
||||||
|
("enum", r.trim_start())
|
||||||
};
|
};
|
||||||
|
|
||||||
let name: String = name_part
|
let name: String = name_part
|
||||||
|
|||||||
+12
-3
@@ -7,7 +7,7 @@
|
|||||||
#
|
#
|
||||||
# Tested with: OrbStack (recommended on macOS), Docker Desktop (slower bind mounts)
|
# Tested with: OrbStack (recommended on macOS), Docker Desktop (slower bind mounts)
|
||||||
|
|
||||||
FROM rust:1.93-bookworm AS base
|
FROM rust:1.94-bookworm AS base
|
||||||
|
|
||||||
# Clippy and rustfmt are needed at runtime for acceptance gates
|
# Clippy and rustfmt are needed at runtime for acceptance gates
|
||||||
RUN rustup component add clippy rustfmt
|
RUN rustup component add clippy rustfmt
|
||||||
@@ -46,8 +46,17 @@ WORKDIR /app
|
|||||||
# build.rs) can produce the release binary with embedded frontend assets.
|
# build.rs) can produce the release binary with embedded frontend assets.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Build frontend deps first (better layer caching)
|
# Build frontend deps first (better layer caching).
|
||||||
RUN cd frontend && npm ci
|
# Cannot use `npm ci` because of npm's optional-dependencies bug
|
||||||
|
# (npm/cli#4828): platform-specific bindings (e.g. rolldown's
|
||||||
|
# linux-arm64-gnu native binary, introduced by 1119's vite 5→8 upgrade)
|
||||||
|
# get listed in package-lock.json for the lockfile author's platform
|
||||||
|
# only, so `npm ci` skips them on every other platform — the build
|
||||||
|
# then fails at runtime with `Cannot find native binding`. Wipe the
|
||||||
|
# lockfile + node_modules and let `npm install` resolve fresh for the
|
||||||
|
# build platform. The lockfile mutation stays inside the container
|
||||||
|
# image and never reaches the host repo.
|
||||||
|
RUN cd frontend && rm -rf node_modules package-lock.json && npm install
|
||||||
|
|
||||||
# Build the release binary (build.rs runs npm run build for the frontend)
|
# Build the release binary (build.rs runs npm run build for the frontend)
|
||||||
RUN cargo build --release \
|
RUN cargo build --release \
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
# huskies-project-base — minimal base for all project containers.
|
||||||
|
#
|
||||||
|
# This image provides git, the huskies server binary, and a non-root user.
|
||||||
|
# It carries no language tooling. Per-stack overlays (docker/stacks/<name>/
|
||||||
|
# Dockerfile.fragment) layer their toolchains on top of this base.
|
||||||
|
#
|
||||||
|
# Prerequisites: build the main `huskies` image first so its binary is
|
||||||
|
# available as a build source.
|
||||||
|
#
|
||||||
|
# docker build -t huskies -f docker/Dockerfile .
|
||||||
|
# docker build -t huskies-project-base -f docker/Dockerfile.base .
|
||||||
|
#
|
||||||
|
# To build a stack image (e.g. rust):
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/rust/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-rust -
|
||||||
|
|
||||||
|
FROM huskies AS huskies-src
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
git \
|
||||||
|
curl \
|
||||||
|
ca-certificates \
|
||||||
|
libssl3 \
|
||||||
|
procps \
|
||||||
|
openssh-server \
|
||||||
|
sudo \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Node.js 22.x from NodeSource — bookworm's apt nodejs is 18.x, which modern
|
||||||
|
# frontend toolchains (vite ≥ 7) refuse to run under. Project builds that
|
||||||
|
# shell out to npm (e.g. build.rs frontend steps) need ≥ 20.
|
||||||
|
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
|
||||||
|
&& apt-get install -y --no-install-recommends nodejs \
|
||||||
|
&& npm install -g @anthropic-ai/claude-code \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Copy the huskies binary and entrypoint from the main image.
|
||||||
|
# The binary lives in /opt/huskies/bin (chowned to the huskies user below) so
|
||||||
|
# the in-process upgrade path (`POST /api/upgrade`) can atomically replace it
|
||||||
|
# without root: rename() needs write permission on the *directory*, not just
|
||||||
|
# the file. /usr/local/bin/huskies stays a symlink so PATH resolution and the
|
||||||
|
# entrypoint CMD are unchanged.
|
||||||
|
COPY --from=huskies-src /usr/local/bin/huskies /opt/huskies/bin/huskies
|
||||||
|
COPY --from=huskies-src /usr/local/bin/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||||
|
|
||||||
|
# Non-root user — Claude Code refuses --dangerously-skip-permissions as root.
|
||||||
|
# -s /bin/bash required for SSH sessions to start a real shell.
|
||||||
|
RUN groupadd -r huskies \
|
||||||
|
&& useradd -r -g huskies -m -d /home/huskies -s /bin/bash huskies \
|
||||||
|
&& mkdir -p /home/huskies/.claude \
|
||||||
|
&& mkdir -p /home/huskies/.ssh \
|
||||||
|
&& chmod 700 /home/huskies/.ssh \
|
||||||
|
&& chown -R huskies:huskies /home/huskies \
|
||||||
|
&& chown -R huskies:huskies /opt/huskies \
|
||||||
|
&& ln -s /opt/huskies/bin/huskies /usr/local/bin/huskies \
|
||||||
|
&& mkdir -p /workspace \
|
||||||
|
&& chown huskies:huskies /workspace \
|
||||||
|
&& git config --global init.defaultBranch master \
|
||||||
|
&& echo "huskies ALL=(root) NOPASSWD: /usr/sbin/sshd" > /etc/sudoers.d/huskies-sshd \
|
||||||
|
&& chmod 0440 /etc/sudoers.d/huskies-sshd \
|
||||||
|
&& mkdir -p /run/sshd \
|
||||||
|
&& sed -i \
|
||||||
|
-e 's/#PasswordAuthentication yes/PasswordAuthentication no/' \
|
||||||
|
-e 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' \
|
||||||
|
-e 's/UsePAM yes/UsePAM no/' \
|
||||||
|
/etc/ssh/sshd_config
|
||||||
|
|
||||||
|
# Shell profile for SSH sessions: land in /workspace and load toolchain paths.
|
||||||
|
RUN printf 'cd /workspace\n[ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env"\n' \
|
||||||
|
> /home/huskies/.profile \
|
||||||
|
&& chown huskies:huskies /home/huskies/.profile
|
||||||
|
|
||||||
|
USER huskies
|
||||||
|
WORKDIR /workspace
|
||||||
|
|
||||||
|
EXPOSE 3001 22
|
||||||
|
|
||||||
|
ENTRYPOINT ["entrypoint.sh"]
|
||||||
|
CMD ["huskies", "/workspace"]
|
||||||
@@ -29,6 +29,9 @@ services:
|
|||||||
- HUSKIES_PORT=3001
|
- HUSKIES_PORT=3001
|
||||||
# Bind to all interfaces so Docker port forwarding works.
|
# Bind to all interfaces so Docker port forwarding works.
|
||||||
- HUSKIES_HOST=0.0.0.0
|
- HUSKIES_HOST=0.0.0.0
|
||||||
|
# Gateway URL so this sled's relay task forwards CRDT events to the gateway.
|
||||||
|
# Uses host.docker.internal so the container can reach the gateway on the host.
|
||||||
|
- HUSKIES_GATEWAY_URL=http://host.docker.internal:3000
|
||||||
# Optional: Matrix bot credentials (if using Matrix integration)
|
# Optional: Matrix bot credentials (if using Matrix integration)
|
||||||
- MATRIX_HOMESERVER=${MATRIX_HOMESERVER:-}
|
- MATRIX_HOMESERVER=${MATRIX_HOMESERVER:-}
|
||||||
- MATRIX_USER=${MATRIX_USER:-}
|
- MATRIX_USER=${MATRIX_USER:-}
|
||||||
|
|||||||
@@ -1,6 +1,32 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
# ── Claude credentials ────────────────────────────────────────────────
|
||||||
|
# The `new project` command bind-mounts the host ~/.claude/.credentials.json
|
||||||
|
# at /run/claude-credentials-src:ro. We copy it here so the huskies user
|
||||||
|
# owns the file and mode 0600 is enforced regardless of host uid/gid.
|
||||||
|
if [ -f /run/claude-credentials-src ]; then
|
||||||
|
mkdir -p /home/huskies/.claude
|
||||||
|
cp /run/claude-credentials-src /home/huskies/.claude/.credentials.json
|
||||||
|
chmod 600 /home/huskies/.claude/.credentials.json
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── SSH authorized key ────────────────────────────────────────────────
|
||||||
|
# HUSKIES_SSH_PUBKEY is set by `new project` when it generates a keypair.
|
||||||
|
# Write it to authorized_keys so the user can connect with the matching
|
||||||
|
# private key stored at ~/.huskies/<project>/id_ed25519 on the host.
|
||||||
|
if [ -n "$HUSKIES_SSH_PUBKEY" ]; then
|
||||||
|
mkdir -p /home/huskies/.ssh
|
||||||
|
chmod 700 /home/huskies/.ssh
|
||||||
|
printf '%s\n' "$HUSKIES_SSH_PUBKEY" > /home/huskies/.ssh/authorized_keys
|
||||||
|
chmod 600 /home/huskies/.ssh/authorized_keys
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── SSH daemon ────────────────────────────────────────────────────────
|
||||||
|
# Start sshd in the background so the container accepts SSH connections.
|
||||||
|
# Uses sudo (huskies has NOPASSWD for /usr/sbin/sshd in sudoers.d).
|
||||||
|
sudo /usr/sbin/sshd -D -e &
|
||||||
|
|
||||||
# ── Git identity ─────────────────────────────────────────────────────
|
# ── Git identity ─────────────────────────────────────────────────────
|
||||||
# Agents commit code inside the container. Without a git identity,
|
# Agents commit code inside the container. Without a git identity,
|
||||||
# commits fail or use garbage defaults. Fail loudly at startup so the
|
# commits fail or use garbage defaults. Fail loudly at startup so the
|
||||||
@@ -25,6 +51,20 @@ export GIT_COMMITTER_NAME="$GIT_USER_NAME"
|
|||||||
export GIT_AUTHOR_EMAIL="$GIT_USER_EMAIL"
|
export GIT_AUTHOR_EMAIL="$GIT_USER_EMAIL"
|
||||||
export GIT_COMMITTER_EMAIL="$GIT_USER_EMAIL"
|
export GIT_COMMITTER_EMAIL="$GIT_USER_EMAIL"
|
||||||
|
|
||||||
|
# ── Git credential helper (HTTPS push) ────────────────────────────────────
|
||||||
|
# If GIT_PUSH_TOKEN is supplied at container creation time, configure git's
|
||||||
|
# built-in credential store so `git push` over HTTPS authenticates without
|
||||||
|
# user interaction. GIT_CLONE_URL provides the host portion of the URL used
|
||||||
|
# as the key in ~/.git-credentials.
|
||||||
|
if [ -n "$GIT_PUSH_TOKEN" ] && [ -n "$GIT_CLONE_URL" ]; then
|
||||||
|
_scheme=$(echo "$GIT_CLONE_URL" | cut -d':' -f1)
|
||||||
|
_host=$(echo "$GIT_CLONE_URL" | sed 's|^https\?://||' | cut -d'/' -f1)
|
||||||
|
git config --global credential.helper store
|
||||||
|
printf '%s://x-access-token:%s@%s\n' "$_scheme" "$GIT_PUSH_TOKEN" "$_host" \
|
||||||
|
> /home/huskies/.git-credentials
|
||||||
|
chmod 600 /home/huskies/.git-credentials
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Frontend native deps ────────────────────────────────────────────
|
# ── Frontend native deps ────────────────────────────────────────────
|
||||||
# The project repo is bind-mounted from the host, so node_modules/
|
# The project repo is bind-mounted from the host, so node_modules/
|
||||||
# may contain native binaries for the wrong platform (e.g. darwin
|
# may contain native binaries for the wrong platform (e.g. darwin
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# Go stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with Go 1.22, gopls (official Go language server), and standard tooling.
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/go/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-go -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Official Go binary distribution — Debian's golang-go package is too old for gopls.
|
||||||
|
# Update GOVERSION to pick up a newer release.
|
||||||
|
ENV GOVERSION="1.22.3"
|
||||||
|
RUN curl -fsSL "https://go.dev/dl/go${GOVERSION}.linux-amd64.tar.gz" \
|
||||||
|
| tar -C /usr/local -xzf -
|
||||||
|
|
||||||
|
ENV PATH="/usr/local/go/bin:${PATH}"
|
||||||
|
|
||||||
|
# gopls: the official Go language server.
|
||||||
|
# GOBIN=/usr/local/bin puts the binary on the system PATH for all users.
|
||||||
|
RUN GOBIN=/usr/local/bin go install golang.org/x/tools/gopls@latest
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
# Stack detection markers for the go stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
go.mod
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
# JVM stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with OpenJDK 21, Maven, and eclipse.jdt.ls (the canonical Java/JVM LSP).
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/jvm/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-jvm -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# OpenJDK 17 (Bookworm's default LTS) and Maven for build support.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
openjdk-17-jdk-headless \
|
||||||
|
maven \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN ln -s /usr/lib/jvm/java-17-openjdk-* /usr/lib/jvm/java-17-openjdk
|
||||||
|
ENV JAVA_HOME="/usr/lib/jvm/java-17-openjdk"
|
||||||
|
|
||||||
|
# Eclipse JDT Language Server — canonical LSP for Java/JVM (Java, Kotlin, Groovy).
|
||||||
|
# Pin to a specific release; update JDTLS_VERSION + JDTLS_BUILD for upgrades.
|
||||||
|
# All releases: https://github.com/eclipse-jdtls/eclipse.jdt.ls/releases
|
||||||
|
ENV JDTLS_VERSION="1.60.0" \
|
||||||
|
JDTLS_BUILD="202606262232"
|
||||||
|
RUN mkdir -p /opt/jdtls \
|
||||||
|
&& curl -fsSL \
|
||||||
|
"https://download.eclipse.org/jdtls/milestones/${JDTLS_VERSION}/jdt-language-server-${JDTLS_VERSION}-${JDTLS_BUILD}.tar.gz" \
|
||||||
|
| tar -xzf - -C /opt/jdtls
|
||||||
|
|
||||||
|
# Wrapper script so `jdtls` is available as a PATH command.
|
||||||
|
RUN { \
|
||||||
|
echo '#!/bin/sh'; \
|
||||||
|
echo 'JAR=$(ls /opt/jdtls/plugins/org.eclipse.equinox.launcher_*.jar 2>/dev/null | head -1)'; \
|
||||||
|
echo 'exec java \'; \
|
||||||
|
echo ' -Declipse.application=org.eclipse.jdt.ls.core.id1 \'; \
|
||||||
|
echo ' -Dosgi.bundles.defaultStartLevel=4 \'; \
|
||||||
|
echo ' -Declipse.product=org.eclipse.jdt.ls.core.product \'; \
|
||||||
|
echo ' -Dlog.protocol=true \'; \
|
||||||
|
echo ' -Dlog.level=ALL \'; \
|
||||||
|
echo ' -jar "$JAR" \'; \
|
||||||
|
echo ' -configuration /opt/jdtls/config_linux \'; \
|
||||||
|
echo ' "$@"'; \
|
||||||
|
} > /usr/local/bin/jdtls \
|
||||||
|
&& chmod +x /usr/local/bin/jdtls
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Stack detection markers for the jvm stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
pom.xml
|
||||||
|
build.gradle
|
||||||
|
build.gradle.kts
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# Node stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with Node.js 22, TypeScript (tsc), and typescript-language-server.
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/node/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-node -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Node.js 22.x (LTS).
|
||||||
|
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
|
||||||
|
&& apt-get install -y --no-install-recommends nodejs \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# TypeScript compiler and language server for LSP-aware agents.
|
||||||
|
# tsc: TypeScript compiler (tsc --version)
|
||||||
|
# typescript-language-server: LSP server used by editors/agents
|
||||||
|
RUN npm install -g typescript typescript-language-server
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
# Stack detection markers for the node stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
# tsconfig.json is listed explicitly so TypeScript-only projects are detected
|
||||||
|
# even without a package.json at the repo root.
|
||||||
|
package.json
|
||||||
|
tsconfig.json
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Python stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with Python 3, pip, and pyright (the Microsoft Python LSP / type checker).
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/python/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-python -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Python 3 runtime and pip.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
python3 \
|
||||||
|
python3-pip \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# pyright: Microsoft's Python language server / static type checker.
|
||||||
|
# --break-system-packages is required on Debian 12+ where pip is externally
|
||||||
|
# managed; the flag is safe inside a Docker container.
|
||||||
|
RUN pip install --no-cache-dir --break-system-packages pyright
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Stack detection markers for the python stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
pyproject.toml
|
||||||
|
requirements.txt
|
||||||
|
setup.py
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# Ruby stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with Ruby, Bundler, and ruby-lsp (the Shopify Ruby language server).
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/ruby/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-ruby -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Ruby runtime, development headers (needed by native gem extensions), and Bundler.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ruby \
|
||||||
|
ruby-dev \
|
||||||
|
bundler \
|
||||||
|
build-essential \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# ruby-lsp: Shopify's Ruby language server (LSP-compliant, actively maintained).
|
||||||
|
# Installed globally so the `ruby-lsp` binary is available on PATH.
|
||||||
|
RUN gem install ruby-lsp
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
# Stack detection markers for the ruby stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
Gemfile
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# Rust stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with a full Rust toolchain, rust-analyzer, and cargo-nextest.
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/rust/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-rust -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Build tools required by rustup and many Rust crates.
|
||||||
|
# libudev-dev: serial/USB device crates (libudev-sys, serialport).
|
||||||
|
# libclang-dev: bindgen-based crates; cargo test is skipped when absent.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
pkg-config \
|
||||||
|
libssl-dev \
|
||||||
|
libudev-dev \
|
||||||
|
libclang-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV RUSTUP_HOME="/home/huskies/.rustup" \
|
||||||
|
CARGO_HOME="/home/huskies/.cargo"
|
||||||
|
|
||||||
|
# Install stable Rust + rust-analyzer component as the huskies user.
|
||||||
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
|
||||||
|
| su huskies -c "sh -s -- -y --no-modify-path --default-toolchain stable" \
|
||||||
|
&& /home/huskies/.cargo/bin/rustup component add rust-analyzer \
|
||||||
|
&& chown -R huskies:huskies /home/huskies/.rustup /home/huskies/.cargo
|
||||||
|
|
||||||
|
# cargo-nextest: fast Rust test runner used by huskies quality gates.
|
||||||
|
RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C /usr/local/bin
|
||||||
|
|
||||||
|
ENV PATH="/home/huskies/.cargo/bin:${PATH}"
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
# Stack detection markers for the rust stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
Cargo.toml
|
||||||
Generated
+945
-1215
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "huskies",
|
"name": "huskies",
|
||||||
"private": true,
|
"private": true,
|
||||||
"version": "0.11.0",
|
"version": "0.14.0",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "vite",
|
"dev": "vite",
|
||||||
@@ -32,11 +32,11 @@
|
|||||||
"@types/node": "^25.0.0",
|
"@types/node": "^25.0.0",
|
||||||
"@types/react": "^19.1.8",
|
"@types/react": "^19.1.8",
|
||||||
"@types/react-dom": "^19.1.6",
|
"@types/react-dom": "^19.1.6",
|
||||||
"@vitejs/plugin-react": "^4.6.0",
|
"@vitejs/plugin-react": "^5.2.0",
|
||||||
"@vitest/coverage-v8": "^2.1.9",
|
"@vitest/coverage-v8": "^4.1.6",
|
||||||
"jsdom": "^28.1.0",
|
"jsdom": "^28.1.0",
|
||||||
"typescript": "~5.8.3",
|
"typescript": "~5.8.3",
|
||||||
"vite": "^5.4.21",
|
"vite": "^8.0.13",
|
||||||
"vitest": "^2.1.4"
|
"vitest": "^4.1.6"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -160,6 +160,7 @@ describe("App", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("shows error when openProject fails", async () => {
|
it("shows error when openProject fails", async () => {
|
||||||
|
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
mockedApi.openProject.mockRejectedValue(new Error("Path does not exist"));
|
mockedApi.openProject.mockRejectedValue(new Error("Path does not exist"));
|
||||||
|
|
||||||
await renderApp();
|
await renderApp();
|
||||||
@@ -182,6 +183,7 @@ describe("App", () => {
|
|||||||
await waitFor(() => {
|
await waitFor(() => {
|
||||||
expect(screen.getByText(/Path does not exist/)).toBeInTheDocument();
|
expect(screen.getByText(/Path does not exist/)).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("shows known projects list", async () => {
|
it("shows known projects list", async () => {
|
||||||
|
|||||||
@@ -266,6 +266,8 @@ describe("subscribeAgentStream", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("handles malformed JSON without throwing", () => {
|
it("handles malformed JSON without throwing", () => {
|
||||||
|
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
|
|
||||||
subscribeAgentStream("42_story_test", "coder", vi.fn());
|
subscribeAgentStream("42_story_test", "coder", vi.fn());
|
||||||
|
|
||||||
expect(() => {
|
expect(() => {
|
||||||
|
|||||||
@@ -50,6 +50,29 @@ export interface AgentAssignment {
|
|||||||
status: string;
|
status: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Display column for a work item — derived server-side from `Stage::pipeline()` (story 1085). */
|
||||||
|
export type Pipeline =
|
||||||
|
| "backlog"
|
||||||
|
| "coding"
|
||||||
|
| "qa"
|
||||||
|
| "merge"
|
||||||
|
| "done"
|
||||||
|
| "closed"
|
||||||
|
| "archived";
|
||||||
|
|
||||||
|
/** Badge/indicator for a work item — derived server-side from `Stage::status()` (story 1085). */
|
||||||
|
export type Status =
|
||||||
|
| "active"
|
||||||
|
| "frozen"
|
||||||
|
| "review-hold"
|
||||||
|
| "blocked"
|
||||||
|
| "merge-failure"
|
||||||
|
| "merge-failure-final"
|
||||||
|
| "abandoned"
|
||||||
|
| "superseded"
|
||||||
|
| "rejected"
|
||||||
|
| "done";
|
||||||
|
|
||||||
/** A single item in any pipeline stage (backlog, current, QA, merge, or done). */
|
/** A single item in any pipeline stage (backlog, current, QA, merge, or done). */
|
||||||
export interface PipelineStageItem {
|
export interface PipelineStageItem {
|
||||||
story_id: string;
|
story_id: string;
|
||||||
@@ -57,6 +80,10 @@ export interface PipelineStageItem {
|
|||||||
error: string | null;
|
error: string | null;
|
||||||
merge_failure: string | null;
|
merge_failure: string | null;
|
||||||
agent: AgentAssignment | null;
|
agent: AgentAssignment | null;
|
||||||
|
/** Display column (story 1085); falls back to the bucket name on legacy servers. */
|
||||||
|
pipeline?: Pipeline;
|
||||||
|
/** Display badge (story 1085); falls back to derived `blocked`/`frozen` on legacy servers. */
|
||||||
|
status?: Status;
|
||||||
review_hold: boolean | null;
|
review_hold: boolean | null;
|
||||||
qa: string | null;
|
qa: string | null;
|
||||||
depends_on: number[] | null;
|
depends_on: number[] | null;
|
||||||
@@ -214,6 +241,8 @@ export interface WorkItemContent {
|
|||||||
stage: string;
|
stage: string;
|
||||||
name: string;
|
name: string;
|
||||||
agent: string | null;
|
agent: string | null;
|
||||||
|
/** Origin JSON string (story 1088), or null for pre-origin items. */
|
||||||
|
origin: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Result for a single test case from the server's test runner. */
|
/** Result for a single test case from the server's test runner. */
|
||||||
|
|||||||
@@ -24,10 +24,38 @@ export interface GatewayInfo {
|
|||||||
projects: GatewayProject[];
|
projects: GatewayProject[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Display column for a work item — derived server-side from `Stage::pipeline()` (story 1085). */
|
||||||
|
export type Pipeline =
|
||||||
|
| "backlog"
|
||||||
|
| "coding"
|
||||||
|
| "qa"
|
||||||
|
| "merge"
|
||||||
|
| "done"
|
||||||
|
| "closed"
|
||||||
|
| "archived";
|
||||||
|
|
||||||
|
/** Badge/indicator for a work item — derived server-side from `Stage::status()` (story 1085). */
|
||||||
|
export type Status =
|
||||||
|
| "active"
|
||||||
|
| "frozen"
|
||||||
|
| "review-hold"
|
||||||
|
| "blocked"
|
||||||
|
| "merge-failure"
|
||||||
|
| "merge-failure-final"
|
||||||
|
| "abandoned"
|
||||||
|
| "superseded"
|
||||||
|
| "rejected"
|
||||||
|
| "done";
|
||||||
|
|
||||||
export interface PipelineItem {
|
export interface PipelineItem {
|
||||||
story_id: string;
|
story_id: string;
|
||||||
name: string;
|
name: string;
|
||||||
|
/** Legacy stage string (kept for back-compat); prefer `pipeline` + `status`. */
|
||||||
stage: string;
|
stage: string;
|
||||||
|
/** Display column (story 1085). Optional until all servers are upgraded. */
|
||||||
|
pipeline?: Pipeline;
|
||||||
|
/** Display badge (story 1085). Optional until all servers are upgraded. */
|
||||||
|
status?: Status;
|
||||||
agent?: { agent_name: string; model: string; status: string } | null;
|
agent?: { agent_name: string; model: string; status: string } | null;
|
||||||
blocked?: boolean;
|
blocked?: boolean;
|
||||||
retry_count?: number;
|
retry_count?: number;
|
||||||
|
|||||||
@@ -472,9 +472,16 @@ describe("Slash command handling (Story 374)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("Story 1058: WebSocket errors do not appear in chat", () => {
|
describe("Story 1058: WebSocket errors do not appear in chat", () => {
|
||||||
|
let consoleSpy: ReturnType<typeof vi.spyOn>;
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
capturedWsHandlers = null;
|
capturedWsHandlers = null;
|
||||||
setupMocks();
|
setupMocks();
|
||||||
|
consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
consoleSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not add a chat message when onError is called", async () => {
|
it("does not add a chat message when onError is called", async () => {
|
||||||
|
|||||||
@@ -69,29 +69,34 @@ describe("StoryRow", () => {
|
|||||||
expect(screen.getByText("awaiting-slot (#2)")).toBeInTheDocument();
|
expect(screen.getByText("awaiting-slot (#2)")).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
// AC2: failure kind labels derived from merge_failure string
|
// Story 1085: failure kind no longer derived from substring. Items in
|
||||||
it("shows ConflictDetected for merge_failure with conflict text", () => {
|
// the merge_failure / merge_failure_final status get a generic FAILED badge;
|
||||||
|
// the kind detail is exposed via the typed `status` field for callers that
|
||||||
|
// need it (instead of being squeezed into the badge text).
|
||||||
|
it("shows ✕ FAILED badge for merge-failure status", () => {
|
||||||
const item: PipelineItem = {
|
const item: PipelineItem = {
|
||||||
story_id: "73_story_conflict",
|
story_id: "73_story_conflict",
|
||||||
name: "Conflict Story",
|
name: "Conflict Story",
|
||||||
stage: "merge",
|
stage: "merge",
|
||||||
blocked: true,
|
pipeline: "merge",
|
||||||
|
status: "merge-failure",
|
||||||
merge_failure: "Merge conflict: conflicts detected",
|
merge_failure: "Merge conflict: conflicts detected",
|
||||||
};
|
};
|
||||||
render(<StoryRow item={item} />);
|
render(<StoryRow item={item} />);
|
||||||
expect(screen.getByText("ConflictDetected")).toBeInTheDocument();
|
expect(screen.getByText("✕ FAILED")).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("shows GatesFailed for merge_failure with quality gates text", () => {
|
it("shows ⛔ FAILED (FINAL) badge for merge-failure-final status", () => {
|
||||||
const item: PipelineItem = {
|
const item: PipelineItem = {
|
||||||
story_id: "74_story_gates",
|
story_id: "74_story_gates",
|
||||||
name: "Gates Failed Story",
|
name: "Gates Failed Story",
|
||||||
stage: "merge",
|
stage: "merge",
|
||||||
blocked: true,
|
pipeline: "merge",
|
||||||
|
status: "merge-failure-final",
|
||||||
merge_failure: "Quality gates failed: cargo test failed",
|
merge_failure: "Quality gates failed: cargo test failed",
|
||||||
};
|
};
|
||||||
render(<StoryRow item={item} />);
|
render(<StoryRow item={item} />);
|
||||||
expect(screen.getByText("GatesFailed")).toBeInTheDocument();
|
expect(screen.getByText("⛔ FAILED (FINAL)")).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("shows RECOVERING badge for merge_failure item with running mergemaster", () => {
|
it("shows RECOVERING badge for merge_failure item with running mergemaster", () => {
|
||||||
@@ -163,4 +168,36 @@ describe("StoryRow", () => {
|
|||||||
render(<StoryRow item={item} />);
|
render(<StoryRow item={item} />);
|
||||||
expect(screen.getByText("⊘ BLOCKED")).toBeInTheDocument();
|
expect(screen.getByText("⊘ BLOCKED")).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Story 1085 AC 4 — Frozen items remain visible in their underlying column
|
||||||
|
// with a frozen indicator. The server hands us `pipeline: "coding"` for a
|
||||||
|
// frozen-while-coding story and the badge is decorated separately.
|
||||||
|
it("shows ❄ FROZEN badge for a frozen item (column stays as underlying pipeline)", () => {
|
||||||
|
const item: PipelineItem = {
|
||||||
|
story_id: "70_story_frozen_coding",
|
||||||
|
name: "Paused Coding Story",
|
||||||
|
stage: "current",
|
||||||
|
pipeline: "coding",
|
||||||
|
status: "frozen",
|
||||||
|
};
|
||||||
|
render(<StoryRow item={item} />);
|
||||||
|
expect(screen.getByText("❄ FROZEN")).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Story 1085 AC 4 (subsumes 1052) — Done items must never get a
|
||||||
|
// MergeFailure indicator, even if a stale `merge_failure` string is present.
|
||||||
|
it("done items render Done badge, never MergeFailure", () => {
|
||||||
|
const item: PipelineItem = {
|
||||||
|
story_id: "71_story_done",
|
||||||
|
name: "Completed Story",
|
||||||
|
stage: "done",
|
||||||
|
pipeline: "done",
|
||||||
|
status: "done",
|
||||||
|
merge_failure: "ignored stale string",
|
||||||
|
};
|
||||||
|
render(<StoryRow item={item} />);
|
||||||
|
expect(screen.getByText("Done")).toBeInTheDocument();
|
||||||
|
expect(screen.queryByText("✕ FAILED")).not.toBeInTheDocument();
|
||||||
|
expect(screen.queryByText(/FAILED/)).not.toBeInTheDocument();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -14,9 +14,42 @@ import {
|
|||||||
type JoinedAgent,
|
type JoinedAgent,
|
||||||
type GatewayProject,
|
type GatewayProject,
|
||||||
type AllProjectsPipeline,
|
type AllProjectsPipeline,
|
||||||
|
type Pipeline,
|
||||||
type PipelineItem,
|
type PipelineItem,
|
||||||
|
type Status,
|
||||||
} from "../api/gateway";
|
} from "../api/gateway";
|
||||||
|
|
||||||
|
/// Resolve an item's pipeline column. Servers running the new (story 1085)
|
||||||
|
/// backend send `pipeline`; older servers only send `stage` so we fall back to
|
||||||
|
/// mapping the bucket name onto the new column vocabulary.
|
||||||
|
function itemPipeline(item: PipelineItem): Pipeline {
|
||||||
|
if (item.pipeline) return item.pipeline;
|
||||||
|
switch (item.stage) {
|
||||||
|
case "current":
|
||||||
|
return "coding";
|
||||||
|
case "qa":
|
||||||
|
return "qa";
|
||||||
|
case "merge":
|
||||||
|
return "merge";
|
||||||
|
case "done":
|
||||||
|
return "done";
|
||||||
|
case "archived":
|
||||||
|
return "archived";
|
||||||
|
default:
|
||||||
|
return "backlog";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve an item's badge. Falls back to `merge_failure`/`blocked` on
|
||||||
|
/// legacy servers that don't yet emit `status`.
|
||||||
|
function itemStatus(item: PipelineItem): Status {
|
||||||
|
if (item.status) return item.status;
|
||||||
|
if (item.merge_failure) return "merge-failure";
|
||||||
|
if (item.blocked) return "blocked";
|
||||||
|
if (item.stage === "done") return "done";
|
||||||
|
return "active";
|
||||||
|
}
|
||||||
|
|
||||||
const { useCallback, useEffect, useRef, useState } = React;
|
const { useCallback, useEffect, useRef, useState } = React;
|
||||||
|
|
||||||
/// Seconds of silence before an agent is considered disconnected.
|
/// Seconds of silence before an agent is considered disconnected.
|
||||||
@@ -48,72 +81,86 @@ const STATUS_LABELS: Record<AgentStatus, string> = {
|
|||||||
disconnected: "Disconnected",
|
disconnected: "Disconnected",
|
||||||
};
|
};
|
||||||
|
|
||||||
const STAGE_COLORS: Record<string, string> = {
|
const PIPELINE_COLORS: Record<Pipeline, string> = {
|
||||||
backlog: "#8b949e",
|
backlog: "#8b949e",
|
||||||
current: "#3fb950",
|
coding: "#3fb950",
|
||||||
qa: "#d2a679",
|
qa: "#d2a679",
|
||||||
merge: "#79c0ff",
|
merge: "#79c0ff",
|
||||||
done: "#6e7681",
|
done: "#6e7681",
|
||||||
|
closed: "#6e7681",
|
||||||
archived: "#6e7681",
|
archived: "#6e7681",
|
||||||
};
|
};
|
||||||
|
|
||||||
const STAGE_LABELS: Record<string, string> = {
|
const PIPELINE_LABELS: Record<Pipeline, string> = {
|
||||||
backlog: "Backlog",
|
backlog: "Backlog",
|
||||||
current: "In Progress",
|
coding: "In Progress",
|
||||||
qa: "QA",
|
qa: "QA",
|
||||||
merge: "Merging",
|
merge: "Merging",
|
||||||
done: "Done",
|
done: "Done",
|
||||||
|
closed: "Closed",
|
||||||
archived: "Archived",
|
archived: "Archived",
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Derive a short label from a merge failure string based on the failure kind.
|
|
||||||
function mergeFailureKindLabel(failure: string): string {
|
|
||||||
if (failure.includes("Merge conflict") || failure.includes("CONFLICT")) {
|
|
||||||
return "ConflictDetected";
|
|
||||||
}
|
|
||||||
if (failure.includes("Quality gates failed") || failure.includes("gates failed")) {
|
|
||||||
return "GatesFailed";
|
|
||||||
}
|
|
||||||
if (failure.includes("no code changes") || failure.includes("empty diff")) {
|
|
||||||
return "EmptyDiff";
|
|
||||||
}
|
|
||||||
if (failure.includes("No commits")) {
|
|
||||||
return "NoCommits";
|
|
||||||
}
|
|
||||||
return "✕ FAILED";
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A single story row inside a project pipeline card.
|
/// A single story row inside a project pipeline card.
|
||||||
/** Render one story row in a gateway-aggregate panel: `#<id> <name>` with stage badge. */
|
/** Render one story row in a gateway-aggregate panel: `#<id> <name>` with status badge. */
|
||||||
export function StoryRow({ item, mergeQueuePos }: { item: PipelineItem; mergeQueuePos?: number }) {
|
export function StoryRow({ item, mergeQueuePos }: { item: PipelineItem; mergeQueuePos?: number }) {
|
||||||
const isStuck = item.merge_failure != null || item.blocked;
|
const pipeline = itemPipeline(item);
|
||||||
const isMergeActive = item.stage === "merge" && !isStuck && item.agent?.status === "running";
|
const status = itemStatus(item);
|
||||||
|
const agentStatus = item.agent?.status;
|
||||||
|
|
||||||
let color: string;
|
let color: string;
|
||||||
let label: string;
|
let label: string;
|
||||||
|
let frozenPrefix = "";
|
||||||
|
|
||||||
if (isMergeActive) {
|
// Frozen items keep their underlying pipeline column but get a ❄️ badge.
|
||||||
color = "#58a6ff";
|
// (AC 4 — story 1085, subsumes the freeze-hides-item bug.)
|
||||||
label = "▶ MERGING";
|
if (status === "frozen") {
|
||||||
} else if (isStuck) {
|
color = "#79c0ff";
|
||||||
const agentStatus = item.agent?.status;
|
label = "❄ FROZEN";
|
||||||
|
frozenPrefix = "❄ ";
|
||||||
|
} else if (status === "merge-failure" || status === "merge-failure-final") {
|
||||||
|
// Done items never reach this branch — `Stage::status()` returns
|
||||||
|
// `Status::Done` for done items (AC 4).
|
||||||
if (agentStatus === "running") {
|
if (agentStatus === "running") {
|
||||||
color = "#e3b341";
|
color = "#e3b341";
|
||||||
label = "⟳ RECOVERING";
|
label = "⟳ RECOVERING";
|
||||||
} else if (agentStatus === "pending") {
|
} else if (agentStatus === "pending") {
|
||||||
color = "#e3b341";
|
color = "#e3b341";
|
||||||
label = "⏳ QUEUED";
|
label = "⏳ QUEUED";
|
||||||
} else if (item.merge_failure != null) {
|
} else {
|
||||||
color = "#f85149";
|
color = "#f85149";
|
||||||
label = mergeFailureKindLabel(item.merge_failure);
|
label = status === "merge-failure-final" ? "⛔ FAILED (FINAL)" : "✕ FAILED";
|
||||||
|
}
|
||||||
|
} else if (status === "blocked") {
|
||||||
|
if (agentStatus === "running") {
|
||||||
|
color = "#e3b341";
|
||||||
|
label = "⟳ RECOVERING";
|
||||||
|
} else if (agentStatus === "pending") {
|
||||||
|
color = "#e3b341";
|
||||||
|
label = "⏳ QUEUED";
|
||||||
} else {
|
} else {
|
||||||
color = "#f85149";
|
color = "#f85149";
|
||||||
label = "⊘ BLOCKED";
|
label = "⊘ BLOCKED";
|
||||||
}
|
}
|
||||||
} else if (item.stage === "merge" && item.agent?.status === "pending") {
|
} else if (status === "review-hold") {
|
||||||
|
color = "#d2a679";
|
||||||
|
label = "REVIEW HOLD";
|
||||||
|
} else if (status === "abandoned") {
|
||||||
|
color = "#6e7681";
|
||||||
|
label = "ABANDONED";
|
||||||
|
} else if (status === "superseded") {
|
||||||
|
color = "#6e7681";
|
||||||
|
label = "SUPERSEDED";
|
||||||
|
} else if (status === "rejected") {
|
||||||
|
color = "#f85149";
|
||||||
|
label = "REJECTED";
|
||||||
|
} else if (pipeline === "merge" && agentStatus === "running") {
|
||||||
|
color = "#58a6ff";
|
||||||
|
label = "▶ MERGING";
|
||||||
|
} else if (pipeline === "merge" && agentStatus === "pending") {
|
||||||
color = "#e3b341";
|
color = "#e3b341";
|
||||||
label = "⏳ QUEUED";
|
label = "⏳ QUEUED";
|
||||||
} else if (item.stage === "merge") {
|
} else if (pipeline === "merge") {
|
||||||
color = "#6e7681";
|
color = "#6e7681";
|
||||||
if (mergeQueuePos === 1) {
|
if (mergeQueuePos === 1) {
|
||||||
label = "NEXT IN QUEUE";
|
label = "NEXT IN QUEUE";
|
||||||
@@ -123,10 +170,11 @@ export function StoryRow({ item, mergeQueuePos }: { item: PipelineItem; mergeQue
|
|||||||
label = "awaiting-slot";
|
label = "awaiting-slot";
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
color = STAGE_COLORS[item.stage] ?? "#8b949e";
|
color = PIPELINE_COLORS[pipeline] ?? "#8b949e";
|
||||||
label = STAGE_LABELS[item.stage] ?? item.stage;
|
label = PIPELINE_LABELS[pipeline] ?? pipeline;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const isMergeActive = pipeline === "merge" && status === "active" && agentStatus === "running";
|
||||||
const idNum = item.story_id.match(/^(\d+)/)?.[1];
|
const idNum = item.story_id.match(/^(\d+)/)?.[1];
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -158,7 +206,7 @@ export function StoryRow({ item, mergeQueuePos }: { item: PipelineItem; mergeQue
|
|||||||
</span>
|
</span>
|
||||||
<span style={{ color: "#e6edf3", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>
|
<span style={{ color: "#e6edf3", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>
|
||||||
{idNum && <span style={{ color: "#8b949e", fontFamily: "monospace" }}>#{idNum}{" "}</span>}
|
{idNum && <span style={{ color: "#8b949e", fontFamily: "monospace" }}>#{idNum}{" "}</span>}
|
||||||
{item.name}
|
{frozenPrefix}{item.name}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
@@ -388,6 +436,8 @@ function aggregateItems(
|
|||||||
story_id: b.story_id,
|
story_id: b.story_id,
|
||||||
name: b.name,
|
name: b.name,
|
||||||
stage: "backlog",
|
stage: "backlog",
|
||||||
|
pipeline: "backlog" as Pipeline,
|
||||||
|
status: "active" as Status,
|
||||||
})),
|
})),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -395,14 +445,14 @@ function aggregateItems(
|
|||||||
return {
|
return {
|
||||||
project,
|
project,
|
||||||
items: (status.active ?? []).filter(
|
items: (status.active ?? []).filter(
|
||||||
(i) => i.stage !== "done",
|
(i) => itemPipeline(i) !== "done",
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (tab === "done") {
|
if (tab === "done") {
|
||||||
return {
|
return {
|
||||||
project,
|
project,
|
||||||
items: (status.active ?? []).filter((i) => i.stage === "done"),
|
items: (status.active ?? []).filter((i) => itemPipeline(i) === "done"),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
// archived
|
// archived
|
||||||
@@ -419,12 +469,12 @@ function tabCount(pipeline: AllProjectsPipeline, tab: TabKey): number {
|
|||||||
if (tab === "in-progress") {
|
if (tab === "in-progress") {
|
||||||
return (
|
return (
|
||||||
sum +
|
sum +
|
||||||
(status.active ?? []).filter((i) => i.stage !== "done").length
|
(status.active ?? []).filter((i) => itemPipeline(i) !== "done").length
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (tab === "done") {
|
if (tab === "done") {
|
||||||
return (
|
return (
|
||||||
sum + (status.active ?? []).filter((i) => i.stage === "done").length
|
sum + (status.active ?? []).filter((i) => itemPipeline(i) === "done").length
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return sum + (status.archived ?? []).length;
|
return sum + (status.archived ?? []).length;
|
||||||
@@ -518,13 +568,16 @@ function ProjectStoryRow({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const IN_PROGRESS_STAGE_LABELS: Record<string, string> = {
|
const IN_PROGRESS_PIPELINE_LABELS: Record<"coding" | "qa" | "merge", string> = {
|
||||||
current: "Coding",
|
coding: "Coding",
|
||||||
qa: "QA",
|
qa: "QA",
|
||||||
merge: "Merging",
|
merge: "Merging",
|
||||||
};
|
};
|
||||||
|
|
||||||
/// In Progress tab content — items grouped by stage (coding / qa / merging).
|
/// In Progress tab content — items grouped by their `pipeline` column.
|
||||||
|
///
|
||||||
|
/// Frozen items appear in the column corresponding to their underlying
|
||||||
|
/// `Stage::resume_to` (server-side), so they always show up in-place.
|
||||||
function InProgressTabContent({
|
function InProgressTabContent({
|
||||||
groups,
|
groups,
|
||||||
}: {
|
}: {
|
||||||
@@ -535,25 +588,22 @@ function InProgressTabContent({
|
|||||||
);
|
);
|
||||||
const multiProject = new Set(allItems.map((x) => x.project)).size > 1;
|
const multiProject = new Set(allItems.map((x) => x.project)).size > 1;
|
||||||
|
|
||||||
const byStage = {
|
const byPipeline = {
|
||||||
current: allItems.filter((x) => x.item.stage === "current"),
|
coding: allItems.filter((x) => itemPipeline(x.item) === "coding"),
|
||||||
qa: allItems.filter((x) => x.item.stage === "qa"),
|
qa: allItems.filter((x) => itemPipeline(x.item) === "qa"),
|
||||||
merge: allItems.filter((x) => x.item.stage === "merge"),
|
merge: allItems.filter((x) => itemPipeline(x.item) === "merge"),
|
||||||
};
|
};
|
||||||
|
|
||||||
const stages = (["current", "qa", "merge"] as const).filter(
|
const pipelines = (["coding", "qa", "merge"] as const).filter(
|
||||||
(s) => byStage[s].length > 0,
|
(p) => byPipeline[p].length > 0,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Compute queue position among clean awaiting merge items (Stage::Merge, no failure, no running agent).
|
// Compute queue position among "clean" awaiting-merge items: pipeline=merge,
|
||||||
|
// status=active, and no agent currently running.
|
||||||
const mergeQueuePosMap = new Map<string, number>();
|
const mergeQueuePosMap = new Map<string, number>();
|
||||||
let queuePos = 0;
|
let queuePos = 0;
|
||||||
for (const { project, item } of byStage.merge) {
|
for (const { project, item } of byPipeline.merge) {
|
||||||
if (
|
if (itemStatus(item) === "active" && item.agent?.status !== "running") {
|
||||||
!item.blocked &&
|
|
||||||
!item.merge_failure &&
|
|
||||||
item.agent?.status !== "running"
|
|
||||||
) {
|
|
||||||
queuePos += 1;
|
queuePos += 1;
|
||||||
mergeQueuePosMap.set(`${project}:${item.story_id}`, queuePos);
|
mergeQueuePosMap.set(`${project}:${item.story_id}`, queuePos);
|
||||||
}
|
}
|
||||||
@@ -569,33 +619,33 @@ function InProgressTabContent({
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div>
|
<div>
|
||||||
{stages.map((stage) => (
|
{pipelines.map((p) => (
|
||||||
<div key={stage} style={{ marginBottom: "20px" }}>
|
<div key={p} style={{ marginBottom: "20px" }}>
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
fontSize: "0.8em",
|
fontSize: "0.8em",
|
||||||
fontWeight: 600,
|
fontWeight: 600,
|
||||||
color: STAGE_COLORS[stage] ?? "#8b949e",
|
color: PIPELINE_COLORS[p] ?? "#8b949e",
|
||||||
textTransform: "uppercase",
|
textTransform: "uppercase",
|
||||||
letterSpacing: "0.06em",
|
letterSpacing: "0.06em",
|
||||||
marginBottom: "8px",
|
marginBottom: "8px",
|
||||||
paddingBottom: "4px",
|
paddingBottom: "4px",
|
||||||
borderBottom: `1px solid ${STAGE_COLORS[stage] ?? "#8b949e"}33`,
|
borderBottom: `1px solid ${PIPELINE_COLORS[p] ?? "#8b949e"}33`,
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{IN_PROGRESS_STAGE_LABELS[stage]}{" "}
|
{IN_PROGRESS_PIPELINE_LABELS[p]}{" "}
|
||||||
<span style={{ color: "#6e7681" }}>
|
<span style={{ color: "#6e7681" }}>
|
||||||
({byStage[stage].length})
|
({byPipeline[p].length})
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
{byStage[stage].map(({ project, item }) => (
|
{byPipeline[p].map(({ project, item }) => (
|
||||||
<ProjectStoryRow
|
<ProjectStoryRow
|
||||||
key={`${project}:${item.story_id}`}
|
key={`${project}:${item.story_id}`}
|
||||||
project={project}
|
project={project}
|
||||||
item={item}
|
item={item}
|
||||||
showProject={multiProject}
|
showProject={multiProject}
|
||||||
mergeQueuePos={
|
mergeQueuePos={
|
||||||
stage === "merge"
|
p === "merge"
|
||||||
? mergeQueuePosMap.get(`${project}:${item.story_id}`)
|
? mergeQueuePosMap.get(`${project}:${item.story_id}`)
|
||||||
: undefined
|
: undefined
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ const DEFAULT_CONTENT = {
|
|||||||
stage: "current",
|
stage: "current",
|
||||||
name: "Big Title Story",
|
name: "Big Title Story",
|
||||||
agent: null,
|
agent: null,
|
||||||
|
origin: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ const DEFAULT_CONTENT = {
|
|||||||
stage: "current",
|
stage: "current",
|
||||||
name: "Big Title Story",
|
name: "Big Title Story",
|
||||||
agent: null,
|
agent: null,
|
||||||
|
origin: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
const sampleTestResults: TestResultsResponse = {
|
const sampleTestResults: TestResultsResponse = {
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ const DEFAULT_CONTENT = {
|
|||||||
stage: "current",
|
stage: "current",
|
||||||
name: "Big Title Story",
|
name: "Big Title Story",
|
||||||
agent: null,
|
agent: null,
|
||||||
|
origin: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
@@ -127,6 +128,7 @@ describe("WorkItemDetailPanel", () => {
|
|||||||
stage: "current",
|
stage: "current",
|
||||||
name: "My Story Name",
|
name: "My Story Name",
|
||||||
agent: null,
|
agent: null,
|
||||||
|
origin: null,
|
||||||
});
|
});
|
||||||
render(
|
render(
|
||||||
<WorkItemDetailPanel
|
<WorkItemDetailPanel
|
||||||
@@ -146,6 +148,7 @@ describe("WorkItemDetailPanel", () => {
|
|||||||
stage: "current",
|
stage: "current",
|
||||||
name: "My Story Name",
|
name: "My Story Name",
|
||||||
agent: null,
|
agent: null,
|
||||||
|
origin: null,
|
||||||
});
|
});
|
||||||
render(
|
render(
|
||||||
<WorkItemDetailPanel
|
<WorkItemDetailPanel
|
||||||
@@ -164,6 +167,7 @@ describe("WorkItemDetailPanel", () => {
|
|||||||
stage: "current",
|
stage: "current",
|
||||||
name: "My Story Name",
|
name: "My Story Name",
|
||||||
agent: null,
|
agent: null,
|
||||||
|
origin: null,
|
||||||
});
|
});
|
||||||
render(
|
render(
|
||||||
<WorkItemDetailPanel
|
<WorkItemDetailPanel
|
||||||
@@ -186,6 +190,7 @@ describe("WorkItemDetailPanel", () => {
|
|||||||
stage: "current",
|
stage: "current",
|
||||||
name: "My Story Name",
|
name: "My Story Name",
|
||||||
agent: null,
|
agent: null,
|
||||||
|
origin: null,
|
||||||
});
|
});
|
||||||
render(
|
render(
|
||||||
<WorkItemDetailPanel
|
<WorkItemDetailPanel
|
||||||
|
|||||||
@@ -20,6 +20,26 @@ import { stripDisplayContent } from "./workItemDetailPanelUtils";
|
|||||||
|
|
||||||
const { useCallback, useEffect, useRef, useState } = React;
|
const { useCallback, useEffect, useRef, useState } = React;
|
||||||
|
|
||||||
|
/** Parse and format an origin JSON string for display. */
|
||||||
|
function formatOrigin(origin: string | null): string {
|
||||||
|
if (!origin) return "unknown";
|
||||||
|
try {
|
||||||
|
const obj = JSON.parse(origin) as {
|
||||||
|
kind?: string;
|
||||||
|
id?: string;
|
||||||
|
ts?: number;
|
||||||
|
};
|
||||||
|
const kind = obj.kind ?? "unknown";
|
||||||
|
const id = obj.id ? ` (${obj.id})` : "";
|
||||||
|
const ts = obj.ts
|
||||||
|
? ` at ${new Date(obj.ts * 1000).toISOString().replace("T", " ").slice(0, 19)}Z`
|
||||||
|
: "";
|
||||||
|
return `${kind}${id}${ts}`;
|
||||||
|
} catch {
|
||||||
|
return origin;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
interface WorkItemDetailPanelProps {
|
interface WorkItemDetailPanelProps {
|
||||||
storyId: string;
|
storyId: string;
|
||||||
pipelineVersion: number;
|
pipelineVersion: number;
|
||||||
@@ -38,6 +58,7 @@ export function WorkItemDetailPanel({
|
|||||||
const [stage, setStage] = useState<string>("");
|
const [stage, setStage] = useState<string>("");
|
||||||
const [name, setName] = useState<string | null>(null);
|
const [name, setName] = useState<string | null>(null);
|
||||||
const [assignedAgent, setAssignedAgent] = useState<string | null>(null);
|
const [assignedAgent, setAssignedAgent] = useState<string | null>(null);
|
||||||
|
const [origin, setOrigin] = useState<string | null>(null);
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const [agentInfo, setAgentInfo] = useState<AgentInfo | null>(null);
|
const [agentInfo, setAgentInfo] = useState<AgentInfo | null>(null);
|
||||||
@@ -63,6 +84,7 @@ export function WorkItemDetailPanel({
|
|||||||
setStage(data.stage);
|
setStage(data.stage);
|
||||||
setName(data.name);
|
setName(data.name);
|
||||||
setAssignedAgent(data.agent);
|
setAssignedAgent(data.agent);
|
||||||
|
setOrigin(data.origin);
|
||||||
})
|
})
|
||||||
.catch((err: unknown) => {
|
.catch((err: unknown) => {
|
||||||
setError(err instanceof Error ? err.message : "Failed to load content");
|
setError(err instanceof Error ? err.message : "Failed to load content");
|
||||||
@@ -289,6 +311,19 @@ export function WorkItemDetailPanel({
|
|||||||
|
|
||||||
<TestResultsSection testResults={testResults} />
|
<TestResultsSection testResults={testResults} />
|
||||||
|
|
||||||
|
{!loading && (
|
||||||
|
<div
|
||||||
|
data-testid="detail-panel-origin"
|
||||||
|
style={{
|
||||||
|
fontSize: "0.75em",
|
||||||
|
color: "#555",
|
||||||
|
fontFamily: "monospace",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
origin: {formatOrigin(origin)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
display: "flex",
|
display: "flex",
|
||||||
|
|||||||
@@ -227,6 +227,7 @@ describe("usePathCompletion hook", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("sets completionError when listDirectoryAbsolute throws an Error", async () => {
|
it("sets completionError when listDirectoryAbsolute throws an Error", async () => {
|
||||||
|
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
mockListDir.mockRejectedValue(new Error("Permission denied"));
|
mockListDir.mockRejectedValue(new Error("Permission denied"));
|
||||||
|
|
||||||
const { result } = renderHook(() =>
|
const { result } = renderHook(() =>
|
||||||
@@ -242,9 +243,13 @@ describe("usePathCompletion hook", () => {
|
|||||||
await waitFor(() => {
|
await waitFor(() => {
|
||||||
expect(result.current.completionError).toBe("Permission denied");
|
expect(result.current.completionError).toBe("Permission denied");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
expect(errorSpy).toHaveBeenCalledWith(new Error("Permission denied"));
|
||||||
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("sets generic completionError when listDirectoryAbsolute throws a non-Error", async () => {
|
it("sets generic completionError when listDirectoryAbsolute throws a non-Error", async () => {
|
||||||
|
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
mockListDir.mockRejectedValue("some string error");
|
mockListDir.mockRejectedValue("some string error");
|
||||||
|
|
||||||
const { result } = renderHook(() =>
|
const { result } = renderHook(() =>
|
||||||
@@ -262,6 +267,9 @@ describe("usePathCompletion hook", () => {
|
|||||||
"Failed to compute suggestion.",
|
"Failed to compute suggestion.",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
expect(errorSpy).toHaveBeenCalledWith("some string error");
|
||||||
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clears suggestionTail when selected match path does not start with input", async () => {
|
it("clears suggestionTail when selected match path does not start with input", async () => {
|
||||||
|
|||||||
Executable
+37
@@ -0,0 +1,37 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Build all project images in dependency order:
|
||||||
|
# huskies → huskies-project-base → huskies-project-<stack> (one per stack fragment)
|
||||||
|
#
|
||||||
|
# Run this after `script/docker_rebuild` or whenever you add a new stack.
|
||||||
|
# Safe to re-run: each step re-tags the image with the latest layers.
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
if [[ -f .env ]]; then
|
||||||
|
set -a
|
||||||
|
source .env
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
|
||||||
|
CACHE_FLAG=""
|
||||||
|
if [[ "${1:-}" == "--no-cache" ]]; then
|
||||||
|
CACHE_FLAG="--no-cache"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Building huskies"
|
||||||
|
docker build $CACHE_FLAG -t huskies -f docker/Dockerfile .
|
||||||
|
|
||||||
|
echo "==> Building huskies-project-base"
|
||||||
|
docker build $CACHE_FLAG -t huskies-project-base -f docker/Dockerfile.base .
|
||||||
|
|
||||||
|
for fragment in docker/stacks/*/Dockerfile.fragment; do
|
||||||
|
stack=$(basename "$(dirname "$fragment")")
|
||||||
|
image="huskies-project-${stack}"
|
||||||
|
echo "==> Building ${image}"
|
||||||
|
(printf 'FROM huskies-project-base\n'; cat "$fragment") \
|
||||||
|
| docker build $CACHE_FLAG -t "$image" -
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "All project images built."
|
||||||
Executable
+121
@@ -0,0 +1,121 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# script/ci-publish-artifact — upload a built sled artifact to a release
|
||||||
|
# channel and verify the channel's manifest reflects the published commit.
|
||||||
|
#
|
||||||
|
# Usage: script/ci-publish-artifact <artifact-path> [git-hash]
|
||||||
|
#
|
||||||
|
# git-hash defaults to `git rev-parse --short HEAD` when omitted.
|
||||||
|
#
|
||||||
|
# Required env:
|
||||||
|
# HUSKIES_CHANNEL_URL Base URL of the release channel (Gitea secret).
|
||||||
|
# HUSKIES_CHANNEL_TOKEN Bearer token authorised to publish (Gitea secret).
|
||||||
|
#
|
||||||
|
# Protocol against the channel host:
|
||||||
|
# POST {HUSKIES_CHANNEL_URL}/<artifact-name>
|
||||||
|
# Headers: Authorization: Bearer <token>, X-Git-Hash: <hash>
|
||||||
|
# Body: raw artifact bytes
|
||||||
|
# GET {HUSKIES_CHANNEL_URL}/manifest.json
|
||||||
|
# Headers: Authorization: Bearer <token>
|
||||||
|
# Body: JSON object with a "git_hash" field
|
||||||
|
#
|
||||||
|
# 5xx responses and network errors are retried with backoff; 4xx responses
|
||||||
|
# (including auth failures) fail immediately since retrying won't fix them.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ARTIFACT_PATH="${1:?Usage: script/ci-publish-artifact <artifact-path> [git-hash]}"
|
||||||
|
GIT_HASH="${2:-$(git rev-parse --short HEAD)}"
|
||||||
|
|
||||||
|
if [ -z "${HUSKIES_CHANNEL_URL:-}" ]; then
|
||||||
|
echo "Error: HUSKIES_CHANNEL_URL is not set." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "${HUSKIES_CHANNEL_TOKEN:-}" ]; then
|
||||||
|
echo "Error: HUSKIES_CHANNEL_TOKEN is not set." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -f "$ARTIFACT_PATH" ]; then
|
||||||
|
echo "Error: artifact not found at $ARTIFACT_PATH" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ARTIFACT_NAME="$(basename "$ARTIFACT_PATH")"
|
||||||
|
CHANNEL_URL="${HUSKIES_CHANNEL_URL%/}"
|
||||||
|
UPLOAD_URL="${CHANNEL_URL}/${ARTIFACT_NAME}"
|
||||||
|
MANIFEST_URL="${CHANNEL_URL}/manifest.json"
|
||||||
|
|
||||||
|
UPLOAD_MAX_ATTEMPTS="${HUSKIES_CI_PUBLISH_MAX_ATTEMPTS:-3}"
|
||||||
|
UPLOAD_BACKOFF_SECS="${HUSKIES_CI_PUBLISH_BACKOFF_SECS:-1}"
|
||||||
|
MANIFEST_MAX_ATTEMPTS=3
|
||||||
|
MANIFEST_BACKOFF_SECS=1
|
||||||
|
|
||||||
|
RESPONSE_FILE="$(mktemp)"
|
||||||
|
trap 'rm -f "$RESPONSE_FILE"' EXIT
|
||||||
|
|
||||||
|
# ── Upload ────────────────────────────────────────────────────────────────
|
||||||
|
attempt=1
|
||||||
|
while :; do
|
||||||
|
echo "==> Uploading ${ARTIFACT_NAME} (${GIT_HASH}), attempt ${attempt}/${UPLOAD_MAX_ATTEMPTS}..."
|
||||||
|
HTTP_CODE=$(curl -sS --connect-timeout 10 --max-time 60 \
|
||||||
|
-o "$RESPONSE_FILE" -w "%{http_code}" \
|
||||||
|
-X POST \
|
||||||
|
-H "Authorization: Bearer ${HUSKIES_CHANNEL_TOKEN}" \
|
||||||
|
-H "X-Git-Hash: ${GIT_HASH}" \
|
||||||
|
--data-binary "@${ARTIFACT_PATH}" \
|
||||||
|
"${UPLOAD_URL}") || HTTP_CODE="000"
|
||||||
|
RESPONSE_BODY="$(cat "$RESPONSE_FILE" 2>/dev/null || true)"
|
||||||
|
|
||||||
|
case "$HTTP_CODE" in
|
||||||
|
2??)
|
||||||
|
echo "==> Upload succeeded (HTTP ${HTTP_CODE})."
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
401|403)
|
||||||
|
echo "Error: upload rejected — authentication failed (HTTP ${HTTP_CODE})." >&2
|
||||||
|
echo "Response: ${RESPONSE_BODY}" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
4??)
|
||||||
|
echo "Error: upload rejected by the channel (HTTP ${HTTP_CODE}); not retrying a client error." >&2
|
||||||
|
echo "Response: ${RESPONSE_BODY}" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
if [ "$attempt" -ge "$UPLOAD_MAX_ATTEMPTS" ]; then
|
||||||
|
echo "Error: upload failed after ${UPLOAD_MAX_ATTEMPTS} attempts (last HTTP ${HTTP_CODE})." >&2
|
||||||
|
echo "Response: ${RESPONSE_BODY}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Transient failure (HTTP ${HTTP_CODE}); retrying in ${UPLOAD_BACKOFF_SECS}s..."
|
||||||
|
sleep "$UPLOAD_BACKOFF_SECS"
|
||||||
|
attempt=$((attempt + 1))
|
||||||
|
UPLOAD_BACKOFF_SECS=$((UPLOAD_BACKOFF_SECS * 2))
|
||||||
|
done
|
||||||
|
|
||||||
|
# ── Verify manifest ──────────────────────────────────────────────────────
|
||||||
|
attempt=1
|
||||||
|
while :; do
|
||||||
|
echo "==> Verifying channel manifest reflects ${GIT_HASH} (attempt ${attempt}/${MANIFEST_MAX_ATTEMPTS})..."
|
||||||
|
MANIFEST_JSON=$(curl -sS --connect-timeout 10 --max-time 30 \
|
||||||
|
-H "Authorization: Bearer ${HUSKIES_CHANNEL_TOKEN}" \
|
||||||
|
"${MANIFEST_URL}") || MANIFEST_JSON=""
|
||||||
|
|
||||||
|
MANIFEST_HASH=$(printf '%s' "$MANIFEST_JSON" \
|
||||||
|
| python3 -c "import sys,json; print(json.load(sys.stdin).get('git_hash',''))" 2>/dev/null || echo "")
|
||||||
|
|
||||||
|
if [ "$MANIFEST_HASH" = "$GIT_HASH" ]; then
|
||||||
|
echo "==> Published ${ARTIFACT_NAME} (${GIT_HASH}) to ${CHANNEL_URL}; manifest verified."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$attempt" -ge "$MANIFEST_MAX_ATTEMPTS" ]; then
|
||||||
|
echo "Error: manifest mismatch — channel reports git_hash '${MANIFEST_HASH}', expected '${GIT_HASH}'." >&2
|
||||||
|
echo "Manifest: ${MANIFEST_JSON}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep "$MANIFEST_BACKOFF_SECS"
|
||||||
|
attempt=$((attempt + 1))
|
||||||
|
MANIFEST_BACKOFF_SECS=$((MANIFEST_BACKOFF_SECS * 2))
|
||||||
|
done
|
||||||
@@ -24,4 +24,6 @@ docker compose -f docker/docker-compose.yml down
|
|||||||
docker compose -f docker/docker-compose.yml build $CACHE_FLAG
|
docker compose -f docker/docker-compose.yml build $CACHE_FLAG
|
||||||
docker compose -f docker/docker-compose.yml up -d
|
docker compose -f docker/docker-compose.yml up -d
|
||||||
|
|
||||||
|
script/build-project-images $CACHE_FLAG
|
||||||
|
|
||||||
echo "Rebuild complete. Logs: docker compose -f docker/docker-compose.yml logs -f"
|
echo "Rebuild complete. Logs: docker compose -f docker/docker-compose.yml logs -f"
|
||||||
|
|||||||
Executable
+165
@@ -0,0 +1,165 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build huskies, install (codesign-heal wrapper + underlying binary), and if a
|
||||||
|
# gateway is running on this host, hot-restart it detached from the current shell
|
||||||
|
# so SSH disconnect — e.g. when redeploying from a phone — doesn't kill it.
|
||||||
|
#
|
||||||
|
# Skips the restart silently if no gateway is running. Errors loudly if more
|
||||||
|
# than one matches, so we don't restart the wrong one.
|
||||||
|
#
|
||||||
|
# Pass --skip-check to bypass `script/check` (useful for docs / build-script
|
||||||
|
# changes you've already verified).
|
||||||
|
#
|
||||||
|
# On relaunch failure the previous binary is restored from
|
||||||
|
# ~/bin/huskies-bin.prev and re-launched, so a bad deploy doesn't leave the
|
||||||
|
# host without a working gateway.
|
||||||
|
#
|
||||||
|
# After a `cp` or download the binary loses its ad-hoc signature and macOS
|
||||||
|
# SIGKILLs it silently on Apple Silicon. The wrapper at ~/bin/huskies re-signs
|
||||||
|
# the underlying binary at ~/bin/huskies-bin whenever codesign validation
|
||||||
|
# fails, then execs it. Normal launches (already signed) are zero-overhead.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
LOG_DIR="${HUSKIES_LOG_DIR:-$PROJECT_ROOT/logs}"
|
||||||
|
GATEWAY_PATTERN='huskies .*--gateway'
|
||||||
|
BIN_DIR="${HOME}/bin"
|
||||||
|
UNDERLYING="${BIN_DIR}/huskies-bin"
|
||||||
|
WRAPPER="${BIN_DIR}/huskies"
|
||||||
|
PREV_BIN="${BIN_DIR}/huskies-bin.prev"
|
||||||
|
NEW_BIN="${PROJECT_ROOT}/target/release/huskies"
|
||||||
|
|
||||||
|
SKIP_CHECK=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--skip-check) SKIP_CHECK=1 ;;
|
||||||
|
-h|--help) sed -n '2,17p' "$0"; exit 0 ;;
|
||||||
|
*) echo "Unknown arg: $arg (use --help)" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$SKIP_CHECK" -eq 0 ] && [ -x "$SCRIPT_DIR/check" ]; then
|
||||||
|
echo "=== Running script/check ==="
|
||||||
|
"$SCRIPT_DIR/check"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "=== Building release binary ==="
|
||||||
|
cd "$PROJECT_ROOT"
|
||||||
|
cargo build --release --bin huskies
|
||||||
|
|
||||||
|
mkdir -p "$BIN_DIR"
|
||||||
|
|
||||||
|
# Snapshot current binary so we can roll back if the relaunch fails.
|
||||||
|
PREV_VERSION=""
|
||||||
|
if [ -x "$UNDERLYING" ]; then
|
||||||
|
PREV_VERSION="$("$UNDERLYING" --version 2>/dev/null || echo unknown)"
|
||||||
|
cp "$UNDERLYING" "$PREV_BIN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cp "$NEW_BIN" "$UNDERLYING"
|
||||||
|
chmod +x "$UNDERLYING"
|
||||||
|
codesign -s - -f "$UNDERLYING" 2>/dev/null
|
||||||
|
NEW_VERSION="$("$UNDERLYING" --version 2>/dev/null || echo unknown)"
|
||||||
|
echo "==> Installed binary: ${UNDERLYING}"
|
||||||
|
if [ -n "$PREV_VERSION" ]; then
|
||||||
|
echo " version: $PREV_VERSION → $NEW_VERSION"
|
||||||
|
else
|
||||||
|
echo " version: $NEW_VERSION (no prior install)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat > "${WRAPPER}" << 'WRAPPER_EOF'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Codesign-heal wrapper — re-signs ~/bin/huskies-bin if the signature is
|
||||||
|
# missing or invalid, then execs the binary. Logs only when it re-signs.
|
||||||
|
BIN="${HOME}/bin/huskies-bin"
|
||||||
|
if ! codesign --verify --quiet "${BIN}" 2>/dev/null; then
|
||||||
|
codesign -s - "${BIN}"
|
||||||
|
echo "[codesign-heal] re-signed ~/bin/huskies-bin" >&2
|
||||||
|
fi
|
||||||
|
exec "${BIN}" "$@"
|
||||||
|
WRAPPER_EOF
|
||||||
|
chmod +x "${WRAPPER}"
|
||||||
|
echo "==> Installed wrapper: ${WRAPPER}"
|
||||||
|
|
||||||
|
# ── Hot-restart gateway if one is running ─────────────────────────────
|
||||||
|
collect_descendants() {
|
||||||
|
local pid="$1" kid
|
||||||
|
for kid in $(pgrep -P "$pid" 2>/dev/null); do
|
||||||
|
collect_descendants "$kid"
|
||||||
|
printf '%s\n' "$kid"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
GATEWAY_PIDS="$(pgrep -f "$GATEWAY_PATTERN" || true)"
|
||||||
|
if [ -z "$GATEWAY_PIDS" ]; then
|
||||||
|
echo "==> No running gateway found; install complete."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$(echo "$GATEWAY_PIDS" | wc -l)" -gt 1 ]; then
|
||||||
|
echo "Error: multiple gateway processes match '${GATEWAY_PATTERN}':" >&2
|
||||||
|
ps -p $GATEWAY_PIDS -o pid,args >&2 || true
|
||||||
|
echo "Refusing to guess which to restart." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
GATEWAY_PID="$GATEWAY_PIDS"
|
||||||
|
GATEWAY_ARGS="$(ps -p "$GATEWAY_PID" -o args= | sed -E 's@^[^ ]*huskies[^ ]* @@')"
|
||||||
|
GATEWAY_CWD="$(lsof -p "$GATEWAY_PID" 2>/dev/null | awk '$4=="cwd"{print $9; exit}')"
|
||||||
|
if [ -z "$GATEWAY_CWD" ]; then GATEWAY_CWD="$PWD"; fi
|
||||||
|
|
||||||
|
LOG_FILE="$LOG_DIR/gateway-$(date +%Y%m%d-%H%M%S).log"
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
|
||||||
|
DESCENDANTS="$(collect_descendants "$GATEWAY_PID" | tr '\n' ' ')"
|
||||||
|
echo "==> Stopping gateway tree (pids: $GATEWAY_PID $DESCENDANTS)"
|
||||||
|
# Kill descendants depth-first so PTY children die before the gateway, then the gateway.
|
||||||
|
for pid in $DESCENDANTS $GATEWAY_PID; do
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
sleep 2
|
||||||
|
|
||||||
|
echo "==> Restarting gateway"
|
||||||
|
echo " log: $LOG_FILE"
|
||||||
|
(
|
||||||
|
cd "$GATEWAY_CWD"
|
||||||
|
nohup "$WRAPPER" $GATEWAY_ARGS >> "$LOG_FILE" 2>&1 < /dev/null &
|
||||||
|
disown
|
||||||
|
)
|
||||||
|
|
||||||
|
# Wait up to 10s for the new gateway to appear AND be a different PID.
|
||||||
|
NEW_PID=""
|
||||||
|
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
||||||
|
sleep 1
|
||||||
|
candidate="$(pgrep -f "$GATEWAY_PATTERN" 2>/dev/null || true)"
|
||||||
|
if [ -n "$candidate" ] && [ "$candidate" != "$GATEWAY_PID" ]; then
|
||||||
|
NEW_PID="$candidate"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -n "$NEW_PID" ]; then
|
||||||
|
echo "==> Gateway restarted as pid $NEW_PID"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Rollback ──────────────────────────────────────────────────────────
|
||||||
|
echo "Error: new gateway failed to come up within 10s; rolling back" >&2
|
||||||
|
if [ -x "$PREV_BIN" ]; then
|
||||||
|
cp "$PREV_BIN" "$UNDERLYING"
|
||||||
|
chmod +x "$UNDERLYING"
|
||||||
|
codesign -s - -f "$UNDERLYING" 2>/dev/null
|
||||||
|
echo "==> Restored previous binary"
|
||||||
|
(
|
||||||
|
cd "$GATEWAY_CWD"
|
||||||
|
nohup "$WRAPPER" $GATEWAY_ARGS >> "$LOG_FILE" 2>&1 < /dev/null &
|
||||||
|
disown
|
||||||
|
)
|
||||||
|
sleep 2
|
||||||
|
if pgrep -f "$GATEWAY_PATTERN" >/dev/null 2>&1; then
|
||||||
|
echo "==> Gateway restored to previous version"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
echo "Error: rollback failed; gateway is DOWN. Inspect $LOG_FILE." >&2
|
||||||
|
exit 1
|
||||||
+37
-7
@@ -124,19 +124,43 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# Categorise merged work items and format names.
|
# Categorise merged work items and format names.
|
||||||
|
# Supports two subject formats (after stripping the "huskies: merge " prefix):
|
||||||
|
# New: "1063 story Human Readable Name"
|
||||||
|
# Old: "1063_story_human_readable_name"
|
||||||
FEATURES=""
|
FEATURES=""
|
||||||
FIXES=""
|
FIXES=""
|
||||||
REFACTORS=""
|
REFACTORS=""
|
||||||
while IFS= read -r item; do
|
while IFS= read -r item; do
|
||||||
[ -z "$item" ] && continue
|
[ -z "$item" ] && continue
|
||||||
# Strip the numeric prefix and type to get the human name.
|
|
||||||
name=$(echo "$item" | sed -E 's/^[0-9]+_(story|bug|refactor|spike)_//' | tr '_' ' ')
|
# Extract the leading numeric ID (present in both formats).
|
||||||
|
id=$(echo "$item" | grep -oE '^[0-9]+')
|
||||||
|
|
||||||
|
# Detect format and extract human name + type word.
|
||||||
|
if echo "$item" | grep -qE '^[0-9]+ (story|bug|refactor|spike|epic) '; then
|
||||||
|
# New format: "1063 story Human Name Here"
|
||||||
|
type_word=$(echo "$item" | sed -E 's/^[0-9]+ ([a-z]+) .*/\1/')
|
||||||
|
name=$(echo "$item" | sed -E 's/^[0-9]+ [a-z]+ //')
|
||||||
|
else
|
||||||
|
# Legacy slug format: "1063_story_human_name_here"
|
||||||
|
type_word=$(echo "$item" | sed -E 's/^[0-9]+_([a-z]+)_.*/\1/')
|
||||||
|
name=$(echo "$item" | sed -E 's/^[0-9]+_(story|bug|refactor|spike|epic)_//' | tr '_' ' ')
|
||||||
|
fi
|
||||||
|
|
||||||
# Capitalise first letter.
|
# Capitalise first letter.
|
||||||
name="$(echo "${name:0:1}" | tr '[:lower:]' '[:upper:]')${name:1}"
|
name="$(echo "${name:0:1}" | tr '[:lower:]' '[:upper:]')${name:1}"
|
||||||
case "$item" in
|
|
||||||
*_bug_*) FIXES="${FIXES}- ${name}\n" ;;
|
# Format as "Name (ID)" when a numeric ID was found, plain name otherwise.
|
||||||
*_refactor_*) REFACTORS="${REFACTORS}- ${name}\n" ;;
|
if [ -n "$id" ]; then
|
||||||
*) FEATURES="${FEATURES}- ${name}\n" ;;
|
entry="${name} (${id})"
|
||||||
|
else
|
||||||
|
entry="${name}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "$type_word" in
|
||||||
|
bug) FIXES="${FIXES}- ${entry}\n" ;;
|
||||||
|
refactor) REFACTORS="${REFACTORS}- ${entry}\n" ;;
|
||||||
|
*) FEATURES="${FEATURES}- ${entry}\n" ;;
|
||||||
esac
|
esac
|
||||||
done <<< "$MERGED_RAW"
|
done <<< "$MERGED_RAW"
|
||||||
|
|
||||||
@@ -257,7 +281,13 @@ echo "$RELEASE_BODY"
|
|||||||
# ── Tag & Push ─────────────────────────────────────────────────
|
# ── Tag & Push ─────────────────────────────────────────────────
|
||||||
echo "==> Tagging ${TAG}..."
|
echo "==> Tagging ${TAG}..."
|
||||||
git tag -a "$TAG" -m "Release ${TAG}"
|
git tag -a "$TAG" -m "Release ${TAG}"
|
||||||
git push origin "$TAG"
|
|
||||||
|
# Push the branch (with the version-bump commit) and the tag together, so
|
||||||
|
# the remote branch never lags the release tag. --atomic means both refs
|
||||||
|
# land or neither does, avoiding a pushed tag pointing at an unpushed commit.
|
||||||
|
BRANCH="$(git rev-parse --abbrev-ref HEAD)"
|
||||||
|
echo "==> Pushing ${BRANCH} and ${TAG}..."
|
||||||
|
git push --atomic origin "$BRANCH" "$TAG"
|
||||||
|
|
||||||
# ── Create Gitea Release ──────────────────────────────────────
|
# ── Create Gitea Release ──────────────────────────────────────
|
||||||
echo "==> Creating release on Gitea..."
|
echo "==> Creating release on Gitea..."
|
||||||
|
|||||||
+28
-11
@@ -11,10 +11,12 @@ export GIT_CONFIG_VALUE_0=master
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
# Ordered fail-fast: cheapest deterministic checks first, slowest builds and
|
# Ordered fail-fast: cheapest deterministic checks first. The frontend build
|
||||||
# test suites last. `set -euo pipefail` aborts at the first failure, so a fmt
|
# must run *before* anything that compiles Rust, because story 1113 introduced
|
||||||
# or clippy drift never wastes time on a frontend build or a multi-minute
|
# a compile-time dependency on `frontend/dist/` via `rust-embed` — a fresh
|
||||||
# test run.
|
# merge worktree without that directory will fail `cargo clippy` on
|
||||||
|
# `EmbeddedAssets::iter()` before the frontend build has a chance to populate
|
||||||
|
# it. `set -euo pipefail` aborts at the first failure.
|
||||||
|
|
||||||
echo "=== Checking Rust formatting ==="
|
echo "=== Checking Rust formatting ==="
|
||||||
if cargo fmt --version &>/dev/null; then
|
if cargo fmt --version &>/dev/null; then
|
||||||
@@ -44,22 +46,37 @@ if [ "$_dup_found" -eq 1 ]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "=== Running cargo clippy ==="
|
|
||||||
cargo clippy --manifest-path "$PROJECT_ROOT/Cargo.toml" --all-targets --all-features -- -D warnings
|
|
||||||
|
|
||||||
echo "=== Checking doc coverage on changed files ==="
|
|
||||||
cargo run --manifest-path "$PROJECT_ROOT/Cargo.toml" -p source-map-gen --bin source-map-check --quiet -- --worktree "$PROJECT_ROOT" --base master
|
|
||||||
|
|
||||||
echo "=== Building frontend ==="
|
echo "=== Building frontend ==="
|
||||||
if [ -d "$PROJECT_ROOT/frontend" ]; then
|
if [ -d "$PROJECT_ROOT/frontend" ]; then
|
||||||
cd "$PROJECT_ROOT/frontend"
|
cd "$PROJECT_ROOT/frontend"
|
||||||
npm install
|
# The merge gate runs in workspaces whose pre-existing `node_modules` was
|
||||||
|
# populated by an earlier `npm install --omit=dev` (or a partial install).
|
||||||
|
# In that state `npm install` reports "up to date, audited N packages"
|
||||||
|
# without actually adding the missing devDependencies, so the subsequent
|
||||||
|
# `tsc && vite build` fails with `sh: 1: tsc: not found`.
|
||||||
|
#
|
||||||
|
# Repair the install when typescript isn't reachable (story 1086 merge gate
|
||||||
|
# regression). We probe the on-disk binary rather than relying on PATH so
|
||||||
|
# this also covers the case where `node_modules/.bin/` is missing.
|
||||||
|
if [ ! -x node_modules/typescript/bin/tsc ]; then
|
||||||
|
echo "[script/test] node_modules missing typescript; performing clean install."
|
||||||
|
rm -rf node_modules
|
||||||
|
npm install --include=dev
|
||||||
|
else
|
||||||
|
npm install --include=dev
|
||||||
|
fi
|
||||||
npm run build
|
npm run build
|
||||||
cd "$PROJECT_ROOT"
|
cd "$PROJECT_ROOT"
|
||||||
else
|
else
|
||||||
echo "Skipping frontend build (no frontend directory)"
|
echo "Skipping frontend build (no frontend directory)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo "=== Running cargo clippy ==="
|
||||||
|
cargo clippy --manifest-path "$PROJECT_ROOT/Cargo.toml" --all-targets --all-features -- -D warnings
|
||||||
|
|
||||||
|
echo "=== Checking doc coverage on changed files ==="
|
||||||
|
cargo run --manifest-path "$PROJECT_ROOT/Cargo.toml" -p source-map-gen --bin source-map-check --quiet -- --worktree "$PROJECT_ROOT" --base master
|
||||||
|
|
||||||
echo "=== Running Rust tests ==="
|
echo "=== Running Rust tests ==="
|
||||||
cargo test --manifest-path "$PROJECT_ROOT/Cargo.toml" --bin huskies
|
cargo test --manifest-path "$PROJECT_ROOT/Cargo.toml" --bin huskies
|
||||||
cargo test --manifest-path "$PROJECT_ROOT/Cargo.toml" -p source-map-gen
|
cargo test --manifest-path "$PROJECT_ROOT/Cargo.toml" -p source-map-gen
|
||||||
|
|||||||
+7
-6
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "huskies"
|
name = "huskies"
|
||||||
version = "0.11.0"
|
version = "0.14.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
build = "build.rs"
|
build = "build.rs"
|
||||||
|
|
||||||
@@ -13,12 +13,10 @@ chrono-tz = { workspace = true }
|
|||||||
futures = { workspace = true }
|
futures = { workspace = true }
|
||||||
homedir = { workspace = true }
|
homedir = { workspace = true }
|
||||||
ignore = { workspace = true }
|
ignore = { workspace = true }
|
||||||
mime_guess = { workspace = true }
|
|
||||||
notify = { workspace = true }
|
notify = { workspace = true }
|
||||||
poem = { workspace = true, features = ["websocket"] }
|
poem = { workspace = true, features = ["websocket"] }
|
||||||
portable-pty = { workspace = true }
|
portable-pty = { workspace = true }
|
||||||
reqwest = { workspace = true, features = ["json", "stream", "form"] }
|
reqwest = { workspace = true, features = ["json", "stream", "form"] }
|
||||||
rust-embed = { workspace = true }
|
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
serde_urlencoded = { workspace = true }
|
serde_urlencoded = { workspace = true }
|
||||||
@@ -42,13 +40,15 @@ tokio-tungstenite = { workspace = true }
|
|||||||
# against a missing system libsqlite3.
|
# against a missing system libsqlite3.
|
||||||
#
|
#
|
||||||
# The 0.35 pin is the ceiling: rusqlite 0.37 (matrix-sdk-sqlite) requires
|
# The 0.35 pin is the ceiling: rusqlite 0.37 (matrix-sdk-sqlite) requires
|
||||||
# 0.35.x exactly, and sqlx-sqlite 0.9.0-alpha.1 requires >=0.30, <0.36. Bumping
|
# 0.35.x exactly, and sqlx-sqlite 0.9.0 requires >=0.30.1, <0.38. Bumping this
|
||||||
# this needs one of those upstreams to widen their range first.
|
# needs matrix-sdk to move to a newer rusqlite first; sqlx then allows up to
|
||||||
|
# 0.37.x.
|
||||||
libsqlite3-sys = { version = "0.35.0", features = ["bundled"] }
|
libsqlite3-sys = { version = "0.35.0", features = ["bundled"] }
|
||||||
sqlx = { workspace = true }
|
sqlx = { workspace = true }
|
||||||
wait-timeout = "0.2.1"
|
wait-timeout = "0.2.1"
|
||||||
bft-json-crdt = { path = "../crates/bft-json-crdt", default-features = false, features = ["bft"] }
|
bft-json-crdt = { path = "../crates/bft-json-crdt", default-features = false, features = ["bft"] }
|
||||||
source-map-gen = { path = "../crates/source-map-gen" }
|
source-map-gen = { path = "../crates/source-map-gen" }
|
||||||
|
release-manifest = { path = "../crates/release-manifest" }
|
||||||
ed25519-dalek = { workspace = true }
|
ed25519-dalek = { workspace = true }
|
||||||
rand = { workspace = true }
|
rand = { workspace = true }
|
||||||
nutype = { workspace = true }
|
nutype = { workspace = true }
|
||||||
@@ -64,5 +64,6 @@ check-cfg = ["cfg(feature, values(\"logging-base\"))"]
|
|||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
mockito = "1"
|
mockito = "1.7.2"
|
||||||
filetime = { workspace = true }
|
filetime = { workspace = true }
|
||||||
|
serde_yaml = { workspace = true }
|
||||||
|
|||||||
@@ -17,6 +17,20 @@ fn run(cmd: &str, args: &[&str], dir: &Path) {
|
|||||||
fn main() {
|
fn main() {
|
||||||
println!("cargo:rerun-if-changed=build.rs");
|
println!("cargo:rerun-if-changed=build.rs");
|
||||||
println!("cargo:rerun-if-env-changed=PROFILE");
|
println!("cargo:rerun-if-env-changed=PROFILE");
|
||||||
|
|
||||||
|
// Embed the current git commit hash at compile time so `get_version` always
|
||||||
|
// reflects the binary that is actually running, not a potentially-stale file.
|
||||||
|
println!("cargo:rerun-if-changed=../.git/HEAD");
|
||||||
|
println!("cargo:rerun-if-changed=../.git/refs/");
|
||||||
|
let git_hash = std::process::Command::new("git")
|
||||||
|
.args(["rev-parse", "--short", "HEAD"])
|
||||||
|
.output()
|
||||||
|
.ok()
|
||||||
|
.filter(|o| o.status.success())
|
||||||
|
.and_then(|o| String::from_utf8(o.stdout).ok())
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
println!("cargo:rustc-env=BUILD_GIT_HASH={git_hash}");
|
||||||
println!("cargo:rerun-if-changed=../frontend/package.json");
|
println!("cargo:rerun-if-changed=../frontend/package.json");
|
||||||
println!("cargo:rerun-if-changed=../frontend/package-lock.json");
|
println!("cargo:rerun-if-changed=../frontend/package-lock.json");
|
||||||
println!("cargo:rerun-if-changed=../frontend/vite.config.ts");
|
println!("cargo:rerun-if-changed=../frontend/vite.config.ts");
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
-- Story 1087: split the legacy `stage` column on `pipeline_items` into a
|
||||||
|
-- `(pipeline, status)` pair so the read side no longer needs to re-derive the
|
||||||
|
-- display column and badge from the stage string.
|
||||||
|
--
|
||||||
|
-- The migration is additive: `stage` is retained for backwards compatibility
|
||||||
|
-- while remaining Step E callers are migrated. The backup of `pipeline.db`
|
||||||
|
-- written by `shadow_write::init` immediately before this migration runs is
|
||||||
|
-- the recovery path if the backfill produces an unexpected projection.
|
||||||
|
|
||||||
|
ALTER TABLE pipeline_items ADD COLUMN pipeline TEXT NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE pipeline_items ADD COLUMN status TEXT NOT NULL DEFAULT '';
|
||||||
|
|
||||||
|
-- Backfill `pipeline` from the existing `stage` column. Every wire-form
|
||||||
|
-- stage string emitted by `stage_dir_name` maps to exactly one of the seven
|
||||||
|
-- Pipeline columns defined in `pipeline_state::types::Pipeline::as_str`.
|
||||||
|
-- Legacy directory strings (`1_backlog`, `2_current`, ...) are also handled
|
||||||
|
-- so that databases predating story 934 migrate cleanly.
|
||||||
|
UPDATE pipeline_items SET pipeline = CASE stage
|
||||||
|
WHEN 'upcoming' THEN 'backlog'
|
||||||
|
WHEN 'backlog' THEN 'backlog'
|
||||||
|
WHEN '1_backlog' THEN 'backlog'
|
||||||
|
WHEN 'coding' THEN 'coding'
|
||||||
|
WHEN 'blocked' THEN 'coding'
|
||||||
|
WHEN '2_current' THEN 'coding'
|
||||||
|
WHEN 'qa' THEN 'qa'
|
||||||
|
WHEN 'review_hold' THEN 'qa'
|
||||||
|
WHEN '3_qa' THEN 'qa'
|
||||||
|
WHEN 'merge' THEN 'merge'
|
||||||
|
WHEN 'merge_failure' THEN 'merge'
|
||||||
|
WHEN 'merge_failure_final' THEN 'merge'
|
||||||
|
WHEN '4_merge' THEN 'merge'
|
||||||
|
WHEN 'done' THEN 'done'
|
||||||
|
WHEN '5_done' THEN 'done'
|
||||||
|
WHEN 'abandoned' THEN 'closed'
|
||||||
|
WHEN 'superseded' THEN 'closed'
|
||||||
|
WHEN 'rejected' THEN 'closed'
|
||||||
|
WHEN 'archived' THEN 'archived'
|
||||||
|
WHEN '6_archived' THEN 'archived'
|
||||||
|
WHEN 'frozen' THEN 'coding'
|
||||||
|
ELSE ''
|
||||||
|
END;
|
||||||
|
|
||||||
|
-- Backfill `status` (badge) from the existing `stage` column.
|
||||||
|
UPDATE pipeline_items SET status = CASE stage
|
||||||
|
WHEN 'frozen' THEN 'frozen'
|
||||||
|
WHEN 'review_hold' THEN 'review-hold'
|
||||||
|
WHEN 'blocked' THEN 'blocked'
|
||||||
|
WHEN 'merge_failure' THEN 'merge-failure'
|
||||||
|
WHEN 'merge_failure_final' THEN 'merge-failure-final'
|
||||||
|
WHEN 'abandoned' THEN 'abandoned'
|
||||||
|
WHEN 'superseded' THEN 'superseded'
|
||||||
|
WHEN 'rejected' THEN 'rejected'
|
||||||
|
WHEN 'done' THEN 'done'
|
||||||
|
WHEN '5_done' THEN 'done'
|
||||||
|
ELSE 'active'
|
||||||
|
END;
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
-- Stores a serialized CRDT state snapshot so startup can skip replaying
|
||||||
|
-- the full op log. Only the single most recent snapshot row is kept.
|
||||||
|
CREATE TABLE IF NOT EXISTS crdt_snapshot (
|
||||||
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||||
|
at_seq INTEGER NOT NULL,
|
||||||
|
max_rowid INTEGER NOT NULL,
|
||||||
|
state_json TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL
|
||||||
|
);
|
||||||
@@ -61,6 +61,11 @@ pub(super) fn build_agent_app_context(
|
|||||||
);
|
);
|
||||||
let (reconciliation_tx, _) = broadcast::channel(64);
|
let (reconciliation_tx, _) = broadcast::channel(64);
|
||||||
let (perm_tx, perm_rx) = tokio::sync::mpsc::unbounded_channel();
|
let (perm_tx, perm_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||||
|
let permission_registry = crate::service::permission_router::ResponderRegistry::new();
|
||||||
|
crate::service::permission_router::spawn_permission_router(
|
||||||
|
perm_rx,
|
||||||
|
Arc::clone(&permission_registry),
|
||||||
|
);
|
||||||
let timer_store = Arc::new(crate::service::timer::TimerStore::load(
|
let timer_store = Arc::new(crate::service::timer::TimerStore::load(
|
||||||
project_root.join(".huskies").join("timers.json"),
|
project_root.join(".huskies").join("timers.json"),
|
||||||
));
|
));
|
||||||
@@ -74,10 +79,11 @@ pub(super) fn build_agent_app_context(
|
|||||||
bot_name: "Agent".to_string(),
|
bot_name: "Agent".to_string(),
|
||||||
bot_user_id: String::new(),
|
bot_user_id: String::new(),
|
||||||
ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||||
perm_rx: Arc::new(tokio::sync::Mutex::new(perm_rx)),
|
permission_registry,
|
||||||
pending_perm_replies: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
pending_perm_replies: crate::service::permission_router::PendingPermReplies::new(),
|
||||||
permission_timeout_secs: 120,
|
permission_timeout_secs: 120,
|
||||||
status: agents.status_broadcaster(),
|
status: agents.status_broadcaster(),
|
||||||
|
chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
|
||||||
});
|
});
|
||||||
crate::http::context::AppContext {
|
crate::http::context::AppContext {
|
||||||
state: Arc::new(state),
|
state: Arc::new(state),
|
||||||
|
|||||||
@@ -159,7 +159,7 @@ pub(super) async fn detect_conflicts(
|
|||||||
our_claims.remove(&story_id);
|
our_claims.remove(&story_id);
|
||||||
|
|
||||||
// Stop any local agent for this story by looking up its name.
|
// Stop any local agent for this story by looking up its name.
|
||||||
if let Ok(agent_list) = agents.list_agents() {
|
if let Ok(agent_list) = agents.list_agents().await {
|
||||||
for info in agent_list {
|
for info in agent_list {
|
||||||
if info.story_id == story_id {
|
if info.story_id == story_id {
|
||||||
let _ = agents
|
let _ = agents
|
||||||
@@ -219,7 +219,7 @@ pub(super) fn reclaim_timed_out_work(_project_root: &Path) {
|
|||||||
/// Check for completed agents, push their feature branches to the remote,
|
/// Check for completed agents, push their feature branches to the remote,
|
||||||
/// and report completion via CRDT.
|
/// and report completion via CRDT.
|
||||||
pub(super) async fn check_completions_and_push(agents: &AgentPool, _project_root: &Path) {
|
pub(super) async fn check_completions_and_push(agents: &AgentPool, _project_root: &Path) {
|
||||||
let Ok(agent_list) = agents.list_agents() else {
|
let Ok(agent_list) = agents.list_agents().await else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -198,10 +198,13 @@ pub async fn run(
|
|||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
// Replay current pipeline state so subscribers (worktree lifecycle, merge-failure
|
// Reconcile subscriber side effects for the current CRDT state without
|
||||||
// auto-spawn) react to any stories already in active stages, then auto-assign.
|
// flooding the broadcast channel (replaces the former replay_current_pipeline_state call).
|
||||||
slog!("[agent-mode] Replaying current pipeline state.");
|
slog!("[agent-mode] Running startup reconcile pass.");
|
||||||
crate::pipeline_state::replay_current_pipeline_state();
|
let done_retention = crate::config::ProjectConfig::load(&project_root)
|
||||||
|
.map(|c| std::time::Duration::from_secs(c.watcher.done_retention_secs))
|
||||||
|
.unwrap_or_else(|_| std::time::Duration::from_secs(4 * 3600));
|
||||||
|
crate::startup::tick_loop::run_reconcile_pass(&project_root, &agents, done_retention).await;
|
||||||
|
|
||||||
// Run initial auto-assign.
|
// Run initial auto-assign.
|
||||||
slog!("[agent-mode] Initial auto-assign scan.");
|
slog!("[agent-mode] Initial auto-assign scan.");
|
||||||
@@ -210,6 +213,15 @@ pub async fn run(
|
|||||||
// Track which stories we've claimed so we can detect conflicts.
|
// Track which stories we've claimed so we can detect conflicts.
|
||||||
let mut our_claims: HashMap<String, f64> = HashMap::new();
|
let mut our_claims: HashMap<String, f64> = HashMap::new();
|
||||||
|
|
||||||
|
// Low-disk-space watchdog (story 1200 AC1): tracks rate-limit/recovery
|
||||||
|
// state across loop iterations. Thresholds come from the config loaded
|
||||||
|
// at startup; host_id identifies this sled in chat messages and the
|
||||||
|
// gateway dedupe key.
|
||||||
|
let mut disk_watch_state = crate::service::disk_watch::DiskWatchState::default();
|
||||||
|
let disk_watch_host_id =
|
||||||
|
crdt_state::our_node_id().unwrap_or_else(|| "unknown-host".to_string());
|
||||||
|
let disk_watch_status = agents.status_broadcaster();
|
||||||
|
|
||||||
// Main loop: heartbeat, scan, claim, detect conflicts.
|
// Main loop: heartbeat, scan, claim, detect conflicts.
|
||||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(SCAN_INTERVAL_SECS));
|
let mut interval = tokio::time::interval(std::time::Duration::from_secs(SCAN_INTERVAL_SECS));
|
||||||
loop {
|
loop {
|
||||||
@@ -218,6 +230,16 @@ pub async fn run(
|
|||||||
// Write heartbeat.
|
// Write heartbeat.
|
||||||
write_heartbeat(&rendezvous_url, port);
|
write_heartbeat(&rendezvous_url, port);
|
||||||
|
|
||||||
|
// Low-disk-space check (story 1200 AC1): every tick period.
|
||||||
|
crate::service::disk_watch::io::check_and_notify(
|
||||||
|
&project_root,
|
||||||
|
&config.disk_watch,
|
||||||
|
&mut disk_watch_state,
|
||||||
|
&watcher_tx,
|
||||||
|
&disk_watch_status,
|
||||||
|
&disk_watch_host_id,
|
||||||
|
);
|
||||||
|
|
||||||
// Scan CRDT for claimable work.
|
// Scan CRDT for claimable work.
|
||||||
scan_and_claim(&agents, &project_root, &mut our_claims).await;
|
scan_and_claim(&agents, &project_root, &mut our_claims).await;
|
||||||
|
|
||||||
|
|||||||
@@ -33,16 +33,28 @@ impl GateFailureKind {
|
|||||||
/// Called once when a gate fails to produce a typed kind. Downstream code
|
/// Called once when a gate fails to produce a typed kind. Downstream code
|
||||||
/// matches on the variant and must not call this on subsequent reads.
|
/// matches on the variant and must not call this on subsequent reads.
|
||||||
pub fn classify(output: &str) -> Self {
|
pub fn classify(output: &str) -> Self {
|
||||||
|
// Strip `test <name> ... ok` lines before checking lint-trigger keywords so
|
||||||
|
// a passing test whose name contains e.g. `missing_doc_comments` or `clippy::`
|
||||||
|
// does not produce a false-positive Lint classification (story 1101).
|
||||||
|
let stripped_for_lint: String = output
|
||||||
|
.lines()
|
||||||
|
.filter(|l| {
|
||||||
|
let t = l.trim();
|
||||||
|
!(t.starts_with("test ") && t.ends_with("... ok"))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
let is_lint = stripped_for_lint.contains("error[clippy::")
|
||||||
|
|| stripped_for_lint.contains("warning[clippy::")
|
||||||
|
|| stripped_for_lint.contains("missing_doc_comments");
|
||||||
|
|
||||||
if output.contains("CONFLICT (content):") || output.contains("Merge conflict:") {
|
if output.contains("CONFLICT (content):") || output.contains("Merge conflict:") {
|
||||||
GateFailureKind::ContentConflict
|
GateFailureKind::ContentConflict
|
||||||
} else if output.contains("Diff in ") || output.contains("would reformat") {
|
} else if output.contains("Diff in ") || output.contains("would reformat") {
|
||||||
GateFailureKind::Fmt
|
GateFailureKind::Fmt
|
||||||
} else if output.contains("missing-docs direction") {
|
} else if output.contains("missing-docs direction") {
|
||||||
GateFailureKind::SourceMapCheck
|
GateFailureKind::SourceMapCheck
|
||||||
} else if output.contains("error[clippy::")
|
} else if is_lint {
|
||||||
|| output.contains("warning[clippy::")
|
|
||||||
|| output.contains("missing_doc_comments")
|
|
||||||
{
|
|
||||||
GateFailureKind::Lint
|
GateFailureKind::Lint
|
||||||
} else if output.contains("error[E") {
|
} else if output.contains("error[E") {
|
||||||
// rustc compile errors (e.g. `error[E0063]: missing field`).
|
// rustc compile errors (e.g. `error[E0063]: missing field`).
|
||||||
@@ -871,6 +883,19 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Story 1101: a passing test whose name contains a lint trigger keyword
|
||||||
|
/// must NOT produce a Lint classification.
|
||||||
|
#[test]
|
||||||
|
fn classify_does_not_false_positive_on_test_name_substring() {
|
||||||
|
let output = "test agents::gates::tests::classify_lint_from_missing_doc_comments ... ok\n\
|
||||||
|
test result: ok. 1 passed; 0 failed";
|
||||||
|
assert_ne!(
|
||||||
|
GateFailureKind::classify(output),
|
||||||
|
GateFailureKind::Lint,
|
||||||
|
"passing test name containing 'missing_doc_comments' must not classify as Lint"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn classify_source_map_check_from_missing_docs_direction() {
|
fn classify_source_map_check_from_missing_docs_direction() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -121,7 +121,7 @@ pub fn move_story_to_done(story_id: &str) -> Result<(), String> {
|
|||||||
Stage::Merge { .. } => PipelineEvent::MergeSucceeded {
|
Stage::Merge { .. } => PipelineEvent::MergeSucceeded {
|
||||||
merge_commit: GitSha("accepted".to_string()),
|
merge_commit: GitSha("accepted".to_string()),
|
||||||
},
|
},
|
||||||
Stage::MergeFailure { .. } => PipelineEvent::Accepted,
|
Stage::MergeFailure { .. } | Stage::MergeFailureFinal { .. } => PipelineEvent::Accepted,
|
||||||
Stage::Coding { .. } | Stage::Qa | Stage::Backlog => PipelineEvent::Close,
|
Stage::Coding { .. } | Stage::Qa | Stage::Backlog => PipelineEvent::Close,
|
||||||
_ => {
|
_ => {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
@@ -627,6 +627,34 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Regression test (story 1178): a story in `Stage::MergeFailureFinal`
|
||||||
|
/// whose merge is later retried and succeeds must be movable to Done.
|
||||||
|
/// Before this fix, `move_story_to_done` had no arm for
|
||||||
|
/// `MergeFailureFinal`, so it always returned an error even after a
|
||||||
|
/// real, successful re-merge — the exact "trap state" this story fixes.
|
||||||
|
#[test]
|
||||||
|
fn move_story_to_done_from_merge_failure_final_succeeds() {
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::db::write_item_with_content(
|
||||||
|
"99952_story_merge_failure_final",
|
||||||
|
"merge_failure_final",
|
||||||
|
"---\nname: Merge Failure Final Test\n---\n# Story\n",
|
||||||
|
crate::db::ItemMeta::named("Merge Failure Final Test"),
|
||||||
|
);
|
||||||
|
|
||||||
|
move_story_to_done("99952_story_merge_failure_final")
|
||||||
|
.expect("move_story_to_done should succeed from MergeFailureFinal");
|
||||||
|
|
||||||
|
let item = crate::pipeline_state::read_typed("99952_story_merge_failure_final")
|
||||||
|
.expect("CRDT read should succeed")
|
||||||
|
.expect("item should exist in CRDT");
|
||||||
|
assert_eq!(
|
||||||
|
item.stage.dir_name(),
|
||||||
|
"done",
|
||||||
|
"item should be in done after move from MergeFailureFinal"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// ── item_type_from_id tests ────────────────────────────────────────────────
|
// ── item_type_from_id tests ────────────────────────────────────────────────
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -10,10 +10,12 @@
|
|||||||
//! - `.huskies/README.md`
|
//! - `.huskies/README.md`
|
||||||
//! - `.huskies/specs/00_CONTEXT.md`
|
//! - `.huskies/specs/00_CONTEXT.md`
|
||||||
//! - `.huskies/AGENT.md`
|
//! - `.huskies/AGENT.md`
|
||||||
//! - `.huskies/source-map.json` (up to 200 KB; truncated with a log if larger)
|
|
||||||
//!
|
//!
|
||||||
//! `STACK.md` is intentionally excluded — it is large and changes often; agents
|
//! `STACK.md` and `.huskies/source-map.json` are intentionally excluded — they
|
||||||
//! should grep it on demand.
|
//! are large and change often; agents should grep on demand instead. Earlier
|
||||||
|
//! versions of this bundle inlined the source map, which ballooned the orientation
|
||||||
|
//! to ~96 KB and drowned out the workflow rules in AGENT.md; the file is still
|
||||||
|
//! kept on disk for the merge-time `source-map-check` doc-coverage gate.
|
||||||
//!
|
//!
|
||||||
//! Behaviour contract:
|
//! Behaviour contract:
|
||||||
//! - Files that are missing or empty are skipped silently (no error, no section).
|
//! - Files that are missing or empty are skipped silently (no error, no section).
|
||||||
@@ -33,12 +35,6 @@ const ORIENTATION_FILES: &[&str] = &[
|
|||||||
".huskies/AGENT.md",
|
".huskies/AGENT.md",
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Path to the source map (relative to project root), appended after AGENT.md.
|
|
||||||
const SOURCE_MAP_REL: &str = ".huskies/source-map.json";
|
|
||||||
|
|
||||||
/// Maximum bytes of source-map content to embed in the prompt.
|
|
||||||
const SOURCE_MAP_BYTE_CAP: usize = 200 * 1024;
|
|
||||||
|
|
||||||
/// Attempt to load the project-local agent prompt by concatenating orientation
|
/// Attempt to load the project-local agent prompt by concatenating orientation
|
||||||
/// files from the project root.
|
/// files from the project root.
|
||||||
///
|
///
|
||||||
@@ -60,14 +56,11 @@ pub fn read_project_local_prompt(project_root: &Path) -> Option<String> {
|
|||||||
sections.push((rel_path, trimmed.to_string()));
|
sections.push((rel_path, trimmed.to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read source-map.json (after AGENT.md) with a byte cap.
|
if sections.is_empty() {
|
||||||
let source_map_content = read_source_map_section(project_root);
|
|
||||||
|
|
||||||
if sections.is_empty() && source_map_content.is_none() {
|
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut included_files: Vec<&str> = sections.iter().map(|(name, _)| *name).collect();
|
let included_files: Vec<&str> = sections.iter().map(|(name, _)| *name).collect();
|
||||||
let mut bundle = String::new();
|
let mut bundle = String::new();
|
||||||
for (i, (name, content)) in sections.iter().enumerate() {
|
for (i, (name, content)) in sections.iter().enumerate() {
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
@@ -77,15 +70,6 @@ pub fn read_project_local_prompt(project_root: &Path) -> Option<String> {
|
|||||||
bundle.push_str(content);
|
bundle.push_str(content);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(sm) = source_map_content {
|
|
||||||
if !bundle.is_empty() {
|
|
||||||
bundle.push('\n');
|
|
||||||
}
|
|
||||||
bundle.push_str(&format!("=== {SOURCE_MAP_REL} ===\n"));
|
|
||||||
bundle.push_str(&sm);
|
|
||||||
included_files.push(SOURCE_MAP_REL);
|
|
||||||
}
|
|
||||||
|
|
||||||
crate::slog!(
|
crate::slog!(
|
||||||
"[agents] orientation bundle: {} bytes, files: [{}]",
|
"[agents] orientation bundle: {} bytes, files: [{}]",
|
||||||
bundle.len(),
|
bundle.len(),
|
||||||
@@ -95,39 +79,6 @@ pub fn read_project_local_prompt(project_root: &Path) -> Option<String> {
|
|||||||
Some(bundle)
|
Some(bundle)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read `.huskies/source-map.json` from `project_root`, applying a byte cap.
|
|
||||||
///
|
|
||||||
/// Returns `None` when the file is absent, unreadable, or empty.
|
|
||||||
/// When the content exceeds [`SOURCE_MAP_BYTE_CAP`], truncates at a char
|
|
||||||
/// boundary and logs the truncation.
|
|
||||||
#[allow(clippy::string_slice)] // cap is walked back to a char boundary before slicing
|
|
||||||
fn read_source_map_section(project_root: &Path) -> Option<String> {
|
|
||||||
let path = project_root.join(SOURCE_MAP_REL);
|
|
||||||
let Ok(content) = std::fs::read_to_string(&path) else {
|
|
||||||
return None;
|
|
||||||
};
|
|
||||||
let trimmed = content.trim();
|
|
||||||
if trimmed.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
if trimmed.len() > SOURCE_MAP_BYTE_CAP {
|
|
||||||
let mut cap = SOURCE_MAP_BYTE_CAP;
|
|
||||||
while cap > 0 && !trimmed.is_char_boundary(cap) {
|
|
||||||
cap -= 1;
|
|
||||||
}
|
|
||||||
crate::slog!(
|
|
||||||
"[agents] source-map.json truncated: {} bytes > {} byte cap; \
|
|
||||||
including first {} bytes",
|
|
||||||
trimmed.len(),
|
|
||||||
SOURCE_MAP_BYTE_CAP,
|
|
||||||
cap
|
|
||||||
);
|
|
||||||
Some(trimmed[..cap].to_string())
|
|
||||||
} else {
|
|
||||||
Some(trimmed.to_string())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -310,10 +261,13 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── source-map.json tests ────────────────────────────────────────────────
|
// ── source-map.json must NOT be inlined into the bundle ──────────────────
|
||||||
|
// The file is kept on disk for the merge-time source-map-check gate, but
|
||||||
|
// inlining it into every agent spawn ballooned the orientation past 96 KB
|
||||||
|
// and drowned out the workflow rules in AGENT.md.
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn source_map_included_after_agent_md() {
|
fn source_map_not_included_even_when_present() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
write_file(tmp.path(), ".huskies/AGENT.md", "agent content");
|
write_file(tmp.path(), ".huskies/AGENT.md", "agent content");
|
||||||
write_file(
|
write_file(
|
||||||
@@ -324,92 +278,12 @@ mod tests {
|
|||||||
|
|
||||||
let result = read_project_local_prompt(tmp.path()).unwrap();
|
let result = read_project_local_prompt(tmp.path()).unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
result.contains("=== .huskies/source-map.json ==="),
|
!result.contains("=== .huskies/source-map.json ==="),
|
||||||
"source-map delimiter must be present: {result}"
|
"source-map must not appear as an orientation section: {result}"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
result.contains(r#""src/lib.rs""#),
|
!result.contains("src/lib.rs"),
|
||||||
"source-map content must be present: {result}"
|
"source-map content must not be inlined: {result}"
|
||||||
);
|
|
||||||
// source-map section must appear after AGENT.md section
|
|
||||||
let agent_pos = result.find("=== .huskies/AGENT.md ===").unwrap();
|
|
||||||
let sm_pos = result.find("=== .huskies/source-map.json ===").unwrap();
|
|
||||||
assert!(
|
|
||||||
sm_pos > agent_pos,
|
|
||||||
"source-map section must come after AGENT.md section"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn source_map_missing_skipped_silently() {
|
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
|
||||||
write_file(tmp.path(), ".huskies/AGENT.md", "agent content");
|
|
||||||
// source-map.json intentionally absent
|
|
||||||
|
|
||||||
let result = read_project_local_prompt(tmp.path()).unwrap();
|
|
||||||
assert!(
|
|
||||||
!result.contains("source-map.json"),
|
|
||||||
"absent source-map must not create a section: {result}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn source_map_empty_skipped_silently() {
|
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
|
||||||
write_file(tmp.path(), ".huskies/AGENT.md", "agent content");
|
|
||||||
write_file(tmp.path(), ".huskies/source-map.json", "");
|
|
||||||
|
|
||||||
let result = read_project_local_prompt(tmp.path()).unwrap();
|
|
||||||
assert!(
|
|
||||||
!result.contains("source-map.json"),
|
|
||||||
"empty source-map must not create a section: {result}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn source_map_only_returns_some() {
|
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
|
||||||
// Only source-map.json present; all orientation files absent.
|
|
||||||
write_file(
|
|
||||||
tmp.path(),
|
|
||||||
".huskies/source-map.json",
|
|
||||||
r#"{"src/main.rs": {}}"#,
|
|
||||||
);
|
|
||||||
|
|
||||||
let result = read_project_local_prompt(tmp.path());
|
|
||||||
assert!(
|
|
||||||
result.is_some(),
|
|
||||||
"source-map alone must produce Some bundle"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
result.unwrap().contains("=== .huskies/source-map.json ==="),
|
|
||||||
"bundle must contain source-map section"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
#[allow(clippy::string_slice)] // sm_start is derived from str::find — always a char boundary
|
|
||||||
fn source_map_truncated_at_byte_cap() {
|
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
|
||||||
write_file(tmp.path(), ".huskies/AGENT.md", "agent");
|
|
||||||
// Build content larger than SOURCE_MAP_BYTE_CAP (200 KB).
|
|
||||||
let big = "x".repeat(SOURCE_MAP_BYTE_CAP + 1024);
|
|
||||||
write_file(tmp.path(), ".huskies/source-map.json", &big);
|
|
||||||
|
|
||||||
let result = read_project_local_prompt(tmp.path()).unwrap();
|
|
||||||
assert!(
|
|
||||||
result.contains("=== .huskies/source-map.json ==="),
|
|
||||||
"truncated source-map must still produce a section: {result}"
|
|
||||||
);
|
|
||||||
// The content length of just the source-map section must be <= SOURCE_MAP_BYTE_CAP.
|
|
||||||
let sm_start = result.find("=== .huskies/source-map.json ===").unwrap()
|
|
||||||
+ "=== .huskies/source-map.json ===\n".len();
|
|
||||||
let sm_content = &result[sm_start..];
|
|
||||||
assert!(
|
|
||||||
sm_content.len() <= SOURCE_MAP_BYTE_CAP,
|
|
||||||
"source-map section content must be <= {} bytes, got {}",
|
|
||||||
SOURCE_MAP_BYTE_CAP,
|
|
||||||
sm_content.len()
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
mod squash;
|
mod squash;
|
||||||
|
|
||||||
pub(crate) use squash::run_squash_merge;
|
pub(crate) use squash::{merge_lock_is_free, run_squash_merge};
|
||||||
|
|
||||||
/// Typed outcome of a completed squash-merge operation.
|
/// Typed outcome of a completed squash-merge operation.
|
||||||
///
|
///
|
||||||
@@ -103,6 +103,9 @@ pub struct MergeJob {
|
|||||||
/// than the current server's boot time. This survives `rebuild_and_restart`
|
/// than the current server's boot time. This survives `rebuild_and_restart`
|
||||||
/// (which re-execs and keeps the same PID).
|
/// (which re-execs and keeps the same PID).
|
||||||
pub server_start_time: f64,
|
pub server_start_time: f64,
|
||||||
|
/// Unix timestamp (seconds) when this merge job started, used to compute
|
||||||
|
/// elapsed time for a still-`Running` job.
|
||||||
|
pub started_at: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Result of a mergemaster merge operation.
|
/// Result of a mergemaster merge operation.
|
||||||
@@ -113,6 +116,11 @@ pub struct MergeReport {
|
|||||||
pub result: MergeResult,
|
pub result: MergeResult,
|
||||||
pub worktree_cleaned_up: bool,
|
pub worktree_cleaned_up: bool,
|
||||||
pub story_archived: bool,
|
pub story_archived: bool,
|
||||||
|
/// Path (relative to the project root) of the full untruncated report
|
||||||
|
/// written by `service::merge::io::write_merge_report`, if the write
|
||||||
|
/// succeeded.
|
||||||
|
#[serde(default)]
|
||||||
|
pub report_path: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -17,6 +17,27 @@ use crate::config::ProjectConfig;
|
|||||||
/// causing `git cherry-pick merge-queue/…` to fail with "bad revision".
|
/// causing `git cherry-pick merge-queue/…` to fail with "bad revision".
|
||||||
static MERGE_LOCK: Mutex<()> = Mutex::new(());
|
static MERGE_LOCK: Mutex<()> = Mutex::new(());
|
||||||
|
|
||||||
|
/// Returns `true` when no squash-merge is currently running, i.e. the merge
|
||||||
|
/// lock is free.
|
||||||
|
///
|
||||||
|
/// Used by the build-directory GC pass (story 1199) to decide whether
|
||||||
|
/// `.huskies/merge_workspace` is safe to reclaim. This is a best-effort,
|
||||||
|
/// momentary check — the lock is not held across the reclaim itself, so a
|
||||||
|
/// merge that starts immediately afterward can still race with GC. That's
|
||||||
|
/// acceptable: the GC pass tolerates races and skips on error rather than
|
||||||
|
/// failing the whole pass.
|
||||||
|
pub(crate) fn merge_lock_is_free() -> bool {
|
||||||
|
MERGE_LOCK.try_lock().is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve the base branch for `project_root` from config, or auto-detect it.
|
||||||
|
fn resolve_base_branch(project_root: &Path) -> String {
|
||||||
|
let configured = crate::config::ProjectConfig::load(project_root)
|
||||||
|
.ok()
|
||||||
|
.and_then(|c| c.base_branch);
|
||||||
|
crate::worktree::resolve_base_branch(project_root, configured.as_deref())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn run_squash_merge(
|
pub(crate) fn run_squash_merge(
|
||||||
project_root: &Path,
|
project_root: &Path,
|
||||||
branch: &str,
|
branch: &str,
|
||||||
@@ -31,10 +52,7 @@ pub(crate) fn run_squash_merge(
|
|||||||
// A zero-commit branch produces an empty squash and a silent "nothing to
|
// A zero-commit branch produces an empty squash and a silent "nothing to
|
||||||
// commit" failure. Catch it early with a grep-able error before any merge
|
// commit" failure. Catch it early with a grep-able error before any merge
|
||||||
// work starts.
|
// work starts.
|
||||||
let base_branch = crate::config::ProjectConfig::load(project_root)
|
let base_branch = resolve_base_branch(project_root);
|
||||||
.ok()
|
|
||||||
.and_then(|c| c.base_branch.clone())
|
|
||||||
.unwrap_or_else(|| "master".to_string());
|
|
||||||
|
|
||||||
let ahead_out = Command::new("git")
|
let ahead_out = Command::new("git")
|
||||||
.args(["rev-list", "--count", &format!("{base_branch}..{branch}")])
|
.args(["rev-list", "--count", &format!("{base_branch}..{branch}")])
|
||||||
@@ -124,7 +142,15 @@ pub(crate) fn run_squash_merge(
|
|||||||
|
|
||||||
// ── Commit in the temporary worktree ──────────────────────────
|
// ── Commit in the temporary worktree ──────────────────────────
|
||||||
all_output.push_str("=== git commit ===\n");
|
all_output.push_str("=== git commit ===\n");
|
||||||
let commit_msg = format!("huskies: merge {story_id}");
|
// Include human-readable name and item type when the CRDT is available.
|
||||||
|
// Falls back to the bare ID when running outside the server (e.g. in tests).
|
||||||
|
let story_label = crate::crdt_state::read_item(story_id)
|
||||||
|
.map(|item| {
|
||||||
|
let type_str = item.item_type().map(|t| t.as_str()).unwrap_or("story");
|
||||||
|
format!(" {} {}", type_str, item.name())
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let commit_msg = format!("huskies: merge {story_id}{story_label}");
|
||||||
let commit = Command::new("git")
|
let commit = Command::new("git")
|
||||||
.args(["commit", "-m", &commit_msg])
|
.args(["commit", "-m", &commit_msg])
|
||||||
.current_dir(&merge_wt_path)
|
.current_dir(&merge_wt_path)
|
||||||
@@ -308,11 +334,6 @@ pub(crate) fn run_squash_merge(
|
|||||||
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
|
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let base_branch = crate::config::ProjectConfig::load(project_root)
|
|
||||||
.ok()
|
|
||||||
.and_then(|c| c.base_branch.clone())
|
|
||||||
.unwrap_or_else(|| "master".to_string());
|
|
||||||
|
|
||||||
if current_branch != base_branch {
|
if current_branch != base_branch {
|
||||||
all_output.push_str(&format!(
|
all_output.push_str(&format!(
|
||||||
"=== VERIFICATION FAILED: expected branch '{base_branch}' but HEAD is on \
|
"=== VERIFICATION FAILED: expected branch '{base_branch}' but HEAD is on \
|
||||||
@@ -507,3 +528,5 @@ fn run_merge_quality_gates(
|
|||||||
mod tests_advanced;
|
mod tests_advanced;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests_basic;
|
mod tests_basic;
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests_changelog;
|
||||||
|
|||||||
@@ -178,6 +178,79 @@ async fn squash_merge_clean_merge_succeeds() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn squash_merge_succeeds_on_main_based_repo_with_base_branch_unset() {
|
||||||
|
use std::fs;
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
|
let tmp = tempdir().unwrap();
|
||||||
|
let repo = tmp.path();
|
||||||
|
|
||||||
|
// Repo whose default branch is `main` — no `master` branch exists at all,
|
||||||
|
// and no `.huskies/project.toml` sets `base_branch`. run_squash_merge must
|
||||||
|
// auto-detect `main` instead of assuming `master` (bug 1176).
|
||||||
|
Command::new("git")
|
||||||
|
.args(["init", "-b", "main"])
|
||||||
|
.current_dir(repo)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
Command::new("git")
|
||||||
|
.args(["config", "user.email", "test@test.com"])
|
||||||
|
.current_dir(repo)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
Command::new("git")
|
||||||
|
.args(["config", "user.name", "Test"])
|
||||||
|
.current_dir(repo)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
Command::new("git")
|
||||||
|
.args(["commit", "--allow-empty", "-m", "init"])
|
||||||
|
.current_dir(repo)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
Command::new("git")
|
||||||
|
.args(["checkout", "-b", "feature/story-main_test"])
|
||||||
|
.current_dir(repo)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
fs::write(repo.join("new_file.txt"), "new content").unwrap();
|
||||||
|
Command::new("git")
|
||||||
|
.args(["add", "."])
|
||||||
|
.current_dir(repo)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
Command::new("git")
|
||||||
|
.args(["commit", "-m", "add new file"])
|
||||||
|
.current_dir(repo)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
Command::new("git")
|
||||||
|
.args(["checkout", "main"])
|
||||||
|
.current_dir(repo)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let result = run_squash_merge(repo, "feature/story-main_test", "main_test").unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
result,
|
||||||
|
super::MergeResult::Success {
|
||||||
|
conflicts_resolved: false,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"clean merge should succeed on a main-based repo; got: {result:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
repo.join("new_file.txt").exists(),
|
||||||
|
"merged file should exist on main"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn squash_merge_nonexistent_branch_fails() {
|
async fn squash_merge_nonexistent_branch_fails() {
|
||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
//! Regression tests for changelog entry parsing — both legacy-slug and new-format
|
||||||
|
//! merge commit subjects must resolve to a human-readable "Name (ID)" entry.
|
||||||
|
|
||||||
|
/// Parse a single merge commit subject (after stripping the `huskies: merge ` prefix)
|
||||||
|
/// into `(id, type_word, human_name)`.
|
||||||
|
///
|
||||||
|
/// Returns `None` for subjects that are not recognised merge items.
|
||||||
|
fn parse_changelog_entry(item: &str) -> Option<(String, String, String)> {
|
||||||
|
let item = item.trim();
|
||||||
|
if item.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract leading numeric ID present in both formats.
|
||||||
|
let id: String = item.chars().take_while(|c| c.is_ascii_digit()).collect();
|
||||||
|
if id.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detect format by the character immediately following the digits.
|
||||||
|
// id contains only ASCII digits so id.len() is a valid char boundary.
|
||||||
|
let rest = item.get(id.len()..).unwrap_or("");
|
||||||
|
if let Some(space_rest) = rest.strip_prefix(' ') {
|
||||||
|
// New format: "1063 story Human Name Here"
|
||||||
|
let mut words = space_rest.splitn(2, ' ');
|
||||||
|
let type_word = words.next().unwrap_or("story").to_string();
|
||||||
|
let name = words.next().unwrap_or("").trim().to_string();
|
||||||
|
if name.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some((id, type_word, name))
|
||||||
|
} else if let Some(slug_rest) = rest.strip_prefix('_') {
|
||||||
|
// Legacy slug format: "1063_story_human_name_here"
|
||||||
|
let mut parts = slug_rest.splitn(2, '_');
|
||||||
|
let type_word = parts.next().unwrap_or("story").to_string();
|
||||||
|
let slug = parts.next().unwrap_or("").replace('_', " ");
|
||||||
|
if slug.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some((id, type_word, slug))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Format a parsed entry as "Human Name (ID)".
|
||||||
|
fn format_entry(id: &str, name: &str) -> String {
|
||||||
|
let mut chars = name.chars();
|
||||||
|
let capitalised = match chars.next() {
|
||||||
|
None => String::new(),
|
||||||
|
Some(c) => c.to_uppercase().collect::<String>() + chars.as_str(),
|
||||||
|
};
|
||||||
|
format!("{capitalised} ({id})")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changelog_new_format_story_resolves_to_name_and_id() {
|
||||||
|
let item = "1063 story Tee pipeline events into gateway context";
|
||||||
|
let (id, _type_word, name) = parse_changelog_entry(item).expect("should parse new format");
|
||||||
|
assert_eq!(id, "1063");
|
||||||
|
assert_eq!(
|
||||||
|
format_entry(&id, &name),
|
||||||
|
"Tee pipeline events into gateway context (1063)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changelog_new_format_bug_resolves_to_name_and_id() {
|
||||||
|
let item = "999 bug Fix the broken auth token";
|
||||||
|
let (id, type_word, name) = parse_changelog_entry(item).expect("should parse new-format bug");
|
||||||
|
assert_eq!(id, "999");
|
||||||
|
assert_eq!(type_word, "bug");
|
||||||
|
assert_eq!(format_entry(&id, &name), "Fix the broken auth token (999)");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changelog_new_format_refactor_resolves_to_name_and_id() {
|
||||||
|
let item = "777 refactor Extract config parsing";
|
||||||
|
let (id, type_word, name) = parse_changelog_entry(item).expect("should parse refactor");
|
||||||
|
assert_eq!(type_word, "refactor");
|
||||||
|
assert_eq!(format_entry(&id, &name), "Extract config parsing (777)");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changelog_legacy_slug_story_resolves_to_name_and_id() {
|
||||||
|
let item = "1063_story_tee_pipeline_events_into_gateway_context";
|
||||||
|
let (id, _type_word, name) = parse_changelog_entry(item).expect("should parse legacy slug");
|
||||||
|
assert_eq!(id, "1063");
|
||||||
|
assert_eq!(
|
||||||
|
format_entry(&id, &name),
|
||||||
|
"Tee pipeline events into gateway context (1063)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changelog_legacy_slug_bug_resolves_to_name_and_id() {
|
||||||
|
let item = "999_bug_fix_the_broken_auth_token";
|
||||||
|
let (id, type_word, name) = parse_changelog_entry(item).expect("should parse legacy bug slug");
|
||||||
|
assert_eq!(id, "999");
|
||||||
|
assert_eq!(type_word, "bug");
|
||||||
|
assert_eq!(format_entry(&id, &name), "Fix the broken auth token (999)");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changelog_mixed_fixture_all_entries_have_human_names() {
|
||||||
|
// Fixture: a mix of legacy-slug and new-format subjects (as they appear
|
||||||
|
// after stripping the "huskies: merge " prefix from the git log).
|
||||||
|
let fixture = [
|
||||||
|
// Legacy slug formats (pre-migration)
|
||||||
|
"1001_story_add_matrix_transport",
|
||||||
|
"1002_bug_fix_crdt_sync_disconnect",
|
||||||
|
"1003_refactor_extract_gateway_config",
|
||||||
|
// New format (post-story-1069)
|
||||||
|
"1050 story Add agent pool auto-assign",
|
||||||
|
"1063 story Tee pipeline events into gateway context",
|
||||||
|
"1064 bug Stop lagged handler re-emitting via same channel",
|
||||||
|
"1065 refactor Move squash merge into own module",
|
||||||
|
];
|
||||||
|
|
||||||
|
for item in &fixture {
|
||||||
|
let result = parse_changelog_entry(item);
|
||||||
|
assert!(result.is_some(), "failed to parse merge subject: {item:?}");
|
||||||
|
let (id, _type_word, name) = result.unwrap();
|
||||||
|
let entry = format_entry(&id, &name);
|
||||||
|
// Every entry must contain the numeric ID in parentheses.
|
||||||
|
assert!(
|
||||||
|
entry.contains(&format!("({id})")),
|
||||||
|
"entry missing numeric ID: {entry:?}"
|
||||||
|
);
|
||||||
|
// Name must not be empty or just whitespace.
|
||||||
|
assert!(
|
||||||
|
!name.trim().is_empty(),
|
||||||
|
"empty human name for item: {item:?}"
|
||||||
|
);
|
||||||
|
// Name must not be a raw slug (contains underscores as word separators).
|
||||||
|
// (Underscores are OK inside words like "auto-assign" but not as spaces.)
|
||||||
|
assert!(
|
||||||
|
!name.contains('_'),
|
||||||
|
"name still contains underscores (slug not decoded): {name:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -161,6 +161,42 @@ pub fn pipeline_stage(agent_name: &str) -> PipelineStage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Map a pipeline [`Stage`] to the canonical [`PipelineStage`] for LLM agent spawning.
|
||||||
|
///
|
||||||
|
/// Returns `None` for stages where no LLM agent should be active (terminal states,
|
||||||
|
/// blocked, frozen, or unclassified merge failures requiring human intervention).
|
||||||
|
/// Returns `Some(stage)` naming the single LLM-agent type that may run on this story.
|
||||||
|
/// Used by `validate_agent_stage` and `reconcile_canonical_agents` to enforce the
|
||||||
|
/// one-agent-per-story invariant (story 1100).
|
||||||
|
pub fn canonical_pipeline_stage(s: &crate::pipeline_state::Stage) -> Option<PipelineStage> {
|
||||||
|
use crate::pipeline_state::{MergeFailureKind, Stage};
|
||||||
|
match s {
|
||||||
|
Stage::Coding { .. } => Some(PipelineStage::Coder),
|
||||||
|
Stage::Qa => Some(PipelineStage::Qa),
|
||||||
|
Stage::Merge { .. } => Some(PipelineStage::Mergemaster),
|
||||||
|
Stage::MergeFailure {
|
||||||
|
kind: MergeFailureKind::ConflictDetected(_),
|
||||||
|
..
|
||||||
|
} => Some(PipelineStage::Mergemaster),
|
||||||
|
Stage::MergeFailure {
|
||||||
|
kind: MergeFailureKind::GatesFailed(_),
|
||||||
|
..
|
||||||
|
} => Some(PipelineStage::Coder),
|
||||||
|
Stage::MergeFailureFinal { .. } => Some(PipelineStage::Mergemaster),
|
||||||
|
Stage::Upcoming
|
||||||
|
| Stage::Backlog
|
||||||
|
| Stage::MergeFailure { .. }
|
||||||
|
| Stage::Done { .. }
|
||||||
|
| Stage::Blocked { .. }
|
||||||
|
| Stage::Archived { .. }
|
||||||
|
| Stage::Frozen { .. }
|
||||||
|
| Stage::ReviewHold { .. }
|
||||||
|
| Stage::Abandoned { .. }
|
||||||
|
| Stage::Superseded { .. }
|
||||||
|
| Stage::Rejected { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Determine the pipeline stage for a configured agent.
|
/// Determine the pipeline stage for a configured agent.
|
||||||
///
|
///
|
||||||
/// Prefers the explicit `stage` config field (added in Bug 150) over the
|
/// Prefers the explicit `stage` config field (added in Bug 150) over the
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ mod tests {
|
|||||||
// task eventually fails.
|
// task eventually fails.
|
||||||
pool.auto_assign_available_work(tmp.path()).await;
|
pool.auto_assign_available_work(tmp.path()).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let has_pending = agents.values().any(|a| {
|
let has_pending = agents.values().any(|a| {
|
||||||
a.agent_name == "coder-1"
|
a.agent_name == "coder-1"
|
||||||
&& matches!(a.status, AgentStatus::Pending | AgentStatus::Running)
|
&& matches!(a.status, AgentStatus::Pending | AgentStatus::Running)
|
||||||
@@ -115,7 +115,7 @@ mod tests {
|
|||||||
pool.auto_assign_available_work(root).await;
|
pool.auto_assign_available_work(root).await;
|
||||||
|
|
||||||
// No agent should have been started for the spike.
|
// No agent should have been started for the spike.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
agents.is_empty(),
|
agents.is_empty(),
|
||||||
"No agents should be assigned to a spike with review_hold"
|
"No agents should be assigned to a spike with review_hold"
|
||||||
@@ -155,7 +155,7 @@ mod tests {
|
|||||||
|
|
||||||
pool.auto_assign_available_work(tmp.path()).await;
|
pool.auto_assign_available_work(tmp.path()).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
// coder-1 must NOT have been assigned to the QA story (wrong stage).
|
// coder-1 must NOT have been assigned to the QA story (wrong stage).
|
||||||
let coder_assigned_to_qa = agents.iter().any(|(key, a)| {
|
let coder_assigned_to_qa = agents.iter().any(|(key, a)| {
|
||||||
key.contains("9930_story_qa1")
|
key.contains("9930_story_qa1")
|
||||||
@@ -209,7 +209,7 @@ mod tests {
|
|||||||
|
|
||||||
pool.auto_assign_available_work(tmp.path()).await;
|
pool.auto_assign_available_work(tmp.path()).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
// coder-1 should have been picked (it matches the stage and is preferred).
|
// coder-1 should have been picked (it matches the stage and is preferred).
|
||||||
let coder1_assigned = agents.values().any(|a| {
|
let coder1_assigned = agents.values().any(|a| {
|
||||||
a.agent_name == "coder-1"
|
a.agent_name == "coder-1"
|
||||||
@@ -262,7 +262,7 @@ mod tests {
|
|||||||
// Must not panic.
|
// Must not panic.
|
||||||
pool.auto_assign_available_work(tmp.path()).await;
|
pool.auto_assign_available_work(tmp.path()).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
// No agent should be assigned to the specific QA story (coder-1 may
|
// No agent should be assigned to the specific QA story (coder-1 may
|
||||||
// be assigned to leaked 2_current items from the global CRDT store).
|
// be assigned to leaked 2_current items from the global CRDT store).
|
||||||
let assigned_to_qa_story = agents.iter().any(|(key, a)| {
|
let assigned_to_qa_story = agents.iter().any(|(key, a)| {
|
||||||
@@ -301,7 +301,7 @@ mod tests {
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.auto_assign_available_work(root).await;
|
pool.auto_assign_available_work(root).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
// Filter to only agents assigned to our specific story to avoid
|
// Filter to only agents assigned to our specific story to avoid
|
||||||
// interference from other tests sharing the global CRDT store.
|
// interference from other tests sharing the global CRDT store.
|
||||||
let assigned_to_our_story = agents.iter().any(|(key, a)| {
|
let assigned_to_our_story = agents.iter().any(|(key, a)| {
|
||||||
@@ -347,7 +347,7 @@ mod tests {
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.auto_assign_available_work(root).await;
|
pool.auto_assign_available_work(root).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let has_pending = agents.values().any(|a| {
|
let has_pending = agents.values().any(|a| {
|
||||||
matches!(
|
matches!(
|
||||||
a.status,
|
a.status,
|
||||||
@@ -553,7 +553,7 @@ mod tests {
|
|||||||
let _ = tokio::join!(t1, t2);
|
let _ = tokio::join!(t1, t2);
|
||||||
|
|
||||||
// At most one Pending/Running entry should exist for coder-1.
|
// At most one Pending/Running entry should exist for coder-1.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let active_coder_count = agents
|
let active_coder_count = agents
|
||||||
.values()
|
.values()
|
||||||
.filter(|a| {
|
.filter(|a| {
|
||||||
@@ -569,14 +569,15 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── AC4: startup event replay + pool reconstruction ──────────────────
|
// ── AC4: startup reconcile + pool reconstruction ──────────────────
|
||||||
|
|
||||||
/// AC4: Simulates a server restart by seeding the CRDT with a story in
|
/// AC4: Simulates a server restart by seeding the CRDT with a story in
|
||||||
/// Coding stage, calling `replay_current_pipeline_state` (the new startup
|
/// Coding stage, then running `auto_assign_available_work` (startup no longer
|
||||||
/// path), then `auto_assign_available_work`. Asserts the pool ends in the
|
/// floods the broadcast channel via replay — it calls reconcile functions
|
||||||
/// expected state: exactly one agent assigned to the story.
|
/// directly). Asserts the pool ends in the expected state: exactly one agent
|
||||||
|
/// assigned to the story, and a second pass does not double-spawn.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn startup_replay_followed_by_auto_assign_assigns_agent_once() {
|
async fn startup_auto_assign_assigns_agent_once() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let sk = tmp.path().join(".huskies");
|
let sk = tmp.path().join(".huskies");
|
||||||
std::fs::create_dir_all(&sk).unwrap();
|
std::fs::create_dir_all(&sk).unwrap();
|
||||||
@@ -597,12 +598,11 @@ mod tests {
|
|||||||
|
|
||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
|
|
||||||
// Simulate startup: replay current state, then auto-assign.
|
// First auto-assign pass.
|
||||||
crate::pipeline_state::replay_current_pipeline_state();
|
|
||||||
pool.auto_assign_available_work(tmp.path()).await;
|
pool.auto_assign_available_work(tmp.path()).await;
|
||||||
|
|
||||||
let count_after_first = {
|
let count_after_first = {
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
agents
|
agents
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(key, a)| {
|
.filter(|(key, a)| {
|
||||||
@@ -612,12 +612,11 @@ mod tests {
|
|||||||
.count()
|
.count()
|
||||||
};
|
};
|
||||||
|
|
||||||
// AC3 (idempotency): replaying twice must not double-spawn agents.
|
// Second pass (idempotency): must not double-spawn agents.
|
||||||
crate::pipeline_state::replay_current_pipeline_state();
|
|
||||||
pool.auto_assign_available_work(tmp.path()).await;
|
pool.auto_assign_available_work(tmp.path()).await;
|
||||||
|
|
||||||
let count_after_second = {
|
let count_after_second = {
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
agents
|
agents
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(key, a)| {
|
.filter(|(key, a)| {
|
||||||
@@ -629,11 +628,11 @@ mod tests {
|
|||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
count_after_first <= 1,
|
count_after_first <= 1,
|
||||||
"after first replay+assign at most one agent must be assigned to {story_id}"
|
"after first auto-assign at most one agent must be assigned to {story_id}"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
count_after_first, count_after_second,
|
count_after_first, count_after_second,
|
||||||
"second replay must not spawn additional agents (idempotency)"
|
"second auto-assign must not spawn additional agents (idempotency)"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,29 +1,39 @@
|
|||||||
//! Backlog promotion: scan `1_backlog/` and promote stories whose `depends_on` are all met.
|
//! Backlog promotion: scan items in `Pipeline::Backlog` and promote stories whose `depends_on` are all met.
|
||||||
|
|
||||||
use crate::pipeline_state::Stage;
|
use crate::pipeline_state::Pipeline;
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
use crate::slog_warn;
|
use crate::slog_warn;
|
||||||
|
|
||||||
use super::super::AgentPool;
|
use super::super::AgentPool;
|
||||||
use super::scan::scan_stage_items;
|
|
||||||
use super::story_checks::{check_archived_dependencies, has_unmet_dependencies};
|
use super::story_checks::{check_archived_dependencies, has_unmet_dependencies};
|
||||||
|
|
||||||
impl AgentPool {
|
impl AgentPool {
|
||||||
/// Scan `1_backlog/` and promote any story whose `depends_on` are all met.
|
/// Scan items in `Pipeline::Backlog` and promote any story whose `depends_on` are all met.
|
||||||
///
|
///
|
||||||
/// A story is only promoted if it explicitly lists `depends_on` AND every
|
/// A story is only promoted if it explicitly lists `depends_on` AND every
|
||||||
/// listed dependency has reached `5_done` or `6_archived`. Stories with no
|
/// listed dependency has reached `Pipeline::Done` or `Pipeline::Archived`.
|
||||||
/// `depends_on` are left in the backlog for human scheduling.
|
/// Stories with no `depends_on` are left in the backlog for human scheduling.
|
||||||
///
|
///
|
||||||
/// **Archived dep semantics:** a dep in `6_archived` counts as satisfied (since
|
/// **Archived dep semantics:** a dep in `Pipeline::Archived` counts as satisfied
|
||||||
/// stories auto-sweep from `5_done` to `6_archived` after 4 hours, and the
|
/// (since stories auto-sweep from `Done` to `Archived` after 4 hours, and the
|
||||||
/// dependent story would normally already be promoted by then). However, if a
|
/// dependent story would normally already be promoted by then). However, if a
|
||||||
/// dep was already in `6_archived` when the dependent story was created (e.g. it
|
/// dep was already archived when the dependent story was created (e.g. it
|
||||||
/// was abandoned/superseded before the dependent existed), a prominent warning is
|
/// was abandoned/superseded before the dependent existed), a prominent warning is
|
||||||
/// logged so the user can see the promotion was triggered by an archived dep, not
|
/// logged so the user can see the promotion was triggered by an archived dep, not
|
||||||
/// a clean completion.
|
/// a clean completion.
|
||||||
pub(super) fn promote_ready_backlog_stories(&self) {
|
pub(super) fn promote_ready_backlog_stories(&self) {
|
||||||
let items = scan_stage_items(&Stage::Backlog);
|
// Story 1086: scan by Pipeline column, not Stage variant. Pipeline::Backlog
|
||||||
|
// covers Stage::Upcoming and Stage::Backlog uniformly.
|
||||||
|
let items: Vec<String> = {
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
let mut ids = BTreeSet::new();
|
||||||
|
for item in crate::pipeline_state::read_all_typed() {
|
||||||
|
if item.stage.pipeline() == Pipeline::Backlog {
|
||||||
|
ids.insert(item.story_id.0.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ids.into_iter().collect()
|
||||||
|
};
|
||||||
for story_id in &items {
|
for story_id in &items {
|
||||||
// Only promote stories that explicitly declare dependencies
|
// Only promote stories that explicitly declare dependencies
|
||||||
// (story 929: read from the CRDT register, not YAML).
|
// (story 929: read from the CRDT register, not YAML).
|
||||||
|
|||||||
@@ -65,9 +65,22 @@ impl AgentPool {
|
|||||||
// AC6: Detect empty-diff stories before starting the merge pipeline.
|
// AC6: Detect empty-diff stories before starting the merge pipeline.
|
||||||
// If the worktree has no commits on the feature branch, block the
|
// If the worktree has no commits on the feature branch, block the
|
||||||
// story immediately via the state machine — no merge job needed.
|
// story immediately via the state machine — no merge job needed.
|
||||||
if let Some(wt_path) = worktree::find_worktree_path(project_root, story_id)
|
//
|
||||||
&& !crate::agents::gates::worktree_has_committed_work(&wt_path)
|
// Bug 1170: worktree_has_committed_work shells out to `git log`
|
||||||
{
|
// synchronously. assign_merge_stage runs on the shared tokio
|
||||||
|
// runtime (it's invoked reactively on every CRDT transition, incl.
|
||||||
|
// unblock), so calling it inline here blocked a runtime worker
|
||||||
|
// thread for the duration of the git subprocess — with a story
|
||||||
|
// whose worktree/agent had crashed, that call could hang
|
||||||
|
// indefinitely and stall /health and the liveness heartbeat along
|
||||||
|
// with it. Run it on the blocking-thread pool instead.
|
||||||
|
if let Some(wt_path) = worktree::find_worktree_path(project_root, story_id) {
|
||||||
|
let has_commits = tokio::task::spawn_blocking(move || {
|
||||||
|
crate::agents::gates::worktree_has_committed_work(&wt_path)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or(false);
|
||||||
|
if !has_commits {
|
||||||
let empty_diff_reason = "Feature branch has no code changes — the coder agent \
|
let empty_diff_reason = "Feature branch has no code changes — the coder agent \
|
||||||
did not produce any commits.";
|
did not produce any commits.";
|
||||||
slog_warn!(
|
slog_warn!(
|
||||||
@@ -77,7 +90,9 @@ impl AgentPool {
|
|||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
crate::agents::lifecycle::transition_to_blocked(story_id, empty_diff_reason)
|
crate::agents::lifecycle::transition_to_blocked(story_id, empty_diff_reason)
|
||||||
{
|
{
|
||||||
slog_error!("[auto-assign] Failed to transition '{story_id}' to Blocked: {e}");
|
slog_error!(
|
||||||
|
"[auto-assign] Failed to transition '{story_id}' to Blocked: {e}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
let _ = self
|
let _ = self
|
||||||
.watcher_tx
|
.watcher_tx
|
||||||
@@ -87,6 +102,7 @@ impl AgentPool {
|
|||||||
});
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Skip if a merge job is already running for this story (e.g. triggered
|
// Skip if a merge job is already running for this story (e.g. triggered
|
||||||
// by a previous auto-assign pass or by pipeline advancement).
|
// by a previous auto-assign pass or by pipeline advancement).
|
||||||
@@ -99,13 +115,7 @@ impl AgentPool {
|
|||||||
// Skip if an explicit mergemaster LLM agent is already running
|
// Skip if an explicit mergemaster LLM agent is already running
|
||||||
// (operator-driven failure recovery path).
|
// (operator-driven failure recovery path).
|
||||||
let has_mergemaster = {
|
let has_mergemaster = {
|
||||||
let agents = match self.agents.lock() {
|
let agents = self.agents.lock().await;
|
||||||
Ok(a) => a,
|
|
||||||
Err(e) => {
|
|
||||||
slog_error!("[auto-assign] Failed to lock agents: {e}");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
is_story_assigned_for_stage(config, &agents, story_id, &PipelineStage::Mergemaster)
|
is_story_assigned_for_stage(config, &agents, story_id, &PipelineStage::Mergemaster)
|
||||||
};
|
};
|
||||||
if has_mergemaster {
|
if has_mergemaster {
|
||||||
@@ -117,3 +127,89 @@ impl AgentPool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::super::super::AgentPool;
|
||||||
|
use crate::config::ProjectConfig;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
|
/// Bug 1170 regression: `assign_merge_stage` used to call
|
||||||
|
/// `worktree_has_committed_work` (which shells out to `git`) directly on
|
||||||
|
/// the async runtime. For a story with a crashed/unassignable agent
|
||||||
|
/// sitting in `4_merge/`, that synchronous subprocess call had no yield
|
||||||
|
/// point, so on a runtime with few worker threads it starved every other
|
||||||
|
/// task — including the liveness heartbeat and `/health` — for the whole
|
||||||
|
/// scan. After wrapping the call in `spawn_blocking`, the executor stays
|
||||||
|
/// free to interleave other work while the git subprocess runs
|
||||||
|
/// off-runtime.
|
||||||
|
///
|
||||||
|
/// This reproduces the unblock → merge-auto-assign path: a story sits in
|
||||||
|
/// `4_merge/` with no active agent entry (the crashed/unassignable case)
|
||||||
|
/// and a worktree directory that isn't a real git repo, forcing every
|
||||||
|
/// `git` invocation in the scan to fail — but only after paying the
|
||||||
|
/// process fork/exec cost, which is what stalls a non-yielding runtime.
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
|
async fn assign_merge_stage_does_not_stall_liveness_heartbeat() {
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path().to_path_buf();
|
||||||
|
std::fs::create_dir_all(root.join(".huskies")).unwrap();
|
||||||
|
std::fs::write(root.join(".huskies/project.toml"), "").unwrap();
|
||||||
|
|
||||||
|
let worktrees_dir = root.join(".huskies/worktrees");
|
||||||
|
std::fs::create_dir_all(&worktrees_dir).unwrap();
|
||||||
|
|
||||||
|
// Simulate several stories stuck in 4_merge/ with a crashed/unassignable
|
||||||
|
// agent: each has a worktree directory (so find_worktree_path succeeds
|
||||||
|
// and the git-shelling check runs) but is not a real git repo and has
|
||||||
|
// no active agent entry in the pool.
|
||||||
|
const STORY_COUNT: usize = 25;
|
||||||
|
for i in 0..STORY_COUNT {
|
||||||
|
let story_id = format!("11700_merge_{i:03}");
|
||||||
|
std::fs::create_dir_all(worktrees_dir.join(&story_id)).unwrap();
|
||||||
|
crate::db::write_item_with_content(
|
||||||
|
&story_id,
|
||||||
|
"4_merge",
|
||||||
|
"---\nname: Crashed Merge\n---\n",
|
||||||
|
crate::db::ItemMeta::named("Crashed Merge"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3200);
|
||||||
|
let config = ProjectConfig::load(&root).unwrap_or_default();
|
||||||
|
|
||||||
|
// Stand in for the liveness heartbeat (tick_loop.rs's
|
||||||
|
// spawn_liveness_tick) and /health polling: a tight-interval task
|
||||||
|
// racing the merge scan on the single-worker-thread runtime.
|
||||||
|
let ticks = Arc::new(AtomicU64::new(0));
|
||||||
|
let ticks_clone = Arc::clone(&ticks);
|
||||||
|
let heartbeat = tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(2)).await;
|
||||||
|
ticks_clone.fetch_add(1, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
pool.assign_merge_stage(&root, &config).await;
|
||||||
|
let elapsed = start.elapsed();
|
||||||
|
|
||||||
|
heartbeat.abort();
|
||||||
|
let observed_ticks = ticks.load(Ordering::SeqCst);
|
||||||
|
|
||||||
|
// With a 2ms heartbeat cadence, an unstalled runtime should have
|
||||||
|
// fired roughly elapsed/2ms ticks. Require at least a quarter of that
|
||||||
|
// as a generous floor — a stalled runtime (pre-fix) produces ~0 ticks
|
||||||
|
// because the single worker thread never yields during the scan.
|
||||||
|
let expected_min_ticks = (elapsed.as_millis() / 2 / 4) as u64;
|
||||||
|
assert!(
|
||||||
|
observed_ticks >= expected_min_ticks,
|
||||||
|
"liveness heartbeat stalled during assign_merge_stage: {observed_ticks} tick(s) \
|
||||||
|
over {elapsed:?} (expected at least ~{expected_min_ticks}); the merge scan likely \
|
||||||
|
blocked the tokio runtime instead of yielding via spawn_blocking"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,52 +1,146 @@
|
|||||||
//! TransitionFired subscriber that auto-blocks stories after N consecutive MergeFailure transitions.
|
//! TransitionFired subscriber that owns the consecutive-MergeFailure budget:
|
||||||
|
//! auto-blocks stories at the threshold and auto-retries `GatesFailed`
|
||||||
|
//! failures below it.
|
||||||
//!
|
//!
|
||||||
//! Listens on the pipeline transition broadcast channel and, for each story,
|
//! Listens on the pipeline transition broadcast channel and, for each story,
|
||||||
//! counts how many times it has entered [`Stage::MergeFailure`] consecutively.
|
//! counts how many times it has entered [`Stage::MergeFailure`] consecutively.
|
||||||
//! When the count reaches the configurable threshold (default 3), the story is
|
//! One counter drives two policies sharing the `merge_failure_block_threshold`
|
||||||
//! transitioned to [`Stage::Blocked`] with a reason that names the failure kind.
|
//! budget (default 3):
|
||||||
//!
|
//!
|
||||||
//! The counter for a story resets whenever a non-`MergeFailure` transition fires
|
//! - **Below the threshold**, a `GatesFailed` failure schedules a delayed
|
||||||
//! for that story (e.g. after a successful merge or a `FixupRequested` demotion
|
//! re-trigger of the deterministic server-side merge (story 1185) — gates
|
||||||
//! back to coding).
|
//! failures are dominated by transients (flaky tests, stale base) that a
|
||||||
|
//! plain re-run fixes. Other kinds still count toward the budget but are
|
||||||
|
//! not retried: `ConflictDetected` has its own mergemaster recovery path via
|
||||||
|
//! [`super::merge_failure_subscriber`]; `EmptyDiff`/`NoCommits`/`Other`
|
||||||
|
//! require human intervention.
|
||||||
|
//! - **At the threshold**, the story is transitioned to [`Stage::Blocked`]
|
||||||
|
//! with a reason naming the failure kind.
|
||||||
|
//!
|
||||||
|
//! The counter resets when the story leaves `MergeFailure` for a real reason
|
||||||
|
//! (successful merge, `FixupRequested`, `Block`), but **not** on
|
||||||
|
//! [`PipelineEvent::MergeRetryStarted`] — that is the `MergeFailure → Merge`
|
||||||
|
//! bounce a retry itself causes. Treating it as a reset made the budget
|
||||||
|
//! unreachable and let a deterministic gates failure retry forever (1185
|
||||||
|
//! review finding 1); counting across the bounce is what makes the budget
|
||||||
|
//! real.
|
||||||
|
//!
|
||||||
|
//! Bug 1025: while a mergemaster is actively running on the story, its
|
||||||
|
//! iteration loop (squash → fail → fix → retry) generates multiple
|
||||||
|
//! MergeFailure transitions. Those are NOT consecutive give-ups — they are
|
||||||
|
//! recovery iterations in progress. We neither count nor schedule retries
|
||||||
|
//! while a mergemaster is in the pool for the story.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::pipeline_state::{MergeFailureKind, PipelineEvent, Stage, StoryId};
|
use crate::io::watcher::WatcherEvent;
|
||||||
|
use crate::pipeline_state::{MergeFailureKind, PipelineEvent, Stage, Status, StoryId};
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
use crate::slog_warn;
|
use crate::slog_warn;
|
||||||
|
|
||||||
use super::super::super::PipelineStage;
|
use super::super::super::PipelineStage;
|
||||||
use super::super::AgentPool;
|
use super::super::AgentPool;
|
||||||
use super::scan::is_story_assigned_for_stage;
|
use super::scan::is_story_assigned_for_stage;
|
||||||
|
use super::story_checks::{
|
||||||
|
has_review_hold, has_unmet_dependencies, is_story_blocked, is_story_frozen,
|
||||||
|
};
|
||||||
|
|
||||||
/// Spawn a background task that blocks stories after N consecutive `MergeFailure` transitions.
|
/// Delay before an auto-retry re-triggers the server-side merge for a
|
||||||
|
/// `GatesFailed` failure. Gives transient conditions (a concurrently landing
|
||||||
|
/// master merge, an exhausted runner) a moment to clear; retrying instantly
|
||||||
|
/// would just replay the same failure.
|
||||||
|
const AUTO_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
/// Per-story scheduling generation, shared between the subscriber loop and the
|
||||||
|
/// delayed retry tasks it spawns.
|
||||||
///
|
///
|
||||||
/// Subscribes to the pipeline transition broadcast channel and tracks a per-story
|
/// Every scheduled retry captures the generation current at schedule time; the
|
||||||
/// consecutive-failure counter. When a story's count reaches the threshold configured
|
/// timer only acts if that generation is still current when it fires. The
|
||||||
/// in `project.toml` (`merge_failure_block_threshold`, default 3), the story is
|
/// subscriber bumps the generation on every (re)schedule and clears the entry
|
||||||
/// transitioned to `Stage::Blocked` with a reason that names the failure kind.
|
/// on counter reset, so stale timers left over from an earlier failure cycle
|
||||||
|
/// become no-ops instead of firing unaccounted retries (1185 review finding 6).
|
||||||
|
type Generations = Arc<Mutex<HashMap<String, u64>>>;
|
||||||
|
|
||||||
|
/// What the subscriber decided to do about one transition. Split out from the
|
||||||
|
/// event loop so the counter/budget policy is synchronous and unit-testable.
|
||||||
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
enum Decision {
|
||||||
|
/// Nothing to do (not a MergeFailure, recovery in progress, retry bounce,
|
||||||
|
/// budget disabled, or a non-retryable kind below the threshold).
|
||||||
|
Nothing,
|
||||||
|
/// Schedule a delayed auto-retry: this is consecutive failure `attempt` of
|
||||||
|
/// a `budget`-sized budget, and the kind is `GatesFailed`.
|
||||||
|
ScheduleRetry { attempt: u32, budget: u32 },
|
||||||
|
/// The budget is exhausted: block the story.
|
||||||
|
Block { count: u32 },
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconcile: no-op for the periodic pass.
|
||||||
///
|
///
|
||||||
/// The counter resets when the story leaves `MergeFailure` (e.g. on `FixupRequested`,
|
/// The consecutive-failure counter is in-memory and cannot be reconstructed
|
||||||
/// `ReQueuedForQa`, or a successful merge via `Unblock → Merge → Done`).
|
/// from CRDT state (only the current stage is stored, not the failure
|
||||||
///
|
/// history). Restart catch-up for stories already parked in
|
||||||
/// Bug 1025: while a mergemaster is actively running on the story, its
|
/// `MergeFailure{GatesFailed}` is handled once, at subscriber startup, by
|
||||||
/// iteration loop (squash → fail → fix → retry) generates multiple
|
/// [`reconcile_stranded_gates_failed`] — running it from the periodic
|
||||||
/// MergeFailure transitions. Those are NOT consecutive give-ups — they are
|
/// reconciler instead would re-schedule retries for budget-exhausted stories
|
||||||
/// recovery iterations in progress. We skip counter increments while a
|
/// on every tick, reintroducing the unbounded-retry bug the startup-only scan
|
||||||
/// mergemaster is in the pool for the story; the counter only increments on
|
/// avoids.
|
||||||
/// transitions that happen with no recovery agent attached.
|
pub(crate) fn reconcile_merge_failure_block() {}
|
||||||
|
|
||||||
|
/// Spawn the background task that owns the consecutive-MergeFailure budget:
|
||||||
|
/// auto-retry for `GatesFailed` below the threshold, auto-block at it.
|
||||||
pub(crate) fn spawn_merge_failure_block_subscriber(pool: Arc<AgentPool>, project_root: PathBuf) {
|
pub(crate) fn spawn_merge_failure_block_subscriber(pool: Arc<AgentPool>, project_root: PathBuf) {
|
||||||
let mut rx = crate::pipeline_state::subscribe_transitions();
|
let mut rx = crate::pipeline_state::subscribe_transitions();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut counters: HashMap<StoryId, (u32, MergeFailureKind)> = HashMap::new();
|
let mut counters: HashMap<StoryId, (u32, MergeFailureKind)> = HashMap::new();
|
||||||
|
let generations: Generations = Generations::default();
|
||||||
|
|
||||||
|
// One-shot restart catch-up: stories already sitting in GatesFailed
|
||||||
|
// when the process starts will never fire another transition on their
|
||||||
|
// own, so without this they'd silently lose auto-retry coverage
|
||||||
|
// (1185 review finding 3).
|
||||||
|
reconcile_stranded_gates_failed(&pool, &project_root, &mut counters, &generations).await;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
match rx.recv().await {
|
match rx.recv().await {
|
||||||
Ok(fired) => {
|
Ok(fired) => {
|
||||||
let recovery_running =
|
let recovery_running =
|
||||||
is_mergemaster_running(&pool, &project_root, &fired.story_id.0);
|
is_mergemaster_running(&pool, &project_root, &fired.story_id.0).await;
|
||||||
on_transition(&project_root, &fired, &mut counters, recovery_running);
|
let threshold = load_threshold(&project_root).await;
|
||||||
|
match decide(&fired, &mut counters, recovery_running, threshold) {
|
||||||
|
Decision::Nothing => {
|
||||||
|
// A real departure from MergeFailure also
|
||||||
|
// invalidates any pending retry timer.
|
||||||
|
if fired.after.status() != Status::MergeFailure
|
||||||
|
&& !matches!(fired.event, PipelineEvent::MergeRetryStarted)
|
||||||
|
{
|
||||||
|
invalidate_generation(&generations, &fired.story_id.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Decision::ScheduleRetry { attempt, budget } => {
|
||||||
|
schedule_auto_retry(
|
||||||
|
Arc::clone(&pool),
|
||||||
|
project_root.clone(),
|
||||||
|
fired.story_id.0.clone(),
|
||||||
|
attempt,
|
||||||
|
budget,
|
||||||
|
Arc::clone(&generations),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Decision::Block { count } => {
|
||||||
|
let kind = counters
|
||||||
|
.get(&fired.story_id)
|
||||||
|
.map(|(_, k)| k.clone())
|
||||||
|
.unwrap_or(MergeFailureKind::Other(String::new()));
|
||||||
|
apply_block(&fired.story_id, count, &kind);
|
||||||
|
counters.remove(&fired.story_id);
|
||||||
|
invalidate_generation(&generations, &fired.story_id.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||||
slog_warn!(
|
slog_warn!(
|
||||||
@@ -60,87 +154,294 @@ pub(crate) fn spawn_merge_failure_block_subscriber(pool: Arc<AgentPool>, project
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return true if a mergemaster agent is currently in the pool for `story_id`.
|
/// Pure budget policy: given a fired transition and the per-story counter,
|
||||||
/// Used to suppress counter increments while recovery is actively iterating
|
/// decide whether to do nothing, schedule a `GatesFailed` auto-retry, or
|
||||||
/// (bug 1025).
|
/// block the story.
|
||||||
fn is_mergemaster_running(pool: &AgentPool, project_root: &Path, story_id: &str) -> bool {
|
|
||||||
let config = match crate::config::ProjectConfig::load(project_root) {
|
|
||||||
Ok(c) => c,
|
|
||||||
Err(_) => return false,
|
|
||||||
};
|
|
||||||
let agents = match pool.agents.lock() {
|
|
||||||
Ok(a) => a,
|
|
||||||
Err(_) => return false,
|
|
||||||
};
|
|
||||||
is_story_assigned_for_stage(&config, &agents, story_id, &PipelineStage::Mergemaster)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Handle a single transition event: update counters and emit Block if threshold is reached.
|
|
||||||
///
|
///
|
||||||
/// `recovery_running`: when `true`, a mergemaster is currently in the pool for
|
/// `threshold == 0` disables both policies (feature off), matching the
|
||||||
/// the story and the failure is part of an in-flight recovery loop. We do NOT
|
/// pre-1185 block-subscriber behaviour.
|
||||||
/// increment the consecutive-failure counter in that case (bug 1025).
|
fn decide(
|
||||||
fn on_transition(
|
|
||||||
project_root: &Path,
|
|
||||||
fired: &crate::pipeline_state::TransitionFired,
|
fired: &crate::pipeline_state::TransitionFired,
|
||||||
counters: &mut HashMap<StoryId, (u32, MergeFailureKind)>,
|
counters: &mut HashMap<StoryId, (u32, MergeFailureKind)>,
|
||||||
recovery_running: bool,
|
recovery_running: bool,
|
||||||
) {
|
threshold: u32,
|
||||||
match &fired.after {
|
) -> Decision {
|
||||||
Stage::MergeFailure { kind, .. } => {
|
// Story 1086: gate on the typed `Status` projection — `Status::MergeFailure`
|
||||||
|
// is precisely the set of stages we count toward the budget.
|
||||||
|
if fired.after.status() != Status::MergeFailure {
|
||||||
|
// The MergeFailure → Merge bounce caused by a retry (auto or manual)
|
||||||
|
// is not a recovery: the budget must survive it, or a deterministic
|
||||||
|
// failure retries forever (1185 review finding 1).
|
||||||
|
if !matches!(fired.event, PipelineEvent::MergeRetryStarted) {
|
||||||
|
counters.remove(&fired.story_id);
|
||||||
|
}
|
||||||
|
return Decision::Nothing;
|
||||||
|
}
|
||||||
|
let Stage::MergeFailure { kind, .. } = &fired.after else {
|
||||||
|
counters.remove(&fired.story_id);
|
||||||
|
return Decision::Nothing;
|
||||||
|
};
|
||||||
|
|
||||||
if recovery_running {
|
if recovery_running {
|
||||||
slog!(
|
slog!(
|
||||||
"[merge-block-sub] Story '{}' MergeFailure while mergemaster is running; \
|
"[merge-block-sub] Story '{}' MergeFailure while mergemaster is running; \
|
||||||
not counting toward block threshold (recovery in progress).",
|
not counting toward budget (recovery in progress).",
|
||||||
fired.story_id.0
|
fired.story_id.0
|
||||||
);
|
);
|
||||||
return;
|
return Decision::Nothing;
|
||||||
}
|
}
|
||||||
|
|
||||||
let entry = counters
|
let entry = counters
|
||||||
.entry(fired.story_id.clone())
|
.entry(fired.story_id.clone())
|
||||||
.or_insert_with(|| (0, kind.clone()));
|
.or_insert_with(|| (0, kind.clone()));
|
||||||
entry.0 += 1;
|
entry.0 += 1;
|
||||||
entry.1 = kind.clone();
|
entry.1 = kind.clone();
|
||||||
|
|
||||||
let count = entry.0;
|
let count = entry.0;
|
||||||
let threshold = load_threshold(project_root);
|
|
||||||
|
|
||||||
if threshold == 0 {
|
if threshold == 0 {
|
||||||
return;
|
return Decision::Nothing;
|
||||||
}
|
}
|
||||||
|
|
||||||
if count >= threshold {
|
if count >= threshold {
|
||||||
|
return Decision::Block { count };
|
||||||
|
}
|
||||||
|
if matches!(kind, MergeFailureKind::GatesFailed(_)) {
|
||||||
|
return Decision::ScheduleRetry {
|
||||||
|
attempt: count,
|
||||||
|
budget: threshold,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Decision::Nothing
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Transition `story_id` to `Blocked` with a reason naming the failure kind.
|
||||||
|
fn apply_block(story_id: &StoryId, count: u32, kind: &MergeFailureKind) {
|
||||||
let kind_str = failure_kind_label(kind);
|
let kind_str = failure_kind_label(kind);
|
||||||
let reason = format!(
|
let reason =
|
||||||
"Auto-blocked after {count} consecutive MergeFailure ({kind_str}) transitions."
|
format!("Auto-blocked after {count} consecutive MergeFailure ({kind_str}) transitions.");
|
||||||
);
|
let story_id = story_id.0.as_str();
|
||||||
let story_id = fired.story_id.0.as_str();
|
|
||||||
slog!(
|
slog!(
|
||||||
"[merge-block-sub] Story '{story_id}' reached {count} consecutive \
|
"[merge-block-sub] Story '{story_id}' reached {count} consecutive \
|
||||||
MergeFailure ({kind_str}); blocking."
|
MergeFailure ({kind_str}); blocking."
|
||||||
);
|
);
|
||||||
if let Err(e) = crate::pipeline_state::apply_transition(
|
if let Err(e) =
|
||||||
story_id,
|
crate::pipeline_state::apply_transition(story_id, PipelineEvent::Block { reason }, None)
|
||||||
PipelineEvent::Block { reason },
|
{
|
||||||
None,
|
|
||||||
) {
|
|
||||||
slog_warn!("[merge-block-sub] Failed to block '{story_id}': {e}");
|
slog_warn!("[merge-block-sub] Failed to block '{story_id}': {e}");
|
||||||
} else {
|
|
||||||
counters.remove(&fired.story_id);
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn the delayed auto-retry task for one `GatesFailed` failure.
|
||||||
|
///
|
||||||
|
/// Bumps the story's scheduling generation so any earlier still-sleeping timer
|
||||||
|
/// for the story becomes stale and skips itself.
|
||||||
|
fn schedule_auto_retry(
|
||||||
|
pool: Arc<AgentPool>,
|
||||||
|
project_root: PathBuf,
|
||||||
|
story_id: String,
|
||||||
|
attempt: u32,
|
||||||
|
budget: u32,
|
||||||
|
generations: Generations,
|
||||||
|
) {
|
||||||
|
let generation = next_generation(&generations, &story_id);
|
||||||
|
slog!(
|
||||||
|
"[merge-block-sub] Story '{story_id}' GatesFailed (attempt {attempt}/{budget}); \
|
||||||
|
scheduling auto-retry in {AUTO_RETRY_DELAY:?}."
|
||||||
|
);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
tokio::time::sleep(AUTO_RETRY_DELAY).await;
|
||||||
|
perform_auto_retry(
|
||||||
|
&pool,
|
||||||
|
&project_root,
|
||||||
|
&story_id,
|
||||||
|
attempt,
|
||||||
|
budget,
|
||||||
|
&generations,
|
||||||
|
generation,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fire one auto-retry: re-check that acting is still correct, start the
|
||||||
|
/// server-side merge, and notify chat only when the merge actually started
|
||||||
|
/// (1185 review finding 7).
|
||||||
|
///
|
||||||
|
/// Guards, in order:
|
||||||
|
/// - the scheduling generation is still current (no newer schedule/reset);
|
||||||
|
/// - the story is still parked in `MergeFailure{GatesFailed}`;
|
||||||
|
/// - the story is not frozen/blocked/on hold/dep-blocked — the same
|
||||||
|
/// eligibility gates `assign_merge_stage` applies (1185 review finding 4);
|
||||||
|
/// - no mergemaster is actively recovering the story (1185 review finding 2).
|
||||||
|
async fn perform_auto_retry(
|
||||||
|
pool: &Arc<AgentPool>,
|
||||||
|
project_root: &Path,
|
||||||
|
story_id: &str,
|
||||||
|
attempt: u32,
|
||||||
|
budget: u32,
|
||||||
|
generations: &Generations,
|
||||||
|
generation: u64,
|
||||||
|
) {
|
||||||
|
if !is_generation_current(generations, story_id, generation) {
|
||||||
|
slog!(
|
||||||
|
"[merge-block-sub] Story '{story_id}' auto-retry ({attempt}/{budget}) is stale \
|
||||||
|
(superseded or reset); skipping."
|
||||||
|
);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let still_gates_failed = matches!(
|
||||||
|
crate::pipeline_state::read_typed(story_id),
|
||||||
|
Ok(Some(item)) if matches!(
|
||||||
|
item.stage,
|
||||||
|
Stage::MergeFailure {
|
||||||
|
kind: MergeFailureKind::GatesFailed(_),
|
||||||
|
..
|
||||||
}
|
}
|
||||||
_ => {
|
)
|
||||||
counters.remove(&fired.story_id);
|
);
|
||||||
|
if !still_gates_failed {
|
||||||
|
slog!(
|
||||||
|
"[merge-block-sub] Story '{story_id}' left GatesFailed before auto-retry \
|
||||||
|
({attempt}/{budget}) fired; skipping."
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if has_review_hold(story_id)
|
||||||
|
|| is_story_frozen(story_id)
|
||||||
|
|| is_story_blocked(story_id)
|
||||||
|
|| has_unmet_dependencies(story_id)
|
||||||
|
{
|
||||||
|
slog!(
|
||||||
|
"[merge-block-sub] Story '{story_id}' is held/frozen/blocked/dep-blocked; \
|
||||||
|
skipping auto-retry ({attempt}/{budget})."
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if is_mergemaster_running(pool, project_root, story_id).await {
|
||||||
|
slog!(
|
||||||
|
"[merge-block-sub] Story '{story_id}' has an active mergemaster; \
|
||||||
|
skipping auto-retry ({attempt}/{budget}) — recovery owns the story."
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
match pool.start_merge_agent_work(project_root, story_id) {
|
||||||
|
Ok(()) => {
|
||||||
|
slog!(
|
||||||
|
"[merge-block-sub] Auto-retrying merge for '{story_id}' \
|
||||||
|
(attempt {attempt}/{budget})."
|
||||||
|
);
|
||||||
|
let _ = pool.watcher_tx.send(WatcherEvent::MergeAutoRetry {
|
||||||
|
story_id: story_id.to_string(),
|
||||||
|
attempt,
|
||||||
|
budget,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
slog_warn!(
|
||||||
|
"[merge-block-sub] Auto-retry for '{story_id}' ({attempt}/{budget}) \
|
||||||
|
could not start: {e}; not notifying."
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load the threshold from project config, falling back to the compiled default.
|
/// One-shot startup scan: schedule a first auto-retry for every story already
|
||||||
fn load_threshold(project_root: &Path) -> u32 {
|
/// parked in `MergeFailure{GatesFailed}`.
|
||||||
crate::config::ProjectConfig::load(project_root)
|
///
|
||||||
|
/// The pre-restart attempt count is unrecoverable, so the counter restarts at
|
||||||
|
/// 1 — worst case a story gets up to `threshold - 1` extra retries across a
|
||||||
|
/// restart, still bounded per process lifetime.
|
||||||
|
async fn reconcile_stranded_gates_failed(
|
||||||
|
pool: &Arc<AgentPool>,
|
||||||
|
project_root: &Path,
|
||||||
|
counters: &mut HashMap<StoryId, (u32, MergeFailureKind)>,
|
||||||
|
generations: &Generations,
|
||||||
|
) {
|
||||||
|
let threshold = load_threshold(project_root).await;
|
||||||
|
if threshold == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for item in crate::pipeline_state::read_all_typed() {
|
||||||
|
let Stage::MergeFailure { kind, .. } = &item.stage else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !matches!(kind, MergeFailureKind::GatesFailed(_)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
counters.insert(item.story_id.clone(), (1, kind.clone()));
|
||||||
|
slog!(
|
||||||
|
"[merge-block-sub] Story '{}' found parked in GatesFailed at startup; \
|
||||||
|
scheduling catch-up auto-retry (attempt 1/{threshold}).",
|
||||||
|
item.story_id.0
|
||||||
|
);
|
||||||
|
schedule_auto_retry(
|
||||||
|
Arc::clone(pool),
|
||||||
|
project_root.to_path_buf(),
|
||||||
|
item.story_id.0.clone(),
|
||||||
|
1,
|
||||||
|
threshold,
|
||||||
|
Arc::clone(generations),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bump and return the scheduling generation for `story_id`.
|
||||||
|
fn next_generation(generations: &Generations, story_id: &str) -> u64 {
|
||||||
|
let mut map = generations.lock().unwrap_or_else(|p| p.into_inner());
|
||||||
|
let entry = map.entry(story_id.to_string()).or_insert(0);
|
||||||
|
*entry += 1;
|
||||||
|
*entry
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop the generation entry for `story_id`, making every pending timer stale.
|
||||||
|
fn invalidate_generation(generations: &Generations, story_id: &str) {
|
||||||
|
generations
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|p| p.into_inner())
|
||||||
|
.remove(story_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True when `expected` is still the current scheduling generation.
|
||||||
|
fn is_generation_current(generations: &Generations, story_id: &str, expected: u64) -> bool {
|
||||||
|
generations
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|p| p.into_inner())
|
||||||
|
.get(story_id)
|
||||||
|
== Some(&expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return true if a mergemaster agent is currently in the pool for `story_id`.
|
||||||
|
/// Used to suppress counting and retries while recovery is actively iterating
|
||||||
|
/// (bug 1025).
|
||||||
|
async fn is_mergemaster_running(pool: &AgentPool, project_root: &Path, story_id: &str) -> bool {
|
||||||
|
let root = project_root.to_path_buf();
|
||||||
|
let config = match tokio::task::spawn_blocking(move || {
|
||||||
|
crate::config::ProjectConfig::load(&root)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Ok(c)) => c,
|
||||||
|
_ => return false,
|
||||||
|
};
|
||||||
|
let agents = pool.agents.lock().await;
|
||||||
|
is_story_assigned_for_stage(&config, &agents, story_id, &PipelineStage::Mergemaster)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load the budget from project config off the async runtime (the read is
|
||||||
|
/// synchronous filesystem I/O — bug 1170 class), falling back to the compiled
|
||||||
|
/// default.
|
||||||
|
async fn load_threshold(project_root: &Path) -> u32 {
|
||||||
|
let root = project_root.to_path_buf();
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
crate::config::ProjectConfig::load(&root)
|
||||||
.map(|c| c.merge_failure_block_threshold)
|
.map(|c| c.merge_failure_block_threshold)
|
||||||
.unwrap_or(3)
|
.unwrap_or(3)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or(3)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Short human-readable label for a [`MergeFailureKind`] variant.
|
/// Short human-readable label for a [`MergeFailureKind`] variant.
|
||||||
@@ -162,11 +463,7 @@ mod tests {
|
|||||||
use crate::pipeline_state::{BranchName, PipelineEvent, Stage, StoryId, TransitionFired};
|
use crate::pipeline_state::{BranchName, PipelineEvent, Stage, StoryId, TransitionFired};
|
||||||
use std::num::NonZeroU32;
|
use std::num::NonZeroU32;
|
||||||
|
|
||||||
fn setup_project(tmp: &tempfile::TempDir) {
|
const THRESHOLD: u32 = 3;
|
||||||
let sk = tmp.path().join(".huskies");
|
|
||||||
std::fs::create_dir_all(&sk).unwrap();
|
|
||||||
std::fs::write(sk.join("project.toml"), "[[agent]]\nname = \"coder\"\n").unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
fn seed_at_merge(story_id: &str) {
|
fn seed_at_merge(story_id: &str) {
|
||||||
crate::crdt_state::init_for_test();
|
crate::crdt_state::init_for_test();
|
||||||
@@ -217,218 +514,232 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// AC3 (threshold-not-reached): 2 consecutive failures below threshold of 3 must NOT block.
|
/// The MergeFailure → Merge bounce a retry causes.
|
||||||
#[test]
|
fn make_retry_started_fired(story_id: &str) -> TransitionFired {
|
||||||
fn below_threshold_does_not_block() {
|
TransitionFired {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
story_id: StoryId(story_id.to_string()),
|
||||||
setup_project(&tmp);
|
before: Stage::MergeFailure {
|
||||||
let story_id = "1018_below";
|
kind: MergeFailureKind::GatesFailed("error".to_string()),
|
||||||
seed_at_merge(story_id);
|
feature_branch: BranchName("feature/test".to_string()),
|
||||||
|
commits_ahead: NonZeroU32::new(1).unwrap(),
|
||||||
// Transition to MergeFailure once to establish the stage.
|
},
|
||||||
crate::agents::lifecycle::transition_to_merge_failure(
|
after: Stage::Merge {
|
||||||
story_id,
|
feature_branch: BranchName("feature/test".to_string()),
|
||||||
MergeFailureKind::GatesFailed("error".to_string()),
|
commits_ahead: NonZeroU32::new(1).unwrap(),
|
||||||
)
|
claim: None,
|
||||||
.expect("initial MergeFailure transition");
|
retries: 1,
|
||||||
|
server_start_time: None,
|
||||||
let mut counters: HashMap<StoryId, (u32, MergeFailureKind)> = HashMap::new();
|
},
|
||||||
let kind = MergeFailureKind::GatesFailed("error".to_string());
|
event: PipelineEvent::MergeRetryStarted,
|
||||||
|
at: chrono::Utc::now(),
|
||||||
// Fire 2 MergeFailure events (default threshold is 3).
|
}
|
||||||
for _ in 0..2 {
|
|
||||||
let fired = make_merge_failure_fired(story_id, kind.clone());
|
|
||||||
on_transition(tmp.path(), &fired, &mut counters, false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Story must still be in MergeFailure (not Blocked).
|
fn gates_failed() -> MergeFailureKind {
|
||||||
let item = crate::pipeline_state::read_typed(story_id)
|
MergeFailureKind::GatesFailed("error".to_string())
|
||||||
.expect("read")
|
}
|
||||||
.expect("item");
|
|
||||||
assert!(
|
/// Below the threshold, GatesFailed schedules a retry with the right
|
||||||
matches!(item.stage, Stage::MergeFailure { .. }),
|
/// attempt numbering.
|
||||||
"story must still be in MergeFailure after 2 failures (threshold 3): {:?}",
|
#[test]
|
||||||
item.stage
|
fn gates_failed_below_threshold_schedules_retry() {
|
||||||
|
let mut counters = HashMap::new();
|
||||||
|
let fired = make_merge_failure_fired("t_sched", gates_failed());
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
decide(&fired, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::ScheduleRetry {
|
||||||
|
attempt: 1,
|
||||||
|
budget: THRESHOLD
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decide(&fired, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::ScheduleRetry {
|
||||||
|
attempt: 2,
|
||||||
|
budget: THRESHOLD
|
||||||
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// AC3 (threshold-reached): 3 consecutive failures at threshold of 3 must block.
|
/// 1185 review finding 1 (regression): the retry's own MergeFailure→Merge
|
||||||
|
/// bounce must NOT reset the counter — the third consecutive failure
|
||||||
|
/// blocks even though retries happened in between.
|
||||||
#[test]
|
#[test]
|
||||||
fn at_threshold_blocks_with_failure_kind_in_reason() {
|
fn merge_retry_started_does_not_reset_counter() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let mut counters = HashMap::new();
|
||||||
setup_project(&tmp);
|
let story = "t_no_reset";
|
||||||
|
let fail = make_merge_failure_fired(story, gates_failed());
|
||||||
|
let bounce = make_retry_started_fired(story);
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
decide(&fail, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::ScheduleRetry { attempt: 1, .. }
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
decide(&bounce, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::Nothing
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
decide(&fail, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::ScheduleRetry { attempt: 2, .. }
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
decide(&bounce, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::Nothing
|
||||||
|
);
|
||||||
|
// Third consecutive failure: budget exhausted despite the bounces.
|
||||||
|
assert_eq!(
|
||||||
|
decide(&fail, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::Block { count: 3 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A real departure (FixupRequested → Coding) still resets the counter.
|
||||||
|
#[test]
|
||||||
|
fn real_departure_resets_counter() {
|
||||||
|
let mut counters = HashMap::new();
|
||||||
|
let story = "t_reset";
|
||||||
|
let fail = make_merge_failure_fired(story, gates_failed());
|
||||||
|
|
||||||
|
decide(&fail, &mut counters, false, THRESHOLD);
|
||||||
|
decide(&fail, &mut counters, false, THRESHOLD);
|
||||||
|
assert_eq!(
|
||||||
|
counters.get(&StoryId(story.to_string())).map(|e| e.0),
|
||||||
|
Some(2)
|
||||||
|
);
|
||||||
|
|
||||||
|
decide(&make_coding_fired(story), &mut counters, false, THRESHOLD);
|
||||||
|
assert!(!counters.contains_key(&StoryId(story.to_string())));
|
||||||
|
|
||||||
|
// Fresh failures start a fresh budget.
|
||||||
|
assert!(matches!(
|
||||||
|
decide(&fail, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::ScheduleRetry { attempt: 1, .. }
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Non-GatesFailed kinds count toward the block budget but never schedule
|
||||||
|
/// a retry (ConflictDetected has its own mergemaster path; the rest need
|
||||||
|
/// humans).
|
||||||
|
#[test]
|
||||||
|
fn non_gates_failed_counts_but_does_not_retry() {
|
||||||
|
let mut counters = HashMap::new();
|
||||||
|
let story = "t_conflict";
|
||||||
|
let conflict = make_merge_failure_fired(story, MergeFailureKind::ConflictDetected(None));
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
decide(&conflict, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::Nothing
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
counters.get(&StoryId(story.to_string())).map(|e| e.0),
|
||||||
|
Some(1)
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
decide(&conflict, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::Nothing
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decide(&conflict, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::Block { count: 3 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mixed kinds share one budget: GatesFailed and ConflictDetected
|
||||||
|
/// interleavings block at the same total count (1185 review finding 5).
|
||||||
|
#[test]
|
||||||
|
fn mixed_kinds_share_one_budget() {
|
||||||
|
let mut counters = HashMap::new();
|
||||||
|
let story = "t_mixed";
|
||||||
|
let fail = make_merge_failure_fired(story, gates_failed());
|
||||||
|
let conflict = make_merge_failure_fired(story, MergeFailureKind::ConflictDetected(None));
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
decide(&fail, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::ScheduleRetry { attempt: 1, .. }
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
decide(&conflict, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::Nothing
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decide(&fail, &mut counters, false, THRESHOLD),
|
||||||
|
Decision::Block { count: 3 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bug 1025: recovery in progress neither counts nor schedules.
|
||||||
|
#[test]
|
||||||
|
fn mergemaster_running_suppresses_counting_and_retry() {
|
||||||
|
let mut counters = HashMap::new();
|
||||||
|
let story = "t_recovery";
|
||||||
|
let fail = make_merge_failure_fired(story, gates_failed());
|
||||||
|
|
||||||
|
for _ in 0..3 {
|
||||||
|
assert_eq!(
|
||||||
|
decide(&fail, &mut counters, true, THRESHOLD),
|
||||||
|
Decision::Nothing
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!counters.contains_key(&StoryId(story.to_string())));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// threshold == 0 disables both policies.
|
||||||
|
#[test]
|
||||||
|
fn threshold_zero_disables_block_and_retry() {
|
||||||
|
let mut counters = HashMap::new();
|
||||||
|
let fail = make_merge_failure_fired("t_disabled", gates_failed());
|
||||||
|
for _ in 0..5 {
|
||||||
|
assert_eq!(decide(&fail, &mut counters, false, 0), Decision::Nothing);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applying a Block decision transitions the story and names the kind.
|
||||||
|
#[test]
|
||||||
|
fn apply_block_blocks_with_failure_kind_in_reason() {
|
||||||
let story_id = "1018_at_threshold";
|
let story_id = "1018_at_threshold";
|
||||||
seed_at_merge(story_id);
|
seed_at_merge(story_id);
|
||||||
|
|
||||||
crate::agents::lifecycle::transition_to_merge_failure(
|
crate::agents::lifecycle::transition_to_merge_failure(
|
||||||
story_id,
|
story_id,
|
||||||
MergeFailureKind::GatesFailed("fmt error".to_string()),
|
MergeFailureKind::GatesFailed("fmt error".to_string()),
|
||||||
)
|
)
|
||||||
.expect("initial MergeFailure transition");
|
.expect("initial MergeFailure transition");
|
||||||
|
|
||||||
let mut counters: HashMap<StoryId, (u32, MergeFailureKind)> = HashMap::new();
|
apply_block(
|
||||||
let kind = MergeFailureKind::GatesFailed("fmt error".to_string());
|
&StoryId(story_id.to_string()),
|
||||||
|
3,
|
||||||
// Fire 3 MergeFailure events — the 3rd must trigger the block.
|
&MergeFailureKind::GatesFailed("fmt error".to_string()),
|
||||||
for _ in 0..3 {
|
);
|
||||||
let fired = make_merge_failure_fired(story_id, kind.clone());
|
|
||||||
on_transition(tmp.path(), &fired, &mut counters, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
let item = crate::pipeline_state::read_typed(story_id)
|
let item = crate::pipeline_state::read_typed(story_id)
|
||||||
.expect("read")
|
.expect("read")
|
||||||
.expect("item");
|
.expect("item");
|
||||||
assert!(
|
match &item.stage {
|
||||||
matches!(item.stage, Stage::Blocked { .. }),
|
Stage::Blocked { reason } => {
|
||||||
"story must be Blocked after 3 consecutive MergeFailures: {:?}",
|
|
||||||
item.stage
|
|
||||||
);
|
|
||||||
|
|
||||||
// The block reason must name the failure kind.
|
|
||||||
if let Stage::Blocked { reason } = &item.stage {
|
|
||||||
assert!(
|
assert!(
|
||||||
reason.contains("GatesFailed"),
|
reason.contains("GatesFailed"),
|
||||||
"block reason must name the failure kind: {reason}"
|
"block reason must name the failure kind: {reason}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
other => panic!("story must be Blocked: {other:?}"),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// AC3 (reset): counter clears after a non-MergeFailure transition.
|
/// 1185 review finding 6 (regression): a newer schedule or a reset makes
|
||||||
///
|
/// earlier timers stale.
|
||||||
/// 2 failures → FixupRequested reset → 2 more failures: still below threshold, no block.
|
|
||||||
#[test]
|
#[test]
|
||||||
fn counter_resets_on_non_merge_failure_transition() {
|
fn stale_generations_are_not_current() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let generations: Generations = Generations::default();
|
||||||
setup_project(&tmp);
|
let g1 = next_generation(&generations, "s");
|
||||||
let story_id = "1018_reset";
|
assert!(is_generation_current(&generations, "s", g1));
|
||||||
seed_at_merge(story_id);
|
|
||||||
|
|
||||||
crate::agents::lifecycle::transition_to_merge_failure(
|
let g2 = next_generation(&generations, "s");
|
||||||
story_id,
|
assert!(!is_generation_current(&generations, "s", g1));
|
||||||
MergeFailureKind::ConflictDetected(None),
|
assert!(is_generation_current(&generations, "s", g2));
|
||||||
)
|
|
||||||
.expect("initial MergeFailure transition");
|
|
||||||
|
|
||||||
let mut counters: HashMap<StoryId, (u32, MergeFailureKind)> = HashMap::new();
|
invalidate_generation(&generations, "s");
|
||||||
let kind = MergeFailureKind::ConflictDetected(None);
|
assert!(!is_generation_current(&generations, "s", g2));
|
||||||
|
|
||||||
// Fire 2 MergeFailure events.
|
|
||||||
for _ in 0..2 {
|
|
||||||
let fired = make_merge_failure_fired(story_id, kind.clone());
|
|
||||||
on_transition(tmp.path(), &fired, &mut counters, false);
|
|
||||||
}
|
|
||||||
assert_eq!(
|
|
||||||
counters.get(&StoryId(story_id.to_string())).map(|e| e.0),
|
|
||||||
Some(2),
|
|
||||||
"counter must be 2 after 2 failures"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Simulate FixupRequested (non-MergeFailure transition).
|
|
||||||
let reset_fired = make_coding_fired(story_id);
|
|
||||||
on_transition(tmp.path(), &reset_fired, &mut counters, false);
|
|
||||||
assert!(
|
|
||||||
!counters.contains_key(&StoryId(story_id.to_string())),
|
|
||||||
"counter must be cleared after non-MergeFailure transition"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Re-seed to MergeFailure so we can apply the block transition.
|
|
||||||
crate::agents::lifecycle::transition_to_merge_failure(
|
|
||||||
story_id,
|
|
||||||
MergeFailureKind::ConflictDetected(None),
|
|
||||||
)
|
|
||||||
.expect("re-enter MergeFailure after reset");
|
|
||||||
|
|
||||||
// Fire 2 more MergeFailure events — still below threshold.
|
|
||||||
for _ in 0..2 {
|
|
||||||
let fired = make_merge_failure_fired(story_id, kind.clone());
|
|
||||||
on_transition(tmp.path(), &fired, &mut counters, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
let item = crate::pipeline_state::read_typed(story_id)
|
|
||||||
.expect("read")
|
|
||||||
.expect("item");
|
|
||||||
assert!(
|
|
||||||
matches!(item.stage, Stage::MergeFailure { .. }),
|
|
||||||
"story must still be in MergeFailure after reset + 2 new failures: {:?}",
|
|
||||||
item.stage
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Bug 1025: while a mergemaster is running, MergeFailure transitions are
|
|
||||||
/// recovery iterations, not consecutive give-ups. 3 failures with
|
|
||||||
/// `recovery_running=true` must NOT block.
|
|
||||||
#[test]
|
|
||||||
fn mergemaster_running_suppresses_block() {
|
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
|
||||||
setup_project(&tmp);
|
|
||||||
let story_id = "1025_recovery_running";
|
|
||||||
seed_at_merge(story_id);
|
|
||||||
|
|
||||||
crate::agents::lifecycle::transition_to_merge_failure(
|
|
||||||
story_id,
|
|
||||||
MergeFailureKind::ConflictDetected(None),
|
|
||||||
)
|
|
||||||
.expect("initial MergeFailure transition");
|
|
||||||
|
|
||||||
let mut counters: HashMap<StoryId, (u32, MergeFailureKind)> = HashMap::new();
|
|
||||||
let kind = MergeFailureKind::ConflictDetected(None);
|
|
||||||
|
|
||||||
// Fire 3 MergeFailure events WHILE a mergemaster is running (gated).
|
|
||||||
for _ in 0..3 {
|
|
||||||
let fired = make_merge_failure_fired(story_id, kind.clone());
|
|
||||||
on_transition(tmp.path(), &fired, &mut counters, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Counter must NOT have incremented at all — recovery in progress.
|
|
||||||
assert!(
|
|
||||||
!counters.contains_key(&StoryId(story_id.to_string())),
|
|
||||||
"counter must not increment while mergemaster is running"
|
|
||||||
);
|
|
||||||
|
|
||||||
// And the story must still be in MergeFailure (not Blocked).
|
|
||||||
let item = crate::pipeline_state::read_typed(story_id)
|
|
||||||
.expect("read")
|
|
||||||
.expect("item");
|
|
||||||
assert!(
|
|
||||||
matches!(item.stage, Stage::MergeFailure { .. }),
|
|
||||||
"story must NOT be blocked while mergemaster is running (recovery in progress): {:?}",
|
|
||||||
item.stage
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Bug 1025 regression guard: the genuinely-stuck case (no mergemaster
|
|
||||||
/// running) still blocks at the threshold, so the original 1018 behaviour
|
|
||||||
/// is preserved.
|
|
||||||
#[test]
|
|
||||||
fn no_mergemaster_still_blocks_at_threshold() {
|
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
|
||||||
setup_project(&tmp);
|
|
||||||
let story_id = "1025_genuine_stuck";
|
|
||||||
seed_at_merge(story_id);
|
|
||||||
|
|
||||||
crate::agents::lifecycle::transition_to_merge_failure(
|
|
||||||
story_id,
|
|
||||||
MergeFailureKind::ConflictDetected(None),
|
|
||||||
)
|
|
||||||
.expect("initial MergeFailure transition");
|
|
||||||
|
|
||||||
let mut counters: HashMap<StoryId, (u32, MergeFailureKind)> = HashMap::new();
|
|
||||||
let kind = MergeFailureKind::ConflictDetected(None);
|
|
||||||
|
|
||||||
// Fire 3 MergeFailure events with NO mergemaster (recovery_running=false).
|
|
||||||
for _ in 0..3 {
|
|
||||||
let fired = make_merge_failure_fired(story_id, kind.clone());
|
|
||||||
on_transition(tmp.path(), &fired, &mut counters, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Story must be Blocked (genuine-stuck case unchanged).
|
|
||||||
let item = crate::pipeline_state::read_typed(story_id)
|
|
||||||
.expect("read")
|
|
||||||
.expect("item");
|
|
||||||
assert!(
|
|
||||||
matches!(item.stage, Stage::Blocked { .. }),
|
|
||||||
"story must still block when no mergemaster is running: {:?}",
|
|
||||||
item.stage
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::pipeline_state::{MergeFailureKind, Stage};
|
use crate::pipeline_state::{MergeFailureKind, Stage, Status};
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
use crate::slog_warn;
|
use crate::slog_warn;
|
||||||
|
|
||||||
@@ -17,6 +17,35 @@ use super::super::super::PipelineStage;
|
|||||||
use super::super::AgentPool;
|
use super::super::AgentPool;
|
||||||
use super::scan::{find_free_agent_for_stage, is_story_assigned_for_stage};
|
use super::scan::{find_free_agent_for_stage, is_story_assigned_for_stage};
|
||||||
|
|
||||||
|
/// Reconcile: for each story currently in `MergeFailure { kind: ConflictDetected }`,
|
||||||
|
/// ensure a mergemaster agent is running.
|
||||||
|
///
|
||||||
|
/// Idempotent — `on_merge_failure_transition` guards against double-spawning via
|
||||||
|
/// `is_story_assigned_for_stage`. Called by the periodic reconciler so that a Lagged
|
||||||
|
/// startup event never leaves a ConflictDetected story without a recovery agent.
|
||||||
|
pub(crate) async fn reconcile_merge_failure(pool: &Arc<AgentPool>, project_root: &Path) {
|
||||||
|
use crate::pipeline_state::{MergeFailureKind, PipelineEvent, Stage, TransitionFired};
|
||||||
|
for item in crate::pipeline_state::read_all_typed() {
|
||||||
|
// Story 1086: scan via the Status projection; the variant pattern is
|
||||||
|
// still needed to read `kind`.
|
||||||
|
if item.stage.status() != Status::MergeFailure {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Stage::MergeFailure { ref kind, .. } = item.stage
|
||||||
|
&& matches!(kind, MergeFailureKind::ConflictDetected(_))
|
||||||
|
{
|
||||||
|
let fired = TransitionFired {
|
||||||
|
story_id: item.story_id.clone(),
|
||||||
|
before: item.stage.clone(),
|
||||||
|
after: item.stage.clone(),
|
||||||
|
event: PipelineEvent::MergeFailed { kind: kind.clone() },
|
||||||
|
at: chrono::Utc::now(),
|
||||||
|
};
|
||||||
|
on_merge_failure_transition(pool, project_root, &fired).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Spawn a background task that auto-spawns mergemaster agents on
|
/// Spawn a background task that auto-spawns mergemaster agents on
|
||||||
/// `Stage::MergeFailure { kind: ConflictDetected(_) }` transitions.
|
/// `Stage::MergeFailure { kind: ConflictDetected(_) }` transitions.
|
||||||
///
|
///
|
||||||
@@ -49,6 +78,11 @@ async fn on_merge_failure_transition(
|
|||||||
project_root: &Path,
|
project_root: &Path,
|
||||||
fired: &crate::pipeline_state::TransitionFired,
|
fired: &crate::pipeline_state::TransitionFired,
|
||||||
) {
|
) {
|
||||||
|
// Story 1086: gate on the typed `Status` projection first; only the
|
||||||
|
// `MergeFailure` kind extraction needs the variant pattern.
|
||||||
|
if fired.after.status() != Status::MergeFailure {
|
||||||
|
return;
|
||||||
|
}
|
||||||
let Stage::MergeFailure { ref kind, .. } = fired.after else {
|
let Stage::MergeFailure { ref kind, .. } = fired.after else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
@@ -66,15 +100,7 @@ async fn on_merge_failure_transition(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let agent_name = {
|
let agent_name = {
|
||||||
let agents = match pool.agents.lock() {
|
let agents = pool.agents.lock().await;
|
||||||
Ok(a) => a,
|
|
||||||
Err(e) => {
|
|
||||||
slog_warn!(
|
|
||||||
"[merge-failure-sub] Failed to lock agent pool for '{story_id}': {e}"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if is_story_assigned_for_stage(
|
if is_story_assigned_for_stage(
|
||||||
&config,
|
&config,
|
||||||
&agents,
|
&agents,
|
||||||
@@ -194,7 +220,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
on_merge_failure_transition(&pool, tmp.path(), &fired).await;
|
on_merge_failure_transition(&pool, tmp.path(), &fired).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.lock().await;
|
||||||
assert!(
|
assert!(
|
||||||
agents.iter().any(|(key, a)| {
|
agents.iter().any(|(key, a)| {
|
||||||
key.contains(story_id)
|
key.contains(story_id)
|
||||||
@@ -225,7 +251,7 @@ mod tests {
|
|||||||
// Give the subscriber time to run (it should do nothing).
|
// Give the subscriber time to run (it should do nothing).
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.lock().await;
|
||||||
let spawned = agents.iter().any(|(key, a)| {
|
let spawned = agents.iter().any(|(key, a)| {
|
||||||
key.contains(story_id)
|
key.contains(story_id)
|
||||||
&& a.agent_name == "mergemaster"
|
&& a.agent_name == "mergemaster"
|
||||||
@@ -253,7 +279,7 @@ mod tests {
|
|||||||
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.lock().await;
|
||||||
let spawned = agents.iter().any(|(key, a)| {
|
let spawned = agents.iter().any(|(key, a)| {
|
||||||
key.contains(story_id)
|
key.contains(story_id)
|
||||||
&& a.agent_name == "mergemaster"
|
&& a.agent_name == "mergemaster"
|
||||||
@@ -281,7 +307,7 @@ mod tests {
|
|||||||
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.lock().await;
|
||||||
let spawned = agents.iter().any(|(key, a)| {
|
let spawned = agents.iter().any(|(key, a)| {
|
||||||
key.contains(story_id)
|
key.contains(story_id)
|
||||||
&& a.agent_name == "mergemaster"
|
&& a.agent_name == "mergemaster"
|
||||||
@@ -309,7 +335,7 @@ mod tests {
|
|||||||
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.lock().await;
|
||||||
let spawned = agents.iter().any(|(key, a)| {
|
let spawned = agents.iter().any(|(key, a)| {
|
||||||
key.contains(story_id)
|
key.contains(story_id)
|
||||||
&& a.agent_name == "mergemaster"
|
&& a.agent_name == "mergemaster"
|
||||||
@@ -340,7 +366,7 @@ mod tests {
|
|||||||
// First call — spawns mergemaster (agent enters Pending).
|
// First call — spawns mergemaster (agent enters Pending).
|
||||||
on_merge_failure_transition(&pool, tmp.path(), &fired).await;
|
on_merge_failure_transition(&pool, tmp.path(), &fired).await;
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.lock().await;
|
||||||
assert!(
|
assert!(
|
||||||
agents.iter().any(|(key, a)| {
|
agents.iter().any(|(key, a)| {
|
||||||
key.contains(story_id)
|
key.contains(story_id)
|
||||||
@@ -354,7 +380,7 @@ mod tests {
|
|||||||
// Second call (self-loop) — agent is still Pending; guard must prevent double-spawn.
|
// Second call (self-loop) — agent is still Pending; guard must prevent double-spawn.
|
||||||
on_merge_failure_transition(&pool, tmp.path(), &fired).await;
|
on_merge_failure_transition(&pool, tmp.path(), &fired).await;
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.lock().await;
|
||||||
let active_count = agents
|
let active_count = agents
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(key, a)| {
|
.filter(|(key, a)| {
|
||||||
|
|||||||
@@ -4,7 +4,8 @@
|
|||||||
mod auto_assign;
|
mod auto_assign;
|
||||||
mod backlog;
|
mod backlog;
|
||||||
mod merge;
|
mod merge;
|
||||||
/// TransitionFired subscriber that auto-blocks stories after N consecutive MergeFailure transitions.
|
/// TransitionFired subscriber owning the consecutive-MergeFailure budget:
|
||||||
|
/// auto-retries GatesFailed below the threshold, auto-blocks at it.
|
||||||
pub(crate) mod merge_failure_block_subscriber;
|
pub(crate) mod merge_failure_block_subscriber;
|
||||||
/// TransitionFired subscriber that auto-spawns mergemaster on ConflictDetected merge failures.
|
/// TransitionFired subscriber that auto-spawns mergemaster on ConflictDetected merge failures.
|
||||||
pub(crate) mod merge_failure_subscriber;
|
pub(crate) mod merge_failure_subscriber;
|
||||||
@@ -17,7 +18,11 @@ pub(crate) mod watchdog;
|
|||||||
// so that pool::lifecycle and pool::pipeline continue to access them unchanged.
|
// so that pool::lifecycle and pool::pipeline continue to access them unchanged.
|
||||||
pub(super) use scan::{find_free_agent_for_stage, is_agent_free};
|
pub(super) use scan::{find_free_agent_for_stage, is_agent_free};
|
||||||
|
|
||||||
|
/// Re-export for `startup::tick_loop`.
|
||||||
|
pub(crate) use merge_failure_block_subscriber::reconcile_merge_failure_block;
|
||||||
/// Re-export for `startup::tick_loop`.
|
/// Re-export for `startup::tick_loop`.
|
||||||
pub(crate) use merge_failure_block_subscriber::spawn_merge_failure_block_subscriber;
|
pub(crate) use merge_failure_block_subscriber::spawn_merge_failure_block_subscriber;
|
||||||
/// Re-export for `startup::tick_loop`.
|
/// Re-export for `startup::tick_loop`.
|
||||||
|
pub(crate) use merge_failure_subscriber::reconcile_merge_failure;
|
||||||
|
/// Re-export for `startup::tick_loop`.
|
||||||
pub(crate) use merge_failure_subscriber::spawn_merge_failure_subscriber;
|
pub(crate) use merge_failure_subscriber::spawn_merge_failure_subscriber;
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ use std::path::Path;
|
|||||||
use crate::config::ProjectConfig;
|
use crate::config::ProjectConfig;
|
||||||
use crate::pipeline_state::Stage;
|
use crate::pipeline_state::Stage;
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
use crate::slog_error;
|
|
||||||
|
|
||||||
use super::super::super::PipelineStage;
|
use super::super::super::PipelineStage;
|
||||||
use super::super::AgentPool;
|
use super::super::AgentPool;
|
||||||
@@ -80,13 +79,7 @@ impl AgentPool {
|
|||||||
if *stage == PipelineStage::Coder
|
if *stage == PipelineStage::Coder
|
||||||
&& let Some(max) = config.max_coders
|
&& let Some(max) = config.max_coders
|
||||||
{
|
{
|
||||||
let agents_lock = match self.agents.lock() {
|
let agents_lock = self.agents.lock().await;
|
||||||
Ok(a) => a,
|
|
||||||
Err(e) => {
|
|
||||||
slog_error!("[auto-assign] Failed to lock agents: {e}");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let active = count_active_agents_for_stage(config, &agents_lock, stage);
|
let active = count_active_agents_for_stage(config, &agents_lock, stage);
|
||||||
if active >= max {
|
if active >= max {
|
||||||
slog!(
|
slog!(
|
||||||
@@ -102,13 +95,7 @@ impl AgentPool {
|
|||||||
// stage_mismatch=true means the preferred agent's stage doesn't match the
|
// stage_mismatch=true means the preferred agent's stage doesn't match the
|
||||||
// pipeline stage, so we fell back to a generic stage agent.
|
// pipeline stage, so we fell back to a generic stage agent.
|
||||||
let (already_assigned, free_agent, preferred_busy, stage_mismatch) = {
|
let (already_assigned, free_agent, preferred_busy, stage_mismatch) = {
|
||||||
let agents = match self.agents.lock() {
|
let agents = self.agents.lock().await;
|
||||||
Ok(a) => a,
|
|
||||||
Err(e) => {
|
|
||||||
slog_error!("[auto-assign] Failed to lock agents: {e}");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let assigned = is_story_assigned_for_stage(config, &agents, story_id, stage);
|
let assigned = is_story_assigned_for_stage(config, &agents, story_id, stage);
|
||||||
if assigned {
|
if assigned {
|
||||||
(true, None, false, false)
|
(true, None, false, false)
|
||||||
|
|||||||
@@ -256,7 +256,7 @@ mod tests {
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent("42_story_foo", "coder-1", AgentStatus::Running);
|
pool.inject_test_agent("42_story_foo", "coder-1", AgentStatus::Running);
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
assert!(is_story_assigned_for_stage(
|
assert!(is_story_assigned_for_stage(
|
||||||
&config,
|
&config,
|
||||||
&agents,
|
&agents,
|
||||||
@@ -285,7 +285,7 @@ mod tests {
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent("42_story_foo", "coder-1", AgentStatus::Completed);
|
pool.inject_test_agent("42_story_foo", "coder-1", AgentStatus::Completed);
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
// Completed agents don't count as assigned
|
// Completed agents don't count as assigned
|
||||||
assert!(!is_story_assigned_for_stage(
|
assert!(!is_story_assigned_for_stage(
|
||||||
&config,
|
&config,
|
||||||
@@ -309,7 +309,7 @@ stage = "qa"
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent("42_story_foo", "qa-2", AgentStatus::Running);
|
pool.inject_test_agent("42_story_foo", "qa-2", AgentStatus::Running);
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
// qa-2 with stage=qa should be recognised as a QA agent
|
// qa-2 with stage=qa should be recognised as a QA agent
|
||||||
assert!(
|
assert!(
|
||||||
is_story_assigned_for_stage(&config, &agents, "42_story_foo", &PipelineStage::Qa),
|
is_story_assigned_for_stage(&config, &agents, "42_story_foo", &PipelineStage::Qa),
|
||||||
@@ -338,7 +338,7 @@ name = "coder-2"
|
|||||||
pool.inject_test_agent("s1", "coder-1", AgentStatus::Running);
|
pool.inject_test_agent("s1", "coder-1", AgentStatus::Running);
|
||||||
pool.inject_test_agent("s2", "coder-2", AgentStatus::Running);
|
pool.inject_test_agent("s2", "coder-2", AgentStatus::Running);
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let free = find_free_agent_for_stage(&config, &agents, &PipelineStage::Coder);
|
let free = find_free_agent_for_stage(&config, &agents, &PipelineStage::Coder);
|
||||||
assert!(free.is_none(), "no free coders should be available");
|
assert!(free.is_none(), "no free coders should be available");
|
||||||
}
|
}
|
||||||
@@ -361,7 +361,7 @@ name = "coder-3"
|
|||||||
// coder-1 is busy, coder-2 is free
|
// coder-1 is busy, coder-2 is free
|
||||||
pool.inject_test_agent("s1", "coder-1", AgentStatus::Running);
|
pool.inject_test_agent("s1", "coder-1", AgentStatus::Running);
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let free = find_free_agent_for_stage(&config, &agents, &PipelineStage::Coder);
|
let free = find_free_agent_for_stage(&config, &agents, &PipelineStage::Coder);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
free,
|
free,
|
||||||
@@ -384,7 +384,7 @@ name = "coder-1"
|
|||||||
// coder-1 completed its previous story — it's free for a new one
|
// coder-1 completed its previous story — it's free for a new one
|
||||||
pool.inject_test_agent("s1", "coder-1", AgentStatus::Completed);
|
pool.inject_test_agent("s1", "coder-1", AgentStatus::Completed);
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let free = find_free_agent_for_stage(&config, &agents, &PipelineStage::Coder);
|
let free = find_free_agent_for_stage(&config, &agents, &PipelineStage::Coder);
|
||||||
assert_eq!(free, Some("coder-1"), "completed coder-1 should be free");
|
assert_eq!(free, Some("coder-1"), "completed coder-1 should be free");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
use crate::agents::pool::StoryAgent;
|
use crate::agents::pool::StoryAgent;
|
||||||
@@ -83,7 +83,7 @@ pub(crate) fn count_turns_in_log(path: &Path) -> u64 {
|
|||||||
/// Turns and budget are counted from the **current session's** log file
|
/// Turns and budget are counted from the **current session's** log file
|
||||||
/// only — prior sessions are excluded so that restart counts from earlier
|
/// only — prior sessions are excluded so that restart counts from earlier
|
||||||
/// runs do not accumulate against the limits.
|
/// runs do not accumulate against the limits.
|
||||||
pub(super) fn check_agent_limits(
|
pub(super) async fn check_agent_limits(
|
||||||
agents: &Mutex<HashMap<String, StoryAgent>>,
|
agents: &Mutex<HashMap<String, StoryAgent>>,
|
||||||
project_root: &Path,
|
project_root: &Path,
|
||||||
) -> Vec<(String, TerminationReason)> {
|
) -> Vec<(String, TerminationReason)> {
|
||||||
@@ -94,10 +94,7 @@ pub(super) fn check_agent_limits(
|
|||||||
|
|
||||||
// Snapshot running agents: (key, story_id, agent_name, tx, log_session_id).
|
// Snapshot running agents: (key, story_id, agent_name, tx, log_session_id).
|
||||||
let running: Vec<RunningAgentSnapshot> = {
|
let running: Vec<RunningAgentSnapshot> = {
|
||||||
let lock = match agents.lock() {
|
let lock = agents.lock().await;
|
||||||
Ok(l) => l,
|
|
||||||
Err(_) => return Vec::new(),
|
|
||||||
};
|
|
||||||
lock.iter()
|
lock.iter()
|
||||||
.filter(|(_, agent)| agent.status == AgentStatus::Running)
|
.filter(|(_, agent)| agent.status == AgentStatus::Running)
|
||||||
.map(|(key, agent)| {
|
.map(|(key, agent)| {
|
||||||
@@ -187,13 +184,14 @@ pub(super) fn check_agent_limits(
|
|||||||
),
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Mark agent as Failed with termination reason.
|
// NOTE: agent status is intentionally NOT updated here. Setting
|
||||||
if let Ok(mut lock) = agents.lock()
|
// `status = Failed` before the kill (the previous behaviour)
|
||||||
&& let Some(agent) = lock.get_mut(key)
|
// opened a window where the `start_agent` idempotency check
|
||||||
{
|
// (which whitelists Running/Pending) would let a fresh spawn
|
||||||
agent.status = AgentStatus::Failed;
|
// through while the prior PTY child was still alive — directly
|
||||||
agent.termination_reason = Some(reason.clone());
|
// causing the concurrent-agents bug we hit on story 1086
|
||||||
}
|
// (2026-05-15). The caller (`run_watchdog_pass`) is responsible
|
||||||
|
// for: (1) verifying the kill, (2) THEN updating the agent record.
|
||||||
|
|
||||||
slog!("[watchdog] Terminating agent '{key}': {reason_str}.");
|
slog!("[watchdog] Terminating agent '{key}': {reason_str}.");
|
||||||
|
|
||||||
|
|||||||
@@ -4,17 +4,23 @@
|
|||||||
mod budget;
|
mod budget;
|
||||||
mod limits;
|
mod limits;
|
||||||
mod orphan;
|
mod orphan;
|
||||||
|
mod reap;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|
||||||
|
use std::collections::HashSet;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
|
use crate::agents::AgentStatus;
|
||||||
use crate::config::ProjectConfig;
|
use crate::config::ProjectConfig;
|
||||||
|
use crate::process_kill::{pids_matching, sigkill_pids_and_verify};
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
|
use crate::slog_warn;
|
||||||
|
|
||||||
use super::super::AgentPool;
|
use super::super::AgentPool;
|
||||||
use limits::check_agent_limits;
|
use limits::check_agent_limits;
|
||||||
use orphan::check_orphaned_agents;
|
use orphan::check_orphaned_agents;
|
||||||
|
use reap::reap_failed_agents;
|
||||||
|
|
||||||
pub(crate) use budget::{compute_budget_from_logs, compute_budget_from_single_log};
|
pub(crate) use budget::{compute_budget_from_logs, compute_budget_from_single_log};
|
||||||
pub(crate) use limits::{count_turns_in_log, resolve_session_log};
|
pub(crate) use limits::{count_turns_in_log, resolve_session_log};
|
||||||
@@ -22,8 +28,8 @@ pub(crate) use limits::{count_turns_in_log, resolve_session_log};
|
|||||||
impl AgentPool {
|
impl AgentPool {
|
||||||
/// Run a single watchdog pass synchronously (test helper).
|
/// Run a single watchdog pass synchronously (test helper).
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub fn run_watchdog_once(&self) {
|
pub async fn run_watchdog_once(&self) {
|
||||||
check_orphaned_agents(&self.agents);
|
check_orphaned_agents(&self.agents).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run one watchdog pass: detect orphans, enforce limits, kill offenders.
|
/// Run one watchdog pass: detect orphans, enforce limits, kill offenders.
|
||||||
@@ -36,21 +42,75 @@ impl AgentPool {
|
|||||||
/// `retry_count` is incremented and the story stays in `2_current/` for
|
/// `retry_count` is incremented and the story stays in `2_current/` for
|
||||||
/// re-attempt. This prevents the original kill-respawn loop (bug 646)
|
/// re-attempt. This prevents the original kill-respawn loop (bug 646)
|
||||||
/// while restoring the `max_retries` semantic for turn/budget overruns.
|
/// while restoring the `max_retries` semantic for turn/budget overruns.
|
||||||
pub fn run_watchdog_pass(&self, project_root: Option<&Path>) -> usize {
|
pub async fn run_watchdog_pass(&self, project_root: Option<&Path>) -> usize {
|
||||||
let orphaned = check_orphaned_agents(&self.agents);
|
let orphaned = check_orphaned_agents(&self.agents).await;
|
||||||
|
|
||||||
if let Some(root) = project_root {
|
if let Some(root) = project_root {
|
||||||
let terminated = check_agent_limits(&self.agents, root);
|
let terminated = check_agent_limits(&self.agents, root).await;
|
||||||
let config = ProjectConfig::load(root).unwrap_or_default();
|
let config = ProjectConfig::load(root).unwrap_or_default();
|
||||||
for (key, _reason) in &terminated {
|
let mut just_terminated: HashSet<String> = HashSet::new();
|
||||||
// Kill the PTY child and abort the task, same as stop_agent.
|
for (key, reason) in &terminated {
|
||||||
self.kill_child_for_key(key);
|
just_terminated.insert(key.clone());
|
||||||
if let Ok(mut lock) = self.agents.lock()
|
// Step 1: snapshot the agent's worktree path so we can find every
|
||||||
&& let Some(agent) = lock.get_mut(key)
|
// process running in it (claude + any subprocesses). This must
|
||||||
&& let Some(handle) = agent.task_handle.take()
|
// happen BEFORE we mutate the agent record so we can read the
|
||||||
|
// worktree info safely.
|
||||||
|
let worktree_path = {
|
||||||
|
let lock = self.agents.lock().await;
|
||||||
|
lock.get(key)
|
||||||
|
.and_then(|a| a.worktree_info.as_ref().map(|wt| wt.path.clone()))
|
||||||
|
};
|
||||||
|
|
||||||
|
// Step 2: SIGKILL every process running in the worktree and
|
||||||
|
// BLOCK until verified gone. The previous mechanism — portable_pty's
|
||||||
|
// `ChildKiller::kill()` — sends SIGHUP, which claude-code
|
||||||
|
// ignores, leaving the process alive while the agent record
|
||||||
|
// was being marked terminated; that gap let a fresh spawn race
|
||||||
|
// in alongside the surviving one. SIGKILL is uncatchable;
|
||||||
|
// [`sigkill_pids_and_verify`] only returns once the kernel has
|
||||||
|
// reaped each pid.
|
||||||
|
if let Some(wt_path) = worktree_path.as_ref() {
|
||||||
|
let pids = pids_matching(&wt_path.display().to_string());
|
||||||
|
if pids.is_empty() {
|
||||||
|
// Nothing in this worktree — agent likely already
|
||||||
|
// exited on its own before the watchdog noticed.
|
||||||
|
} else {
|
||||||
|
match sigkill_pids_and_verify(&pids) {
|
||||||
|
Ok(n) => slog!(
|
||||||
|
"[watchdog] SIGKILL'd {n} process(es) in worktree {} for '{key}'.",
|
||||||
|
wt_path.display()
|
||||||
|
),
|
||||||
|
Err(survivors) => slog_warn!(
|
||||||
|
"[watchdog] SIGKILL incomplete for '{key}': pids still alive: {survivors:?}. \
|
||||||
|
Proceeding with cleanup; concurrent spawn protection may be weakened."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
slog_warn!(
|
||||||
|
"[watchdog] No worktree path recorded for '{key}'; cannot tree-kill, \
|
||||||
|
falling back to portable_pty SIGHUP (likely no-op for claude-code)."
|
||||||
|
);
|
||||||
|
self.kill_child_for_key(key).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: NOW update the agent record. The process is verified
|
||||||
|
// gone (or we logged that SIGKILL didn't take effect, which is
|
||||||
|
// exceptional), so flipping status away from Running can no
|
||||||
|
// longer open a window for a concurrent spawn.
|
||||||
{
|
{
|
||||||
|
let mut lock = self.agents.lock().await;
|
||||||
|
if let Some(agent) = lock.get_mut(key) {
|
||||||
|
agent.status = AgentStatus::Failed;
|
||||||
|
agent.termination_reason = Some(reason.clone());
|
||||||
|
if let Some(handle) = agent.task_handle.take() {
|
||||||
|
// Best-effort abort of the outer tokio task. The PTY
|
||||||
|
// blocking thread already returned (claude is dead),
|
||||||
|
// so this is bookkeeping rather than load-bearing.
|
||||||
handle.abort();
|
handle.abort();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Use the retry mechanism: increment retry_count and only block
|
// Use the retry mechanism: increment retry_count and only block
|
||||||
// when the limit is exceeded, matching the pipeline's behaviour.
|
// when the limit is exceeded, matching the pipeline's behaviour.
|
||||||
@@ -77,7 +137,16 @@ impl AgentPool {
|
|||||||
if !terminated.is_empty() {
|
if !terminated.is_empty() {
|
||||||
Self::notify_agent_state_changed(&self.watcher_tx);
|
Self::notify_agent_state_changed(&self.watcher_tx);
|
||||||
}
|
}
|
||||||
return orphaned + terminated.len();
|
|
||||||
|
// Bug 1198: reap any other Failed pool entry with no live process
|
||||||
|
// — orphan-detected above, or left behind by a spawn error
|
||||||
|
// (inactivity-watchdog kill, worktree timeout, runtime error)
|
||||||
|
// that never routed through the retry/respawn path. Entries the
|
||||||
|
// limits loop above just processed are excluded so their retry
|
||||||
|
// count isn't bumped twice.
|
||||||
|
let reaped = reap_failed_agents(self, root, &config, &just_terminated).await;
|
||||||
|
|
||||||
|
return orphaned + terminated.len() + reaped;
|
||||||
}
|
}
|
||||||
|
|
||||||
orphaned
|
orphaned
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
//! Orphan detection: marks running agents whose backing task has exited.
|
//! Orphan detection: marks running agents whose backing task has exited.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
use crate::agents::pool::StoryAgent;
|
use crate::agents::pool::StoryAgent;
|
||||||
@@ -15,11 +15,8 @@ use crate::slog;
|
|||||||
/// without updating the agent status — for example when the process is killed
|
/// without updating the agent status — for example when the process is killed
|
||||||
/// externally and the PTY master fd returns EOF before our inactivity timeout
|
/// externally and the PTY master fd returns EOF before our inactivity timeout
|
||||||
/// fires, but some other edge case prevents the normal cleanup path from running.
|
/// fires, but some other edge case prevents the normal cleanup path from running.
|
||||||
pub(super) fn check_orphaned_agents(agents: &Mutex<HashMap<String, StoryAgent>>) -> usize {
|
pub(super) async fn check_orphaned_agents(agents: &Mutex<HashMap<String, StoryAgent>>) -> usize {
|
||||||
let mut lock = match agents.lock() {
|
let mut lock = agents.lock().await;
|
||||||
Ok(l) => l,
|
|
||||||
Err(_) => return 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Collect orphaned entries: Running or Pending agents whose task handle is finished.
|
// Collect orphaned entries: Running or Pending agents whose task handle is finished.
|
||||||
// Pending agents can be orphaned if worktree creation panics before setting status.
|
// Pending agents can be orphaned if worktree creation panics before setting status.
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
//! Reap: removes stale `Failed` pool entries left behind by orphan detection,
|
||||||
|
//! watchdog kills, or internal spawn failures, and respawns the story's agent
|
||||||
|
//! (or blocks the story once its retry budget is exhausted).
|
||||||
|
//!
|
||||||
|
//! `check_orphaned_agents` only scans `Running`/`Pending` entries, so once an
|
||||||
|
//! entry is marked `Failed` it becomes invisible to every later watchdog
|
||||||
|
//! pass. Without this step a `Failed` entry with no live process sits in the
|
||||||
|
//! pool forever: `list_agents` keeps showing it, and the story never gets a
|
||||||
|
//! new agent unless some unrelated CRDT transition happens to trigger a
|
||||||
|
//! system-wide auto-assign scan.
|
||||||
|
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use crate::agents::AgentStatus;
|
||||||
|
use crate::agents::pool::AgentPool;
|
||||||
|
use crate::agents::pool::pipeline::should_block_story;
|
||||||
|
use crate::config::ProjectConfig;
|
||||||
|
use crate::io::watcher::WatcherEvent;
|
||||||
|
use crate::{slog, slog_warn};
|
||||||
|
|
||||||
|
/// Reap every `Failed` pool entry with no live process, except keys in
|
||||||
|
/// `exclude_keys` (already handled by the caller's own retry/block logic in
|
||||||
|
/// this same pass — e.g. limit-exceeded kills).
|
||||||
|
///
|
||||||
|
/// For each reaped entry: removes it from the pool (so `list_agents` stops
|
||||||
|
/// showing it), increments the story's retry count via [`should_block_story`]
|
||||||
|
/// and, unless that blocks the story, respawns the agent by name.
|
||||||
|
/// `start_agent`'s own session-store lookup resumes the prior session
|
||||||
|
/// automatically whenever one was recorded — no explicit session plumbing
|
||||||
|
/// needed here.
|
||||||
|
pub(super) async fn reap_failed_agents(
|
||||||
|
pool: &AgentPool,
|
||||||
|
project_root: &Path,
|
||||||
|
config: &ProjectConfig,
|
||||||
|
exclude_keys: &HashSet<String>,
|
||||||
|
) -> usize {
|
||||||
|
let dead: Vec<(String, String, String)> = {
|
||||||
|
let mut agents = pool.agents.lock().await;
|
||||||
|
let keys: Vec<String> = agents
|
||||||
|
.iter()
|
||||||
|
.filter(|(key, agent)| {
|
||||||
|
agent.status == AgentStatus::Failed
|
||||||
|
&& !exclude_keys.contains(*key)
|
||||||
|
&& agent
|
||||||
|
.task_handle
|
||||||
|
.as_ref()
|
||||||
|
.map(|h| h.is_finished())
|
||||||
|
.unwrap_or(true)
|
||||||
|
})
|
||||||
|
.map(|(key, _)| key.clone())
|
||||||
|
.collect();
|
||||||
|
keys.into_iter()
|
||||||
|
.filter_map(|key| {
|
||||||
|
agents.remove(&key).map(|agent| {
|
||||||
|
let story_id = key
|
||||||
|
.rsplit_once(':')
|
||||||
|
.map(|(s, _)| s.to_string())
|
||||||
|
.unwrap_or_else(|| key.clone());
|
||||||
|
(key, story_id, agent.agent_name)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
|
||||||
|
let count = dead.len();
|
||||||
|
for (key, story_id, agent_name) in dead {
|
||||||
|
if let Some(block_reason) = should_block_story(&story_id, config.max_retries, "watchdog") {
|
||||||
|
let _ = pool.watcher_tx.send(WatcherEvent::StoryBlocked {
|
||||||
|
story_id: story_id.clone(),
|
||||||
|
reason: block_reason,
|
||||||
|
});
|
||||||
|
slog!(
|
||||||
|
"[watchdog] Story '{story_id}' blocked after exceeding retry limit \
|
||||||
|
(reaped dead pool entry '{key}')."
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
slog!(
|
||||||
|
"[watchdog] Reaping dead pool entry '{key}'; respawning '{agent_name}' \
|
||||||
|
for '{story_id}'."
|
||||||
|
);
|
||||||
|
if let Err(e) = pool
|
||||||
|
.start_agent(project_root, &story_id, Some(&agent_name), None, None)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
slog_warn!(
|
||||||
|
"[watchdog] Failed to respawn '{agent_name}' for '{story_id}' after reap: {e}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if count > 0 {
|
||||||
|
AgentPool::notify_agent_state_changed(&pool.watcher_tx);
|
||||||
|
}
|
||||||
|
|
||||||
|
count
|
||||||
|
}
|
||||||
@@ -10,8 +10,8 @@ use crate::agents::{AgentEvent, AgentStatus, TerminationReason};
|
|||||||
|
|
||||||
// ── Limit enforcement integration tests (bug 624) ────────────────────────
|
// ── Limit enforcement integration tests (bug 624) ────────────────────────
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn watchdog_terminates_agent_exceeding_turn_limit() {
|
async fn watchdog_terminates_agent_exceeding_turn_limit() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let root = tmp.path();
|
let root = tmp.path();
|
||||||
|
|
||||||
@@ -37,12 +37,12 @@ max_turns = 10
|
|||||||
);
|
);
|
||||||
let mut rx = tx.subscribe();
|
let mut rx = tx.subscribe();
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert!(found >= 1, "watchdog should detect the over-limit agent");
|
assert!(found >= 1, "watchdog should detect the over-limit agent");
|
||||||
|
|
||||||
// Agent should now be Failed with TurnLimit reason.
|
// Agent should now be Failed with TurnLimit reason.
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("story_a", "coder-1");
|
let key = composite_key("story_a", "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(agent.status, AgentStatus::Failed);
|
assert_eq!(agent.status, AgentStatus::Failed);
|
||||||
@@ -60,8 +60,8 @@ max_turns = 10
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn watchdog_terminates_agent_exceeding_budget_limit() {
|
async fn watchdog_terminates_agent_exceeding_budget_limit() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let root = tmp.path();
|
let root = tmp.path();
|
||||||
|
|
||||||
@@ -87,11 +87,11 @@ max_budget_usd = 5.00
|
|||||||
);
|
);
|
||||||
let mut rx = tx.subscribe();
|
let mut rx = tx.subscribe();
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert!(found >= 1, "watchdog should detect the over-budget agent");
|
assert!(found >= 1, "watchdog should detect the over-budget agent");
|
||||||
|
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("story_b", "coder-1");
|
let key = composite_key("story_b", "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(agent.status, AgentStatus::Failed);
|
assert_eq!(agent.status, AgentStatus::Failed);
|
||||||
@@ -106,8 +106,8 @@ max_budget_usd = 5.00
|
|||||||
assert!(matches!(event, AgentEvent::Error { .. }));
|
assert!(matches!(event, AgentEvent::Error { .. }));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn watchdog_does_not_terminate_agent_under_limits() {
|
async fn watchdog_does_not_terminate_agent_under_limits() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let root = tmp.path();
|
let root = tmp.path();
|
||||||
|
|
||||||
@@ -133,11 +133,11 @@ max_budget_usd = 10.00
|
|||||||
// has 25 turns < 50 so no violation).
|
// has 25 turns < 50 so no violation).
|
||||||
pool.inject_test_agent_with_session("story_c", "coder-1", AgentStatus::Running, "sess-ok");
|
pool.inject_test_agent_with_session("story_c", "coder-1", AgentStatus::Running, "sess-ok");
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert_eq!(found, 0, "agent under limits should not be terminated");
|
assert_eq!(found, 0, "agent under limits should not be terminated");
|
||||||
|
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("story_c", "coder-1");
|
let key = composite_key("story_c", "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -153,8 +153,8 @@ max_budget_usd = 10.00
|
|||||||
/// coder-1 with max_turns=50, max_budget_usd=5.00 ran 5.6× over the turn
|
/// coder-1 with max_turns=50, max_budget_usd=5.00 ran 5.6× over the turn
|
||||||
/// limit (280 turns). The watchdog must terminate at the turn limit (turns
|
/// limit (280 turns). The watchdog must terminate at the turn limit (turns
|
||||||
/// hit first in the observed trace), with reason TurnLimit.
|
/// hit first in the observed trace), with reason TurnLimit.
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn regression_bug624_coder1_story623_trajectory() {
|
async fn regression_bug624_coder1_story623_trajectory() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let root = tmp.path();
|
let root = tmp.path();
|
||||||
|
|
||||||
@@ -183,11 +183,11 @@ max_budget_usd = 5.00
|
|||||||
);
|
);
|
||||||
let mut rx = tx.subscribe();
|
let mut rx = tx.subscribe();
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert!(found >= 1, "watchdog must catch the turn-limit violation");
|
assert!(found >= 1, "watchdog must catch the turn-limit violation");
|
||||||
|
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("story_623", "coder-1");
|
let key = composite_key("story_623", "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(agent.status, AgentStatus::Failed);
|
assert_eq!(agent.status, AgentStatus::Failed);
|
||||||
@@ -218,8 +218,8 @@ max_budget_usd = 5.00
|
|||||||
///
|
///
|
||||||
/// This test seeds a single session that legitimately exceeds the limit
|
/// This test seeds a single session that legitimately exceeds the limit
|
||||||
/// and uses `max_retries = 1` so that the first violation blocks.
|
/// and uses `max_retries = 1` so that the first violation blocks.
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn watchdog_marks_story_blocked_after_limit_termination() {
|
async fn watchdog_marks_story_blocked_after_limit_termination() {
|
||||||
crate::db::ensure_content_store();
|
crate::db::ensure_content_store();
|
||||||
crate::crdt_state::init_for_test();
|
crate::crdt_state::init_for_test();
|
||||||
|
|
||||||
@@ -263,7 +263,7 @@ max_turns = 10
|
|||||||
"sess-runaway",
|
"sess-runaway",
|
||||||
);
|
);
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert!(found >= 1, "watchdog should detect the over-limit agent");
|
assert!(found >= 1, "watchdog should detect the over-limit agent");
|
||||||
|
|
||||||
// With max_retries=1, the first violation blocks immediately via the state machine.
|
// With max_retries=1, the first violation blocks immediately via the state machine.
|
||||||
@@ -278,7 +278,7 @@ max_turns = 10
|
|||||||
|
|
||||||
// Sanity: the agent itself is also Failed with the right reason.
|
// Sanity: the agent itself is also Failed with the right reason.
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key(story_id, "coder-1");
|
let key = composite_key(story_id, "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(agent.status, AgentStatus::Failed);
|
assert_eq!(agent.status, AgentStatus::Failed);
|
||||||
@@ -297,8 +297,8 @@ max_turns = 10
|
|||||||
/// fresh session_id whose log has fewer events than `max_turns`.
|
/// fresh session_id whose log has fewer events than `max_turns`.
|
||||||
/// Assert the agent is NOT terminated (per-session count is under the
|
/// Assert the agent is NOT terminated (per-session count is under the
|
||||||
/// limit) AND the story is NOT marked blocked.
|
/// limit) AND the story is NOT marked blocked.
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn per_session_counting_does_not_terminate_under_limit() {
|
async fn per_session_counting_does_not_terminate_under_limit() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let root = tmp.path();
|
let root = tmp.path();
|
||||||
|
|
||||||
@@ -323,14 +323,14 @@ max_turns = 10
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent_with_session("story_d", "coder-1", AgentStatus::Running, "new-sess");
|
pool.inject_test_agent_with_session("story_d", "coder-1", AgentStatus::Running, "new-sess");
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
found, 0,
|
found, 0,
|
||||||
"agent under per-session limit should NOT be terminated"
|
"agent under per-session limit should NOT be terminated"
|
||||||
);
|
);
|
||||||
|
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("story_d", "coder-1");
|
let key = composite_key("story_d", "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -345,8 +345,8 @@ max_turns = 10
|
|||||||
/// Same setup as per_session_counting_does_not_terminate_under_limit, but
|
/// Same setup as per_session_counting_does_not_terminate_under_limit, but
|
||||||
/// the new agent's own session log exceeds `max_turns`. Assert the agent
|
/// the new agent's own session log exceeds `max_turns`. Assert the agent
|
||||||
/// IS terminated AND (with max_retries=1) the story IS marked blocked.
|
/// IS terminated AND (with max_retries=1) the story IS marked blocked.
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn per_session_counting_terminates_over_limit() {
|
async fn per_session_counting_terminates_over_limit() {
|
||||||
crate::db::ensure_content_store();
|
crate::db::ensure_content_store();
|
||||||
crate::crdt_state::init_for_test();
|
crate::crdt_state::init_for_test();
|
||||||
|
|
||||||
@@ -390,14 +390,14 @@ max_turns = 10
|
|||||||
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "new-sess");
|
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "new-sess");
|
||||||
let mut rx = tx.subscribe();
|
let mut rx = tx.subscribe();
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert!(
|
assert!(
|
||||||
found >= 1,
|
found >= 1,
|
||||||
"agent over per-session limit must be terminated"
|
"agent over per-session limit must be terminated"
|
||||||
);
|
);
|
||||||
|
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key(story_id, "coder-1");
|
let key = composite_key(story_id, "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(agent.status, AgentStatus::Failed);
|
assert_eq!(agent.status, AgentStatus::Failed);
|
||||||
@@ -423,8 +423,8 @@ max_turns = 10
|
|||||||
/// `max_turns`. After session 1: retry_count=1, NOT blocked. After
|
/// `max_turns`. After session 1: retry_count=1, NOT blocked. After
|
||||||
/// session 2: retry_count=2, NOT blocked. After session 3:
|
/// session 2: retry_count=2, NOT blocked. After session 3:
|
||||||
/// retry_count=3 >= max_retries, story IS blocked.
|
/// retry_count=3 >= max_retries, story IS blocked.
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn watchdog_retry_semantic_blocks_after_max_retries() {
|
async fn watchdog_retry_semantic_blocks_after_max_retries() {
|
||||||
crate::db::ensure_content_store();
|
crate::db::ensure_content_store();
|
||||||
|
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
@@ -453,7 +453,7 @@ max_turns = 10
|
|||||||
write_fake_session_log(root, story_id, "coder-1", "session-1", 12);
|
write_fake_session_log(root, story_id, "coder-1", "session-1", 12);
|
||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-1");
|
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-1");
|
||||||
pool.run_watchdog_pass(Some(root));
|
pool.run_watchdog_pass(Some(root)).await;
|
||||||
|
|
||||||
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -473,7 +473,7 @@ max_turns = 10
|
|||||||
write_fake_session_log(root, story_id, "coder-1", "session-2", 12);
|
write_fake_session_log(root, story_id, "coder-1", "session-2", 12);
|
||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-2");
|
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-2");
|
||||||
pool.run_watchdog_pass(Some(root));
|
pool.run_watchdog_pass(Some(root)).await;
|
||||||
|
|
||||||
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -493,7 +493,7 @@ max_turns = 10
|
|||||||
write_fake_session_log(root, story_id, "coder-1", "session-3", 12);
|
write_fake_session_log(root, story_id, "coder-1", "session-3", 12);
|
||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-3");
|
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-3");
|
||||||
pool.run_watchdog_pass(Some(root));
|
pool.run_watchdog_pass(Some(root)).await;
|
||||||
|
|
||||||
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -518,8 +518,8 @@ max_turns = 10
|
|||||||
/// must not count against the watchdog's turn budget. A session log with
|
/// must not count against the watchdog's turn budget. A session log with
|
||||||
/// 5 tool turns and 30 narration turns reports turns_used == 5, so an
|
/// 5 tool turns and 30 narration turns reports turns_used == 5, so an
|
||||||
/// agent with max_tool_turns = 10 stays Running.
|
/// agent with max_tool_turns = 10 stays Running.
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn watchdog_does_not_count_narration_only_turns() {
|
async fn watchdog_does_not_count_narration_only_turns() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let root = tmp.path();
|
let root = tmp.path();
|
||||||
|
|
||||||
@@ -542,13 +542,13 @@ max_turns = 200
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent_with_session("story_923", "coder-1", AgentStatus::Running, "sess-narr");
|
pool.inject_test_agent_with_session("story_923", "coder-1", AgentStatus::Running, "sess-narr");
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
found, 0,
|
found, 0,
|
||||||
"agent must not be terminated: only 5 tool turns of a 10-turn budget"
|
"agent must not be terminated: only 5 tool turns of a 10-turn budget"
|
||||||
);
|
);
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("story_923", "coder-1");
|
let key = composite_key("story_923", "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(agent.status, AgentStatus::Running);
|
assert_eq!(agent.status, AgentStatus::Running);
|
||||||
@@ -558,8 +558,8 @@ max_turns = 200
|
|||||||
/// Story 923: max_tool_turns takes precedence over max_turns when both are
|
/// Story 923: max_tool_turns takes precedence over max_turns when both are
|
||||||
/// set. With max_tool_turns = 3 and max_turns = 200, an agent that has 4
|
/// set. With max_tool_turns = 3 and max_turns = 200, an agent that has 4
|
||||||
/// tool turns is killed even though total turns (4) is far below max_turns.
|
/// tool turns is killed even though total turns (4) is far below max_turns.
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn watchdog_max_tool_turns_overrides_max_turns() {
|
async fn watchdog_max_tool_turns_overrides_max_turns() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let root = tmp.path();
|
let root = tmp.path();
|
||||||
|
|
||||||
@@ -585,13 +585,13 @@ max_turns = 200
|
|||||||
);
|
);
|
||||||
let mut rx = tx.subscribe();
|
let mut rx = tx.subscribe();
|
||||||
|
|
||||||
let found = pool.run_watchdog_pass(Some(root));
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
assert!(
|
assert!(
|
||||||
found >= 1,
|
found >= 1,
|
||||||
"watchdog must terminate when tool turns exceed max_tool_turns"
|
"watchdog must terminate when tool turns exceed max_tool_turns"
|
||||||
);
|
);
|
||||||
|
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("story_923b", "coder-1");
|
let key = composite_key("story_923b", "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(agent.status, AgentStatus::Failed);
|
assert_eq!(agent.status, AgentStatus::Failed);
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ use std::path::Path;
|
|||||||
|
|
||||||
mod limits_tests;
|
mod limits_tests;
|
||||||
mod orphan_tests;
|
mod orphan_tests;
|
||||||
|
mod reap_tests;
|
||||||
|
|
||||||
/// Write a fake session log file with `n` tool-using assistant turn entries.
|
/// Write a fake session log file with `n` tool-using assistant turn entries.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -20,18 +20,18 @@ async fn check_orphaned_agents_returns_count_of_orphaned_agents() {
|
|||||||
pool.inject_test_agent_with_handle("story_a", "coder", AgentStatus::Running, h1);
|
pool.inject_test_agent_with_handle("story_a", "coder", AgentStatus::Running, h1);
|
||||||
pool.inject_test_agent_with_handle("story_b", "coder", AgentStatus::Running, h2);
|
pool.inject_test_agent_with_handle("story_b", "coder", AgentStatus::Running, h2);
|
||||||
|
|
||||||
let found = check_orphaned_agents(&pool.agents);
|
let found = check_orphaned_agents(&pool.agents).await;
|
||||||
assert_eq!(found, 2, "should detect both orphaned agents");
|
assert_eq!(found, 2, "should detect both orphaned agents");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn check_orphaned_agents_returns_zero_when_no_orphans() {
|
async fn check_orphaned_agents_returns_zero_when_no_orphans() {
|
||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
// Inject agents in terminal states — not orphaned.
|
// Inject agents in terminal states — not orphaned.
|
||||||
pool.inject_test_agent("story_a", "coder", AgentStatus::Completed);
|
pool.inject_test_agent("story_a", "coder", AgentStatus::Completed);
|
||||||
pool.inject_test_agent("story_b", "qa", AgentStatus::Failed);
|
pool.inject_test_agent("story_b", "qa", AgentStatus::Failed);
|
||||||
|
|
||||||
let found = check_orphaned_agents(&pool.agents);
|
let found = check_orphaned_agents(&pool.agents).await;
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
found, 0,
|
found, 0,
|
||||||
"no orphans should be detected for terminal agents"
|
"no orphans should be detected for terminal agents"
|
||||||
@@ -53,10 +53,10 @@ async fn watchdog_detects_orphaned_running_agent() {
|
|||||||
pool.inject_test_agent_with_handle("orphan_story", "coder", AgentStatus::Running, handle);
|
pool.inject_test_agent_with_handle("orphan_story", "coder", AgentStatus::Running, handle);
|
||||||
let mut rx = tx.subscribe();
|
let mut rx = tx.subscribe();
|
||||||
|
|
||||||
pool.run_watchdog_once();
|
pool.run_watchdog_once().await;
|
||||||
|
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("orphan_story", "coder");
|
let key = composite_key("orphan_story", "coder");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -87,13 +87,13 @@ async fn watchdog_orphan_detection_returns_nonzero_enabling_auto_assign() {
|
|||||||
|
|
||||||
// Before watchdog: agent is Running.
|
// Before watchdog: agent is Running.
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("orphan_story", "coder");
|
let key = composite_key("orphan_story", "coder");
|
||||||
assert_eq!(agents.get(&key).unwrap().status, AgentStatus::Running);
|
assert_eq!(agents.get(&key).unwrap().status, AgentStatus::Running);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run watchdog pass — should return 1 (orphan found).
|
// Run watchdog pass — should return 1 (orphan found).
|
||||||
let found = check_orphaned_agents(&pool.agents);
|
let found = check_orphaned_agents(&pool.agents).await;
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
found, 1,
|
found, 1,
|
||||||
"watchdog must return 1 for a single orphaned agent"
|
"watchdog must return 1 for a single orphaned agent"
|
||||||
@@ -101,7 +101,7 @@ async fn watchdog_orphan_detection_returns_nonzero_enabling_auto_assign() {
|
|||||||
|
|
||||||
// After watchdog: agent is Failed.
|
// After watchdog: agent is Failed.
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = composite_key("orphan_story", "coder");
|
let key = composite_key("orphan_story", "coder");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
agents.get(&key).unwrap().status,
|
agents.get(&key).unwrap().status,
|
||||||
|
|||||||
@@ -0,0 +1,238 @@
|
|||||||
|
//! Regression tests for the reap pass (bug 1198): a Failed pool entry with
|
||||||
|
//! no live process must be respawned by the next watchdog pass, with its
|
||||||
|
//! story's retry count bumped, and must stop appearing in list_agents.
|
||||||
|
|
||||||
|
use super::super::super::super::{AgentPool, composite_key};
|
||||||
|
use super::{write_fake_session_log, write_project_config};
|
||||||
|
use crate::agents::AgentStatus;
|
||||||
|
|
||||||
|
/// AC1 + AC4: a Failed entry with no live process (simulating e.g. the
|
||||||
|
/// inactivity watchdog kill landing in spawn.rs's generic Err arm) is
|
||||||
|
/// respawned by the very next `run_watchdog_pass` — no manual
|
||||||
|
/// stop_agent/start_agent needed.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reap_respawns_stale_failed_agent() {
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::crdt_state::init_for_test();
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
write_project_config(
|
||||||
|
root,
|
||||||
|
r#"
|
||||||
|
[[agent]]
|
||||||
|
name = "coder-1"
|
||||||
|
runtime = "claude-code"
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
let story_id = "1198_story_stuck";
|
||||||
|
crate::db::write_content(
|
||||||
|
crate::db::ContentKey::Story(story_id),
|
||||||
|
"---\nname: Stuck Story\n---\n",
|
||||||
|
);
|
||||||
|
crate::crdt_state::write_item_str(story_id, "2_current", Some("Stuck Story"), None, None, None);
|
||||||
|
|
||||||
|
// Inject a Failed entry with no task_handle — exactly what spawn.rs's
|
||||||
|
// generic Err arm leaves behind after e.g. an inactivity-watchdog kill.
|
||||||
|
let pool = AgentPool::new_test(3001);
|
||||||
|
pool.inject_test_agent(story_id, "coder-1", AgentStatus::Failed);
|
||||||
|
|
||||||
|
let found = pool.run_watchdog_pass(Some(root)).await;
|
||||||
|
assert!(found >= 1, "reap should count the stale Failed entry");
|
||||||
|
|
||||||
|
// A fresh entry must exist for the same story — the agent respawned.
|
||||||
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
|
let key = composite_key(story_id, "coder-1");
|
||||||
|
let agent = agents.get(&key).expect("agent must have been respawned");
|
||||||
|
assert_ne!(
|
||||||
|
agent.status,
|
||||||
|
AgentStatus::Failed,
|
||||||
|
"respawned entry must not still be Failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// AC2: reaping a stale Failed entry bumps the story's retry count via the
|
||||||
|
/// existing should_block_story path.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reap_increments_retry_count() {
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::crdt_state::init_for_test();
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
write_project_config(
|
||||||
|
root,
|
||||||
|
r#"
|
||||||
|
max_retries = 5
|
||||||
|
|
||||||
|
[[agent]]
|
||||||
|
name = "coder-1"
|
||||||
|
runtime = "claude-code"
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
let story_id = "1198_story_retry";
|
||||||
|
crate::db::write_content(
|
||||||
|
crate::db::ContentKey::Story(story_id),
|
||||||
|
"---\nname: Retry Story\n---\n",
|
||||||
|
);
|
||||||
|
crate::crdt_state::write_item_str(story_id, "2_current", Some("Retry Story"), None, None, None);
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3001);
|
||||||
|
pool.inject_test_agent(story_id, "coder-1", AgentStatus::Failed);
|
||||||
|
|
||||||
|
pool.run_watchdog_pass(Some(root)).await;
|
||||||
|
|
||||||
|
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
||||||
|
assert_eq!(
|
||||||
|
item.retry_count(),
|
||||||
|
1,
|
||||||
|
"reaping a stale Failed entry must bump retry_count exactly once"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// AC2: exhausting max_retries via a reap blocks the story via the existing
|
||||||
|
/// should_block_story path (same mechanism as the limits watchdog).
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reap_blocks_story_after_max_retries() {
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::crdt_state::init_for_test();
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
write_project_config(
|
||||||
|
root,
|
||||||
|
r#"
|
||||||
|
max_retries = 1
|
||||||
|
|
||||||
|
[[agent]]
|
||||||
|
name = "coder-1"
|
||||||
|
runtime = "claude-code"
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
let story_id = "1198_story_block";
|
||||||
|
crate::db::write_content(
|
||||||
|
crate::db::ContentKey::Story(story_id),
|
||||||
|
"---\nname: Block Story\n---\n",
|
||||||
|
);
|
||||||
|
crate::crdt_state::write_item_str(story_id, "2_current", Some("Block Story"), None, None, None);
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3001);
|
||||||
|
pool.inject_test_agent(story_id, "coder-1", AgentStatus::Failed);
|
||||||
|
|
||||||
|
pool.run_watchdog_pass(Some(root)).await;
|
||||||
|
|
||||||
|
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
||||||
|
assert_eq!(
|
||||||
|
item.stage().dir_name(),
|
||||||
|
"blocked",
|
||||||
|
"story must be blocked after exhausting max_retries=1 via reap"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// AC3: list_agents never shows a Failed entry after the next watchdog pass
|
||||||
|
/// reaps it — regardless of whether the story blocks or respawns.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reap_removes_failed_entry_from_list_agents() {
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::crdt_state::init_for_test();
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
write_project_config(
|
||||||
|
root,
|
||||||
|
r#"
|
||||||
|
max_retries = 1
|
||||||
|
|
||||||
|
[[agent]]
|
||||||
|
name = "coder-1"
|
||||||
|
runtime = "claude-code"
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
let story_id = "1198_story_listing";
|
||||||
|
crate::db::write_content(
|
||||||
|
crate::db::ContentKey::Story(story_id),
|
||||||
|
"---\nname: Listing Story\n---\n",
|
||||||
|
);
|
||||||
|
crate::crdt_state::write_item_str(
|
||||||
|
story_id,
|
||||||
|
"2_current",
|
||||||
|
Some("Listing Story"),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3001);
|
||||||
|
pool.inject_test_agent(story_id, "coder-1", AgentStatus::Failed);
|
||||||
|
|
||||||
|
pool.run_watchdog_pass(Some(root)).await;
|
||||||
|
|
||||||
|
let listed = pool.list_agents().await.unwrap();
|
||||||
|
assert!(
|
||||||
|
!listed
|
||||||
|
.iter()
|
||||||
|
.any(|a| a.story_id == story_id && a.status == AgentStatus::Failed),
|
||||||
|
"list_agents must not show a Failed entry for '{story_id}' after the next watchdog pass"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The limits-termination path (turn/budget overrun) must not have its
|
||||||
|
/// retry_count double-bumped by the reap pass running in the same
|
||||||
|
/// `run_watchdog_pass` call.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reap_does_not_double_bump_limits_terminated_agent() {
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::crdt_state::init_for_test();
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
write_project_config(
|
||||||
|
root,
|
||||||
|
r#"
|
||||||
|
max_retries = 5
|
||||||
|
|
||||||
|
[[agent]]
|
||||||
|
name = "coder-1"
|
||||||
|
runtime = "claude-code"
|
||||||
|
max_turns = 10
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
let story_id = "1198_story_no_double_bump";
|
||||||
|
crate::db::write_content(
|
||||||
|
crate::db::ContentKey::Story(story_id),
|
||||||
|
"---\nname: No Double Bump\n---\n",
|
||||||
|
);
|
||||||
|
crate::crdt_state::write_item_str(
|
||||||
|
story_id,
|
||||||
|
"2_current",
|
||||||
|
Some("No Double Bump"),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
write_fake_session_log(root, story_id, "coder-1", "sess-overrun", 12);
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3001);
|
||||||
|
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "sess-overrun");
|
||||||
|
|
||||||
|
pool.run_watchdog_pass(Some(root)).await;
|
||||||
|
|
||||||
|
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
|
||||||
|
assert_eq!(
|
||||||
|
item.retry_count(),
|
||||||
|
1,
|
||||||
|
"a single limit-termination pass must bump retry_count by exactly 1, \
|
||||||
|
not twice (once from the limits branch, once from reap)"
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -9,10 +9,19 @@
|
|||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use crate::pipeline_state::Stage;
|
use crate::pipeline_state::{Pipeline, Stage, Status};
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
use crate::slog_warn;
|
use crate::slog_warn;
|
||||||
|
|
||||||
|
/// Reconcile: re-populate the CostRollup register from disk for all known stories.
|
||||||
|
///
|
||||||
|
/// Idempotent — `init_from_disk` scans all existing token-usage JSONL files and
|
||||||
|
/// overwrites the in-memory register. Called by the periodic reconciler so that
|
||||||
|
/// a Lagged event can never leave a story with a stale or absent cost entry.
|
||||||
|
pub(crate) fn reconcile_cost_rollup(project_root: &Path) {
|
||||||
|
crate::service::agents::cost_rollup::init_from_disk(project_root);
|
||||||
|
}
|
||||||
|
|
||||||
/// Spawn a background task that maintains the CostRollup register.
|
/// Spawn a background task that maintains the CostRollup register.
|
||||||
///
|
///
|
||||||
/// On every terminal stage transition (Done, Archived, Abandoned, Superseded,
|
/// On every terminal stage transition (Done, Archived, Abandoned, Superseded,
|
||||||
@@ -41,17 +50,15 @@ pub(crate) fn spawn_cost_rollup_subscriber(project_root: PathBuf) {
|
|||||||
/// Returns `true` if `stage` is a terminal pipeline stage.
|
/// Returns `true` if `stage` is a terminal pipeline stage.
|
||||||
///
|
///
|
||||||
/// Terminal stages are those from which no further work is expected:
|
/// Terminal stages are those from which no further work is expected:
|
||||||
/// Done, Archived, Abandoned, Superseded, Rejected.
|
/// Done, Archived, Abandoned, Superseded, Rejected. Story 1086 routes the
|
||||||
/// MergeFailure variants are NOT terminal — stories can recover from them.
|
/// classification through the [`Status`] / [`Pipeline`] projection so future
|
||||||
|
/// Stage variants automatically participate. MergeFailure variants are NOT
|
||||||
|
/// terminal — stories can recover from them.
|
||||||
fn is_terminal(stage: &Stage) -> bool {
|
fn is_terminal(stage: &Stage) -> bool {
|
||||||
matches!(
|
matches!(
|
||||||
stage,
|
stage.status(),
|
||||||
Stage::Done { .. }
|
Status::Done | Status::Abandoned | Status::Superseded | Status::Rejected
|
||||||
| Stage::Archived { .. }
|
) || matches!(stage.pipeline(), Pipeline::Archived)
|
||||||
| Stage::Abandoned { .. }
|
|
||||||
| Stage::Superseded { .. }
|
|
||||||
| Stage::Rejected { .. }
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Snapshot the cost data for `fired.story_id` into the register when
|
/// Snapshot the cost data for `fired.story_id` into the register when
|
||||||
|
|||||||
@@ -18,10 +18,9 @@ mod test_helpers;
|
|||||||
|
|
||||||
use crate::io::watcher::WatcherEvent;
|
use crate::io::watcher::WatcherEvent;
|
||||||
use crate::service::status::StatusBroadcaster;
|
use crate::service::status::StatusBroadcaster;
|
||||||
use portable_pty::ChildKiller;
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::Arc;
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::{Mutex, broadcast};
|
||||||
|
|
||||||
// Bring pool-internal types into pool's namespace so that sub-modules
|
// Bring pool-internal types into pool's namespace so that sub-modules
|
||||||
// (auto_assign, pipeline, etc.) can access them via `use super::...`.
|
// (auto_assign, pipeline, etc.) can access them via `use super::...`.
|
||||||
@@ -31,10 +30,6 @@ use types::{StoryAgent, composite_key};
|
|||||||
pub struct AgentPool {
|
pub struct AgentPool {
|
||||||
agents: Arc<Mutex<HashMap<String, StoryAgent>>>,
|
agents: Arc<Mutex<HashMap<String, StoryAgent>>>,
|
||||||
port: u16,
|
port: u16,
|
||||||
/// Registry of active PTY child process killers, keyed by "{story_id}:{agent_name}".
|
|
||||||
/// Used to terminate child processes on server shutdown or agent stop, preventing
|
|
||||||
/// orphaned Claude Code processes from running after the server exits.
|
|
||||||
child_killers: Arc<Mutex<HashMap<String, Box<dyn ChildKiller + Send + Sync>>>>,
|
|
||||||
/// Broadcast channel for notifying WebSocket clients of agent state changes.
|
/// Broadcast channel for notifying WebSocket clients of agent state changes.
|
||||||
/// When an agent transitions state (Pending, Running, Completed, Failed, Stopped),
|
/// When an agent transitions state (Pending, Running, Completed, Failed, Stopped),
|
||||||
/// an `AgentStateChanged` event is emitted so the frontend can refresh the
|
/// an `AgentStateChanged` event is emitted so the frontend can refresh the
|
||||||
@@ -56,7 +51,6 @@ impl AgentPool {
|
|||||||
let pool = Self {
|
let pool = Self {
|
||||||
agents: Arc::new(Mutex::new(HashMap::new())),
|
agents: Arc::new(Mutex::new(HashMap::new())),
|
||||||
port,
|
port,
|
||||||
child_killers: Arc::new(Mutex::new(HashMap::new())),
|
|
||||||
watcher_tx: watcher_tx.clone(),
|
watcher_tx: watcher_tx.clone(),
|
||||||
status_broadcaster: Arc::new(StatusBroadcaster::new()),
|
status_broadcaster: Arc::new(StatusBroadcaster::new()),
|
||||||
};
|
};
|
||||||
@@ -93,10 +87,12 @@ impl AgentPool {
|
|||||||
_ => continue,
|
_ => continue,
|
||||||
};
|
};
|
||||||
let key = composite_key(&story_id, &agent_name);
|
let key = composite_key(&story_id, &agent_name);
|
||||||
if let Ok(mut agents) = agents_clone.lock()
|
|
||||||
&& let Some(agent) = agents.get_mut(&key)
|
|
||||||
{
|
{
|
||||||
agent.throttled = Some(crate::agents::AgentExecution::Throttled { until });
|
let mut agents = agents_clone.lock().await;
|
||||||
|
if let Some(agent) = agents.get_mut(&key) {
|
||||||
|
agent.throttled =
|
||||||
|
Some(crate::agents::AgentExecution::Throttled { until });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
let _ = watcher_tx_clone.send(WatcherEvent::AgentStateChanged);
|
let _ = watcher_tx_clone.send(WatcherEvent::AgentStateChanged);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::Arc;
|
||||||
|
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
@@ -16,7 +16,7 @@ use std::path::Path;
|
|||||||
/// type cycle between `start_agent` and `run_server_owned_completion`.
|
/// type cycle between `start_agent` and `run_server_owned_completion`.
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(crate) fn spawn_pipeline_advance(
|
pub(crate) fn spawn_pipeline_advance(
|
||||||
agents: Arc<Mutex<HashMap<String, StoryAgent>>>,
|
agents: Arc<tokio::sync::Mutex<HashMap<String, StoryAgent>>>,
|
||||||
port: u16,
|
port: u16,
|
||||||
story_id: &str,
|
story_id: &str,
|
||||||
agent_name: &str,
|
agent_name: &str,
|
||||||
@@ -33,7 +33,6 @@ pub(crate) fn spawn_pipeline_advance(
|
|||||||
let pool = AgentPool {
|
let pool = AgentPool {
|
||||||
agents,
|
agents,
|
||||||
port,
|
port,
|
||||||
child_killers: Arc::new(Mutex::new(HashMap::new())),
|
|
||||||
watcher_tx,
|
watcher_tx,
|
||||||
status_broadcaster: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
status_broadcaster: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -78,21 +78,34 @@ impl AgentPool {
|
|||||||
// The coder exited with uncommitted content but no commits
|
// The coder exited with uncommitted content but no commits
|
||||||
// (typical "claude-code session boundary mid-sweep" pattern).
|
// (typical "claude-code session boundary mid-sweep" pattern).
|
||||||
// Use a PROGRESS-AWARE retry cap: the agent gets unlimited
|
// Use a PROGRESS-AWARE retry cap: the agent gets unlimited
|
||||||
// respawns as long as file edits keep growing between
|
// respawns as long as progress is being made between attempts.
|
||||||
// attempts; only when the worktree diff is byte-identical
|
// Progress is satisfied if EITHER (a) the worktree diff grew,
|
||||||
// to the previous attempt do we count it as "no progress".
|
// OR (b) the set of files the agent read grew. Raw tool-call
|
||||||
// After NO_PROGRESS_CAP consecutive no-progress respawns,
|
// count does NOT count — a looping agent can produce many calls.
|
||||||
// block for human attention.
|
// Only self-exited sessions with no file or read progress count
|
||||||
|
// toward the cap; forced exits (API error, network, budget
|
||||||
|
// exhaustion) are excluded (story 1089).
|
||||||
|
// After NO_PROGRESS_CAP consecutive qualifying no-progress
|
||||||
|
// respawns, block for human attention.
|
||||||
//
|
//
|
||||||
// TOTAL_ATTEMPTS_CAP is the OUTER bound: even if the agent
|
// TOTAL_ATTEMPTS_CAP is the OUTER bound: even if the agent
|
||||||
// keeps making file-edit progress every session, after this
|
// keeps making file-edit progress every session, after this
|
||||||
// many total respawns without a commit we escalate — caught
|
// many total respawns without a commit we escalate — catches
|
||||||
// the "agent flaps between different edits but never
|
// the "agent flaps between different edits but never commits"
|
||||||
// commits" pattern that the progress-aware counter would
|
// pattern that the progress-aware counter would never trigger.
|
||||||
// never trigger.
|
|
||||||
const NO_PROGRESS_CAP: u32 = 3;
|
const NO_PROGRESS_CAP: u32 = 3;
|
||||||
const TOTAL_ATTEMPTS_CAP: u32 = 8;
|
const TOTAL_ATTEMPTS_CAP: u32 = 8;
|
||||||
|
|
||||||
|
// AC1: consume the forced-exit flag written by spawn.rs when
|
||||||
|
// the agent process exited with a non-zero code.
|
||||||
|
let forced_exit = crate::db::read_content(
|
||||||
|
crate::db::ContentKey::CommitRecoveryForcedExit(story_id),
|
||||||
|
)
|
||||||
|
.is_some();
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryForcedExit(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
|
|
||||||
let current_fingerprint = worktree_path.as_deref().and_then(|p| {
|
let current_fingerprint = worktree_path.as_deref().and_then(|p| {
|
||||||
std::process::Command::new("git")
|
std::process::Command::new("git")
|
||||||
.args(["diff", "master"])
|
.args(["diff", "master"])
|
||||||
@@ -104,18 +117,31 @@ impl AgentPool {
|
|||||||
let stored_fingerprint = crate::db::read_content(
|
let stored_fingerprint = crate::db::read_content(
|
||||||
crate::db::ContentKey::CommitRecoveryDiffFingerprint(story_id),
|
crate::db::ContentKey::CommitRecoveryDiffFingerprint(story_id),
|
||||||
);
|
);
|
||||||
let made_progress = current_fingerprint.is_some()
|
let diff_progress = current_fingerprint.is_some()
|
||||||
&& stored_fingerprint.as_ref() != current_fingerprint.as_ref();
|
&& stored_fingerprint.as_ref() != current_fingerprint.as_ref();
|
||||||
let no_progress_count = if made_progress || stored_fingerprint.is_none() {
|
|
||||||
|
// AC2: check read-file set progress as an additional signal.
|
||||||
|
let read_progress = previous_session_id.as_deref().is_some_and(|session_id| {
|
||||||
|
collect_read_progress(&project_root, story_id, agent_name, session_id)
|
||||||
|
});
|
||||||
|
|
||||||
|
let made_progress = diff_progress || read_progress;
|
||||||
|
|
||||||
|
let prev_no_progress_count = crate::db::read_content(
|
||||||
|
crate::db::ContentKey::CommitRecoveryPending(story_id),
|
||||||
|
)
|
||||||
|
.and_then(|s| s.trim().parse::<u32>().ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
// AC1: forced exits do not increment the stuck-respawn counter.
|
||||||
|
let no_progress_count = if forced_exit {
|
||||||
|
prev_no_progress_count
|
||||||
|
} else if made_progress || stored_fingerprint.is_none() {
|
||||||
1
|
1
|
||||||
} else {
|
} else {
|
||||||
crate::db::read_content(crate::db::ContentKey::CommitRecoveryPending(
|
prev_no_progress_count + 1
|
||||||
story_id,
|
|
||||||
))
|
|
||||||
.and_then(|s| s.trim().parse::<u32>().ok())
|
|
||||||
.unwrap_or(0)
|
|
||||||
+ 1
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let total_attempts = crate::db::read_content(
|
let total_attempts = crate::db::read_content(
|
||||||
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
||||||
)
|
)
|
||||||
@@ -136,13 +162,17 @@ impl AgentPool {
|
|||||||
crate::db::delete_content(
|
crate::db::delete_content(
|
||||||
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
||||||
);
|
);
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
slog!(
|
slog!(
|
||||||
"[pipeline] Coder '{agent_name}' for '{story_id}' hit total \
|
"[pipeline] Coder '{agent_name}' for '{story_id}' hit total \
|
||||||
commit-recovery cap ({total_attempts}/{TOTAL_ATTEMPTS_CAP}) \
|
commit-recovery cap ({total_attempts}/{TOTAL_ATTEMPTS_CAP}) \
|
||||||
without a commit. Blocking story."
|
without a commit. Blocking story."
|
||||||
);
|
);
|
||||||
let reason = format!(
|
let reason = format!(
|
||||||
"agent flapped — {total_attempts} respawns without ever committing"
|
"commit absent after {total_attempts} respawns — \
|
||||||
|
agent kept making edits but never committed"
|
||||||
);
|
);
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
crate::agents::lifecycle::transition_to_blocked(story_id, &reason)
|
crate::agents::lifecycle::transition_to_blocked(story_id, &reason)
|
||||||
@@ -167,14 +197,18 @@ impl AgentPool {
|
|||||||
crate::db::delete_content(
|
crate::db::delete_content(
|
||||||
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
||||||
);
|
);
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
slog!(
|
slog!(
|
||||||
"[pipeline] Coder '{agent_name}' for '{story_id}' made no \
|
"[pipeline] Coder '{agent_name}' for '{story_id}' made no \
|
||||||
file-edit progress over {no_progress_count} consecutive \
|
file or read progress over {no_progress_count} consecutive \
|
||||||
commit-recovery respawns. Blocking story."
|
self-exit commit-recovery respawns. Blocking story."
|
||||||
);
|
);
|
||||||
|
// AC4: block message names the specific cause.
|
||||||
let reason = format!(
|
let reason = format!(
|
||||||
"agent stuck — {no_progress_count} respawns without commits or \
|
"stuck-respawn cap reached: {NO_PROGRESS_CAP} consecutive \
|
||||||
new file edits"
|
self-exits with no file or read progress"
|
||||||
);
|
);
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
crate::agents::lifecycle::transition_to_blocked(story_id, &reason)
|
crate::agents::lifecycle::transition_to_blocked(story_id, &reason)
|
||||||
@@ -206,7 +240,8 @@ impl AgentPool {
|
|||||||
"[pipeline] Coder '{agent_name}' exited with uncommitted work \
|
"[pipeline] Coder '{agent_name}' exited with uncommitted work \
|
||||||
for '{story_id}' (no-progress {no_progress_count}/\
|
for '{story_id}' (no-progress {no_progress_count}/\
|
||||||
{NO_PROGRESS_CAP}, total {total_attempts}/\
|
{NO_PROGRESS_CAP}, total {total_attempts}/\
|
||||||
{TOTAL_ATTEMPTS_CAP}; progress_made={made_progress}). \
|
{TOTAL_ATTEMPTS_CAP}; diff_progress={diff_progress}, \
|
||||||
|
read_progress={read_progress}, forced_exit={forced_exit}). \
|
||||||
Issuing commit-only respawn."
|
Issuing commit-only respawn."
|
||||||
);
|
);
|
||||||
let addendum = "\n\nYou have uncommitted work in this worktree. \
|
let addendum = "\n\nYou have uncommitted work in this worktree. \
|
||||||
@@ -302,10 +337,13 @@ impl AgentPool {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else if completion.gates_passed {
|
} else if completion.gates_passed {
|
||||||
// Clear any stale recovery key when the coder succeeds normally.
|
// Clear any stale recovery keys when the coder succeeds normally.
|
||||||
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
|
||||||
story_id,
|
story_id,
|
||||||
));
|
));
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
// Determine effective QA mode for this story.
|
// Determine effective QA mode for this story.
|
||||||
let qa_mode = {
|
let qa_mode = {
|
||||||
let item_type = crate::agents::lifecycle::item_type_from_id(story_id);
|
let item_type = crate::agents::lifecycle::item_type_from_id(story_id);
|
||||||
@@ -361,11 +399,14 @@ impl AgentPool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Clear any stale recovery key when gates fail normally (agent committed
|
// Clear any stale recovery keys when gates fail normally (agent committed
|
||||||
// but the build is broken — treat as a standard retry, not a recovery).
|
// but the build is broken — treat as a standard retry, not a recovery).
|
||||||
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
|
||||||
story_id,
|
story_id,
|
||||||
));
|
));
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
// Bug 645 / 668: Before retry/block, check if the agent left committed
|
// Bug 645 / 668: Before retry/block, check if the agent left committed
|
||||||
// work AND the agent had a passing run_tests result captured during its
|
// work AND the agent had a passing run_tests result captured during its
|
||||||
// session. An agent may crash mid-output (e.g. Claude Code CLI PTY write
|
// session. An agent may crash mid-output (e.g. Claude Code CLI PTY write
|
||||||
@@ -653,7 +694,7 @@ impl AgentPool {
|
|||||||
if let Err(e) = crate::agents::lifecycle::move_story_to_done(story_id) {
|
if let Err(e) = crate::agents::lifecycle::move_story_to_done(story_id) {
|
||||||
slog_error!("[pipeline] Failed to move '{story_id}' to done: {e}");
|
slog_error!("[pipeline] Failed to move '{story_id}' to done: {e}");
|
||||||
}
|
}
|
||||||
self.remove_agents_for_story(story_id);
|
self.remove_agents_for_story(story_id).await;
|
||||||
crate::crdt_state::delete_merge_job(story_id);
|
crate::crdt_state::delete_merge_job(story_id);
|
||||||
// TODO: Re-enable worktree cleanup once we have persistent agent logs.
|
// TODO: Re-enable worktree cleanup once we have persistent agent logs.
|
||||||
// Removing worktrees destroys evidence needed to debug empty-commit agents.
|
// Removing worktrees destroys evidence needed to debug empty-commit agents.
|
||||||
@@ -724,6 +765,109 @@ mod helpers;
|
|||||||
use helpers::{resolve_qa_mode_from_store, write_review_hold_to_store};
|
use helpers::{resolve_qa_mode_from_store, write_review_hold_to_store};
|
||||||
pub(crate) use helpers::{should_block_story, spawn_pipeline_advance};
|
pub(crate) use helpers::{should_block_story, spawn_pipeline_advance};
|
||||||
|
|
||||||
|
/// Parse a huskies agent log and return the set of file paths passed to the
|
||||||
|
/// Read tool in that session. Returns an empty set if the log cannot be read.
|
||||||
|
///
|
||||||
|
/// Used by [`collect_read_progress`] to detect read-exploration progress even
|
||||||
|
/// when the worktree diff did not grow (story 1089, AC2).
|
||||||
|
fn collect_read_files_from_log(
|
||||||
|
project_root: &std::path::Path,
|
||||||
|
story_id: &str,
|
||||||
|
agent_name: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> std::collections::HashSet<String> {
|
||||||
|
let log_path = crate::agent_log::log_file_path(project_root, story_id, agent_name, session_id);
|
||||||
|
let mut files = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
let log_text = match std::fs::read_to_string(&log_path) {
|
||||||
|
Ok(t) => t,
|
||||||
|
Err(_) => return files,
|
||||||
|
};
|
||||||
|
|
||||||
|
for line in log_text.lines() {
|
||||||
|
let trimmed = line.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let entry: serde_json::Value = match serde_json::from_str(trimmed) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
// Only look at agent_json events where data.type == "assistant".
|
||||||
|
if entry.get("type").and_then(|t| t.as_str()) != Some("agent_json") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let data = match entry.get("data") {
|
||||||
|
Some(d) => d,
|
||||||
|
None => continue,
|
||||||
|
};
|
||||||
|
if data.get("type").and_then(|t| t.as_str()) != Some("assistant") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let content = match data.pointer("/message/content").and_then(|c| c.as_array()) {
|
||||||
|
Some(c) => c,
|
||||||
|
None => continue,
|
||||||
|
};
|
||||||
|
for item in content {
|
||||||
|
if item.get("type").and_then(|t| t.as_str()) != Some("tool_use") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if item.get("name").and_then(|n| n.as_str()) != Some("Read") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(path) = item.pointer("/input/file_path").and_then(|p| p.as_str()) {
|
||||||
|
files.insert(path.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
files
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return `true` if the agent read any files in `session_id` that were not in
|
||||||
|
/// the cumulative read set for `story_id`. Updates the stored cumulative set
|
||||||
|
/// when new files are found (story 1089, AC2).
|
||||||
|
fn collect_read_progress(
|
||||||
|
project_root: &std::path::Path,
|
||||||
|
story_id: &str,
|
||||||
|
agent_name: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> bool {
|
||||||
|
let session_files = collect_read_files_from_log(project_root, story_id, agent_name, session_id);
|
||||||
|
if session_files.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let stored_set: std::collections::HashSet<String> =
|
||||||
|
crate::db::read_content(crate::db::ContentKey::CommitRecoveryReadSet(story_id))
|
||||||
|
.map(|s| {
|
||||||
|
s.lines()
|
||||||
|
.filter(|l| !l.is_empty())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let union: std::collections::HashSet<String> =
|
||||||
|
stored_set.union(&session_files).cloned().collect();
|
||||||
|
|
||||||
|
if union.len() > stored_set.len() {
|
||||||
|
let mut sorted: Vec<&String> = union.iter().collect();
|
||||||
|
sorted.sort();
|
||||||
|
crate::db::write_content(
|
||||||
|
crate::db::ContentKey::CommitRecoveryReadSet(story_id),
|
||||||
|
&sorted
|
||||||
|
.into_iter()
|
||||||
|
.map(String::as_str)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n"),
|
||||||
|
);
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ async fn mergemaster_blocks_and_sends_story_blocked_when_no_commits_ahead() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// No mergemaster agent should have been started.
|
// No mergemaster agent should have been started.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let mergemaster_started = agents
|
let mergemaster_started = agents
|
||||||
.values()
|
.values()
|
||||||
.any(|a| a.agent_name.contains("mergemaster"));
|
.any(|a| a.agent_name.contains("mergemaster"));
|
||||||
@@ -162,7 +162,7 @@ stage = "qa"
|
|||||||
|
|
||||||
// Verify that 293 cannot get a QA agent right now (QA is busy).
|
// Verify that 293 cannot get a QA agent right now (QA is busy).
|
||||||
{
|
{
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
!is_agent_free(&agents, "qa"),
|
!is_agent_free(&agents, "qa"),
|
||||||
"qa should be busy on story 292"
|
"qa should be busy on story 292"
|
||||||
@@ -172,7 +172,7 @@ stage = "qa"
|
|||||||
// Simulate QA completing on story 292: remove the agent from the pool
|
// Simulate QA completing on story 292: remove the agent from the pool
|
||||||
// (as run_server_owned_completion does) then run pipeline advance.
|
// (as run_server_owned_completion does) then run pipeline advance.
|
||||||
{
|
{
|
||||||
let mut agents = pool.agents.lock().unwrap();
|
let mut agents = pool.agents.try_lock().unwrap();
|
||||||
agents.remove(&composite_key("292_story_first", "qa"));
|
agents.remove(&composite_key("292_story_first", "qa"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -193,7 +193,7 @@ stage = "qa"
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
// After pipeline advance, auto_assign should have started QA on story 293.
|
// After pipeline advance, auto_assign should have started QA on story 293.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let qa_on_293 = agents.values().any(|a| {
|
let qa_on_293 = agents.values().any(|a| {
|
||||||
a.agent_name == "qa" && matches!(a.status, AgentStatus::Pending | AgentStatus::Running)
|
a.agent_name == "qa" && matches!(a.status, AgentStatus::Pending | AgentStatus::Running)
|
||||||
});
|
});
|
||||||
@@ -278,7 +278,7 @@ async fn stale_mergemaster_advance_for_done_story_is_noop() {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
// No agents should have been started.
|
// No agents should have been started.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
agents.is_empty(),
|
agents.is_empty(),
|
||||||
"No agents should be started for a stale advance on a done story. \
|
"No agents should be started for a stale advance on a done story. \
|
||||||
@@ -871,7 +871,7 @@ stage = "coder"
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
// The coder must be re-spawned — Pending or Running.
|
// The coder must be re-spawned — Pending or Running.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let coder_restarted = agents.values().any(|a| {
|
let coder_restarted = agents.values().any(|a| {
|
||||||
a.agent_name == "coder-1" && matches!(a.status, AgentStatus::Pending | AgentStatus::Running)
|
a.agent_name == "coder-1" && matches!(a.status, AgentStatus::Pending | AgentStatus::Running)
|
||||||
});
|
});
|
||||||
@@ -957,7 +957,7 @@ stage = "coder"
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
// The recovery respawn must have been issued — coder-1 should be Pending/Running.
|
// The recovery respawn must have been issued — coder-1 should be Pending/Running.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let coder_restarted = agents.values().any(|a| {
|
let coder_restarted = agents.values().any(|a| {
|
||||||
a.agent_name == "coder-1" && matches!(a.status, AgentStatus::Pending | AgentStatus::Running)
|
a.agent_name == "coder-1" && matches!(a.status, AgentStatus::Pending | AgentStatus::Running)
|
||||||
});
|
});
|
||||||
@@ -1077,7 +1077,7 @@ stage = "coder"
|
|||||||
"Story must be blocked after NO_PROGRESS_CAP consecutive no-progress respawns"
|
"Story must be blocked after NO_PROGRESS_CAP consecutive no-progress respawns"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
block_reason.contains("without commits or new file edits"),
|
block_reason.contains("self-exits with no file or read progress"),
|
||||||
"Block reason should describe the no-progress condition, got: {block_reason}"
|
"Block reason should describe the no-progress condition, got: {block_reason}"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1193,7 +1193,7 @@ stage = "coder"
|
|||||||
"Story must be blocked once total commit-recovery attempts hits the outer cap"
|
"Story must be blocked once total commit-recovery attempts hits the outer cap"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
block_reason.contains("flapped") && block_reason.contains("without ever committing"),
|
block_reason.contains("commit absent") && block_reason.contains("never committed"),
|
||||||
"Block reason should describe the flapping pattern, got: {block_reason}"
|
"Block reason should describe the flapping pattern, got: {block_reason}"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1328,7 +1328,7 @@ async fn coder_completion_with_test_evidence_and_zero_commits_does_not_advance()
|
|||||||
);
|
);
|
||||||
|
|
||||||
// No QA or merge agent should have been started.
|
// No QA or merge agent should have been started.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let qa_or_merge_started = agents
|
let qa_or_merge_started = agents
|
||||||
.values()
|
.values()
|
||||||
.any(|a| a.agent_name.contains("qa") || a.agent_name.contains("merge"));
|
.any(|a| a.agent_name.contains("qa") || a.agent_name.contains("merge"));
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ impl AgentPool {
|
|||||||
|
|
||||||
// Verify agent exists, is Running, and grab its worktree path.
|
// Verify agent exists, is Running, and grab its worktree path.
|
||||||
let worktree_path = {
|
let worktree_path = {
|
||||||
let agents = self.agents.lock().map_err(|e| e.to_string())?;
|
let agents = self.agents.lock().await;
|
||||||
let agent = agents
|
let agent = agents
|
||||||
.get(&key)
|
.get(&key)
|
||||||
.ok_or_else(|| format!("No agent '{agent_name}' for story '{story_id}'"))?;
|
.ok_or_else(|| format!("No agent '{agent_name}' for story '{story_id}'"))?;
|
||||||
@@ -82,7 +82,7 @@ impl AgentPool {
|
|||||||
merge_failure_reported_for_advance,
|
merge_failure_reported_for_advance,
|
||||||
session_id_for_advance,
|
session_id_for_advance,
|
||||||
) = {
|
) = {
|
||||||
let mut agents = self.agents.lock().map_err(|e| e.to_string())?;
|
let mut agents = self.agents.lock().await;
|
||||||
let agent = agents.get_mut(&key).ok_or_else(|| {
|
let agent = agents.get_mut(&key).ok_or_else(|| {
|
||||||
format!("Agent '{agent_name}' for story '{story_id}' disappeared during gate check")
|
format!("Agent '{agent_name}' for story '{story_id}' disappeared during gate check")
|
||||||
})?;
|
})?;
|
||||||
@@ -111,7 +111,6 @@ impl AgentPool {
|
|||||||
let pool_clone = Self {
|
let pool_clone = Self {
|
||||||
agents: Arc::clone(&self.agents),
|
agents: Arc::clone(&self.agents),
|
||||||
port: self.port,
|
port: self.port,
|
||||||
child_killers: Arc::clone(&self.child_killers),
|
|
||||||
watcher_tx: self.watcher_tx.clone(),
|
watcher_tx: self.watcher_tx.clone(),
|
||||||
status_broadcaster: Arc::clone(&self.status_broadcaster),
|
status_broadcaster: Arc::clone(&self.status_broadcaster),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -2,7 +2,8 @@
|
|||||||
use crate::io::watcher::WatcherEvent;
|
use crate::io::watcher::WatcherEvent;
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::Mutex;
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
use super::super::super::super::{AgentEvent, CompletionReport, PipelineStage, pipeline_stage};
|
use super::super::super::super::{AgentEvent, CompletionReport, PipelineStage, pipeline_stage};
|
||||||
@@ -45,10 +46,7 @@ pub(in crate::agents::pool) async fn run_server_owned_completion(
|
|||||||
|
|
||||||
// Guard: skip if completion was already recorded (legacy path).
|
// Guard: skip if completion was already recorded (legacy path).
|
||||||
{
|
{
|
||||||
let lock = match agents.lock() {
|
let lock = agents.lock().await;
|
||||||
Ok(a) => a,
|
|
||||||
Err(_) => return,
|
|
||||||
};
|
|
||||||
match lock.get(&key) {
|
match lock.get(&key) {
|
||||||
Some(agent) if agent.completion.is_some() => {
|
Some(agent) if agent.completion.is_some() => {
|
||||||
slog!(
|
slog!(
|
||||||
@@ -64,35 +62,18 @@ pub(in crate::agents::pool) async fn run_server_owned_completion(
|
|||||||
|
|
||||||
// Get worktree path for running gates.
|
// Get worktree path for running gates.
|
||||||
let worktree_path = {
|
let worktree_path = {
|
||||||
let lock = match agents.lock() {
|
let lock = agents.lock().await;
|
||||||
Ok(a) => a,
|
|
||||||
Err(_) => return,
|
|
||||||
};
|
|
||||||
lock.get(&key)
|
lock.get(&key)
|
||||||
.and_then(|a| a.worktree_info.as_ref().map(|wt| wt.path.clone()))
|
.and_then(|a| a.worktree_info.as_ref().map(|wt| wt.path.clone()))
|
||||||
};
|
};
|
||||||
|
|
||||||
// Kill any in-flight cargo test processes for this worktree so they don't
|
// Kill any in-flight cargo test processes for this worktree so they don't
|
||||||
// hold the build lock while gates try to run.
|
// hold the build lock while gates try to run.
|
||||||
if let Some(wt_path) = worktree_path.as_ref()
|
if let Some(wt_path) = worktree_path.as_ref() {
|
||||||
&& let Ok(output) = std::process::Command::new("pgrep")
|
let pattern = format!("--manifest-path {}/Cargo.toml", wt_path.display());
|
||||||
.args([
|
let _ = crate::process_kill::sigkill_pids_and_verify(&crate::process_kill::pids_matching(
|
||||||
"-f",
|
&pattern,
|
||||||
&format!("--manifest-path {}/Cargo.toml", wt_path.display()),
|
));
|
||||||
])
|
|
||||||
.output()
|
|
||||||
{
|
|
||||||
let pids = String::from_utf8_lossy(&output.stdout);
|
|
||||||
for pid_str in pids.lines() {
|
|
||||||
if let Ok(pid) = pid_str.trim().parse::<i32>() {
|
|
||||||
crate::slog!(
|
|
||||||
"[agents] Killing stale cargo process (pid {pid}) for '{story_id}' before running gates"
|
|
||||||
);
|
|
||||||
unsafe {
|
|
||||||
libc::kill(pid, libc::SIGKILL);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run acceptance gates. Third element of the tuple is `needs_commit_recovery`:
|
// Run acceptance gates. Third element of the tuple is `needs_commit_recovery`:
|
||||||
@@ -206,10 +187,7 @@ pub(in crate::agents::pool) async fn run_server_owned_completion(
|
|||||||
// Store completion report, extract data for pipeline advance, then
|
// Store completion report, extract data for pipeline advance, then
|
||||||
// remove the entry so completed agents never appear in list_agents.
|
// remove the entry so completed agents never appear in list_agents.
|
||||||
let (tx, project_root_for_advance, wt_path_for_advance, merge_failure_reported_for_advance) = {
|
let (tx, project_root_for_advance, wt_path_for_advance, merge_failure_reported_for_advance) = {
|
||||||
let mut lock = match agents.lock() {
|
let mut lock = agents.lock().await;
|
||||||
Ok(a) => a,
|
|
||||||
Err(_) => return,
|
|
||||||
};
|
|
||||||
let agent = match lock.get_mut(&key) {
|
let agent = match lock.get_mut(&key) {
|
||||||
Some(a) => a,
|
Some(a) => a,
|
||||||
None => return,
|
None => return,
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ async fn server_owned_completion_skips_when_already_completed() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Subscribe before calling so we can check if Done event was emitted.
|
// Subscribe before calling so we can check if Done event was emitted.
|
||||||
let mut rx = pool.subscribe("s10", "coder-1").unwrap();
|
let mut rx = pool.subscribe("s10", "coder-1").await.unwrap();
|
||||||
|
|
||||||
run_server_owned_completion(
|
run_server_owned_completion(
|
||||||
&pool.agents,
|
&pool.agents,
|
||||||
@@ -121,7 +121,7 @@ async fn server_owned_completion_skips_when_already_completed() {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
// Status should remain Completed (unchanged) — no gate re-run.
|
// Status should remain Completed (unchanged) — no gate re-run.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = super::super::super::composite_key("s10", "coder-1");
|
let key = super::super::super::composite_key("s10", "coder-1");
|
||||||
let agent = agents.get(&key).unwrap();
|
let agent = agents.get(&key).unwrap();
|
||||||
assert_eq!(agent.status, AgentStatus::Completed);
|
assert_eq!(agent.status, AgentStatus::Completed);
|
||||||
@@ -147,7 +147,7 @@ async fn server_owned_completion_runs_gates_on_clean_worktree() {
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent_with_path("s11", "coder-1", AgentStatus::Running, repo.to_path_buf());
|
pool.inject_test_agent_with_path("s11", "coder-1", AgentStatus::Running, repo.to_path_buf());
|
||||||
|
|
||||||
let mut rx = pool.subscribe("s11", "coder-1").unwrap();
|
let mut rx = pool.subscribe("s11", "coder-1").await.unwrap();
|
||||||
|
|
||||||
run_server_owned_completion(
|
run_server_owned_completion(
|
||||||
&pool.agents,
|
&pool.agents,
|
||||||
@@ -160,7 +160,7 @@ async fn server_owned_completion_runs_gates_on_clean_worktree() {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
// Agent entry should be removed from the map after completion.
|
// Agent entry should be removed from the map after completion.
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = super::super::super::composite_key("s11", "coder-1");
|
let key = super::super::super::composite_key("s11", "coder-1");
|
||||||
assert!(
|
assert!(
|
||||||
agents.get(&key).is_none(),
|
agents.get(&key).is_none(),
|
||||||
@@ -192,7 +192,7 @@ async fn server_owned_completion_fails_on_dirty_worktree() {
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent_with_path("s12", "coder-1", AgentStatus::Running, repo.to_path_buf());
|
pool.inject_test_agent_with_path("s12", "coder-1", AgentStatus::Running, repo.to_path_buf());
|
||||||
|
|
||||||
let mut rx = pool.subscribe("s12", "coder-1").unwrap();
|
let mut rx = pool.subscribe("s12", "coder-1").await.unwrap();
|
||||||
|
|
||||||
run_server_owned_completion(
|
run_server_owned_completion(
|
||||||
&pool.agents,
|
&pool.agents,
|
||||||
@@ -205,7 +205,7 @@ async fn server_owned_completion_fails_on_dirty_worktree() {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
// Agent entry should be removed from the map after completion (even on failure).
|
// Agent entry should be removed from the map after completion (even on failure).
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = super::super::super::composite_key("s12", "coder-1");
|
let key = super::super::super::composite_key("s12", "coder-1");
|
||||||
assert!(
|
assert!(
|
||||||
agents.get(&key).is_none(),
|
agents.get(&key).is_none(),
|
||||||
@@ -307,7 +307,7 @@ async fn server_owned_completion_is_noop_for_mergemaster() {
|
|||||||
|
|
||||||
// The agent entry should remain in the pool (lifecycle cleanup is the
|
// The agent entry should remain in the pool (lifecycle cleanup is the
|
||||||
// caller's responsibility, not run_server_owned_completion's).
|
// caller's responsibility, not run_server_owned_completion's).
|
||||||
let agents = pool.agents.lock().unwrap();
|
let agents = pool.agents.try_lock().unwrap();
|
||||||
let key = super::super::super::composite_key("99_story_merge445", "mergemaster");
|
let key = super::super::super::composite_key("99_story_merge445", "mergemaster");
|
||||||
assert!(
|
assert!(
|
||||||
agents.get(&key).is_some(),
|
agents.get(&key).is_some(),
|
||||||
@@ -361,7 +361,7 @@ async fn server_owned_completion_preserves_dirty_worktree_with_committed_work()
|
|||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.inject_test_agent_with_path("645_test", "coder-1", AgentStatus::Running, wt_path.clone());
|
pool.inject_test_agent_with_path("645_test", "coder-1", AgentStatus::Running, wt_path.clone());
|
||||||
|
|
||||||
let mut rx = pool.subscribe("645_test", "coder-1").unwrap();
|
let mut rx = pool.subscribe("645_test", "coder-1").await.unwrap();
|
||||||
|
|
||||||
run_server_owned_completion(
|
run_server_owned_completion(
|
||||||
&pool.agents,
|
&pool.agents,
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ impl AgentPool {
|
|||||||
let pool = Arc::new(Self {
|
let pool = Arc::new(Self {
|
||||||
agents: Arc::clone(&self.agents),
|
agents: Arc::clone(&self.agents),
|
||||||
port: self.port,
|
port: self.port,
|
||||||
child_killers: Arc::clone(&self.child_killers),
|
|
||||||
watcher_tx: self.watcher_tx.clone(),
|
watcher_tx: self.watcher_tx.clone(),
|
||||||
status_broadcaster: Arc::clone(&self.status_broadcaster),
|
status_broadcaster: Arc::clone(&self.status_broadcaster),
|
||||||
});
|
});
|
||||||
@@ -35,9 +34,8 @@ impl AgentPool {
|
|||||||
/// If the agent was already removed from the pool (race: `remove_agents_for_story`
|
/// If the agent was already removed from the pool (race: `remove_agents_for_story`
|
||||||
/// ran first) this is a no-op; the `ContentKey::MergeSuccess` DB key written
|
/// ran first) this is a no-op; the `ContentKey::MergeSuccess` DB key written
|
||||||
/// by the caller acts as the authoritative fallback in that case.
|
/// by the caller acts as the authoritative fallback in that case.
|
||||||
pub fn set_merge_success_reported(&self, story_id: &str) {
|
pub async fn set_merge_success_reported(&self, story_id: &str) {
|
||||||
match self.agents.lock() {
|
let mut lock = self.agents.lock().await;
|
||||||
Ok(mut lock) => {
|
|
||||||
let found = lock.iter_mut().find(|(key, agent)| {
|
let found = lock.iter_mut().find(|(key, agent)| {
|
||||||
let key_story_id = key
|
let key_story_id = key
|
||||||
.rsplit_once(':')
|
.rsplit_once(':')
|
||||||
@@ -62,11 +60,6 @@ impl AgentPool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
|
||||||
slog_error!("[pipeline] set_merge_success_reported: could not lock agents: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Record that the mergemaster agent for `story_id` explicitly reported a
|
/// Record that the mergemaster agent for `story_id` explicitly reported a
|
||||||
/// merge failure via the `report_merge_failure` MCP tool.
|
/// merge failure via the `report_merge_failure` MCP tool.
|
||||||
@@ -75,9 +68,8 @@ impl AgentPool {
|
|||||||
/// that `run_pipeline_advance` can block advancement to `5_done/` even when
|
/// that `run_pipeline_advance` can block advancement to `5_done/` even when
|
||||||
/// the server-owned gate check returns `gates_passed=true` (those gates run
|
/// the server-owned gate check returns `gates_passed=true` (those gates run
|
||||||
/// in the feature-branch worktree, not on master).
|
/// in the feature-branch worktree, not on master).
|
||||||
pub fn set_merge_failure_reported(&self, story_id: &str) {
|
pub async fn set_merge_failure_reported(&self, story_id: &str) {
|
||||||
match self.agents.lock() {
|
let mut lock = self.agents.lock().await;
|
||||||
Ok(mut lock) => {
|
|
||||||
let found = lock.iter_mut().find(|(key, agent)| {
|
let found = lock.iter_mut().find(|(key, agent)| {
|
||||||
let key_story_id = key
|
let key_story_id = key
|
||||||
.rsplit_once(':')
|
.rsplit_once(':')
|
||||||
@@ -102,9 +94,4 @@ impl AgentPool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
|
||||||
slog_error!("[pipeline] set_merge_failure_reported: could not lock agents: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user