From bb5d7879ff04016043fd65149f64a0a1bf372a8d Mon Sep 17 00:00:00 2001 From: Huskies Agent Date: Fri, 17 Jul 2026 13:26:13 +0000 Subject: [PATCH] huskies: merge 1189 story Gitea Actions workflow: build the sled artifact on master merges and publish to the dev channel --- .gitea/workflows/README.md | 53 +++++ .gitea/workflows/release-artifact.yml | 33 +++ Cargo.lock | 20 ++ Cargo.toml | 1 + script/ci-publish-artifact | 121 ++++++++++ server/Cargo.toml | 1 + server/src/ci_publish_artifact.rs | 307 ++++++++++++++++++++++++++ server/src/main.rs | 2 + 8 files changed, 538 insertions(+) create mode 100644 .gitea/workflows/README.md create mode 100644 .gitea/workflows/release-artifact.yml create mode 100755 script/ci-publish-artifact create mode 100644 server/src/ci_publish_artifact.rs diff --git a/.gitea/workflows/README.md b/.gitea/workflows/README.md new file mode 100644 index 00000000..bb56abc5 --- /dev/null +++ b/.gitea/workflows/README.md @@ -0,0 +1,53 @@ +# Gitea Actions workflows + +## `release-artifact.yml` + +Triggers on every push to `master`. Builds the `linux-arm64` sled binary and +publishes it to the "dev" release channel via `script/ci-publish-artifact`. + +### Runner registration + +The job targets the `arm64-mac` runner label. Register an `act_runner` on an +Apple Silicon macOS host that has `cargo`/`rustc` and `curl` on `PATH`: + +```sh +act_runner register \ + --instance https://code.crashlabs.io \ + --token \ + --labels arm64-mac +act_runner daemon +``` + +The registration token comes from the repo's **Settings → Actions → +Runners → Create new Runner** page in Gitea. Without a runner carrying the +`arm64-mac` label, jobs from this workflow queue indefinitely. + +### Secrets + +Configure these under the repo's **Settings → Actions → Secrets**. Never +commit credentials — the workflow only ever references them via +`${{ secrets.* }}`. + +| Secret | Purpose | +| --- | --- | +| `HUSKIES_CHANNEL_URL` | Base URL of the dev release channel host. | +| `HUSKIES_CHANNEL_TOKEN` | Bearer token authorised to publish artifacts to that channel. | + +### Channel host contract + +`script/ci-publish-artifact` expects the channel host at +`HUSKIES_CHANNEL_URL` to implement: + +- `POST {HUSKIES_CHANNEL_URL}/` — accepts the raw artifact + bytes as the request body. Requires `Authorization: Bearer ` and + `X-Git-Hash: ` headers. Non-2xx responses in the 4xx range + (including 401/403) are treated as permanent failures; 5xx responses and + network errors are retried with backoff. +- `GET {HUSKIES_CHANNEL_URL}/manifest.json` — returns a JSON object with a + `git_hash` field reflecting the most recently published artifact. + Requires `Authorization: Bearer `. + +This is a separate, unsigned channel distinct from the Ed25519-signed +release channels the `pull ` gateway command consumes (see +`server/src/service/gateway/release_manifest.rs`) — CI has no safe place to +hold a channel signing key, so the dev channel trusts the bearer token alone. diff --git a/.gitea/workflows/release-artifact.yml b/.gitea/workflows/release-artifact.yml new file mode 100644 index 00000000..79653c60 --- /dev/null +++ b/.gitea/workflows/release-artifact.yml @@ -0,0 +1,33 @@ +name: Publish sled artifact + +on: + push: + branches: + - master + +jobs: + publish-dev-artifact: + # Registered on an arm64 macOS act_runner host — see + # .gitea/workflows/README.md for registration instructions. + runs-on: [arm64-mac] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build linux-arm64 sled binary + env: + # Dedicated target dir so this CI build never clobbers a developer's + # incremental target/release build on the shared runner host, mirroring + # SLED_TARGET_DIR in server/src/chat/transport/matrix/release.rs. + CARGO_TARGET_DIR: target/ci-release + run: cargo build --release -p huskies + + - name: Stage artifact + run: cp target/ci-release/release/huskies target/ci-release/release/huskies-linux-arm64 + + - name: Publish to dev channel + env: + HUSKIES_CHANNEL_URL: ${{ secrets.HUSKIES_CHANNEL_URL }} + HUSKIES_CHANNEL_TOKEN: ${{ secrets.HUSKIES_CHANNEL_TOKEN }} + run: script/ci-publish-artifact target/ci-release/release/huskies-linux-arm64 "$(git rev-parse --short HEAD)" diff --git a/Cargo.lock b/Cargo.lock index 61cd2592..6232344d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1958,6 +1958,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "serde_yaml", "sha1 0.11.0", "sha2 0.11.0", "source-map-gen", @@ -4368,6 +4369,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serial2" version = "0.2.37" @@ -5371,6 +5385,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 0d75b2c2..a807dfbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,3 +65,4 @@ sqlx = { version = "0.9.0", default-features = false, features = [ "macros", "migrate", ] } +serde_yaml = "0.9.34" diff --git a/script/ci-publish-artifact b/script/ci-publish-artifact new file mode 100755 index 00000000..0d68469e --- /dev/null +++ b/script/ci-publish-artifact @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# script/ci-publish-artifact — upload a built sled artifact to a release +# channel and verify the channel's manifest reflects the published commit. +# +# Usage: script/ci-publish-artifact [git-hash] +# +# git-hash defaults to `git rev-parse --short HEAD` when omitted. +# +# Required env: +# HUSKIES_CHANNEL_URL Base URL of the release channel (Gitea secret). +# HUSKIES_CHANNEL_TOKEN Bearer token authorised to publish (Gitea secret). +# +# Protocol against the channel host: +# POST {HUSKIES_CHANNEL_URL}/ +# Headers: Authorization: Bearer , X-Git-Hash: +# Body: raw artifact bytes +# GET {HUSKIES_CHANNEL_URL}/manifest.json +# Headers: Authorization: Bearer +# Body: JSON object with a "git_hash" field +# +# 5xx responses and network errors are retried with backoff; 4xx responses +# (including auth failures) fail immediately since retrying won't fix them. +set -euo pipefail + +ARTIFACT_PATH="${1:?Usage: script/ci-publish-artifact [git-hash]}" +GIT_HASH="${2:-$(git rev-parse --short HEAD)}" + +if [ -z "${HUSKIES_CHANNEL_URL:-}" ]; then + echo "Error: HUSKIES_CHANNEL_URL is not set." >&2 + exit 1 +fi +if [ -z "${HUSKIES_CHANNEL_TOKEN:-}" ]; then + echo "Error: HUSKIES_CHANNEL_TOKEN is not set." >&2 + exit 1 +fi +if [ ! -f "$ARTIFACT_PATH" ]; then + echo "Error: artifact not found at $ARTIFACT_PATH" >&2 + exit 1 +fi + +ARTIFACT_NAME="$(basename "$ARTIFACT_PATH")" +CHANNEL_URL="${HUSKIES_CHANNEL_URL%/}" +UPLOAD_URL="${CHANNEL_URL}/${ARTIFACT_NAME}" +MANIFEST_URL="${CHANNEL_URL}/manifest.json" + +UPLOAD_MAX_ATTEMPTS="${HUSKIES_CI_PUBLISH_MAX_ATTEMPTS:-3}" +UPLOAD_BACKOFF_SECS="${HUSKIES_CI_PUBLISH_BACKOFF_SECS:-1}" +MANIFEST_MAX_ATTEMPTS=3 +MANIFEST_BACKOFF_SECS=1 + +RESPONSE_FILE="$(mktemp)" +trap 'rm -f "$RESPONSE_FILE"' EXIT + +# ── Upload ──────────────────────────────────────────────────────────────── +attempt=1 +while :; do + echo "==> Uploading ${ARTIFACT_NAME} (${GIT_HASH}), attempt ${attempt}/${UPLOAD_MAX_ATTEMPTS}..." + HTTP_CODE=$(curl -sS --connect-timeout 10 --max-time 60 \ + -o "$RESPONSE_FILE" -w "%{http_code}" \ + -X POST \ + -H "Authorization: Bearer ${HUSKIES_CHANNEL_TOKEN}" \ + -H "X-Git-Hash: ${GIT_HASH}" \ + --data-binary "@${ARTIFACT_PATH}" \ + "${UPLOAD_URL}") || HTTP_CODE="000" + RESPONSE_BODY="$(cat "$RESPONSE_FILE" 2>/dev/null || true)" + + case "$HTTP_CODE" in + 2??) + echo "==> Upload succeeded (HTTP ${HTTP_CODE})." + break + ;; + 401|403) + echo "Error: upload rejected — authentication failed (HTTP ${HTTP_CODE})." >&2 + echo "Response: ${RESPONSE_BODY}" >&2 + exit 1 + ;; + 4??) + echo "Error: upload rejected by the channel (HTTP ${HTTP_CODE}); not retrying a client error." >&2 + echo "Response: ${RESPONSE_BODY}" >&2 + exit 1 + ;; + esac + + if [ "$attempt" -ge "$UPLOAD_MAX_ATTEMPTS" ]; then + echo "Error: upload failed after ${UPLOAD_MAX_ATTEMPTS} attempts (last HTTP ${HTTP_CODE})." >&2 + echo "Response: ${RESPONSE_BODY}" >&2 + exit 1 + fi + + echo "==> Transient failure (HTTP ${HTTP_CODE}); retrying in ${UPLOAD_BACKOFF_SECS}s..." + sleep "$UPLOAD_BACKOFF_SECS" + attempt=$((attempt + 1)) + UPLOAD_BACKOFF_SECS=$((UPLOAD_BACKOFF_SECS * 2)) +done + +# ── Verify manifest ────────────────────────────────────────────────────── +attempt=1 +while :; do + echo "==> Verifying channel manifest reflects ${GIT_HASH} (attempt ${attempt}/${MANIFEST_MAX_ATTEMPTS})..." + MANIFEST_JSON=$(curl -sS --connect-timeout 10 --max-time 30 \ + -H "Authorization: Bearer ${HUSKIES_CHANNEL_TOKEN}" \ + "${MANIFEST_URL}") || MANIFEST_JSON="" + + MANIFEST_HASH=$(printf '%s' "$MANIFEST_JSON" \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('git_hash',''))" 2>/dev/null || echo "") + + if [ "$MANIFEST_HASH" = "$GIT_HASH" ]; then + echo "==> Published ${ARTIFACT_NAME} (${GIT_HASH}) to ${CHANNEL_URL}; manifest verified." + exit 0 + fi + + if [ "$attempt" -ge "$MANIFEST_MAX_ATTEMPTS" ]; then + echo "Error: manifest mismatch — channel reports git_hash '${MANIFEST_HASH}', expected '${GIT_HASH}'." >&2 + echo "Manifest: ${MANIFEST_JSON}" >&2 + exit 1 + fi + + sleep "$MANIFEST_BACKOFF_SECS" + attempt=$((attempt + 1)) + MANIFEST_BACKOFF_SECS=$((MANIFEST_BACKOFF_SECS * 2)) +done diff --git a/server/Cargo.toml b/server/Cargo.toml index bc150ca3..7c5da01f 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -66,3 +66,4 @@ check-cfg = ["cfg(feature, values(\"logging-base\"))"] tempfile = { workspace = true } mockito = "1.7.2" filetime = { workspace = true } +serde_yaml = { workspace = true } diff --git a/server/src/ci_publish_artifact.rs b/server/src/ci_publish_artifact.rs new file mode 100644 index 00000000..c7156e86 --- /dev/null +++ b/server/src/ci_publish_artifact.rs @@ -0,0 +1,307 @@ +//! Tests for `script/ci-publish-artifact` and `.gitea/workflows/release-artifact.yml` +//! (story 1189). Exercises the publish script end-to-end as a subprocess against a +//! minimal local HTTP stub rather than a live channel host — this crate has no +//! production code path that talks to the dev channel itself. + +use std::collections::{HashMap, VecDeque}; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::process::Command; +use std::sync::{Arc, Mutex}; +use std::thread; + +/// One request as observed by the stub: method, path, and the two headers +/// the publish script is required to send. +struct ObservedRequest { + method: String, + path: String, + authorization: Option, + git_hash: Option, +} + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..") +} + +fn script_path() -> PathBuf { + repo_root().join("script/ci-publish-artifact") +} + +/// Read one HTTP/1.1 request off `stream`: request line, headers (lower-cased +/// keys), and body (read exactly `Content-Length` bytes, defaulting to none). +fn read_request(stream: &mut TcpStream) -> (String, String, HashMap, Vec) { + let mut header_bytes = Vec::new(); + let mut byte = [0u8; 1]; + loop { + stream.read_exact(&mut byte).expect("read request byte"); + header_bytes.push(byte[0]); + if header_bytes.ends_with(b"\r\n\r\n") { + break; + } + } + let header_text = String::from_utf8_lossy(&header_bytes); + let mut lines = header_text.lines(); + let request_line = lines.next().unwrap_or(""); + let mut parts = request_line.split_whitespace(); + let method = parts.next().unwrap_or("").to_string(); + let path = parts.next().unwrap_or("").to_string(); + + let mut headers = HashMap::new(); + for line in lines { + if let Some((k, v)) = line.split_once(':') { + headers.insert(k.trim().to_lowercase(), v.trim().to_string()); + } + } + + let content_length: usize = headers + .get("content-length") + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + let mut body = vec![0u8; content_length]; + if content_length > 0 { + stream.read_exact(&mut body).expect("read request body"); + } + (method, path, headers, body) +} + +fn write_response(stream: &mut TcpStream, status: u16, body: &str) { + let reason = match status { + 200 => "OK", + 401 => "Unauthorized", + 403 => "Forbidden", + 404 => "Not Found", + 500 => "Internal Server Error", + _ => "Unknown", + }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); +} + +/// Spawn a background thread serving `responses` in order, one per accepted +/// connection; once exhausted, further requests get a 500 fallback (a test +/// bug, not an expected script behaviour). Returns the port and a log of +/// every request observed, in arrival order. +fn spawn_stub(responses: Vec<(u16, &'static str)>) -> (u16, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub listener"); + let port = listener.local_addr().expect("stub local addr").port(); + let queue = Arc::new(Mutex::new(responses.into_iter().collect::>())); + let log: Arc>> = Arc::new(Mutex::new(Vec::new())); + let log_writer = Arc::clone(&log); + + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let (method, path, headers, _body) = read_request(&mut stream); + log_writer.lock().unwrap().push(ObservedRequest { + method, + path, + authorization: headers.get("authorization").cloned(), + git_hash: headers.get("x-git-hash").cloned(), + }); + let (status, body) = queue + .lock() + .unwrap() + .pop_front() + .unwrap_or((500, "no more stub responses queued")); + write_response(&mut stream, status, body); + } + }); + + (port, log) +} + +struct RunResult { + success: bool, + stderr: String, +} + +fn run_publish_script(port: u16, git_hash: &str) -> RunResult { + let dir = tempfile::tempdir().expect("tempdir"); + let artifact_path = dir.path().join("huskies-linux-arm64"); + std::fs::write(&artifact_path, b"fake sled binary").expect("write fake artifact"); + + let output = Command::new("bash") + .arg(script_path()) + .arg(&artifact_path) + .arg(git_hash) + .env("HUSKIES_CHANNEL_URL", format!("http://127.0.0.1:{port}")) + .env("HUSKIES_CHANNEL_TOKEN", "test-token") + .env("HUSKIES_CI_PUBLISH_BACKOFF_SECS", "1") + .output() + .expect("run script/ci-publish-artifact"); + + RunResult { + success: output.status.success(), + stderr: String::from_utf8_lossy(&output.stderr).to_string(), + } +} + +// ── Script behaviour ───────────────────────────────────────────────────── + +#[test] +fn publish_succeeds_and_verifies_manifest() { + let (port, log) = spawn_stub(vec![ + (200, r#"{"status":"ok"}"#), + (200, r#"{"git_hash":"abc1234"}"#), + ]); + + let result = run_publish_script(port, "abc1234"); + assert!(result.success, "expected success: {}", result.stderr); + + let requests = log.lock().unwrap(); + assert_eq!(requests.len(), 2, "expected one upload + one manifest GET"); + assert_eq!(requests[0].method, "POST"); + assert_eq!( + requests[0].authorization.as_deref(), + Some("Bearer test-token") + ); + assert_eq!(requests[0].git_hash.as_deref(), Some("abc1234")); + assert_eq!(requests[1].method, "GET"); + assert_eq!(requests[1].path, "/manifest.json"); +} + +#[test] +fn publish_fails_immediately_on_auth_failure_without_retry() { + let (port, log) = spawn_stub(vec![(401, r#"{"error":"bad token"}"#)]); + + let result = run_publish_script(port, "abc1234"); + assert!(!result.success, "auth failure must not succeed"); + assert!( + result.stderr.contains("authentication failed"), + "stderr should explain the auth failure: {}", + result.stderr + ); + + let requests = log.lock().unwrap(); + assert_eq!( + requests.len(), + 1, + "a 401 must not be retried, and must not reach the manifest check" + ); +} + +#[test] +fn publish_retries_transient_failure_then_succeeds() { + let (port, log) = spawn_stub(vec![ + (500, "boom"), + (200, r#"{"status":"ok"}"#), + (200, r#"{"git_hash":"abc1234"}"#), + ]); + + let result = run_publish_script(port, "abc1234"); + assert!( + result.success, + "expected eventual success: {}", + result.stderr + ); + + let requests = log.lock().unwrap(); + assert_eq!( + requests.len(), + 3, + "expected the failed upload, the retried upload, and the manifest GET" + ); + assert_eq!(requests[0].method, "POST"); + assert_eq!(requests[1].method, "POST"); + assert_eq!(requests[2].method, "GET"); +} + +#[test] +fn publish_fails_on_manifest_mismatch() { + let (port, log) = spawn_stub(vec![ + (200, r#"{"status":"ok"}"#), + (200, r#"{"git_hash":"deadbee"}"#), + (200, r#"{"git_hash":"deadbee"}"#), + (200, r#"{"git_hash":"deadbee"}"#), + ]); + + let result = run_publish_script(port, "abc1234"); + assert!(!result.success, "manifest mismatch must fail"); + assert!( + result.stderr.contains("manifest mismatch"), + "stderr should explain the mismatch: {}", + result.stderr + ); + + let requests = log.lock().unwrap(); + assert_eq!( + requests.len(), + 4, + "expected the upload plus 3 manifest-check attempts" + ); +} + +// ── Workflow yaml structure ────────────────────────────────────────────── + +#[test] +fn workflow_yaml_triggers_on_master_push_and_targets_arm64_mac() { + let path = repo_root().join(".gitea/workflows/release-artifact.yml"); + let content = std::fs::read_to_string(&path).expect("read release-artifact.yml"); + let value: serde_yaml::Value = serde_yaml::from_str(&content).expect("valid yaml"); + + let branches = value["on"]["push"]["branches"] + .as_sequence() + .expect("on.push.branches sequence") + .iter() + .map(|v| v.as_str().unwrap_or_default().to_string()) + .collect::>(); + assert_eq!(branches, vec!["master".to_string()]); + + let jobs = value["jobs"].as_mapping().expect("jobs mapping"); + let (_, job) = jobs.iter().next().expect("at least one job defined"); + + let runs_on = job["runs-on"].as_sequence().expect("runs-on sequence"); + let runs_on: Vec<&str> = runs_on.iter().filter_map(|v| v.as_str()).collect(); + assert_eq!(runs_on, vec!["arm64-mac"]); + + let steps = job["steps"].as_sequence().expect("steps sequence"); + let calls_publish_script = steps.iter().any(|step| { + step.get("run") + .and_then(|r| r.as_str()) + .is_some_and(|s| s.contains("script/ci-publish-artifact")) + }); + assert!( + calls_publish_script, + "a step must invoke script/ci-publish-artifact" + ); + + let references_channel_secrets = steps.iter().any(|step| { + step.get("env") + .and_then(|e| e.as_mapping()) + .is_some_and(|env| { + env.values().any(|v| { + v.as_str().is_some_and(|s| { + s.contains("secrets.HUSKIES_CHANNEL_URL") + || s.contains("secrets.HUSKIES_CHANNEL_TOKEN") + }) + }) + }) + }); + assert!( + references_channel_secrets, + "workflow must reference the channel secrets via ${{{{ secrets.* }}}}" + ); +} + +#[test] +fn readme_documents_runner_label_and_secrets() { + let path = repo_root().join(".gitea/workflows/README.md"); + let content = std::fs::read_to_string(&path).expect("read .gitea/workflows/README.md"); + assert!( + content.contains("arm64-mac"), + "README must document the runner label" + ); + assert!( + content.contains("HUSKIES_CHANNEL_URL"), + "README must document HUSKIES_CHANNEL_URL" + ); + assert!( + content.contains("HUSKIES_CHANNEL_TOKEN"), + "README must document HUSKIES_CHANNEL_TOKEN" + ); +} diff --git a/server/src/main.rs b/server/src/main.rs index cecd31fa..88016603 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -10,6 +10,8 @@ mod agent_log; mod agent_mode; mod agents; mod chat; +#[cfg(test)] +mod ci_publish_artifact; mod config; /// CRDT snapshot — serialisation and restore of the full pipeline CRDT state. pub mod crdt_snapshot;