huskies: merge 1202 bug Flaky test: matrix pull pull_rejects_tampered_artifact_sha256_mismatch fails nondeterministically in merge gates

server/src/chat/transport/matrix/pull.rs

@@ -306,10 +306,23 @@ mod tests {
     /// Spawn a minimal HTTP server serving fixed responses for
     /// `/manifest.json` and `/<artifact>` so `run_pull` can be exercised
     /// end-to-end without a real network.
+    ///
+    /// `run_pull` issues its manifest fetch and artifact download
+    /// sequentially over one shared `reqwest::Client`, whose HTTP/1.1 pool
+    /// assumes keep-alive by default. Every response here closes its
+    /// connection after a single request, so we send `Connection: close` to
+    /// stop the client from ever trying to reuse a connection this server
+    /// has already torn down (a real source of intermittent "cannot reach"
+    /// fetch errors that raced ahead of the sha256 check). We also block on
+    /// a readiness signal fired from inside the accept loop so the returned
+    /// port is guaranteed to have an active acceptor before any caller
+    /// connects to it.
     async fn spawn_mock_channel(manifest_body: Vec<u8>, artifact_body: Vec<u8>) -> u16 {
         let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
         let port = listener.local_addr().unwrap().port();
+        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
         tokio::spawn(async move {
+            let _ = ready_tx.send(());
             loop {
                 let Ok((mut stream, _)) = listener.accept().await else {
                     break;
@@ -327,13 +340,17 @@ mod tests {
                     } else {
                         artifact_body
                     };
-                    let response =
-                        format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len());
+                    let response = format!(
+                        "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+                        body.len()
+                    );
                     let _ = stream.write_all(response.as_bytes()).await;
                     let _ = stream.write_all(&body).await;
+                    let _ = stream.shutdown().await;
                 });
             }
         });
+        let _ = ready_rx.await;
         port
     }
 
@@ -468,10 +485,64 @@ mod tests {
             msg.contains("sha256 mismatch"),
             "expected sha256 failure: {msg}"
         );
+        assert!(
+            !msg.contains("could not fetch manifest")
+                && !msg.contains("could not download artifact"),
+            "manifest fetch and artifact download must succeed so the sha256 check is reached, \
+             not preempted by a fetch/download error: {msg}"
+        );
         assert!(msg.contains("Nothing was installed"));
         assert!(!dir.path().join("huskies-linux-arm64").exists());
     }
 
+    /// Stress variant of [`pull_rejects_tampered_artifact_sha256_mismatch`]:
+    /// repeats the same scenario against many independent mock channels to
+    /// catch any reintroduction of the fetch-ordering race under load.
+    #[tokio::test]
+    async fn pull_rejects_tampered_artifact_sha256_mismatch_stress() {
+        for _ in 0..50 {
+            let kp = make_keypair();
+            let pubkey = crate::node_identity::public_key_hex(&kp);
+
+            use sha2::{Digest, Sha256};
+            let mut hasher = Sha256::new();
+            hasher.update(b"the real bytes");
+            let real_sha256: String = hasher
+                .finalize()
+                .iter()
+                .map(|b| format!("{b:02x}"))
+                .collect();
+
+            let manifest = ReleaseManifest {
+                artifact: "huskies-linux-arm64".to_string(),
+                sha256: real_sha256,
+                version: "abc1234".to_string(),
+                channel: "stable".to_string(),
+                timestamp: 1,
+            };
+            let signed_manifest = signed(&kp, manifest);
+            let manifest_body = serde_json::to_vec(&signed_manifest).unwrap();
+
+            let port = spawn_mock_channel(manifest_body, b"tampered bytes".to_vec()).await;
+            let dir = tempfile::tempdir().unwrap();
+
+            let msg = run_pull(
+                "stable",
+                &format!("http://127.0.0.1:{port}"),
+                &pubkey,
+                None,
+                dir.path(),
+                "huskies-linux-arm64",
+            )
+            .await;
+            assert!(
+                msg.contains("sha256 mismatch"),
+                "expected sha256 failure on every iteration: {msg}"
+            );
+            assert!(!dir.path().join("huskies-linux-arm64").exists());
+        }
+    }
+
     #[tokio::test]
     async fn pull_missing_manifest_endpoint_is_loud_error() {
         let dir = tempfile::tempdir().unwrap();