Story 27: Coverage tracking (full-stack)
Add end-to-end coverage tracking: backend collects vitest coverage, records metrics with threshold/baseline tracking, and blocks acceptance on regression. Frontend displays coverage in gate/review panels with a "Collect Coverage" button. Includes 20 Rust tests, 17 Vitest tests, and 14 Playwright E2E tests. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dave
@@ -1,13 +1,13 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
test.describe("App boot smoke test", () => {
|
||||
test("renders the project selection screen", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
test("renders the project selection screen", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
|
||||
await expect(page.getByText("StorkIt")).toBeVisible();
|
||||
await expect(page.getByPlaceholder("/path/to/project")).toBeVisible();
|
||||
await expect(
|
||||
page.getByRole("button", { name: "Open Project" }),
|
||||
).toBeVisible();
|
||||
});
|
||||
await expect(page.getByText("StorkIt")).toBeVisible();
|
||||
await expect(page.getByPlaceholder("/path/to/project")).toBeVisible();
|
||||
await expect(
|
||||
page.getByRole("button", { name: "Open Project" }),
|
||||
).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
247
frontend/tests/e2e/test-protection.spec.ts
Normal file
247
frontend/tests/e2e/test-protection.spec.ts
Normal file
@@ -0,0 +1,247 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import type {
|
||||
AcceptanceResponse,
|
||||
ReviewListResponse,
|
||||
} from "../../src/api/workflow";
|
||||
|
||||
/**
|
||||
* Helper: mock all API routes needed to render the Chat view.
|
||||
*/
|
||||
function mockChatApis(
|
||||
page: import("@playwright/test").Page,
|
||||
overrides: {
|
||||
acceptance?: AcceptanceResponse;
|
||||
reviewQueue?: ReviewListResponse;
|
||||
} = {},
|
||||
) {
|
||||
const acceptance: AcceptanceResponse = overrides.acceptance ?? {
|
||||
can_accept: false,
|
||||
reasons: ["No test results recorded for the story."],
|
||||
warning: null,
|
||||
summary: { total: 0, passed: 0, failed: 0 },
|
||||
missing_categories: ["unit", "integration"],
|
||||
};
|
||||
|
||||
const reviewQueue: ReviewListResponse = overrides.reviewQueue ?? {
|
||||
stories: [],
|
||||
};
|
||||
|
||||
return Promise.all([
|
||||
page.route("**/api/projects", (route) =>
|
||||
route.fulfill({ json: ["/tmp/test-project"] }),
|
||||
),
|
||||
page.route("**/api/io/fs/home", (route) => route.fulfill({ json: "/tmp" })),
|
||||
page.route("**/api/project", (route) => {
|
||||
if (route.request().method() === "POST") {
|
||||
return route.fulfill({ json: "/tmp/test-project" });
|
||||
}
|
||||
if (route.request().method() === "DELETE") {
|
||||
return route.fulfill({ json: true });
|
||||
}
|
||||
return route.fulfill({ json: null });
|
||||
}),
|
||||
page.route("**/api/ollama/models**", (route) =>
|
||||
route.fulfill({ json: ["llama3.1"] }),
|
||||
),
|
||||
page.route("**/api/anthropic/key/exists", (route) =>
|
||||
route.fulfill({ json: false }),
|
||||
),
|
||||
page.route("**/api/anthropic/models", (route) =>
|
||||
route.fulfill({ json: [] }),
|
||||
),
|
||||
page.route("**/api/model", (route) => {
|
||||
if (route.request().method() === "POST") {
|
||||
return route.fulfill({ json: true });
|
||||
}
|
||||
return route.fulfill({ json: null });
|
||||
}),
|
||||
page.route("**/api/workflow/acceptance", (route) => {
|
||||
if (route.request().url().includes("/ensure")) return route.fallback();
|
||||
return route.fulfill({ json: acceptance });
|
||||
}),
|
||||
page.route("**/api/workflow/review/all", (route) =>
|
||||
route.fulfill({ json: reviewQueue }),
|
||||
),
|
||||
page.route("**/api/workflow/acceptance/ensure", (route) =>
|
||||
route.fulfill({ json: true }),
|
||||
),
|
||||
page.route("**/api/io/fs/list/absolute**", (route) =>
|
||||
route.fulfill({ json: [] }),
|
||||
),
|
||||
]);
|
||||
}
|
||||
|
||||
async function openProject(page: import("@playwright/test").Page) {
|
||||
await page.goto("/");
|
||||
await page.getByPlaceholder("/path/to/project").fill("/tmp/test-project");
|
||||
await page.getByRole("button", { name: "Open Project" }).click();
|
||||
await expect(page.getByText("Workflow Gates", { exact: true })).toBeVisible();
|
||||
}
|
||||
|
||||
test.describe("Coverage threshold (AC1)", () => {
|
||||
test("shows blocked status when coverage is below threshold", async ({
|
||||
page,
|
||||
}) => {
|
||||
await mockChatApis(page, {
|
||||
acceptance: {
|
||||
can_accept: false,
|
||||
reasons: ["Coverage below threshold (55.0% < 80.0%)."],
|
||||
warning: null,
|
||||
summary: { total: 3, passed: 3, failed: 0 },
|
||||
missing_categories: [],
|
||||
coverage_report: {
|
||||
current_percent: 55.0,
|
||||
threshold_percent: 80.0,
|
||||
baseline_percent: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await openProject(page);
|
||||
|
||||
await expect(page.getByText("Blocked").first()).toBeVisible();
|
||||
await expect(page.getByText(/Coverage: 55\.0%/)).toBeVisible();
|
||||
await expect(page.getByText(/threshold: 80\.0%/)).toBeVisible();
|
||||
await expect(
|
||||
page.getByText("Coverage below threshold (55.0% < 80.0%)."),
|
||||
).toBeVisible();
|
||||
});
|
||||
|
||||
test("shows green coverage when above threshold", async ({ page }) => {
|
||||
await mockChatApis(page, {
|
||||
acceptance: {
|
||||
can_accept: true,
|
||||
reasons: [],
|
||||
warning: null,
|
||||
summary: { total: 5, passed: 5, failed: 0 },
|
||||
missing_categories: [],
|
||||
coverage_report: {
|
||||
current_percent: 92.0,
|
||||
threshold_percent: 80.0,
|
||||
baseline_percent: 90.0,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await openProject(page);
|
||||
|
||||
await expect(page.getByText("Ready to accept")).toBeVisible();
|
||||
await expect(page.getByText(/Coverage: 92\.0%/)).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Coverage regression reporting (AC2)", () => {
|
||||
test("review panel shows coverage regression reason", async ({ page }) => {
|
||||
await mockChatApis(page, {
|
||||
reviewQueue: {
|
||||
stories: [
|
||||
{
|
||||
story_id: "27_protect_tests_and_coverage",
|
||||
can_accept: false,
|
||||
reasons: ["Coverage regression: 90.0% → 82.0% (threshold: 80.0%)."],
|
||||
warning: null,
|
||||
summary: { total: 4, passed: 4, failed: 0 },
|
||||
missing_categories: [],
|
||||
coverage_report: {
|
||||
current_percent: 82.0,
|
||||
threshold_percent: 80.0,
|
||||
baseline_percent: 90.0,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
await openProject(page);
|
||||
|
||||
await expect(
|
||||
page.getByText(/Coverage regression.*90\.0%.*82\.0%/),
|
||||
).toBeVisible();
|
||||
await expect(page.getByText(/Coverage: 82\.0%/)).toBeVisible();
|
||||
|
||||
const blockedButton = page.getByRole("button", { name: "Blocked" });
|
||||
await expect(blockedButton).toBeDisabled();
|
||||
});
|
||||
|
||||
test("blocked acceptance with coverage and test failures combined", async ({
|
||||
page,
|
||||
}) => {
|
||||
await mockChatApis(page, {
|
||||
acceptance: {
|
||||
can_accept: false,
|
||||
reasons: [
|
||||
"1 test(s) are failing; acceptance is blocked.",
|
||||
"Coverage below threshold (65.0% < 80.0%).",
|
||||
],
|
||||
warning: null,
|
||||
summary: { total: 4, passed: 3, failed: 1 },
|
||||
missing_categories: [],
|
||||
coverage_report: {
|
||||
current_percent: 65.0,
|
||||
threshold_percent: 80.0,
|
||||
baseline_percent: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await openProject(page);
|
||||
|
||||
await expect(page.getByText("Blocked").first()).toBeVisible();
|
||||
await expect(
|
||||
page.getByText("Coverage below threshold (65.0% < 80.0%)."),
|
||||
).toBeVisible();
|
||||
await expect(page.getByText(/Coverage: 65\.0%/)).toBeVisible();
|
||||
await expect(page.getByText(/1 test\(s\) are failing/)).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Coverage collection (E2E)", () => {
|
||||
test("collect coverage button triggers collection and displays result", async ({
|
||||
page,
|
||||
}) => {
|
||||
await mockChatApis(page);
|
||||
|
||||
// Mock the collect coverage endpoint
|
||||
await page.route("**/api/workflow/coverage/collect", (route) =>
|
||||
route.fulfill({
|
||||
json: {
|
||||
current_percent: 85.0,
|
||||
threshold_percent: 80.0,
|
||||
baseline_percent: null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await openProject(page);
|
||||
|
||||
// Click the Collect Coverage button
|
||||
const collectButton = page.getByRole("button", {
|
||||
name: "Collect Coverage",
|
||||
});
|
||||
await expect(collectButton).toBeVisible();
|
||||
|
||||
// Override acceptance to return coverage data after collection
|
||||
await page.unroute("**/api/workflow/acceptance");
|
||||
await page.route("**/api/workflow/acceptance", (route) => {
|
||||
if (route.request().url().includes("/ensure")) return route.fallback();
|
||||
return route.fulfill({
|
||||
json: {
|
||||
can_accept: true,
|
||||
reasons: [],
|
||||
warning: null,
|
||||
summary: { total: 5, passed: 5, failed: 0 },
|
||||
missing_categories: [],
|
||||
coverage_report: {
|
||||
current_percent: 85.0,
|
||||
threshold_percent: 80.0,
|
||||
baseline_percent: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
await collectButton.click();
|
||||
|
||||
await expect(page.getByText(/Coverage: 85\.0%/)).toBeVisible();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user