diff --git a/cmd/coveragecli/main.go b/cmd/coveragecli/main.go index 298922c..1045d10 100644 --- a/cmd/coveragecli/main.go +++ b/cmd/coveragecli/main.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/json" + "encoding/xml" "errors" "flag" "fmt" @@ -12,6 +13,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "regexp" "sort" "strconv" @@ -114,6 +116,57 @@ type metricAgg struct { Total float64 } +// JUnit XML structs — shared between Playwright and Appium JUnit reports. +// JUnitTestSuites represents the root element in JUnit XML. +type JUnitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Name string `xml:"name,attr,omitempty"` + Tests int `xml:"tests,attr,omitempty"` + Failures int `xml:"failures,attr,omitempty"` + Errors int `xml:"errors,attr,omitempty"` + Time float64 `xml:"time,attr,omitempty"` + TestSuites []JUnitTestSuite `xml:"testsuite"` +} + +// JUnitTestSuite represents a single element in JUnit XML. +type JUnitTestSuite struct { + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr,omitempty"` + Failures int `xml:"failures,attr,omitempty"` + Errors int `xml:"errors,attr,omitempty"` + Skipped int `xml:"skipped,attr,omitempty"` + Time float64 `xml:"time,attr,omitempty"` + Timestamp string `xml:"timestamp,attr,omitempty"` + Hostname string `xml:"hostname,attr,omitempty"` + Properties []JUnitProperty `xml:"properties>property,omitempty"` + TestCases []JUnitTestCase `xml:"testcase"` + SystemOut string `xml:"system-out,omitempty"` +} + +type JUnitTestCase struct { + Classname string `xml:"classname,attr,omitempty"` + Name string `xml:"name,attr"` + Time float64 `xml:"time,attr,omitempty"` + Status string `xml:"status,attr,omitempty"` + Failure *JUnitFailure `xml:"failure,omitempty"` + Skipped *JUnitSkipped `xml:"skipped,omitempty"` +} + +type JUnitFailure struct { + Message string `xml:"message,attr,omitempty"` + Type string `xml:"type,attr,omitempty"` + Body string `xml:",chardata"` +} + +type JUnitSkipped struct { + Message string `xml:"message,attr,omitempty"` +} + +type JUnitProperty struct { + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + func main() { if len(os.Args) > 1 { switch os.Args[1] { @@ -437,7 +490,7 @@ func runE2EUpload(args []string) { triggerType := fs.String("trigger-type", "manual", "Trigger type: push|pr|manual") environment := fs.String("environment", "", "Environment: test|stage|prod (optional)") runTimestamp := fs.String("run-timestamp", time.Now().UTC().Format(time.RFC3339), "Run timestamp (RFC3339)") - platformType := fs.String("platform-type", "web", "Platform type: web|android|ios") + if err := fs.Parse(args); err != nil { exitErr("parse flags", err) } @@ -457,9 +510,16 @@ func runE2EUpload(args []string) { exitErr("read e2e report", err) } - var report map[string]any - if err := json.Unmarshal(rawReport, &report); err != nil { - exitErr("parse e2e report json", err) + // Detect file format from extension + ext := strings.ToLower(filepath.Ext(*reportPath)) + var isXML bool + switch ext { + case ".xml": + isXML = true + case ".json": + isXML = false + default: + exitErr("validate input", fmt.Errorf("unsupported file extension %q: expected .json or .xml", ext)) } var group *string @@ -475,17 +535,35 @@ func runE2EUpload(args []string) { env = environment } - // Normalize report structure based on report type - var normalizeReport map[string]any - switch *reportType { - case "playwright": - normalizeReport = normalizePlaywrightReport(report) - case "appium": - normalizeReport = normalizeAppiumReport(report) - default: - exitErr("validate input", fmt.Errorf("unsupported report type: %s", *reportType)) + // Normalize report structure based on report type and file format + var normalizedReport map[string]any + if isXML { + var junitData JUnitTestSuites + if err := xml.Unmarshal(rawReport, &junitData); err != nil { + exitErr("parse e2e report xml", err) + } + switch *reportType { + case "playwright": + normalizedReport = normalizePlaywrightJUnit(junitData) + case "appium": + normalizedReport = normalizeAppiumJUnit(junitData) + default: + exitErr("validate input", fmt.Errorf("unsupported report type: %s", *reportType)) + } + } else { + var report map[string]any + if err := json.Unmarshal(rawReport, &report); err != nil { + exitErr("parse e2e report json", err) + } + switch *reportType { + case "playwright": + normalizedReport = normalizePlaywrightReport(report) + case "appium": + exitErr("validate input", fmt.Errorf("appium JSON report format is not yet supported; use JUnit XML (.xml)")) + default: + exitErr("validate input", fmt.Errorf("unsupported report type: %s", *reportType)) + } } - normalizeReport["platformType"] = *platformType payload := e2ePayload{ ProjectKey: *projectKey, @@ -498,7 +576,7 @@ func runE2EUpload(args []string) { TriggerType: *triggerType, RunTimestamp: *runTimestamp, Environment: env, - TestReport: normalizeReport, + TestReport: normalizedReport, } body, err := json.MarshalIndent(payload, "", " ") @@ -630,6 +708,8 @@ func normalizePlaywrightReport(raw map[string]any) map[string]any { // Use the last test result (accounts for retries). tests := firstSlice(specMap, "tests") + file := firstString(suiteMap, "file") + spec_type := "happyPath" state := "skipped" runTime := 0.0 var failureBlock map[string]any @@ -668,6 +748,24 @@ func normalizePlaywrightReport(raw map[string]any) map[string]any { } } } + + // spec_type can be either setup, happyPath or negativePath + // checks if the file name contains "setup", "happyPath" or "negativePath" to determine the spec_type + // fall back to checking the projectId + projectID := firstString(testMap, "projectId") + + switch { + case strings.Contains(file, "setup"): + spec_type = "setup" + case strings.Contains(file, "happyPath"): + spec_type = "happyPath" + case strings.Contains(file, "negativePath"): + spec_type = "negativePath" + case projectID == "happypath" || projectID == "negativePath" || projectID == "setup": + spec_type = projectID + default: + spec_type = "happyPath" + } } } @@ -682,6 +780,8 @@ func normalizePlaywrightReport(raw map[string]any) map[string]any { "containerHierarchyTexts": hierarchyCopy, "state": state, "runTime": runTime, + "suite_type": firstString(suiteMap, "type"), + "specType": spec_type, } if failureBlock != nil { normalized["failure"] = failureBlock @@ -695,12 +795,115 @@ func normalizePlaywrightReport(raw map[string]any) map[string]any { return result } -func normalizeAppiumReport(raw map[string]any) map[string]any { - // not implemented yet - exitErr("normalize report", fmt.Errorf("appium report normalization not implemented yet")) +// normalizePlaywrightJUnit converts a Playwright JUnit XML report into the normalized map[string]any structure. +// Playwright JUnit uses classname format: "file › Suite Title › Nested Suite" +func normalizePlaywrightJUnit(data JUnitTestSuites) map[string]any { + fmt.Errorf("Playwright JUnit XML normalization is not yet implemented") return nil } +// normalizeAppiumJUnit converts an Appium JUnit XML report into the normalized map[string]any structure. +// Appium JUnit uses classname format: "com.package.ClassName" (dot-separated) +func normalizeAppiumJUnit(data JUnitTestSuites) map[string]any { + result := make(map[string]any) + testFramework := "appium" + result["reportType"] = &testFramework + result["testFramework"] = &testFramework + + // Use top-level testsuites name as suiteDescription + if data.Name != "" { + result["suiteDescription"] = data.Name + } + result["suitePath"] = data.TestSuites[0].TestCases[0].Classname + result["frameworkVersion"] = "" + + // Extract platform metadata from first testsuite's properties + // Set default platform type for Appium + platformType := "android" + if len(data.TestSuites) > 0 { + for _, prop := range data.TestSuites[0].Properties { + switch prop.Name { + case "platformName": + platformType = strings.ToLower(prop.Value) + case "automationName": + result["frameworkVersion"] = prop.Value + } + } + } + result["platformType"] = platformType + fmt.Println("normalize: ", result["platformType"]) + + var specReports []map[string]any + for _, suite := range data.TestSuites { + for _, tc := range suite.TestCases { + // Appium classname format: "com.package.tests.Login.LoginPass" (split on ".") + var hierarchy []any + if tc.Classname != "" { + parts := strings.Split(tc.Classname, ".") + for _, p := range parts { + if p != "" { + hierarchy = append(hierarchy, p) + } + } + } + + // Determine state from failure/skipped elements or status attribute + state := "passed" + if tc.Failure != nil { + state = "failed" + } else if tc.Skipped != nil { + state = "skipped" + } else if tc.Status != "" { + // Some Appium/TestNG reporters include a status attribute + switch strings.ToLower(tc.Status) { + case "passed": + state = "passed" + case "failed": + state = "failed" + case "skipped": + state = "skipped" + } + } + + // Determine specType from classname keywords + specType := "happyPath" + classLower := strings.ToLower(tc.Classname) + switch { + case strings.Contains(classLower, "setup"): + specType = "setup" + case strings.Contains(classLower, "happyPath"): + specType = "happyPath" + case strings.Contains(classLower, "negativepath"): + specType = "negativePath" + default: + specType = "happyPath" + } + + spec := map[string]any{ + "leafNodeText": tc.Name, + "containerHierarchyTexts": hierarchy, + "state": state, + "runTime": tc.Time, + "suite_type": suite.Name, + "specType": specType, + } + + if tc.Failure != nil { + failure := map[string]any{ + "message": tc.Failure.Message, + } + if tc.Failure.Body != "" { + failure["stackTrace"] = strings.TrimSpace(tc.Failure.Body) + } + spec["failure"] = failure + } + specReports = append(specReports, spec) + } + } + result["specReports"] = specReports + return result +} + // stripANSI removes ANSI escape codes from a string. // This is useful to clean up error messages from Playwright which may include ANSI codes for coloring. func stripANSI(s string) string { diff --git a/cmd/coveragecli/main_test.go b/cmd/coveragecli/main_test.go index 1b68754..cb2c400 100644 --- a/cmd/coveragecli/main_test.go +++ b/cmd/coveragecli/main_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "encoding/xml" "os" "path/filepath" "testing" @@ -405,3 +406,91 @@ func TestStripANSI(t *testing.T) { }) } } + +func TestNormalizeAppiumJUnit(t *testing.T) { + t.Parallel() + + raw, err := os.ReadFile(filepath.Join("testdata", "appium-junit-report.xml")) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + var data JUnitTestSuites + if err := xml.Unmarshal(raw, &data); err != nil { + t.Fatalf("unmarshal xml: %v", err) + } + + result := normalizeAppiumJUnit(data) + + // Verify metadata + if rt, ok := result["reportType"].(*string); !ok || *rt != "appium" { + t.Fatalf("reportType = %v, want *appium", result["reportType"]) + } + if result["suiteDescription"] != "Appium Android Test Suite" { + t.Fatalf("suiteDescription = %v, want 'Appium Android Test Suite'", result["suiteDescription"]) + } + if result["platformType"] != "android" { + t.Fatalf("platformType = %v, want android", result["platformType"]) + } + if result["frameworkVersion"] != "UiAutomator2" { + t.Fatalf("frameworkVersion = %v, want UiAutomator2", result["frameworkVersion"]) + } + + specs, ok := result["specReports"].([]map[string]any) + if !ok { + t.Fatalf("specReports is not []map[string]any") + } + if len(specs) != 3 { + t.Fatalf("specReports count = %d, want 3", len(specs)) + } + + tests := []struct { + name string + state string + specType string + time float64 + }{ + {"Test1", "passed", "happyPath", 12.542}, + {"Test2", "failed", "happyPath", 9.118}, + {"Test3", "passed", "happyPath", 20.658}, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := specs[i] + if spec["leafNodeText"] != tt.name { + t.Errorf("spec[%d] leafNodeText = %v, want %v", i, spec["leafNodeText"], tt.name) + } + if spec["state"] != tt.state { + t.Errorf("spec[%d] state = %v, want %v", i, spec["state"], tt.state) + } + if spec["specType"] != tt.specType { + t.Errorf("spec[%d] specType = %v, want %v", i, spec["specType"], tt.specType) + } + if spec["runTime"] != tt.time { + t.Errorf("spec[%d] runTime = %v, want %v", i, spec["runTime"], tt.time) + } + // Verify hierarchy + hier, ok := spec["containerHierarchyTexts"].([]any) + if !ok || len(hier) == 0 { + t.Errorf("spec[%d] containerHierarchyTexts empty", i) + } + if len(hier) > 0 && hier[0] != "Tests" { + t.Errorf("spec[%d] hierarchy[0] = %v, want Tests", i, hier[0]) + } + }) + } + + // Verify failure block on the failed spec + failedSpec := specs[1] + failure, ok := failedSpec["failure"].(map[string]any) + if !ok { + t.Fatalf("failed spec has no failure block") + } + if failure["message"] != "Assertion failed: Expected error message not displayed" { + t.Errorf("failure message = %v", failure["message"]) + } + if _, ok := failure["stackTrace"]; !ok { + t.Errorf("failure missing stackTrace") + } +} diff --git a/cmd/coveragecli/testdata/appium-junit-report.xml b/cmd/coveragecli/testdata/appium-junit-report.xml new file mode 100644 index 0000000..bd1fbe4 --- /dev/null +++ b/cmd/coveragecli/testdata/appium-junit-report.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cmd/frontend/web/assets/e2e.js b/cmd/frontend/web/assets/e2e.js index e570868..43d0de3 100644 --- a/cmd/frontend/web/assets/e2e.js +++ b/cmd/frontend/web/assets/e2e.js @@ -12,6 +12,7 @@ const e2eBranchFilter = document.getElementById('e2eBranchFilter'); const e2eStatusFilter = document.getElementById('e2eStatusFilter'); const e2eEnvironmentFilter = document.getElementById('e2eEnvironmentFilter'); const e2ePlatformFilter = document.getElementById('e2ePlatformFilter'); +const e2eSpecTypeFilter = document.getElementById('e2eSpecTypeFilter'); const e2eReload = document.getElementById('e2eReload'); const e2eAutoRefreshInterval = document.getElementById('e2eAutoRefreshInterval'); const e2eAutoRefreshStatus = document.getElementById('e2eAutoRefreshStatus'); @@ -24,6 +25,7 @@ const closeE2EHeatmap = document.getElementById('closeE2EHeatmap'); const e2eHeatmapOverlay = document.getElementById('e2eHeatmapOverlay'); const heatmapBranchFilter = document.getElementById('heatmapBranchFilter'); const heatmapStatusFilter = document.getElementById('heatmapStatusFilter'); +const heatmapSpecTypeFilter = document.getElementById('heatmapSpecTypeFilter'); const heatmapReload = document.getElementById('heatmapReload'); const e2eHeatmap = document.getElementById('e2eHeatmap'); const appShell = document.getElementById('appShell'); @@ -82,6 +84,9 @@ e2eEnvironmentFilter.addEventListener('change', async () => { e2ePlatformFilter.addEventListener('change', async () => { await loadE2ERuns(selectedProjectId); }); +e2eSpecTypeFilter.addEventListener('change', async () => { + await loadE2ERuns(selectedProjectId); +}); e2eReload.addEventListener('click', async () => { await runWithButtonBusy(e2eReload, 'Reload', 'Reloading...', async () => { await loadE2EScreen(selectedProjectId, { preferredRunId: selectedE2ERunId }); @@ -101,6 +106,9 @@ heatmapBranchFilter.addEventListener('change', async () => { heatmapStatusFilter.addEventListener('change', async () => { await loadHeatmap(); }); +heatmapSpecTypeFilter.addEventListener('change', async () => { + await loadHeatmap(); +}); heatmapReload.addEventListener('click', async () => { await runWithButtonBusy(heatmapReload, 'Reload', 'Reloading...', async () => { await loadHeatmap(); @@ -375,7 +383,7 @@ async function loadProjects() { e2eRunChain.innerHTML = '

No projects match current filters.

'; e2eRunsBody.innerHTML = 'No projects match current filters.'; } - e2eFailedSpecsBody.innerHTML = 'No run selected.'; + e2eFailedSpecsBody.innerHTML = 'No run selected.'; e2eStatus.textContent = '-'; e2eStatus.className = 'value'; e2ePassRate.textContent = '-'; @@ -486,7 +494,7 @@ async function ensureSelectedProjectIsVisible() { e2eScreenProjectMeta.textContent = 'Adjust group and search filters to select a project.'; e2eRunChain.innerHTML = '

No projects match current filters.

'; e2eRunsBody.innerHTML = 'No projects match current filters.'; - e2eFailedSpecsBody.innerHTML = 'No run selected.'; + e2eFailedSpecsBody.innerHTML = 'No run selected.'; e2eStatus.textContent = '-'; e2eStatus.className = 'value'; e2ePassRate.textContent = '-'; @@ -561,7 +569,7 @@ async function loadE2EScreen(projectId, options = {}) { if (!projectId) { e2eRunChain.innerHTML = '

Select a project to view its run chain.

'; e2eRunsBody.innerHTML = 'Select a project first.'; - e2eFailedSpecsBody.innerHTML = 'No run selected.'; + e2eFailedSpecsBody.innerHTML = 'No run selected.'; return; } @@ -610,6 +618,7 @@ async function loadE2ERuns(projectId, preferredRunId = null) { const selectedStatus = e2eStatusFilter.value || ''; const selectedEnvironment = e2eEnvironmentFilter.value || ''; const selectedPlatform = e2ePlatformFilter.value || ''; + const selectedSpecType = e2eSpecTypeFilter.value || ''; url.searchParams.set('branch', selectedBranch); if (selectedStatus) { url.searchParams.set('status', selectedStatus); @@ -620,6 +629,9 @@ async function loadE2ERuns(projectId, preferredRunId = null) { if (selectedPlatform) { url.searchParams.set('platform', selectedPlatform); } + if (selectedSpecType) { + url.searchParams.set('specType', selectedSpecType); + } const res = await fetch(url.toString()); if (!res.ok) throw new Error(`failed to load E2E runs (${res.status})`); @@ -632,7 +644,8 @@ async function loadE2ERuns(projectId, preferredRunId = null) { const currentStatus = e2eStatusFilter.value || ''; const currentEnvironment = e2eEnvironmentFilter.value || ''; const currentPlatform = e2ePlatformFilter.value || ''; - if (currentBranch !== selectedBranch || currentStatus !== selectedStatus || currentEnvironment !== selectedEnvironment || currentPlatform !== selectedPlatform) return; + const currentSpecType = e2eSpecTypeFilter.value || ''; + if (currentBranch !== selectedBranch || currentStatus !== selectedStatus || currentEnvironment !== selectedEnvironment || currentPlatform !== selectedPlatform || currentSpecType !== selectedSpecType) return; currentE2ERunItems = items; const passedRuns = items.filter((run) => run.status === 'passed').length; @@ -640,9 +653,9 @@ async function loadE2ERuns(projectId, preferredRunId = null) { if (passedRuns === 0 && failedRuns === 0) { e2ePassRate.textContent = '-'; } else if (failedRuns === 0) { - e2ePassRate.textContent = '∞%'; + e2ePassRate.textContent = '100%'; } else { - e2ePassRate.textContent = `${((passedRuns / failedRuns) * 100).toFixed(2)}%`; + e2ePassRate.textContent = `${((passedRuns / (passedRuns + failedRuns)) * 100).toFixed(2)}%`; } if (items.length === 0) { @@ -652,7 +665,7 @@ async function loadE2ERuns(projectId, preferredRunId = null) { e2eFailedSpecsCount.textContent = '-'; e2eRunChain.innerHTML = '

No E2E runs found for current filters.

'; e2eRunsBody.innerHTML = 'No E2E runs found.'; - e2eFailedSpecsBody.innerHTML = 'No run selected.'; + e2eFailedSpecsBody.innerHTML = 'No run selected.'; return; } @@ -698,7 +711,7 @@ async function loadE2ERuns(projectId, preferredRunId = null) { selectedE2ERunId = null; e2eRunChain.innerHTML = `

${err.message}

`; e2eRunsBody.innerHTML = `${err.message}`; - e2eFailedSpecsBody.innerHTML = 'Failed to load selected run details.'; + e2eFailedSpecsBody.innerHTML = 'Failed to load selected run details.'; e2ePassRate.textContent = '-'; } } @@ -779,16 +792,16 @@ async function loadE2ERunDetails(projectId, runId) { if (!res.ok) throw new Error(`failed to load E2E run details (${res.status})`); const data = await res.json(); const failedSpecs = data.failedSpecs || []; - if (failedSpecs.length === 0) { - e2eFailedSpecsBody.innerHTML = 'No failed specs for this run.'; + e2eFailedSpecsBody.innerHTML = 'No failed specs for this run.'; return; } - for (const failed of failedSpecs) { + for (const failed of failedSpecs) { const tr = document.createElement('tr'); tr.innerHTML = ` ${escapeHtml(failed.specPath || '-')} + ${escapeHtml(failed.specType || '-')} ${escapeHtml(failed.failureMessage || '-')} ${escapeHtml(failed.file || '-')} ${failed.line || '-'} @@ -796,7 +809,7 @@ async function loadE2ERunDetails(projectId, runId) { e2eFailedSpecsBody.appendChild(tr); } } catch (err) { - e2eFailedSpecsBody.innerHTML = `${err.message}`; + e2eFailedSpecsBody.innerHTML = `${err.message}`; } } @@ -844,6 +857,7 @@ async function loadHeatmap() { url.searchParams.set('runsPerProject', '10'); if (heatmapBranchFilter.value) url.searchParams.set('branch', heatmapBranchFilter.value); if (heatmapStatusFilter.value) url.searchParams.set('status', heatmapStatusFilter.value); + if (heatmapSpecTypeFilter.value) url.searchParams.set('specType', heatmapSpecTypeFilter.value); const res = await fetch(url.toString()); if (!res.ok) throw new Error(`heatmap request failed (${res.status})`); diff --git a/cmd/frontend/web/e2eTest.html b/cmd/frontend/web/e2eTest.html index e1dcaa3..9cf7b2b 100644 --- a/cmd/frontend/web/e2eTest.html +++ b/cmd/frontend/web/e2eTest.html @@ -106,6 +106,11 @@

Select a project

+ @@ -152,6 +157,7 @@

Failed Specs (Selected Run)

Spec Path + Spec Type Message File Line @@ -181,6 +187,11 @@

E2E Heatmap

+ diff --git a/internal/adapters/http/handlers.go b/internal/adapters/http/handlers.go index 192683e..3aac20e 100644 --- a/internal/adapters/http/handlers.go +++ b/internal/adapters/http/handlers.go @@ -386,6 +386,7 @@ func (h *Handler) ListE2ERuns(w http.ResponseWriter, r *http.Request) { Branch: q.Get("branch"), Status: q.Get("status"), Environment: q.Get("environment"), + SpecType: q.Get("specType"), From: from, To: to, Page: page, @@ -559,6 +560,7 @@ func (h *Handler) GetE2EHeatmap(w http.ResponseWriter, r *http.Request) { out, err := h.getE2EHeatmap.Execute(r.Context(), application.E2EHeatmapInput{ Branch: q.Get("branch"), Status: q.Get("status"), + SpecType: q.Get("specType"), RunsPerProject: runsPerProject, }) if err != nil { diff --git a/internal/adapters/postgres/e2e_spec_result_repository.go b/internal/adapters/postgres/e2e_spec_result_repository.go index bfa9da7..9fbc37c 100644 --- a/internal/adapters/postgres/e2e_spec_result_repository.go +++ b/internal/adapters/postgres/e2e_spec_result_repository.go @@ -26,9 +26,9 @@ func (r *E2ESpecResultRepository) CreateBatch(ctx context.Context, specs []domai _, err := q.Exec(ctx, ` INSERT INTO e2e_test_spec_results ( id, e2e_run_id, spec_path, leaf_node_text, state, duration_ms, - failure_message, failure_location_file, failure_location_line + failure_message, failure_location_file, failure_location_line, spec_type ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) `, spec.ID, spec.E2ETestRunID, @@ -39,6 +39,7 @@ func (r *E2ESpecResultRepository) CreateBatch(ctx context.Context, specs []domai spec.FailureMessage, spec.FailureLocationFile, spec.FailureLocationLine, + spec.SpecType, ) if err != nil { return fmt.Errorf("insert e2e spec result: %w", err) @@ -52,7 +53,7 @@ func (r *E2ESpecResultRepository) ListByRunID(ctx context.Context, runID string) q := getQuerier(ctx, r.pool) rows, err := q.Query(ctx, ` SELECT id, e2e_run_id, spec_path, leaf_node_text, state, duration_ms, - failure_message, failure_location_file, failure_location_line + failure_message, failure_location_file, failure_location_line, spec_type FROM e2e_test_spec_results WHERE e2e_run_id = $1 ORDER BY spec_path ASC @@ -75,6 +76,7 @@ func (r *E2ESpecResultRepository) ListByRunID(ctx context.Context, runID string) &spec.FailureMessage, &spec.FailureLocationFile, &spec.FailureLocationLine, + &spec.SpecType, ); err != nil { return nil, fmt.Errorf("scan e2e spec result: %w", err) } @@ -92,7 +94,7 @@ func (r *E2ESpecResultRepository) ListFailedByRunID(ctx context.Context, runID s q := getQuerier(ctx, r.pool) rows, err := q.Query(ctx, ` SELECT id, e2e_run_id, spec_path, leaf_node_text, state, duration_ms, - failure_message, failure_location_file, failure_location_line + failure_message, failure_location_file, failure_location_line, spec_type FROM e2e_test_spec_results WHERE e2e_run_id = $1 AND state IN ('failed', 'flaky') ORDER BY spec_path ASC @@ -115,6 +117,7 @@ func (r *E2ESpecResultRepository) ListFailedByRunID(ctx context.Context, runID s &spec.FailureMessage, &spec.FailureLocationFile, &spec.FailureLocationLine, + &spec.SpecType, ); err != nil { return nil, fmt.Errorf("scan failed e2e spec result: %w", err) } diff --git a/internal/adapters/postgres/e2e_test_run_repository.go b/internal/adapters/postgres/e2e_test_run_repository.go index b8ccb79..2f45b43 100644 --- a/internal/adapters/postgres/e2e_test_run_repository.go +++ b/internal/adapters/postgres/e2e_test_run_repository.go @@ -208,7 +208,7 @@ func (r *E2ETestRunRepository) GetByID(ctx context.Context, projectID string, ru return run, nil } -func (r *E2ETestRunRepository) ListByProject(ctx context.Context, projectID string, branch string, status string, environment string, from *time.Time, to *time.Time, page int, pageSize int) ([]domain.E2ETestRun, int, error) { +func (r *E2ETestRunRepository) ListByProject(ctx context.Context, projectID string, branch string, status string, environment string, specType string, from *time.Time, to *time.Time, page int, pageSize int) ([]domain.E2ETestRun, int, error) { q := getQuerier(ctx, r.pool) offset := (page - 1) * pageSize @@ -235,6 +235,11 @@ func (r *E2ETestRunRepository) ListByProject(ctx context.Context, projectID stri idx++ } } + if specType != "" { + where += fmt.Sprintf(" AND EXISTS (SELECT 1 FROM e2e_test_spec_results s WHERE s.e2e_run_id = e2e_test_runs.id AND s.spec_type = $%d)", idx) + args = append(args, specType) + idx++ + } if from != nil { where += fmt.Sprintf(" AND run_timestamp >= $%d", idx) args = append(args, *from) @@ -311,7 +316,7 @@ func (r *E2ETestRunRepository) ListByProject(ctx context.Context, projectID stri return runs, total, nil } -func (r *E2ETestRunRepository) HeatmapData(ctx context.Context, branch string, status string, runsPerProject int) ([]application.TestHeatmapRow, error) { +func (r *E2ETestRunRepository) HeatmapData(ctx context.Context, branch string, status string, specType string, runsPerProject int) ([]application.TestHeatmapRow, error) { q := getQuerier(ctx, r.pool) where := "WHERE 1=1" @@ -328,6 +333,11 @@ func (r *E2ETestRunRepository) HeatmapData(ctx context.Context, branch string, s args = append(args, status) idx++ } + if specType != "" { + where += fmt.Sprintf(" AND EXISTS (SELECT 1 FROM e2e_test_spec_results s WHERE s.e2e_run_id = itr.id AND s.spec_type = $%d)", idx) + args = append(args, specType) + idx++ + } args = append(args, runsPerProject) diff --git a/internal/application/e2e_usecase.go b/internal/application/e2e_usecase.go index 511de61..11476da 100644 --- a/internal/application/e2e_usecase.go +++ b/internal/application/e2e_usecase.go @@ -41,6 +41,7 @@ type IngestSpecReport struct { LeafNodeText string `json:"leafNodeText"` ContainerHierarchyTexts []string `json:"containerHierarchyTexts"` State string `json:"state"` + SpecType string `json:"specType,omitempty"` RunTime float64 `json:"runTime"` Failure *IngestTestFailure `json:"failure,omitempty"` } @@ -118,6 +119,13 @@ func NewIngestE2ERunUseCase( } } +var validSpecTypes = map[string]bool{ + "": true, + "happyPath": true, + "negativePath": true, + "setup": true, +} + func (uc *IngestE2ERunUseCase) Execute(ctx context.Context, in IngestE2ERunInput) (IngestE2ERunOutput, error) { if err := validateE2EIngestInput(in); err != nil { return IngestE2ERunOutput{}, err @@ -292,6 +300,7 @@ func (uc *IngestE2ERunUseCase) buildE2EEntities(projectID string, in IngestE2ERu SpecPath: specPath, LeafNodeText: spec.LeafNodeText, State: normalizedState, + SpecType: spec.SpecType, DurationMS: durationMS, FailureMessage: failureMessage, FailureLocationFile: failureFile, @@ -375,6 +384,11 @@ func validateE2EIngestInput(in IngestE2ERunInput) error { if normalizeTestState(spec.State) == domain.E2ESpecStateFailed && (spec.Failure == nil || strings.TrimSpace(spec.Failure.Message) == "") { return NewInvalidArgument("failure.message is required when state is failed", map[string]any{"field": fmt.Sprintf("testReport.specReports[%d].failure.message", i)}) } + + specType := strings.TrimSpace(spec.SpecType) + if !validSpecTypes[specType] { + return NewInvalidArgument("specType must be happyPath, negativePath, or setup", map[string]any{"field": fmt.Sprintf("testReport.specReports[%d].specType", i)}) + } } return nil @@ -446,7 +460,7 @@ func failedE2ESpecsFromResults(specs []domain.E2ESpecResult) []FailedSpecRespons if spec.State != domain.E2ESpecStateFailed && spec.State != domain.E2ESpecStateFlaky { continue } - failed := FailedSpecResponse{SpecPath: spec.SpecPath} + failed := FailedSpecResponse{SpecPath: spec.SpecPath, SpecType: string(spec.SpecType)} if spec.FailureMessage != nil { failed.FailureMessage = *spec.FailureMessage } @@ -467,6 +481,7 @@ type ListE2ERunsInput struct { Branch string Status string Environment string + SpecType string From *time.Time To *time.Time Page int @@ -518,8 +533,13 @@ func (uc *ListE2ERunsUseCase) Execute(ctx context.Context, in ListE2ERunsInput) if environment != "" && environment != "test" && environment != "stage" && environment != "prod" { return ListE2ERunsOutput{}, NewInvalidArgument("environment must be one of: test, stage, prod", map[string]any{"field": "environment"}) } + specType := strings.TrimSpace(in.SpecType) - runs, total, err := uc.runs.ListByProject(ctx, in.ProjectID, in.Branch, status, environment, in.From, in.To, page, pageSize) + if !validSpecTypes[specType] { + return ListE2ERunsOutput{}, NewInvalidArgument("specType must be happyPath, negativePath, or setup", map[string]any{"field": "specType"}) + } + + runs, total, err := uc.runs.ListByProject(ctx, in.ProjectID, in.Branch, status, environment, specType, in.From, in.To, page, pageSize) if err != nil { return ListE2ERunsOutput{}, NewInternal("failed to list E2E runs", err) } @@ -660,6 +680,7 @@ func (uc *GetE2ERunUseCase) Execute(ctx context.Context, projectID string, runID type E2EHeatmapInput struct { Branch string Status string + SpecType string RunsPerProject int } @@ -689,7 +710,12 @@ func (uc *GetE2EHeatmapUseCase) Execute(ctx context.Context, in E2EHeatmapInput) return GetE2EHeatmapOutput{}, NewInvalidArgument("status must be passed or failed", map[string]any{"field": "status"}) } - rows, err := uc.runs.HeatmapData(ctx, in.Branch, status, runsPerProject) + specType := strings.TrimSpace(in.SpecType) + if !validSpecTypes[specType] { + return GetE2EHeatmapOutput{}, NewInvalidArgument("specType must be one of: happyPath, negativePath, setup (or empty)", map[string]any{"field": "specType"}) + } + + rows, err := uc.runs.HeatmapData(ctx, in.Branch, status, specType, runsPerProject) if err != nil { return GetE2EHeatmapOutput{}, NewInternal("failed to load heatmap data", err) } diff --git a/internal/application/e2e_usecase_test.go b/internal/application/e2e_usecase_test.go index a43a29b..17ea771 100644 --- a/internal/application/e2e_usecase_test.go +++ b/internal/application/e2e_usecase_test.go @@ -81,6 +81,7 @@ func TestIngestE2ERunUseCaseExecute(t *testing.T) { ContainerHierarchyTexts: []string{"Auth Failure"}, State: "Passed", RunTime: 2.00, + SpecType: "happyPath", Failure: &IngestTestFailure{ Message: "Auth Failure", Location: &IngestTestLocation{ @@ -491,6 +492,26 @@ func TestIngestE2ERunBuildE2EEntities(t *testing.T) { t.Fatalf("expected spec run ID %s, got %s", run.ID, specs[0].E2ETestRunID) } }) + + t.Run("Build entities with explicit specType", func(t *testing.T) { + uc := NewIngestE2ERunUseCase(nil, nil, nil, nil, &stubIDGenerator{}, &stubClock{}) + input := runInput + input.TestReport.SpecReports = []IngestSpecReport{ + { + LeafNodeText: "Error handling", + State: "Passed", + SpecType: "negativePath", + RunTime: 1.0, + }, + } + _, specs := uc.buildE2EEntities("project-id", input, time.Date(2026, 4, 1, 12, 0, 0, 0, time.UTC)) + if len(specs) != 1 { + t.Fatalf("expected 1 spec, got %d", len(specs)) + } + if specs[0].SpecType != "negativePath" { + t.Fatalf("expected spec type negativePath, got %s", specs[0].SpecType) + } + }) } func TestValidateE2EIngestInput(t *testing.T) { @@ -578,6 +599,42 @@ func TestValidateE2EIngestInput(t *testing.T) { wantErr: true, wantField: "testReport.specReports[0].failure.message", }, + { + name: "invalid specType returns error", + mutate: func(in *IngestE2ERunInput) { + in.TestReport.SpecReports[0].SpecType = "invalid" + }, + wantErr: true, + wantField: "testReport.specReports[0].specType", + }, + { + name: "happyPath specType passes validation", + mutate: func(in *IngestE2ERunInput) { + in.TestReport.SpecReports[0].SpecType = "happyPath" + }, + wantErr: false, + }, + { + name: "negativePath specType passes validation", + mutate: func(in *IngestE2ERunInput) { + in.TestReport.SpecReports[0].SpecType = "negativePath" + }, + wantErr: false, + }, + { + name: "empty specType passes validation", + mutate: func(in *IngestE2ERunInput) { + in.TestReport.SpecReports[0].SpecType = "" + }, + wantErr: false, + }, + { + name: "setup specType passes validation", + mutate: func(in *IngestE2ERunInput) { + in.TestReport.SpecReports[0].SpecType = "setup" + }, + wantErr: false, + }, } for _, tt := range test { t.Run(tt.name, func(t *testing.T) { @@ -691,6 +748,7 @@ func TestListE2ERunsExecute(t *testing.T) { Branch: "main", Status: "passed", Environment: "test", + SpecType: "happyPath", From: &from, To: &to, Page: 1, @@ -760,6 +818,7 @@ func TestListE2ERunsExecute(t *testing.T) { _, err := uc.Execute(context.Background(), ListE2ERunsInput{ ProjectID: "proj1", Status: "invalid", + SpecType: "happyPath", }) if err == nil { t.Fatalf("expected error, got nil") @@ -798,6 +857,28 @@ func TestListE2ERunsExecute(t *testing.T) { t.Fatalf("expected field environment, got %s", field) } }) + t.Run("Returns an error when specType is invalid", func(t *testing.T) { + runRepo := &stubE2ETestRunRepository{} + uc := NewListE2ERunsUseCase(runRepo) + _, err := uc.Execute(context.Background(), ListE2ERunsInput{ + ProjectID: "proj1", + SpecType: "invalid", + }) + if err == nil { + t.Fatalf("expected error, got nil") + } + var appErr *AppError + if !errors.As(err, &appErr) { + t.Fatalf("expected AppError, got %T", err) + } + if appErr.Code != CodeInvalidArgument { + t.Fatalf("expected code to be INVALID_ARGUMENT, got %s", appErr.Code) + } + field, _ := appErr.Details["field"].(string) + if field != "specType" { + t.Fatalf("expected field specType, got %s", field) + } + }) t.Run("Returns an error when fails to list the runs", func(t *testing.T) { runRepo := &stubE2ETestRunRepository{listErr: fmt.Errorf("db error")} uc := NewListE2ERunsUseCase(runRepo) @@ -1076,6 +1157,7 @@ func TestGetE2EHeatmapExecute(t *testing.T) { out, err := uc.Execute(context.Background(), E2EHeatmapInput{ Branch: "main", Status: "failed", + SpecType: "happyPath", RunsPerProject: 10, }) if err != nil { diff --git a/internal/application/integration_usecase.go b/internal/application/integration_usecase.go index 61990db..2f1e4aa 100644 --- a/internal/application/integration_usecase.go +++ b/internal/application/integration_usecase.go @@ -83,6 +83,7 @@ type IntegrationComparisonResponse struct { type FailedSpecResponse struct { SpecPath string `json:"specPath"` + SpecType string `json:"specType,omitempty"` FailureMessage string `json:"failureMessage"` File string `json:"file,omitempty"` Line int `json:"line,omitempty"` diff --git a/internal/application/mock_application.go b/internal/application/mock_application.go index 0c28523..091abb4 100644 --- a/internal/application/mock_application.go +++ b/internal/application/mock_application.go @@ -73,7 +73,7 @@ func (s *stubE2ETestRunRepository) GetByID(ctx context.Context, projectID string return *s.byID, nil } -func (s *stubE2ETestRunRepository) ListByProject(ctx context.Context, projectID string, branch string, status string, environment string, from *time.Time, to *time.Time, page int, pageSize int) ([]domain.E2ETestRun, int, error) { +func (s *stubE2ETestRunRepository) ListByProject(ctx context.Context, projectID string, branch string, status string, environment string, specType string, from *time.Time, to *time.Time, page int, pageSize int) ([]domain.E2ETestRun, int, error) { s.capturedBranch = branch s.capturedStatus = status if s.listErr != nil { @@ -82,7 +82,7 @@ func (s *stubE2ETestRunRepository) ListByProject(ctx context.Context, projectID return s.listed, s.listTotal, nil } -func (s *stubE2ETestRunRepository) HeatmapData(ctx context.Context, branch string, status string, runsPerProject int) ([]TestHeatmapRow, error) { +func (s *stubE2ETestRunRepository) HeatmapData(ctx context.Context, branch string, status string, specType string, runsPerProject int) ([]TestHeatmapRow, error) { s.capturedBranch = branch s.capturedStatus = status if s.heatmapErr != nil { @@ -124,4 +124,4 @@ func (s *stubE2ESpecResultRepository) ListFailedByRunID(ctx context.Context, run return nil, s.failedByRunIDErr } return s.failedByRunID, nil -} \ No newline at end of file +} diff --git a/internal/application/ports.go b/internal/application/ports.go index 1fca483..6cda445 100644 --- a/internal/application/ports.go +++ b/internal/application/ports.go @@ -64,8 +64,8 @@ type E2ETestRunRepository interface { GetLatestByProjectAndBranch(ctx context.Context, projectID string, branch string) (domain.E2ETestRun, error) GetLatestByProject(ctx context.Context, projectID string) (domain.E2ETestRun, error) GetByID(ctx context.Context, projectID string, runID string) (domain.E2ETestRun, error) - ListByProject(ctx context.Context, projectID string, branch string, status string, environment string, from *time.Time, to *time.Time, page int, pageSize int) ([]domain.E2ETestRun, int, error) - HeatmapData(ctx context.Context, branch string, status string, runsPerProject int) ([]TestHeatmapRow, error) + ListByProject(ctx context.Context, projectID string, branch string, status string, environment string, specType string, from *time.Time, to *time.Time, page int, pageSize int) ([]domain.E2ETestRun, int, error) + HeatmapData(ctx context.Context, branch string, status string, specType string, runsPerProject int) ([]TestHeatmapRow, error) } type E2ESpecResultRepository interface { diff --git a/internal/domain/e2e.go b/internal/domain/e2e.go index 1e9bc55..4bad7f6 100644 --- a/internal/domain/e2e.go +++ b/internal/domain/e2e.go @@ -52,6 +52,7 @@ type E2ESpecResult struct { SpecPath string LeafNodeText string State E2ESpecState + SpecType string DurationMS int64 FailureMessage *string FailureLocationFile *string diff --git a/migrations/004_add_spec_type.sql b/migrations/004_add_spec_type.sql new file mode 100644 index 0000000..c83cfd3 --- /dev/null +++ b/migrations/004_add_spec_type.sql @@ -0,0 +1,14 @@ +-- +goose Up + +ALTER TABLE e2e_test_spec_results + ADD COLUMN spec_type TEXT NOT NULL DEFAULT '' CHECK (spec_type IN ('setup', 'happyPath', 'negativePath', '')) ; + +CREATE INDEX IF NOT EXISTS e2e_test_spec_results_spec_type_idx + ON e2e_test_spec_results(e2e_run_id, spec_type) ; + +-- +goose Down + +DROP INDEX IF EXISTS e2e_test_spec_results_spec_type_idx; + +ALTER TABLE e2e_test_spec_results + DROP COLUMN IF EXISTS spec_type; \ No newline at end of file