agentic-data-stack/.github/workflows/e2e.yml at main · ClickHouse/agentic-data-stack · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
name: E2E tests

# Heavy, browser-level suite. It runs DAILY + on manual dispatch only — never on
# PRs. The smoke test (smoke-test.yml) stays the PR-time launch gate; this suite
# catches `:latest` image drift and functional regressions on a schedule.
on:
  schedule:
    - cron: '37 4 * * *'   # daily ~04:37 UTC (offset from the smoke test's 13:00)
  workflow_dispatch:

# Avoid piling up runs on the same ref.
concurrency:
  group: e2e-${{ github.ref }}
  cancel-in-progress: true

jobs:
  e2e:
    runs-on: ubuntu-latest
    timeout-minutes: 40
    steps:
      - name: Checkout
        uses: actions/checkout@v5

      - name: Generate .env
        run: bash scripts/generate-env.sh

      # --wait blocks until every healthchecked service is healthy (mock-llm,
      # ClickHouse MCP, LibreChat, Langfuse, ...) and fails if any is unhealthy.
      - name: Launch stack (prod + e2e override) and wait for health
        run: docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --wait --wait-timeout 600

      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: npm
          cache-dependency-path: e2e/package-lock.json

      - name: Install Playwright and browser
        working-directory: e2e
        run: |
          npm ci
          npx playwright install --with-deps chromium

      - name: Run E2E tests
        working-directory: e2e
        run: npx playwright test

      - name: Upload Playwright report
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: playwright-report
          path: e2e/playwright-report
          retention-days: 14

      - name: Dump diagnostics on failure
        if: failure()
        run: |
          docker compose -f docker-compose.yml -f docker-compose.e2e.yml ps -a
          docker compose -f docker-compose.yml -f docker-compose.e2e.yml logs --no-color --tail=200

      - name: Tear down
        if: always()
        run: docker compose -f docker-compose.yml -f docker-compose.e2e.yml down -v

  # On a failed DAILY run, open (or comment on) a de-duped tracking issue. Manual
  # (workflow_dispatch) failures surface directly to the person who triggered
  # them, so they don't need an issue. Mirrors smoke-test.yml's report job.
  report-daily-failure:
    needs: e2e
    if: contains(fromJSON('["failure", "cancelled"]'), needs.e2e.result) && github.event_name == 'schedule'
    runs-on: ubuntu-latest
    permissions:
      issues: write
    steps:
      - name: Open or update tracking issue
        uses: actions/github-script@v7
        with:
          script: |
            const label = 'e2e-test-failure';
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const body = `The daily E2E test suite failed.\n\nRun: ${runUrl}`;

            // Ensure the tracking label exists so the de-dupe search is reliable
            // in a repo that has never had it.
            try {
              await github.rest.issues.getLabel({
                owner: context.repo.owner,
                repo: context.repo.repo,
                name: label,
              });
            } catch (e) {
              if (e.status !== 404) throw e;
              await github.rest.issues.createLabel({
                owner: context.repo.owner,
                repo: context.repo.repo,
                name: label,
                color: 'd73a4a',
              });
            }

            const existing = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              labels: label,
            });

            // listForRepo returns PRs as well as issues; exclude PRs so we never
            // comment on a mislabeled pull request.
            const openIssue = existing.data.find(i => !i.pull_request);

            if (openIssue) {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: openIssue.number,
                body,
              });
            } else {
              await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title: 'Daily E2E test suite failed',
                body,
                labels: [label],
              });
            }