From ce5fd5e7e69264438a91ecebd1497fa91e9660d8 Mon Sep 17 00:00:00 2001 From: mouliangyu <21963576+mouliangyu@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:36:53 +0800 Subject: [PATCH 1/4] ci: warn on slow ci-sim runs --- .github/scripts/ci_sim_duration_warning.py | 269 ++++++++++++++++++++ .github/workflows/ci.yml | 16 ++ .github/workflows/watchdog.yml | 48 +++- test/python/test_ci_sim_duration_warning.py | 152 +++++++++++ 4 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/ci_sim_duration_warning.py create mode 100644 test/python/test_ci_sim_duration_warning.py diff --git a/.github/scripts/ci_sim_duration_warning.py b/.github/scripts/ci_sim_duration_warning.py new file mode 100644 index 0000000000..5159683a50 --- /dev/null +++ b/.github/scripts/ci_sim_duration_warning.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +"""Post a non-blocking PR warning when ci-sim exceeds its runtime budget.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime +import json +import os +from pathlib import Path +import re +import sys +from typing import Any, Dict, List, Optional, Tuple, Union +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + + +COMMENT_MARKER = "" +RESOLVED_PREFIX = "Resolved:" +BOT_LOGIN = "github-actions[bot]" +JOB_NAME = "vpto-sim-validation" +SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") + + +@dataclass(frozen=True) +class RunConfig: + repo: str + run_id: int + run_head_sha: str + run_url: str + soft_timeout_minutes: int + context_dir: Path + dry_run: bool = False + + +class GitHubClient: + def __init__(self, token: str, api_url: str = "https://api.github.com") -> None: + self._token = token + self._api_url = api_url.rstrip("/") + + def request( + self, + method: str, + path: str, + payload: Optional[Dict[str, Any]] = None, + query: Optional[Dict[str, Union[str, int]]] = None, + ) -> Tuple[Any, Dict[str, str]]: + url = f"{self._api_url}/{path.lstrip('/')}" + if query: + url = f"{url}?{urlencode(query)}" + data = json.dumps(payload).encode() if payload is not None else None + request = Request( + url, + data=data, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self._token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urlopen(request, timeout=30) as response: + body = response.read() + return (json.loads(body) if body else None), dict(response.headers) + except HTTPError as error: + detail = error.read().decode(errors="replace") + raise RuntimeError(f"GitHub API {method} {path} failed: {error.code}: {detail}") from error + + def get(self, path: str, query: Optional[Dict[str, Union[str, int]]] = None) -> Any: + return self.request("GET", path, query=query)[0] + + def post(self, path: str, payload: Dict[str, Any]) -> Any: + return self.request("POST", path, payload=payload)[0] + + def patch(self, path: str, payload: Dict[str, Any]) -> Any: + return self.request("PATCH", path, payload=payload)[0] + + def list_all(self, path: str) -> List[Any]: + items: List[Any] = [] + page = 1 + while True: + batch = self.get(path, query={"per_page": 100, "page": page}) + if not isinstance(batch, list): + raise RuntimeError(f"Expected a list from GitHub API path {path}") + items.extend(batch) + if len(batch) < 100: + return items + page += 1 + + +def _read_context(context_dir: Path) -> Optional[Tuple[int, str]]: + number_path = context_dir / "pr-number" + sha_path = context_dir / "pr-head-sha" + if not number_path.is_file() or not sha_path.is_file(): + return None + number_text = number_path.read_text(encoding="utf-8").strip() + head_sha = sha_path.read_text(encoding="utf-8").strip() + if not number_text.isdigit() or int(number_text) <= 0 or not SHA_RE.fullmatch(head_sha): + raise ValueError(f"Invalid PR context in {context_dir}") + return int(number_text), head_sha + + +def _parse_timestamp(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _format_duration(seconds: int) -> str: + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + if hours: + return f"{hours}h {minutes}m {seconds}s" + return f"{minutes}m {seconds}s" + + +def _format_budget(minutes: int) -> str: + hours, minutes = divmod(minutes, 60) + if hours and minutes: + return f"{hours}h {minutes}m" + if hours: + return f"{hours}h" + return f"{minutes}m" + + +def _warning_body(author: str, elapsed: str, budget: str, conclusion: str, run_url: str) -> str: + return "\n".join( + [ + COMMENT_MARKER, + f"Warning: @{author}, ci-sim exceeded its soft runtime budget.", + "", + f"- `{JOB_NAME}` runtime: **{elapsed}**", + f"- Soft budget: **{budget}**", + f"- Job conclusion: **{conclusion}**", + f"- [Workflow run]({run_url})", + "", + "This warning is advisory only and does not affect required checks. " + "Please inspect the step timings for an unexpected regression.", + ] + ) + + +def _resolved_body(elapsed: str, budget: str, run_url: str) -> str: + return "\n".join( + [ + COMMENT_MARKER, + "Resolved: ci-sim runtime is back within its soft budget.", + "", + f"- Latest `{JOB_NAME}` runtime: **{elapsed}**", + f"- Soft budget: **{budget}**", + f"- [Workflow run]({run_url})", + "", + "The previous duration warning is resolved. This status is advisory only.", + ] + ) + + +def observe_run(client: GitHubClient, config: RunConfig) -> str: + context = _read_context(config.context_dir) + if context is None: + return f"No PR context found for workflow run {config.run_id}; skipping duration warning." + pr_number, context_head_sha = context + + pull = client.get(f"repos/{config.repo}/pulls/{pr_number}") + if pull["state"] != "open": + return f"PR #{pr_number} is {pull['state']}; skipping duration warning." + current_head_sha = pull["head"]["sha"] + if context_head_sha != config.run_head_sha or current_head_sha != config.run_head_sha: + return f"Workflow run {config.run_id} is stale for PR #{pr_number}; skipping duration warning." + + jobs_response = client.get( + f"repos/{config.repo}/actions/runs/{config.run_id}/jobs", + query={"filter": "latest", "per_page": 100}, + ) + jobs = [ + job + for job in jobs_response.get("jobs", []) + if job.get("name") == JOB_NAME and job.get("started_at") and job.get("completed_at") + ] + if not jobs: + return f"No completed {JOB_NAME} job found in workflow run {config.run_id}." + job = jobs[0] + elapsed_seconds = int((_parse_timestamp(job["completed_at"]) - _parse_timestamp(job["started_at"])).total_seconds()) + if elapsed_seconds < 0: + raise ValueError(f"Invalid job timestamps: {job['started_at']} to {job['completed_at']}") + + elapsed = _format_duration(elapsed_seconds) + budget = _format_budget(config.soft_timeout_minutes) + comments = client.list_all(f"repos/{config.repo}/issues/{pr_number}/comments") + existing = next( + ( + comment + for comment in reversed(comments) + if comment.get("user", {}).get("login") == BOT_LOGIN + and COMMENT_MARKER in comment.get("body", "") + ), + None, + ) + + if elapsed_seconds > config.soft_timeout_minutes * 60: + body = _warning_body(pull["user"]["login"], elapsed, budget, job.get("conclusion", "unknown"), config.run_url) + if config.dry_run: + return f"Dry run: would {'update' if existing else 'create'} slow CI warning on PR #{pr_number}.\n{body}" + if existing: + client.patch(f"repos/{config.repo}/issues/comments/{existing['id']}", {"body": body}) + return f"Updated slow CI warning on PR #{pr_number}." + client.post(f"repos/{config.repo}/issues/{pr_number}/comments", {"body": body}) + return f"Created slow CI warning on PR #{pr_number}." + + if existing: + if RESOLVED_PREFIX in existing.get("body", ""): + return f"CI runtime {elapsed} remains within the {budget} budget." + body = _resolved_body(elapsed, budget, config.run_url) + if config.dry_run: + return f"Dry run: would resolve slow CI warning on PR #{pr_number}.\n{body}" + client.patch(f"repos/{config.repo}/issues/comments/{existing['id']}", {"body": body}) + return f"Resolved slow CI warning on PR #{pr_number}." + return f"CI runtime {elapsed} is within the {budget} budget." + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("value must be a positive integer") + return parsed + + +def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True) + parser.add_argument("--run-id", required=True, type=_positive_int) + parser.add_argument("--run-head-sha", required=True) + parser.add_argument("--run-url", required=True) + parser.add_argument("--soft-timeout-minutes", type=_positive_int, default=90) + parser.add_argument("--pr-context-dir", type=Path, default=Path("pr-context")) + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> int: + args = _parse_args(argv) + if not SHA_RE.fullmatch(args.run_head_sha): + raise ValueError(f"Invalid run head SHA: {args.run_head_sha}") + token = os.environ.get("GH_TOKEN") + if not token: + raise RuntimeError("GH_TOKEN is required") + config = RunConfig( + repo=args.repo, + run_id=args.run_id, + run_head_sha=args.run_head_sha, + run_url=args.run_url, + soft_timeout_minutes=args.soft_timeout_minutes, + context_dir=args.pr_context_dir, + dry_run=args.dry_run, + ) + print(observe_run(GitHubClient(token), config)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad780e9c0a..31d76dd9bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,22 @@ permissions: pull-requests: read jobs: + watchdog-script-test: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Test ci-sim duration warning + run: >- + python3 -m unittest discover + -s test/python + -p 'test_ci_sim_duration_warning.py' + -v + license-header-check: if: ${{ github.event_name == 'pull_request' }} runs-on: ubuntu-22.04 diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml index b6e51d9bbb..8f2bdb4249 100644 --- a/.github/workflows/watchdog.yml +++ b/.github/workflows/watchdog.yml @@ -14,11 +14,53 @@ on: types: [completed] permissions: - actions: write + actions: read contents: read pull-requests: read jobs: + warn-slow-ci-sim: + if: ${{ github.event.workflow_run.event == 'pull_request' }} + runs-on: ubuntu-22.04 + continue-on-error: true + permissions: + actions: read + contents: read + issues: write + pull-requests: read + env: + SOFT_TIMEOUT_MINUTES: ${{ vars.CI_SIM_SOFT_TIMEOUT_MINUTES || '90' }} + steps: + # workflow_run executes this workflow from the default branch, so this + # checkout never runs a script supplied by the pull request. + - name: Checkout trusted watchdog scripts + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Download PR context + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: pr-context + path: pr-context + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Update slow CI warning + env: + GH_TOKEN: ${{ github.token }} + run: >- + python3 .github/scripts/ci_sim_duration_warning.py + --repo "${{ github.repository }}" + --run-id "${{ github.event.workflow_run.id }}" + --run-head-sha "${{ github.event.workflow_run.head_sha }}" + --run-url "${{ github.event.workflow_run.html_url }}" + --soft-timeout-minutes "${SOFT_TIMEOUT_MINUTES}" + --pr-context-dir pr-context + rerun-self-hosted-git-failure: if: >- ${{ @@ -26,6 +68,10 @@ jobs: github.event.workflow_run.run_attempt <= 8 }} runs-on: ubuntu-22.04 + permissions: + actions: write + contents: read + pull-requests: read steps: - name: Download PR context uses: actions/download-artifact@v4 diff --git a/test/python/test_ci_sim_duration_warning.py b/test/python/test_ci_sim_duration_warning.py new file mode 100644 index 0000000000..c92806546e --- /dev/null +++ b/test/python/test_ci_sim_duration_warning.py @@ -0,0 +1,152 @@ +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys +import tempfile +import unittest + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = _REPO_ROOT / ".github" / "scripts" / "ci_sim_duration_warning.py" +_SPEC = importlib.util.spec_from_file_location("ci_sim_duration_warning", _SCRIPT) +assert _SPEC is not None and _SPEC.loader is not None +warning = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = warning +_SPEC.loader.exec_module(warning) + + +class FakeGitHubClient: + def __init__(self, *, elapsed_seconds=3600, pr_state="open", current_sha="a" * 40, comments=None): + self.elapsed_seconds = elapsed_seconds + self.pr_state = pr_state + self.current_sha = current_sha + self.comments = list(comments or []) + self.writes = [] + + def get(self, path, query=None): + if "/pulls/" in path: + return { + "state": self.pr_state, + "head": {"sha": self.current_sha}, + "user": {"login": "test-author"}, + } + if path.endswith("/jobs"): + return { + "jobs": [ + { + "name": warning.JOB_NAME, + "started_at": "2026-07-28T10:00:00Z", + "completed_at": _timestamp_after(self.elapsed_seconds), + "conclusion": "success", + } + ] + } + raise AssertionError(f"Unexpected GET {path} {query}") + + def list_all(self, path): + self.comment_path = path + return self.comments + + def post(self, path, payload): + self.writes.append(("POST", path, payload)) + + def patch(self, path, payload): + self.writes.append(("PATCH", path, payload)) + + +def _timestamp_after(seconds): + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + return f"2026-07-28T{10 + hours:02d}:{minutes:02d}:{seconds:02d}Z" + + +class DurationWarningTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + context_dir = Path(self.temp_dir.name) + (context_dir / "pr-number").write_text("42\n", encoding="utf-8") + (context_dir / "pr-head-sha").write_text(f"{'a' * 40}\n", encoding="utf-8") + self.config = warning.RunConfig( + repo="test/repo", + run_id=99, + run_head_sha="a" * 40, + run_url="https://example.invalid/run/99", + soft_timeout_minutes=90, + context_dir=context_dir, + ) + + def tearDown(self): + self.temp_dir.cleanup() + + def test_within_budget_does_not_comment(self): + client = FakeGitHubClient(elapsed_seconds=90 * 60) + result = warning.observe_run(client, self.config) + self.assertIn("within the 1h 30m budget", result) + self.assertEqual(client.writes, []) + + def test_over_budget_creates_warning(self): + client = FakeGitHubClient(elapsed_seconds=2 * 60 * 60 + 1) + result = warning.observe_run(client, self.config) + self.assertEqual(result, "Created slow CI warning on PR #42.") + method, path, payload = client.writes[0] + self.assertEqual((method, path), ("POST", "repos/test/repo/issues/42/comments")) + self.assertIn("@test-author", payload["body"]) + self.assertIn("2h 0m 1s", payload["body"]) + self.assertIn(warning.COMMENT_MARKER, payload["body"]) + + def test_over_budget_updates_existing_bot_comment(self): + comments = [ + {"id": 7, "user": {"login": warning.BOT_LOGIN}, "body": warning.COMMENT_MARKER}, + ] + client = FakeGitHubClient(elapsed_seconds=2 * 60 * 60, comments=comments) + result = warning.observe_run(client, self.config) + self.assertEqual(result, "Updated slow CI warning on PR #42.") + self.assertEqual(client.writes[0][0:2], ("PATCH", "repos/test/repo/issues/comments/7")) + + def test_recovery_resolves_existing_warning(self): + comments = [ + {"id": 8, "user": {"login": warning.BOT_LOGIN}, "body": warning.COMMENT_MARKER}, + ] + client = FakeGitHubClient(elapsed_seconds=60 * 60, comments=comments) + result = warning.observe_run(client, self.config) + self.assertEqual(result, "Resolved slow CI warning on PR #42.") + self.assertIn("Resolved:", client.writes[0][2]["body"]) + + def test_resolved_comment_is_not_updated_again(self): + comments = [ + { + "id": 8, + "user": {"login": warning.BOT_LOGIN}, + "body": f"{warning.COMMENT_MARKER}\n{warning.RESOLVED_PREFIX}", + }, + ] + client = FakeGitHubClient(elapsed_seconds=60 * 60, comments=comments) + result = warning.observe_run(client, self.config) + self.assertIn("remains within", result) + self.assertEqual(client.writes, []) + + def test_stale_run_skips_before_reading_jobs(self): + client = FakeGitHubClient(current_sha="b" * 40, elapsed_seconds=2 * 60 * 60) + result = warning.observe_run(client, self.config) + self.assertIn("is stale", result) + self.assertEqual(client.writes, []) + + def test_dry_run_does_not_write(self): + client = FakeGitHubClient(elapsed_seconds=2 * 60 * 60) + config = warning.RunConfig(**{**self.config.__dict__, "dry_run": True}) + result = warning.observe_run(client, config) + self.assertIn("Dry run: would create", result) + self.assertEqual(client.writes, []) + + +if __name__ == "__main__": + unittest.main() From fa19652a03921c168063762fa5176976aec0f34b Mon Sep 17 00:00:00 2001 From: mouliangyu <21963576+mouliangyu@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:07:41 +0800 Subject: [PATCH 2/4] ci: drop dedicated watchdog tests --- .github/workflows/ci.yml | 16 --- test/python/test_ci_sim_duration_warning.py | 152 -------------------- 2 files changed, 168 deletions(-) delete mode 100644 test/python/test_ci_sim_duration_warning.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31d76dd9bd..ad780e9c0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,22 +69,6 @@ permissions: pull-requests: read jobs: - watchdog-script-test: - if: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Test ci-sim duration warning - run: >- - python3 -m unittest discover - -s test/python - -p 'test_ci_sim_duration_warning.py' - -v - license-header-check: if: ${{ github.event_name == 'pull_request' }} runs-on: ubuntu-22.04 diff --git a/test/python/test_ci_sim_duration_warning.py b/test/python/test_ci_sim_duration_warning.py deleted file mode 100644 index c92806546e..0000000000 --- a/test/python/test_ci_sim_duration_warning.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from __future__ import annotations - -import importlib.util -from pathlib import Path -import sys -import tempfile -import unittest - - -_REPO_ROOT = Path(__file__).resolve().parents[2] -_SCRIPT = _REPO_ROOT / ".github" / "scripts" / "ci_sim_duration_warning.py" -_SPEC = importlib.util.spec_from_file_location("ci_sim_duration_warning", _SCRIPT) -assert _SPEC is not None and _SPEC.loader is not None -warning = importlib.util.module_from_spec(_SPEC) -sys.modules[_SPEC.name] = warning -_SPEC.loader.exec_module(warning) - - -class FakeGitHubClient: - def __init__(self, *, elapsed_seconds=3600, pr_state="open", current_sha="a" * 40, comments=None): - self.elapsed_seconds = elapsed_seconds - self.pr_state = pr_state - self.current_sha = current_sha - self.comments = list(comments or []) - self.writes = [] - - def get(self, path, query=None): - if "/pulls/" in path: - return { - "state": self.pr_state, - "head": {"sha": self.current_sha}, - "user": {"login": "test-author"}, - } - if path.endswith("/jobs"): - return { - "jobs": [ - { - "name": warning.JOB_NAME, - "started_at": "2026-07-28T10:00:00Z", - "completed_at": _timestamp_after(self.elapsed_seconds), - "conclusion": "success", - } - ] - } - raise AssertionError(f"Unexpected GET {path} {query}") - - def list_all(self, path): - self.comment_path = path - return self.comments - - def post(self, path, payload): - self.writes.append(("POST", path, payload)) - - def patch(self, path, payload): - self.writes.append(("PATCH", path, payload)) - - -def _timestamp_after(seconds): - hours, remainder = divmod(seconds, 3600) - minutes, seconds = divmod(remainder, 60) - return f"2026-07-28T{10 + hours:02d}:{minutes:02d}:{seconds:02d}Z" - - -class DurationWarningTest(unittest.TestCase): - def setUp(self): - self.temp_dir = tempfile.TemporaryDirectory() - context_dir = Path(self.temp_dir.name) - (context_dir / "pr-number").write_text("42\n", encoding="utf-8") - (context_dir / "pr-head-sha").write_text(f"{'a' * 40}\n", encoding="utf-8") - self.config = warning.RunConfig( - repo="test/repo", - run_id=99, - run_head_sha="a" * 40, - run_url="https://example.invalid/run/99", - soft_timeout_minutes=90, - context_dir=context_dir, - ) - - def tearDown(self): - self.temp_dir.cleanup() - - def test_within_budget_does_not_comment(self): - client = FakeGitHubClient(elapsed_seconds=90 * 60) - result = warning.observe_run(client, self.config) - self.assertIn("within the 1h 30m budget", result) - self.assertEqual(client.writes, []) - - def test_over_budget_creates_warning(self): - client = FakeGitHubClient(elapsed_seconds=2 * 60 * 60 + 1) - result = warning.observe_run(client, self.config) - self.assertEqual(result, "Created slow CI warning on PR #42.") - method, path, payload = client.writes[0] - self.assertEqual((method, path), ("POST", "repos/test/repo/issues/42/comments")) - self.assertIn("@test-author", payload["body"]) - self.assertIn("2h 0m 1s", payload["body"]) - self.assertIn(warning.COMMENT_MARKER, payload["body"]) - - def test_over_budget_updates_existing_bot_comment(self): - comments = [ - {"id": 7, "user": {"login": warning.BOT_LOGIN}, "body": warning.COMMENT_MARKER}, - ] - client = FakeGitHubClient(elapsed_seconds=2 * 60 * 60, comments=comments) - result = warning.observe_run(client, self.config) - self.assertEqual(result, "Updated slow CI warning on PR #42.") - self.assertEqual(client.writes[0][0:2], ("PATCH", "repos/test/repo/issues/comments/7")) - - def test_recovery_resolves_existing_warning(self): - comments = [ - {"id": 8, "user": {"login": warning.BOT_LOGIN}, "body": warning.COMMENT_MARKER}, - ] - client = FakeGitHubClient(elapsed_seconds=60 * 60, comments=comments) - result = warning.observe_run(client, self.config) - self.assertEqual(result, "Resolved slow CI warning on PR #42.") - self.assertIn("Resolved:", client.writes[0][2]["body"]) - - def test_resolved_comment_is_not_updated_again(self): - comments = [ - { - "id": 8, - "user": {"login": warning.BOT_LOGIN}, - "body": f"{warning.COMMENT_MARKER}\n{warning.RESOLVED_PREFIX}", - }, - ] - client = FakeGitHubClient(elapsed_seconds=60 * 60, comments=comments) - result = warning.observe_run(client, self.config) - self.assertIn("remains within", result) - self.assertEqual(client.writes, []) - - def test_stale_run_skips_before_reading_jobs(self): - client = FakeGitHubClient(current_sha="b" * 40, elapsed_seconds=2 * 60 * 60) - result = warning.observe_run(client, self.config) - self.assertIn("is stale", result) - self.assertEqual(client.writes, []) - - def test_dry_run_does_not_write(self): - client = FakeGitHubClient(elapsed_seconds=2 * 60 * 60) - config = warning.RunConfig(**{**self.config.__dict__, "dry_run": True}) - result = warning.observe_run(client, config) - self.assertIn("Dry run: would create", result) - self.assertEqual(client.writes, []) - - -if __name__ == "__main__": - unittest.main() From c272f0f33550270ed95fe34d89a7727efcca239b Mon Sep 17 00:00:00 2001 From: mouliangyu <21963576+mouliangyu@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:25:58 +0800 Subject: [PATCH 3/4] ci: label pull requests with slow sim runs --- .github/scripts/ci_sim_duration_warning.py | 53 +++++++++++++++++----- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/.github/scripts/ci_sim_duration_warning.py b/.github/scripts/ci_sim_duration_warning.py index 5159683a50..d4b88e1153 100644 --- a/.github/scripts/ci_sim_duration_warning.py +++ b/.github/scripts/ci_sim_duration_warning.py @@ -20,7 +20,7 @@ import sys from typing import Any, Dict, List, Optional, Tuple, Union from urllib.error import HTTPError -from urllib.parse import urlencode +from urllib.parse import quote, urlencode from urllib.request import Request, urlopen @@ -28,6 +28,7 @@ RESOLVED_PREFIX = "Resolved:" BOT_LOGIN = "github-actions[bot]" JOB_NAME = "vpto-sim-validation" +SLOW_LABEL = "ci-slow" SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") @@ -85,6 +86,9 @@ def post(self, path: str, payload: Dict[str, Any]) -> Any: def patch(self, path: str, payload: Dict[str, Any]) -> Any: return self.request("PATCH", path, payload=payload)[0] + def delete(self, path: str) -> Any: + return self.request("DELETE", path)[0] + def list_all(self, path: str) -> List[Any]: items: List[Any] = [] page = 1 @@ -175,6 +179,7 @@ def observe_run(client: GitHubClient, config: RunConfig) -> str: current_head_sha = pull["head"]["sha"] if context_head_sha != config.run_head_sha or current_head_sha != config.run_head_sha: return f"Workflow run {config.run_id} is stale for PR #{pr_number}; skipping duration warning." + has_slow_label = any(label.get("name") == SLOW_LABEL for label in pull.get("labels", [])) jobs_response = client.get( f"repos/{config.repo}/actions/runs/{config.run_id}/jobs", @@ -208,21 +213,45 @@ def observe_run(client: GitHubClient, config: RunConfig) -> str: if elapsed_seconds > config.soft_timeout_minutes * 60: body = _warning_body(pull["user"]["login"], elapsed, budget, job.get("conclusion", "unknown"), config.run_url) if config.dry_run: - return f"Dry run: would {'update' if existing else 'create'} slow CI warning on PR #{pr_number}.\n{body}" + label_action = "keep" if has_slow_label else "add" + return ( + f"Dry run: would {label_action} {SLOW_LABEL} and " + f"{'update' if existing else 'create'} slow CI warning on PR #{pr_number}.\n{body}" + ) + if not has_slow_label: + client.post(f"repos/{config.repo}/issues/{pr_number}/labels", {"labels": [SLOW_LABEL]}) if existing: client.patch(f"repos/{config.repo}/issues/comments/{existing['id']}", {"body": body}) - return f"Updated slow CI warning on PR #{pr_number}." - client.post(f"repos/{config.repo}/issues/{pr_number}/comments", {"body": body}) - return f"Created slow CI warning on PR #{pr_number}." - - if existing: - if RESOLVED_PREFIX in existing.get("body", ""): - return f"CI runtime {elapsed} remains within the {budget} budget." + comment_action = "Updated" + else: + client.post(f"repos/{config.repo}/issues/{pr_number}/comments", {"body": body}) + comment_action = "Created" + label_action = "Kept" if has_slow_label else "Applied" + return f"{label_action} {SLOW_LABEL}; {comment_action.lower()} slow CI warning on PR #{pr_number}." + + comment_needs_resolution = bool(existing and RESOLVED_PREFIX not in existing.get("body", "")) + if config.dry_run and (has_slow_label or comment_needs_resolution): + actions = [] + if has_slow_label: + actions.append(f"remove {SLOW_LABEL}") + if comment_needs_resolution: + actions.append("resolve slow CI warning") + body = _resolved_body(elapsed, budget, config.run_url) + return f"Dry run: would {' and '.join(actions)} on PR #{pr_number}.\n{body}" + + actions = [] + if has_slow_label: + encoded_label = quote(SLOW_LABEL, safe="") + client.delete(f"repos/{config.repo}/issues/{pr_number}/labels/{encoded_label}") + actions.append(f"Removed {SLOW_LABEL}") + if comment_needs_resolution: body = _resolved_body(elapsed, budget, config.run_url) - if config.dry_run: - return f"Dry run: would resolve slow CI warning on PR #{pr_number}.\n{body}" client.patch(f"repos/{config.repo}/issues/comments/{existing['id']}", {"body": body}) - return f"Resolved slow CI warning on PR #{pr_number}." + actions.append("resolved slow CI warning") + if actions: + return f"{'; '.join(actions)} on PR #{pr_number}." + if existing: + return f"CI runtime {elapsed} remains within the {budget} budget." return f"CI runtime {elapsed} is within the {budget} budget." From 1b02eac9ccc661fee7a5cbb580bd9c56760ef336 Mon Sep 17 00:00:00 2001 From: mouliangyu <21963576+mouliangyu@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:49:23 +0800 Subject: [PATCH 4/4] ci: use github-script for slow run warnings --- .github/scripts/ci_sim_duration_warning.js | 201 ++++++++++++++ .github/scripts/ci_sim_duration_warning.py | 298 --------------------- .github/workflows/watchdog.yml | 18 +- 3 files changed, 209 insertions(+), 308 deletions(-) create mode 100644 .github/scripts/ci_sim_duration_warning.js delete mode 100644 .github/scripts/ci_sim_duration_warning.py diff --git a/.github/scripts/ci_sim_duration_warning.js b/.github/scripts/ci_sim_duration_warning.js new file mode 100644 index 0000000000..fccc6b82b5 --- /dev/null +++ b/.github/scripts/ci_sim_duration_warning.js @@ -0,0 +1,201 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +const fs = require('fs'); +const path = require('path'); + +const COMMENT_MARKER = ''; +const RESOLVED_PREFIX = 'Resolved:'; +const BOT_LOGIN = 'github-actions[bot]'; +const JOB_NAME = 'vpto-sim-validation'; +const SLOW_LABEL = 'ci-slow'; +const SHA_RE = /^[0-9a-fA-F]{40}$/; + +function readPrContext(contextDir) { + const numberPath = path.join(contextDir, 'pr-number'); + const shaPath = path.join(contextDir, 'pr-head-sha'); + if (!fs.existsSync(numberPath) || !fs.existsSync(shaPath)) { + return null; + } + + const numberText = fs.readFileSync(numberPath, 'utf8').trim(); + const headSha = fs.readFileSync(shaPath, 'utf8').trim(); + const number = Number(numberText); + if (!/^\d+$/.test(numberText) || !Number.isSafeInteger(number) || number <= 0 || !SHA_RE.test(headSha)) { + throw new Error(`Invalid PR context in ${contextDir}`); + } + return {number, headSha}; +} + +function formatDuration(totalSeconds) { + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + return hours ? `${hours}h ${minutes}m ${seconds}s` : `${minutes}m ${seconds}s`; +} + +function formatBudget(totalMinutes) { + const hours = Math.floor(totalMinutes / 60); + const minutes = totalMinutes % 60; + if (hours && minutes) return `${hours}h ${minutes}m`; + if (hours) return `${hours}h`; + return `${minutes}m`; +} + +function warningBody(author, elapsed, budget, conclusion, runUrl) { + return [ + COMMENT_MARKER, + `Warning: @${author}, ci-sim exceeded its soft runtime budget.`, + '', + `- \`${JOB_NAME}\` runtime: **${elapsed}**`, + `- Soft budget: **${budget}**`, + `- Job conclusion: **${conclusion}**`, + `- [Workflow run](${runUrl})`, + '', + 'This warning is advisory only and does not affect required checks. ' + + 'Please inspect the step timings for an unexpected regression.', + ].join('\n'); +} + +function resolvedBody(elapsed, budget, runUrl) { + return [ + COMMENT_MARKER, + 'Resolved: ci-sim runtime is back within its soft budget.', + '', + `- Latest \`${JOB_NAME}\` runtime: **${elapsed}**`, + `- Soft budget: **${budget}**`, + `- [Workflow run](${runUrl})`, + '', + 'The previous duration warning is resolved. This status is advisory only.', + ].join('\n'); +} + +function positiveInteger(value, name) { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return parsed; +} + +module.exports = async function observeRun({github, context, config = {}}) { + const owner = config.owner || context.repo.owner; + const repo = config.repo || context.repo.repo; + const workflowRun = context.payload.workflow_run || {}; + const runId = positiveInteger(config.runId || workflowRun.id, 'run ID'); + const runHeadSha = config.runHeadSha || workflowRun.head_sha; + const runUrl = config.runUrl || workflowRun.html_url; + const softTimeoutMinutes = positiveInteger( + config.softTimeoutMinutes || process.env.SOFT_TIMEOUT_MINUTES || 90, + 'soft timeout', + ); + const contextDir = config.contextDir || process.env.PR_CONTEXT_DIR || 'pr-context'; + const dryRun = config.dryRun === true; + + if (!SHA_RE.test(runHeadSha || '')) { + throw new Error(`Invalid run head SHA: ${runHeadSha}`); + } + + const prContext = readPrContext(contextDir); + if (!prContext) { + return `No PR context found for workflow run ${runId}; skipping duration warning.`; + } + + const pullResponse = await github.rest.pulls.get({owner, repo, pull_number: prContext.number}); + const pull = pullResponse.data; + if (pull.state !== 'open') { + return `PR #${prContext.number} is ${pull.state}; skipping duration warning.`; + } + if (prContext.headSha !== runHeadSha || pull.head.sha !== runHeadSha) { + return `Workflow run ${runId} is stale for PR #${prContext.number}; skipping duration warning.`; + } + const hasSlowLabel = pull.labels.some(label => label.name === SLOW_LABEL); + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner, + repo, + run_id: runId, + filter: 'latest', + per_page: 100, + }); + const job = jobs.find(item => item.name === JOB_NAME && item.started_at && item.completed_at); + if (!job) { + return `No completed ${JOB_NAME} job found in workflow run ${runId}.`; + } + + const elapsedSeconds = Math.floor((Date.parse(job.completed_at) - Date.parse(job.started_at)) / 1000); + if (!Number.isFinite(elapsedSeconds) || elapsedSeconds < 0) { + throw new Error(`Invalid job timestamps: ${job.started_at} to ${job.completed_at}`); + } + + const elapsed = formatDuration(elapsedSeconds); + const budget = formatBudget(softTimeoutMinutes); + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number: prContext.number, + per_page: 100, + }); + const existing = comments + .slice() + .reverse() + .find(comment => comment.user?.login === BOT_LOGIN && comment.body?.includes(COMMENT_MARKER)); + + if (elapsedSeconds > softTimeoutMinutes * 60) { + const body = warningBody(pull.user.login, elapsed, budget, job.conclusion || 'unknown', runUrl); + if (dryRun) { + const labelAction = hasSlowLabel ? 'keep' : 'add'; + const commentAction = existing ? 'update' : 'create'; + return `Dry run: would ${labelAction} ${SLOW_LABEL} and ${commentAction} slow CI warning ` + + `on PR #${prContext.number}.\n${body}`; + } + + if (!hasSlowLabel) { + await github.rest.issues.addLabels({owner, repo, issue_number: prContext.number, labels: [SLOW_LABEL]}); + } + if (existing) { + await github.rest.issues.updateComment({owner, repo, comment_id: existing.id, body}); + } else { + await github.rest.issues.createComment({owner, repo, issue_number: prContext.number, body}); + } + const labelAction = hasSlowLabel ? 'Kept' : 'Applied'; + const commentAction = existing ? 'updated' : 'created'; + return `${labelAction} ${SLOW_LABEL}; ${commentAction} slow CI warning on PR #${prContext.number}.`; + } + + const commentNeedsResolution = Boolean(existing && !existing.body?.includes(RESOLVED_PREFIX)); + if (dryRun && (hasSlowLabel || commentNeedsResolution)) { + const actions = []; + if (hasSlowLabel) actions.push(`remove ${SLOW_LABEL}`); + if (commentNeedsResolution) actions.push('resolve slow CI warning'); + return `Dry run: would ${actions.join(' and ')} on PR #${prContext.number}.\n` + + resolvedBody(elapsed, budget, runUrl); + } + + const actions = []; + if (hasSlowLabel) { + await github.rest.issues.removeLabel({owner, repo, issue_number: prContext.number, name: SLOW_LABEL}); + actions.push(`Removed ${SLOW_LABEL}`); + } + if (commentNeedsResolution) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: existing.id, + body: resolvedBody(elapsed, budget, runUrl), + }); + actions.push('resolved slow CI warning'); + } + if (actions.length) { + return `${actions.join('; ')} on PR #${prContext.number}.`; + } + if (existing) { + return `CI runtime ${elapsed} remains within the ${budget} budget.`; + } + return `CI runtime ${elapsed} is within the ${budget} budget.`; +}; diff --git a/.github/scripts/ci_sim_duration_warning.py b/.github/scripts/ci_sim_duration_warning.py deleted file mode 100644 index d4b88e1153..0000000000 --- a/.github/scripts/ci_sim_duration_warning.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. -"""Post a non-blocking PR warning when ci-sim exceeds its runtime budget.""" - -from __future__ import annotations - -import argparse -from dataclasses import dataclass -from datetime import datetime -import json -import os -from pathlib import Path -import re -import sys -from typing import Any, Dict, List, Optional, Tuple, Union -from urllib.error import HTTPError -from urllib.parse import quote, urlencode -from urllib.request import Request, urlopen - - -COMMENT_MARKER = "" -RESOLVED_PREFIX = "Resolved:" -BOT_LOGIN = "github-actions[bot]" -JOB_NAME = "vpto-sim-validation" -SLOW_LABEL = "ci-slow" -SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") - - -@dataclass(frozen=True) -class RunConfig: - repo: str - run_id: int - run_head_sha: str - run_url: str - soft_timeout_minutes: int - context_dir: Path - dry_run: bool = False - - -class GitHubClient: - def __init__(self, token: str, api_url: str = "https://api.github.com") -> None: - self._token = token - self._api_url = api_url.rstrip("/") - - def request( - self, - method: str, - path: str, - payload: Optional[Dict[str, Any]] = None, - query: Optional[Dict[str, Union[str, int]]] = None, - ) -> Tuple[Any, Dict[str, str]]: - url = f"{self._api_url}/{path.lstrip('/')}" - if query: - url = f"{url}?{urlencode(query)}" - data = json.dumps(payload).encode() if payload is not None else None - request = Request( - url, - data=data, - method=method, - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {self._token}", - "X-GitHub-Api-Version": "2022-11-28", - }, - ) - try: - with urlopen(request, timeout=30) as response: - body = response.read() - return (json.loads(body) if body else None), dict(response.headers) - except HTTPError as error: - detail = error.read().decode(errors="replace") - raise RuntimeError(f"GitHub API {method} {path} failed: {error.code}: {detail}") from error - - def get(self, path: str, query: Optional[Dict[str, Union[str, int]]] = None) -> Any: - return self.request("GET", path, query=query)[0] - - def post(self, path: str, payload: Dict[str, Any]) -> Any: - return self.request("POST", path, payload=payload)[0] - - def patch(self, path: str, payload: Dict[str, Any]) -> Any: - return self.request("PATCH", path, payload=payload)[0] - - def delete(self, path: str) -> Any: - return self.request("DELETE", path)[0] - - def list_all(self, path: str) -> List[Any]: - items: List[Any] = [] - page = 1 - while True: - batch = self.get(path, query={"per_page": 100, "page": page}) - if not isinstance(batch, list): - raise RuntimeError(f"Expected a list from GitHub API path {path}") - items.extend(batch) - if len(batch) < 100: - return items - page += 1 - - -def _read_context(context_dir: Path) -> Optional[Tuple[int, str]]: - number_path = context_dir / "pr-number" - sha_path = context_dir / "pr-head-sha" - if not number_path.is_file() or not sha_path.is_file(): - return None - number_text = number_path.read_text(encoding="utf-8").strip() - head_sha = sha_path.read_text(encoding="utf-8").strip() - if not number_text.isdigit() or int(number_text) <= 0 or not SHA_RE.fullmatch(head_sha): - raise ValueError(f"Invalid PR context in {context_dir}") - return int(number_text), head_sha - - -def _parse_timestamp(value: str) -> datetime: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - - -def _format_duration(seconds: int) -> str: - hours, remainder = divmod(seconds, 3600) - minutes, seconds = divmod(remainder, 60) - if hours: - return f"{hours}h {minutes}m {seconds}s" - return f"{minutes}m {seconds}s" - - -def _format_budget(minutes: int) -> str: - hours, minutes = divmod(minutes, 60) - if hours and minutes: - return f"{hours}h {minutes}m" - if hours: - return f"{hours}h" - return f"{minutes}m" - - -def _warning_body(author: str, elapsed: str, budget: str, conclusion: str, run_url: str) -> str: - return "\n".join( - [ - COMMENT_MARKER, - f"Warning: @{author}, ci-sim exceeded its soft runtime budget.", - "", - f"- `{JOB_NAME}` runtime: **{elapsed}**", - f"- Soft budget: **{budget}**", - f"- Job conclusion: **{conclusion}**", - f"- [Workflow run]({run_url})", - "", - "This warning is advisory only and does not affect required checks. " - "Please inspect the step timings for an unexpected regression.", - ] - ) - - -def _resolved_body(elapsed: str, budget: str, run_url: str) -> str: - return "\n".join( - [ - COMMENT_MARKER, - "Resolved: ci-sim runtime is back within its soft budget.", - "", - f"- Latest `{JOB_NAME}` runtime: **{elapsed}**", - f"- Soft budget: **{budget}**", - f"- [Workflow run]({run_url})", - "", - "The previous duration warning is resolved. This status is advisory only.", - ] - ) - - -def observe_run(client: GitHubClient, config: RunConfig) -> str: - context = _read_context(config.context_dir) - if context is None: - return f"No PR context found for workflow run {config.run_id}; skipping duration warning." - pr_number, context_head_sha = context - - pull = client.get(f"repos/{config.repo}/pulls/{pr_number}") - if pull["state"] != "open": - return f"PR #{pr_number} is {pull['state']}; skipping duration warning." - current_head_sha = pull["head"]["sha"] - if context_head_sha != config.run_head_sha or current_head_sha != config.run_head_sha: - return f"Workflow run {config.run_id} is stale for PR #{pr_number}; skipping duration warning." - has_slow_label = any(label.get("name") == SLOW_LABEL for label in pull.get("labels", [])) - - jobs_response = client.get( - f"repos/{config.repo}/actions/runs/{config.run_id}/jobs", - query={"filter": "latest", "per_page": 100}, - ) - jobs = [ - job - for job in jobs_response.get("jobs", []) - if job.get("name") == JOB_NAME and job.get("started_at") and job.get("completed_at") - ] - if not jobs: - return f"No completed {JOB_NAME} job found in workflow run {config.run_id}." - job = jobs[0] - elapsed_seconds = int((_parse_timestamp(job["completed_at"]) - _parse_timestamp(job["started_at"])).total_seconds()) - if elapsed_seconds < 0: - raise ValueError(f"Invalid job timestamps: {job['started_at']} to {job['completed_at']}") - - elapsed = _format_duration(elapsed_seconds) - budget = _format_budget(config.soft_timeout_minutes) - comments = client.list_all(f"repos/{config.repo}/issues/{pr_number}/comments") - existing = next( - ( - comment - for comment in reversed(comments) - if comment.get("user", {}).get("login") == BOT_LOGIN - and COMMENT_MARKER in comment.get("body", "") - ), - None, - ) - - if elapsed_seconds > config.soft_timeout_minutes * 60: - body = _warning_body(pull["user"]["login"], elapsed, budget, job.get("conclusion", "unknown"), config.run_url) - if config.dry_run: - label_action = "keep" if has_slow_label else "add" - return ( - f"Dry run: would {label_action} {SLOW_LABEL} and " - f"{'update' if existing else 'create'} slow CI warning on PR #{pr_number}.\n{body}" - ) - if not has_slow_label: - client.post(f"repos/{config.repo}/issues/{pr_number}/labels", {"labels": [SLOW_LABEL]}) - if existing: - client.patch(f"repos/{config.repo}/issues/comments/{existing['id']}", {"body": body}) - comment_action = "Updated" - else: - client.post(f"repos/{config.repo}/issues/{pr_number}/comments", {"body": body}) - comment_action = "Created" - label_action = "Kept" if has_slow_label else "Applied" - return f"{label_action} {SLOW_LABEL}; {comment_action.lower()} slow CI warning on PR #{pr_number}." - - comment_needs_resolution = bool(existing and RESOLVED_PREFIX not in existing.get("body", "")) - if config.dry_run and (has_slow_label or comment_needs_resolution): - actions = [] - if has_slow_label: - actions.append(f"remove {SLOW_LABEL}") - if comment_needs_resolution: - actions.append("resolve slow CI warning") - body = _resolved_body(elapsed, budget, config.run_url) - return f"Dry run: would {' and '.join(actions)} on PR #{pr_number}.\n{body}" - - actions = [] - if has_slow_label: - encoded_label = quote(SLOW_LABEL, safe="") - client.delete(f"repos/{config.repo}/issues/{pr_number}/labels/{encoded_label}") - actions.append(f"Removed {SLOW_LABEL}") - if comment_needs_resolution: - body = _resolved_body(elapsed, budget, config.run_url) - client.patch(f"repos/{config.repo}/issues/comments/{existing['id']}", {"body": body}) - actions.append("resolved slow CI warning") - if actions: - return f"{'; '.join(actions)} on PR #{pr_number}." - if existing: - return f"CI runtime {elapsed} remains within the {budget} budget." - return f"CI runtime {elapsed} is within the {budget} budget." - - -def _positive_int(value: str) -> int: - parsed = int(value) - if parsed <= 0: - raise argparse.ArgumentTypeError("value must be a positive integer") - return parsed - - -def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repo", required=True) - parser.add_argument("--run-id", required=True, type=_positive_int) - parser.add_argument("--run-head-sha", required=True) - parser.add_argument("--run-url", required=True) - parser.add_argument("--soft-timeout-minutes", type=_positive_int, default=90) - parser.add_argument("--pr-context-dir", type=Path, default=Path("pr-context")) - parser.add_argument("--dry-run", action="store_true") - return parser.parse_args(argv) - - -def main(argv: Optional[List[str]] = None) -> int: - args = _parse_args(argv) - if not SHA_RE.fullmatch(args.run_head_sha): - raise ValueError(f"Invalid run head SHA: {args.run_head_sha}") - token = os.environ.get("GH_TOKEN") - if not token: - raise RuntimeError("GH_TOKEN is required") - config = RunConfig( - repo=args.repo, - run_id=args.run_id, - run_head_sha=args.run_head_sha, - run_url=args.run_url, - soft_timeout_minutes=args.soft_timeout_minutes, - context_dir=args.pr_context_dir, - dry_run=args.dry_run, - ) - print(observe_run(GitHubClient(token), config)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml index 8f2bdb4249..da243ce0be 100644 --- a/.github/workflows/watchdog.yml +++ b/.github/workflows/watchdog.yml @@ -30,6 +30,7 @@ jobs: pull-requests: read env: SOFT_TIMEOUT_MINUTES: ${{ vars.CI_SIM_SOFT_TIMEOUT_MINUTES || '90' }} + PR_CONTEXT_DIR: pr-context steps: # workflow_run executes this workflow from the default branch, so this # checkout never runs a script supplied by the pull request. @@ -50,16 +51,13 @@ jobs: github-token: ${{ github.token }} - name: Update slow CI warning - env: - GH_TOKEN: ${{ github.token }} - run: >- - python3 .github/scripts/ci_sim_duration_warning.py - --repo "${{ github.repository }}" - --run-id "${{ github.event.workflow_run.id }}" - --run-head-sha "${{ github.event.workflow_run.head_sha }}" - --run-url "${{ github.event.workflow_run.html_url }}" - --soft-timeout-minutes "${SOFT_TIMEOUT_MINUTES}" - --pr-context-dir pr-context + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + retries: 3 + script: | + const observeRun = require('./.github/scripts/ci_sim_duration_warning.js'); + core.info(await observeRun({github, context})); rerun-self-hosted-git-failure: if: >-