Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .jules/bolt.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,9 @@
## 2026-07-09 - Avoid N+1 API blocking in SBOM aggregator
**Learning:** The `collect_inventories` function in `scripts/ci/sbom_inventory_aggregator.py` was fetching SBOMs from the GitHub dependency graph synchronously for every repository in the organization. For large organizations (up to 500 repos), this N+1 network/CLI bottleneck significantly stalled the aggregation workflow.
**Action:** Use `concurrent.futures.ThreadPoolExecutor` to fetch SBOMs concurrently when multiple repositories are provided, bounded by a `max_workers` limit (e.g., 10) to avoid overwhelming the CLI/API, while preserving the fast serial path for single-item inputs.
## 2026-07-08 - [Log Redaction Performance Optimization]
**Learning:** O(N^2) complexity in Python string manipulation (character-by-character append) and multiple regex substitute passes significantly degrade performance when processing massive CI logs.
**Action:** When implementing parsing or string manipulation in tight linear loops, use string slicing to batch-process chunks (`output.append(text[start:cursor])`). When applying multiple unrelated regex replacements to the same string fragment, combine them using the `|` alternation operator into a single compiled pattern to prevent redundant parsing over the text.
## 2026-08-05 - [Regex Algorithmic Complexity Issue Fix]
**Learning:** An unbounded while loop processing input characters to match a pattern inside a string parser can lead to O(N^2) complexity if the parser fails after matching a very long invalid string and falls back to check the next index. This can trigger timeouts in CI jobs on adversarial input logs.
**Action:** Always add an upper bound (e.g., `MAX_KEY_LENGTH`) in linear parsers when scanning for known patterns in untrusted input.
24 changes: 15 additions & 9 deletions scripts/ci/redact_sensitive_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from typing import Any

REDACTED = "[REDACTED]"
MAX_KEY_LENGTH = 64
KEY_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-")
SENSITIVE_KEY_RE = re.compile(
r"(?:token|secret|password|passwd|credential|authorization|jwt|"
Expand All @@ -24,11 +25,12 @@
r"[^\s\"'\\]+",
re.IGNORECASE,
)
PROVIDER_TOKEN_RES = (
re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b"),
re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"),
re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"),
re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
# Optimizing: using `|` alternation operator to combine regexes into a single compiled pattern to avoid redundant passes over the string
PROVIDER_TOKEN_RE = re.compile(
r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b|"
r"\bsk-[A-Za-z0-9_-]{20,}\b|"
r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b|"
r"\bAKIA[0-9A-Z]{16}\b"
)


Expand All @@ -54,7 +56,7 @@ def _consume_sensitive_assignment(text: str, start: int) -> tuple[str, int] | No
key_start = cursor
if cursor >= len(text) or text[cursor] not in KEY_CHARS or text[cursor].isdigit():
return None
while cursor < len(text) and text[cursor] in KEY_CHARS:
while cursor < len(text) and text[cursor] in KEY_CHARS and (cursor - key_start) < MAX_KEY_LENGTH:
cursor += 1
key = text[key_start:cursor]
if key_quote:
Expand Down Expand Up @@ -99,14 +101,19 @@ def _redact_assignments(text: str) -> str:
"""Redact sensitive key/value assignments without backtracking regexes."""
output: list[str] = []
cursor = 0
start = 0
while cursor < len(text):
match = _consume_sensitive_assignment(text, cursor)
if match is None:
output.append(text[cursor])
cursor += 1
continue
if cursor > start:
output.append(text[start:cursor])
replacement, cursor = match
output.append(replacement)
start = cursor
if cursor > start:
output.append(text[start:cursor])
return "".join(output)


Expand All @@ -115,8 +122,7 @@ def _redact_unstructured(text: str) -> str:
cleaned = _redact_assignments(text)
cleaned = BEARER_RE.sub(lambda match: f"{match.group('prefix')}{REDACTED}", cleaned)
cleaned = JWT_RE.sub(REDACTED, cleaned)
for pattern in PROVIDER_TOKEN_RES:
cleaned = pattern.sub(REDACTED, cleaned)
cleaned = PROVIDER_TOKEN_RE.sub(REDACTED, cleaned)
return cleaned


Expand Down
Loading