-
Notifications
You must be signed in to change notification settings - Fork 4
Expand file tree
/
Copy pathMakefile
More file actions
378 lines (338 loc) · 14.3 KB
/
Copy pathMakefile
File metadata and controls
378 lines (338 loc) · 14.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
# @cpt-algo:cpt-studio-spec-init-structure-change-infrastructure:p1
.PHONY: test test-verbose test-quick test-coverage test-coverage-diff validate validate-examples validate-feature validate-code validate-code-feature self-check validate-kits validate-kits-sdlc vulture vulture-ci pylint install install-pipx install-proxy install-prompt-tests clean help check-pytest check-pytest-cov check-pipx check-vulture check-pylint check-versions check-prompt-tests bootstrap-init bootstrap-repair update update-local seed-cache ensure-bootstrap generate-agents spec-coverage ci lint-ci test-prompts test-prompts-view
# Detect container architecture for act (arm64 on Apple Silicon, amd64 otherwise)
UNAME_M := $(shell uname -m)
ifeq ($(UNAME_M),arm64)
ACT_ARCH := linux/arm64
else ifeq ($(UNAME_M),aarch64)
ACT_ARCH := linux/arm64
else
ACT_ARCH := linux/amd64
endif
ACT_FLAGS ?= --container-architecture $(ACT_ARCH)
PYTHON ?= python3
PIPX ?= pipx
PIPX_BIN_DIR ?= $(HOME)/.local/bin
CFS ?= cfs
BOOTSTRAP_STUDIO ?= .bootstrap/.core/skills/studio/scripts/studio.py
SOURCE_STUDIO ?= skills/studio/scripts/studio.py
PYTEST_PIPX ?= $(PIPX_BIN_DIR)/pytest
PYTEST_PIPX_COV ?= $(PYTEST_PIPX)
VULTURE_PIPX ?= $(PIPX) run --spec vulture vulture
PYLINT_PIPX ?= $(PIPX) run --spec pylint pylint
DIFF_COVER_PIPX ?= $(PIPX) run --spec diff-cover diff-cover
DIFF_COVER_COMPARE ?= main
DIFF_COVER_MIN ?= 80
VULTURE_MIN_CONF ?= 0
PYLINT_TARGETS ?= src/studio_proxy skills/studio/scripts/studio
# Prompt-tests (cf-skill UX pilot, tests/prompts/cf-ux/)
PROMPTFOO_VERSION ?= latest
PROMPTFOO ?= npx -y promptfoo@$(PROMPTFOO_VERSION)
PROMPT_TESTS_DIR ?= tests/prompts/cf-ux
PROMPT_TESTS_TIMEOUT_MS ?= 900000
# Default target
help:
@echo "Constructor Studio Makefile"
@echo ""
@echo "Available targets:"
@echo " make test - Run all tests in parallel (-n 6)"
@echo " make test-verbose - Run tests with verbose output"
@echo " make test-quick - Run fast tests only (skip slow integration tests)"
@echo " make test-coverage - Run tests with coverage report"
@echo " make test-coverage-diff - Coverage for diff vs main (≥$(DIFF_COVER_MIN)%)"
@echo " make validate-examples - Validate requirements examples under examples/requirements"
@echo " make validate - Validate core methodology spec"
@echo " make self-check - Validate SDLC examples against their templates"
@echo " make validate-kits - Validate all registered kits"
@echo " make validate-kits-sdlc - Validate kits/sdlc kit by path"
@echo " make check-versions - Check version consistency across components"
@echo " make spec-coverage - Check spec coverage (≥90% overall, ≥60% per file, ≥0.46 granularity)"
@echo " make vulture - Scan python code for dead code (report only, does not fail)"
@echo " make vulture-ci - Scan python code for dead code (fails if findings)"
@echo " make ci - Run full CI pipeline locally"
@echo " make lint-ci - Lint GitHub Actions workflow files"
@echo " make install - Install Python dependencies"
@echo " make install-proxy - Reinstall cfs proxy from local source"
@echo " make install-prompt-tests - Pre-cache promptfoo for cf-skill UX tests"
@echo " make test-prompts - Run cf-skill UX pilot (claude + codex)"
@echo " make test-prompts-view - Open promptfoo HTML report for last run"
@echo " make bootstrap-init - Initialize .bootstrap from local source"
@echo " make bootstrap-repair - Repair generated .bootstrap runtime files"
@echo " make update-local - Update .bootstrap from local source"
@echo " make update - Alias for update-local"
@echo " make generate-agents - Generate all local agent integrations"
@echo " make clean - Remove Python cache files"
@echo " make help - Show this help message"
# Run all tests
check-pipx:
@command -v $(PIPX) >/dev/null 2>&1 || { \
echo ""; \
echo "ERROR: pipx not found"; \
echo ""; \
echo "Install it with:"; \
echo " brew install pipx"; \
echo " pipx ensurepath"; \
echo ""; \
exit 1; \
}
check-pytest: check-pipx
@test -x "$(PYTEST_PIPX)" || { \
echo ""; \
echo "ERROR: pytest binary not found at $(PYTEST_PIPX)"; \
echo ""; \
echo "Install it with:"; \
echo " make install"; \
echo ""; \
exit 1; \
}
@$(PYTEST_PIPX) --version >/dev/null 2>&1 || { \
echo ""; \
echo "ERROR: pytest is not runnable via pipx"; \
echo ""; \
echo "Install it with:"; \
echo " make install"; \
echo ""; \
exit 1; \
}
@$(PYTEST_PIPX) --help 2>/dev/null | grep -q -- '--numprocesses' || { \
echo ""; \
echo "ERROR: pytest-xdist not available (missing -n support)"; \
echo ""; \
echo "Install it with:"; \
echo " make install"; \
echo ""; \
exit 1; \
}
check-pytest-cov: check-pytest
@$(PYTEST_PIPX_COV) --help 2>/dev/null | grep -q -- '--cov' || { \
echo ""; \
echo "ERROR: pytest-cov not available (missing --cov option)"; \
echo ""; \
echo "Install it with:"; \
echo " make install"; \
echo ""; \
exit 1; \
}
check-vulture: check-pipx
@$(VULTURE_PIPX) --version >/dev/null 2>&1 || { \
echo ""; \
echo "ERROR: vulture is not runnable via pipx"; \
echo ""; \
echo "Install it with:"; \
echo " pipx install vulture"; \
echo "or just run: make vulture (pipx run will download it)"; \
echo ""; \
exit 1; \
}
check-pylint: check-pipx
@$(PYLINT_PIPX) --version >/dev/null 2>&1 || { \
echo ""; \
echo "ERROR: pylint is not runnable via pipx"; \
echo ""; \
echo "Install it with:"; \
echo " pipx install pylint"; \
echo "or just run: make pylint (pipx run will download it)"; \
echo ""; \
exit 1; \
}
test: check-pytest
@echo "Running Constructor Studio tests with pipx..."
$(PYTEST_PIPX) tests/ -n 6 -v --tb=short
# Run tests with verbose output
test-verbose: check-pytest
@echo "Running Constructor Studio tests (verbose) with pipx..."
$(PYTEST_PIPX) tests/ -vv
# Run quick tests only
test-quick: check-pytest
@echo "Running quick tests with pipx..."
$(PYTEST_PIPX) tests/ -v -m "not slow"
# Run tests with coverage
test-coverage: check-pytest-cov
@echo "Running tests with coverage..."
$(PYTEST_PIPX_COV) tests/ \
-n 6 \
--cov=skills/studio/scripts/studio \
--cov-report=term-missing \
--cov-report=json:coverage.json \
--cov-report=xml:coverage.xml \
--cov-report=html \
-v --tb=short
@$(PYTHON) scripts/check_coverage.py coverage.json --root skills/studio/scripts/studio --min 90 --exclude vendor/
@echo ""
@echo "Coverage report generated:"
@echo " HTML: htmlcov/index.html"
@echo " XML: coverage.xml"
@echo " Open with: open htmlcov/index.html"
@if git rev-parse --verify $(DIFF_COVER_COMPARE) >/dev/null 2>&1 && \
! git diff --quiet $(DIFF_COVER_COMPARE) -- 2>/dev/null; then \
echo ""; \
echo "Checking diff coverage vs $(DIFF_COVER_COMPARE) (min $(DIFF_COVER_MIN)%)..."; \
$(DIFF_COVER_PIPX) coverage.xml \
--compare-branch=$(DIFF_COVER_COMPARE) \
--fail-under=$(DIFF_COVER_MIN) \
--diff-range-notation='..' \
--show-uncovered; \
else \
echo ""; \
echo "Skipping diff coverage (no diff vs $(DIFF_COVER_COMPARE) or branch not found)"; \
fi
# Run diff-coverage standalone (reuses existing coverage.xml)
test-coverage-diff:
@if [ ! -f coverage.xml ]; then \
echo "ERROR: coverage.xml not found. Run 'make test-coverage' first."; \
exit 1; \
fi
@if git rev-parse --verify $(DIFF_COVER_COMPARE) >/dev/null 2>&1 && \
! git diff --quiet $(DIFF_COVER_COMPARE) -- 2>/dev/null; then \
echo "Checking diff coverage vs $(DIFF_COVER_COMPARE) (min $(DIFF_COVER_MIN)%)..."; \
$(DIFF_COVER_PIPX) coverage.xml \
--compare-branch=$(DIFF_COVER_COMPARE) \
--fail-under=$(DIFF_COVER_MIN) \
--diff-range-notation='..' \
--show-uncovered; \
else \
echo "Skipping diff coverage (no diff vs $(DIFF_COVER_COMPARE) or branch not found)"; \
fi
vulture: check-vulture
@echo "Running vulture dead-code scan (excluding tests by scanning only skills/studio/scripts/studio)..."
@echo "Tip: raise/lower VULTURE_MIN_CONF to reduce false positives (current: $(VULTURE_MIN_CONF))."
@$(VULTURE_PIPX) skills/studio/scripts/studio vulture_whitelist.py --exclude '*/vendor/*' --min-confidence $(VULTURE_MIN_CONF) || true
vulture-ci: check-vulture
@echo "Running vulture dead-code scan (CI mode, fails if findings)..."
$(VULTURE_PIPX) skills/studio/scripts/studio vulture_whitelist.py --exclude '*/vendor/*' --min-confidence $(VULTURE_MIN_CONF)
pylint: check-pylint
@echo "Running pylint..."
PYTHONPATH=src:skills/studio/scripts $(PYLINT_PIPX) --jobs=6 $(PYLINT_TARGETS)
# Spec coverage check (Constructor Studio system only)
spec-coverage: ensure-bootstrap
@echo "Checking spec coverage (Constructor Studio system)..."
$(PYTHON) $(BOOTSTRAP_STUDIO) spec-coverage --system studio --min-coverage 90 --min-file-coverage 60 --min-granularity 0.46
# Check version consistency
check-versions:
@$(PYTHON) scripts/check_versions.py
# Initialize .bootstrap from local source. Repeat runs repair generated runtime files.
bootstrap-init: seed-cache
@echo "Initializing .bootstrap from local source..."
$(PYTHON) $(SOURCE_STUDIO) init --project-root . --install-dir .bootstrap --kit-tracking tracked --kit-tracking sdlc=ignored --yes
# Repair generated .bootstrap runtime files without updating tracked kit files.
bootstrap-repair: seed-cache
@echo "Repairing generated .bootstrap runtime files from local source..."
$(PYTHON) $(SOURCE_STUDIO) init --project-root . --install-dir .bootstrap --kit-tracking tracked --kit-tracking sdlc=ignored --yes
# Backward-compatible alias: update means local self-hosted update in this repo.
update: update-local
# Update .bootstrap from local source. Kit files are skipped by default by cfs update.
update-local: seed-cache
@if [ ! -f "$(BOOTSTRAP_STUDIO)" ]; then \
$(PYTHON) $(SOURCE_STUDIO) init --project-root . --install-dir .bootstrap --kit-tracking tracked --kit-tracking sdlc=ignored --yes; \
else \
$(PYTHON) $(SOURCE_STUDIO) update --project-root . -y; \
fi
seed-cache:
@echo "Seeding Constructor Studio cache from tracked source..."
PYTHONPATH=src $(PYTHON) scripts/seed_local_cache.py .
ensure-bootstrap:
@if [ ! -f "$(BOOTSTRAP_STUDIO)" ]; then \
echo "Bootstrap studio entrypoint missing; running make bootstrap-init..."; \
$(MAKE) bootstrap-init; \
fi
generate-agents: ensure-bootstrap
@echo "Generating all Constructor Studio agent integrations..."
$(PYTHON) $(BOOTSTRAP_STUDIO) generate-agents -y
# Validate core methodology spec
validate: ensure-bootstrap
$(CFS) validate
# Validate SDLC examples against templates
self-check: ensure-bootstrap
@echo "Running self-check: validating SDLC examples against templates..."
$(CFS) self-check
# Validate all registered kits
validate-kits: ensure-bootstrap
@echo "Validating all registered kits..."
$(CFS) validate-kits
# Validate kits/sdlc kit by path
validate-kits-sdlc: ensure-bootstrap
@echo "Validating kits/sdlc..."
$(CFS) validate-kits kits/sdlc
# Install Python dependencies
install-pipx: check-pipx
@echo "Installing pytest + pytest-cov via pipx..."
@$(PIPX) install pytest >/dev/null 2>&1 || $(PIPX) upgrade pytest
@$(PIPX) inject pytest pytest-cov pytest-xdist
@echo "Done. If pytest is not found, run: pipx ensurepath (then restart your shell)."
install: install-pipx
# Reinstall cfs/constructor-studio proxy from local source
install-proxy: check-pipx
$(PIPX) install --force .
# cf-skill UX prompt-tests (tests/prompts/cf-ux/) -----------------------------
#
# Dependencies the tests need at runtime:
# - node / npx (for promptfoo)
# - claude CLI (provider + grader)
# - codex CLI (provider)
# - cfs CLI (sandbox init via the local studio engine — already
# installed by `make install-proxy`)
#
# Test runs spin up fresh `cfs init`-ed tmpdir sandboxes and consume real
# Claude / Codex API tokens. Defaults to cheap models (Haiku 4.5, gpt-5.4-mini
# at low effort, 128k context); override via CF_UX_* env vars — see
# tests/prompts/cf-ux/README.md.
check-prompt-tests:
@command -v node >/dev/null 2>&1 || { \
echo ""; \
echo "ERROR: node not found (needed for npx promptfoo)"; \
echo " Install via nvm: https://github.com/nvm-sh/nvm"; \
echo ""; \
exit 1; \
}
@command -v claude >/dev/null 2>&1 || { \
echo ""; \
echo "ERROR: claude CLI not found"; \
echo " Install: https://docs.claude.com/en/docs/claude-code"; \
echo ""; \
exit 1; \
}
@command -v codex >/dev/null 2>&1 || { \
echo ""; \
echo "ERROR: codex CLI not found"; \
echo " Install: https://developers.openai.com/codex/cli"; \
echo ""; \
exit 1; \
}
@command -v $(CFS) >/dev/null 2>&1 || { \
echo ""; \
echo "ERROR: cfs not found — run \`make install-proxy\` first"; \
echo ""; \
exit 1; \
}
install-prompt-tests: check-prompt-tests
@echo "Pre-caching promptfoo@$(PROMPTFOO_VERSION) via npx..."
@$(PROMPTFOO) --version >/dev/null
@echo "Done. Run prompt tests with: make test-prompts"
test-prompts: check-prompt-tests
@echo "Running cf-skill UX pilot in $(PROMPT_TESTS_DIR)..."
@cd $(PROMPT_TESTS_DIR) && \
REQUEST_TIMEOUT_MS=$(PROMPT_TESTS_TIMEOUT_MS) $(PROMPTFOO) eval --no-cache
test-prompts-view:
@cd $(PROMPT_TESTS_DIR) && $(PROMPTFOO) view
# --------------------------------------------------------------------------
# Lint CI workflow files
lint-ci:
@echo "Linting GitHub Actions workflows..."
actionlint
# Run CI via act in Docker (mirrors .github/workflows/ci.yml exactly)
# Runs jobs sequentially — stops on first failure.
# Auto-detects arm64/amd64. Override: make ci ACT_FLAGS="--your-flags"
ci: lint-ci
@for job in $$(act push --list $(ACT_FLAGS) 2>/dev/null | tail -n +2 | awk '{print $$2}' | grep -v '^sonarqube$$'); do \
echo "▶ Running job: $$job"; \
act push -j $$job $(ACT_FLAGS) || exit 1; \
done
@echo ""
@echo "✓ All CI jobs passed."
# Clean Python cache
clean:
@echo "Cleaning Python cache files..."
find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
find . -type f -name "*.pyc" -delete
find . -type d -name ".pytest_cache" -exec rm -rf {} + 2>/dev/null || true
@echo "Clean complete"