diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index fedeb159..b42e5c74 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -6,16 +6,14 @@ name: python # the core checks, and so a green run proves the bindings build cold from a # checkout (the paper's reproducibility requirement). on: - # Push runs only on main (post-merge); feature branches are covered by the - # pull_request trigger below, so a branch with an open PR doesn't double-run. + # Push runs on EVERY commit to main (no paths filter): release-python.yml's + # require-ci-green gate asserts a successful python.yml run exists for the exact + # release SHA on main, so a docs-only or workflow-only release tip must still + # produce a run here or the wheel gate would be unsatisfiable. Feature branches + # stay path-filtered via the pull_request trigger below (a PR that doesn't touch + # the binding skips this), and main's concurrency cancels superseded runs. push: branches: [main] - paths: - - "ordvec-python/**" - - "src/**" - - "Cargo.toml" - - "Cargo.lock" - - ".github/workflows/python.yml" pull_request: paths: - "ordvec-python/**" diff --git a/.github/workflows/release-crate.yml b/.github/workflows/release-crate.yml index 72fe54d2..bab21b57 100644 --- a/.github/workflows/release-crate.yml +++ b/.github/workflows/release-crate.yml @@ -50,13 +50,19 @@ jobs: run: cargo publish -p ordvec --locked --dry-run require-ci-green: - # A manual release dispatch must not ship a commit whose full CI matrix has - # not gone green. The `verify` job above re-runs tests + the publish - # dry-run, but it does NOT cover lint, the no-default/experimental configs, - # MSRV 1.89, the deps + cargo-deny supply-chain gates, or the AVX-512-SDE / - # wasm-simd128 runtime coverage. Rather than duplicate (and drift from) all - # of ci.yml, require that the `ci` workflow concluded `success` for this - # exact SHA before publish runs. + # A manual release dispatch must not ship a commit whose CI has not gone + # green. The `verify` job above re-runs tests + the publish dry-run, but it + # does NOT cover lint, the no-default/experimental configs, MSRV 1.89, the + # deps + cargo-deny supply-chain gates, the AVX-512-SDE / wasm-simd128 + # runtime coverage (ci.yml), the cargo-fuzz loader/FastScan smoke (fuzz.yml), + # or the CodeQL static scan of the shipped Rust source (codeql.yml). Rather + # than duplicate (and drift from) those workflows, require that EACH of them + # concluded `success` for this exact SHA on main before publish runs. + # + # Only per-push-to-main workflows are gated, so a run for the release SHA is + # guaranteed to exist: audit.yml is schedule-only (no per-SHA run to assert); + # coverage*/scorecard are advisory or external-service-flaky; zizmor/ + # actionlint are pre-merge CI hygiene, not artifact safety. name: require full CI green for this commit runs-on: ubuntu-latest permissions: @@ -67,24 +73,26 @@ jobs: uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - - name: assert ci.yml is green for this commit + - name: assert ci.yml, fuzz.yml and codeql.yml are green for this commit env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} SHA: ${{ github.sha }} run: | set -euo pipefail - # Require a SUCCESSFUL ci.yml run for this SHA *on main*. Filtering on - # the branch as well as head_sha stops a green run for the same commit - # on an unrelated branch from satisfying the gate. - ok="$(gh api \ - "repos/${REPO}/actions/workflows/ci.yml/runs?head_sha=${SHA}&branch=main&per_page=20" \ - --jq '[.workflow_runs[] | select(.head_branch == "main" and .conclusion == "success")] | length')" - echo "successful ci.yml runs for ${SHA} on main: ${ok}" - if [ "${ok}" -lt 1 ]; then - echo "::error::no successful ci.yml run for ${SHA} on main. Push to main, let CI pass, then re-run this release." - exit 1 - fi + # Require a SUCCESSFUL run for this SHA *on main* for each workflow. + # Filtering on the branch as well as head_sha stops a green run for the + # same commit on an unrelated branch from satisfying the gate. + for wf in ci.yml fuzz.yml codeql.yml; do + ok="$(gh api \ + "repos/${REPO}/actions/workflows/${wf}/runs?head_sha=${SHA}&branch=main&status=success&per_page=20" \ + --jq '[.workflow_runs[] | select(.head_branch == "main" and .conclusion == "success")] | length')" + echo "successful ${wf} runs for ${SHA} on main: ${ok}" + if [ "${ok}" -lt 1 ]; then + echo "::error::no successful ${wf} run for ${SHA} on main. Push to main, let CI pass, then re-run this release." + exit 1 + fi + done publish: name: publish to crates.io diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 5671d6ef..70f30e83 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -139,11 +139,19 @@ jobs: if-no-files-found: error require-ci-green: - # The wheel must not ship from a commit whose core (`ci.yml`) or binding - # (`python.yml`) matrices have not gone green. The build-wheels job above + # The wheel must not ship from a commit whose core (`ci.yml`), binding + # (`python.yml`), fuzz-smoke (`fuzz.yml`), or CodeQL static-scan + # (`codeql.yml`) gates have not gone green. The build-wheels job above # builds + pytest-checks each platform wheel, but it does not cover the core - # crate's gates or the binding's full OS/Python matrix and lint. Require - # both workflows to have concluded `success` for this exact SHA. + # crate's gates, the binding's full OS/Python matrix and lint, the loader/ + # FastScan fuzz smoke, or the CodeQL scan of the shipped Rust + Python + # source. Require each workflow to have concluded `success` for this exact + # SHA on main. + # + # Only per-push-to-main workflows are gated, so a run for the release SHA is + # guaranteed to exist: audit.yml is schedule-only (no per-SHA run to assert); + # coverage*/scorecard are advisory or external-service-flaky; zizmor/ + # actionlint are pre-merge CI hygiene, not artifact safety. name: require core + binding CI green for this commit runs-on: ubuntu-latest permissions: @@ -154,7 +162,7 @@ jobs: uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - - name: assert ci.yml and python.yml are green for this commit + - name: assert ci.yml, python.yml, fuzz.yml and codeql.yml are green for this commit env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} @@ -164,9 +172,9 @@ jobs: # Require a SUCCESSFUL run for this SHA *on main* for each workflow. # Filtering on the branch as well as head_sha stops a green run for the # same commit on an unrelated branch from satisfying the gate. - for wf in ci.yml python.yml; do + for wf in ci.yml python.yml fuzz.yml codeql.yml; do ok="$(gh api \ - "repos/${REPO}/actions/workflows/${wf}/runs?head_sha=${SHA}&branch=main&per_page=20" \ + "repos/${REPO}/actions/workflows/${wf}/runs?head_sha=${SHA}&branch=main&status=success&per_page=20" \ --jq '[.workflow_runs[] | select(.head_branch == "main" and .conclusion == "success")] | length')" echo "successful ${wf} runs for ${SHA} on main: ${ok}" if [ "${ok}" -lt 1 ]; then diff --git a/GOVERNANCE.md b/GOVERNANCE.md index b90783f6..3d984d08 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -8,7 +8,7 @@ describes how it is run. - **Maintainer.** ordvec is currently maintained by Nelson Spence ([@Fieldnote-Echo](https://github.com/Fieldnote-Echo)), the project lead and final decision-maker on technical direction, releases, and scope. -- **Code owners.** Listed in [`.github/CODEOWNERS`](.github/CODEOWNERS); they +- **Code owners.** Listed in [`.github/CODEOWNERS`](https://github.com/Fieldnote-Echo/ordvec/blob/main/.github/CODEOWNERS); they review and approve changes. ## Decision-making diff --git a/README.md b/README.md index a46926f9..096ba353 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,8 @@ [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/12977/badge)](https://www.bestpractices.dev/projects/12977) [![codecov](https://codecov.io/gh/Fieldnote-Echo/ordvec/graph/badge.svg)](https://codecov.io/gh/Fieldnote-Echo/ordvec) - Training-free ordinal & sign quantization for vector retrieval. @@ -88,12 +86,12 @@ accompanying the paper. Details in ## Quickstart -The crate is being prepared for its first crates.io release. Until then, -add it as a git dependency: - ```toml [dependencies] -ordvec = { git = "https://github.com/Fieldnote-Echo/ordvec" } +ordvec = "0.2" + +# Or, to track unreleased `main`, use a git dependency instead: +# ordvec = { git = "https://github.com/Fieldnote-Echo/ordvec" } ``` ```rust @@ -112,28 +110,32 @@ let top_ids = results.indices_for_query(0); // top-10 doc ids for query 0 let top_scores = results.scores_for_query(0); ``` -For the sub-linear two-stage path (`Bitmap` / `SignBitmap` candidate +For the two-stage compressed-scan path (`Bitmap` / `SignBitmap` candidate generation → `RankQuant` rerank) and the full mode comparison, see [`docs/RANK_MODES.md`](docs/RANK_MODES.md). ### Python -PyO3/maturin bindings are **in progress** — the same `Rank` / `RankQuant` / -`Bitmap` / `SignBitmap` API from Python, shipping to PyPI as `ordvec` (the -coordinated PyPI release is pending). Until then, build from source with -`maturin develop` in [`ordvec-python/`](ordvec-python/). +The same `Rank` / `RankQuant` / `Bitmap` / `SignBitmap` API is available from +Python — the bindings ship to PyPI as `ordvec`: + +```bash +pip install ordvec +``` + +Wheels target CPython 3.10+ (abi3); to build from source instead, see +[`ordvec-python/`](https://github.com/Fieldnote-Echo/ordvec/tree/main/ordvec-python). ## Documentation - **Design deep-dive & reproducible benchmark tables:** [`docs/RANK_MODES.md`](docs/RANK_MODES.md) - **Design alternatives evaluated and cut:** - [`docs/ALTERNATIVES_CONSIDERED.md`](docs/ALTERNATIVES_CONSIDERED.md) + [`docs/ALTERNATIVES_CONSIDERED.md`](https://github.com/Fieldnote-Echo/ordvec/blob/main/docs/ALTERNATIVES_CONSIDERED.md) - **Index-file trust model:** [`docs/INDEX_PROVENANCE.md`](docs/INDEX_PROVENANCE.md), [`THREAT_MODEL.md`](THREAT_MODEL.md) -- **API docs:** *(available after the first - crates.io release)* +- **API docs:** - **Paper (OrdVec / RankQuant):** _link TBD — see [Research collaboration](#research-collaboration)._ @@ -213,8 +215,9 @@ for mathematical assistance and mentorship. ordvec is the reference implementation for an in-progress paper on **ordinal retrieval** — using the rank and sign structure of embeddings, rather than their floating-point magnitudes, as the retrieval signal. The repository is -open specifically to grow a group of collaborators, **including named -co-authorship on the paper** — a different invitation than "send a PR." +open specifically to grow a group of collaborators, **including potential +named co-authorship where contributions meet the paper's authorship bar** — +a different invitation than "send a PR." Collaboration we're actively seeking: - **Real-corpus evaluation** — running the modes against public corpora diff --git a/RELEASING.md b/RELEASING.md index 5166dbb3..3c4bcca7 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -14,8 +14,9 @@ workflows. Nothing ships on a tag push or a merge. Both `release-crate.yml` and `release-python.yml`: - are **`workflow_dispatch`-only** (no `push` / tag trigger); -- run a **`require-ci-green`** gate confirming `ci.yml` (and, for the wheel, - `python.yml`) are green for the target commit on `main`; +- run a **`require-ci-green`** gate confirming the per-commit CI is green for the + target commit on `main` — `ci.yml`, `fuzz.yml`, and `codeql.yml` for the crate, + plus `python.yml` for the wheel (a *successful* run for that exact SHA on `main`); - publish via **OIDC trusted publishing** (no long-lived crates.io / PyPI tokens in the repo); - emit **SLSA build provenance** (`actions/attest-build-provenance`) **before** @@ -55,8 +56,27 @@ Trusted Publishing step. sync (`cargo build --locked`). 2. Bump the version (crate `Cargo.toml`, and `ordvec-python` if the wheel changed) and update `CHANGELOG.md`. Commit on `main`. -3. Confirm CI is **green for that exact `main` SHA** (the dispatch ref must be - `main` — the environment will refuse any other branch). +3. Confirm CI is **green for current `main` HEAD**. A release dispatches from + `main` (the environment refuses any other ref), so `require-ci-green` always + checks `main` HEAD's SHA — which needs a **completed, successful** (not + cancelled, not in-progress) run of `ci.yml`, `fuzz.yml`, `codeql.yml` (and + `python.yml` for the wheel). + - **Do not merge another PR between the release commit and the dispatch.** + `ci.yml` / `python.yml` use `cancel-in-progress`, so merging again moves + `main` HEAD and cancels the previous commit's in-flight CI. The superseded + commit is no longer the release target: **release from the new HEAD once its + own CI has completed green** — never from, or by re-validating, the older + commit. + - If HEAD's *own* run shows `cancelled` (superseded, but you have since + stopped pushing), re-run **that HEAD run** from the Actions UI and wait for + it to finish green before dispatching. The SHA you re-run must be the exact + SHA you publish; do not hand-clear the gate on any other commit. + - Release only from a commit on `main` with a **successful push-to-main run** + of each gated workflow — in practice the tip the merge produced (a squash + commit, a rebased tip, or a merge commit), whatever the merge strategy. An + interior commit that exists in history only from a PR branch has no + push-to-main run (its CI ran as a `pull_request` on the branch) and so is + not releasable. 4. Get the maintainer's explicit go to publish. 5. Dispatch `release-crate.yml` (crate) and/or `release-python.yml` (wheel) from **`main`**. diff --git a/docs/RANK_MODES.md b/docs/RANK_MODES.md index 80b7a3e4..1f88caed 100644 --- a/docs/RANK_MODES.md +++ b/docs/RANK_MODES.md @@ -35,7 +35,7 @@ Real-corpus evaluation appears in the accompanying paper (link TBD). The bitmap two-stage path (`Bitmap` candidate gen → `RankQuant` exact subset rerank) is the operating point that -turns RankQuant from a slow exact scan into a sub-linear retriever: +turns RankQuant from a slow exact scan into a fast two-stage retriever: the bitmap probe is the cheap candidate generator, and `search_asymmetric_subset` reruns the exact RankQuant kernel on only the surviving M candidates. The bench reports this path as its