diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 81ac47a..8eab8ee 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -29,6 +29,9 @@ jobs: - name: Assemble walk_graph.bin from chunks run: bash scripts/assemble-walk-graph.sh + - name: Assemble pois.bin from chunks + run: bash scripts/assemble-pois.sh + - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-unknown-unknown diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 38a06df..310403c 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -16,3 +16,18 @@ jobs: node-version: "20" - name: Run node --test unit tests run: node --test tests/unit/**/*.test.mjs + + pipeline-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 + - name: Assemble walk_graph.bin (needed by walk_graph_reader test) + run: bash scripts/assemble-walk-graph.sh + - name: Install pipeline deps (incl. dev for pytest) + run: uv sync --extra dev + working-directory: pipelines + - name: Run pytest + run: uv run pytest -v + working-directory: pipelines diff --git a/.gitignore b/.gitignore index 560aed9..e7c96cc 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ Cargo.lock # Built routing blobs (reassembled from tiles/walk-graph.part-* by CI / dev server) tiles/walk_graph.bin +tiles/pois.bin # Raw OSM extracts (downloaded by build-walk-graph.sh, not committed) pipelines/cache/ diff --git a/pipelines/poi_emit.py b/pipelines/poi_emit.py new file mode 100644 index 0000000..b1fc79c --- /dev/null +++ b/pipelines/poi_emit.py @@ -0,0 +1,94 @@ +""" +Pack POIs into the `tiles/pois.bin` binary format documented in the spec. + +Header (24 bytes, little-endian): + magic : 4 bytes = "POI1" + version : u32 = 1 + walk_graph_version : u32 + n_pois : u32 + names_off : u32 (byte offset to NAMES section) + reserved : u32 + +Records: n_pois × 20 bytes, fixed stride. lon/lat as i32×1e7. +Names: variable-length UTF-8, no terminator. Records index by (name_off, name_len). +""" +from __future__ import annotations + +import struct +from dataclasses import dataclass + +MAGIC = b"POI1" +HEADER_FMT = "<4sIIIII" +HEADER_SIZE = struct.calcsize(HEADER_FMT) # 24 +RECORD_FMT = " bytes: + names_buf = bytearray() + name_index: dict[str, tuple[int, int]] = {} + + def store_name(name: str) -> tuple[int, int]: + if not name: + return (0, 0) + if name in name_index: + return name_index[name] + encoded = name.encode("utf-8") + if len(encoded) > MAX_NAME_LEN: + raise ValueError(f"name too long ({len(encoded)} bytes): {name!r}") + off = len(names_buf) + names_buf.extend(encoded) + name_index[name] = (off, len(encoded)) + return (off, len(encoded)) + + records = bytearray() + for p in pois: + off, n_len = store_name(p.name) + flags = 0 if p.name else FLAG_UNNAMED + records.extend(struct.pack( + RECORD_FMT, + int(round(p.lon * 1e7)), + int(round(p.lat * 1e7)), + p.walk_node, + off, + n_len, + p.category, + flags, + )) + + names_off = HEADER_SIZE + len(records) + header = struct.pack( + HEADER_FMT, + MAGIC, + 1, + walk_graph_version, + len(pois), + names_off, + 0, + ) + return bytes(header + records + names_buf) + + +def read_poi_header(blob: bytes) -> dict: + magic, version, walk_graph_version, n_pois, names_off, _ = struct.unpack_from( + HEADER_FMT, blob, 0 + ) + if magic != MAGIC: + raise ValueError(f"bad magic: {magic!r}") + return { + "version": version, + "walk_graph_version": walk_graph_version, + "n_pois": n_pois, + "names_off": names_off, + } diff --git a/pipelines/pois.py b/pipelines/pois.py new file mode 100644 index 0000000..3884cb8 --- /dev/null +++ b/pipelines/pois.py @@ -0,0 +1,223 @@ +""" +Extract POIs from an OSM PBF and emit `pois.bin`. + + uv run python pipelines/pois.py + +Pipeline: + 1. Pass 0 (ways): record way-node refs for ways with category tags. + 2. Pass 1 (nodes): resolve coords; emit standalone-node POIs immediately, + and collect coords for way nodes we'll need for centroids. + 3. Compute way centroids (bbox center) for buffered ways. + 4. Pre-snap each POI to a walk-graph node in the LCC; drop unsnappable. + 5. Clip to bbox. + 6. Emit pois.bin (sorted by (category, name) for cache-friendly category scans). +""" +from __future__ import annotations + +import sys +from dataclasses import dataclass +from pathlib import Path + +import osmium + +# Dual-import: package-style for pytest (runs from repo root with pipelines/ +# on sys.path), sibling-style for direct script invocation matching the +# walk_graph.py convention (`uv run --directory pipelines python pois.py ...`). +try: + from pipelines.poi_emit import POI, write_poi_blob + from pipelines.walk_graph_reader import WalkGraphReader +except ImportError: + from poi_emit import POI, write_poi_blob + from walk_graph_reader import WalkGraphReader + + +# Category code constants. Order = priority (lower wins on multi-tagged POIs). +CATEGORIES: dict[str, int] = { + "food": 1, + "transit": 2, + "park": 3, + "culture": 4, + "attraction": 5, + "shop": 6, + "school": 7, + "health": 8, + "service": 9, + "worship": 10, +} + +SHOP_ALLOWED = { + "supermarket", "convenience", "bakery", "books", "clothes", + "department_store", "mall", "hardware", +} + + +def classify_tags(tags: dict[str, str]) -> int | None: + """Map an OSM tag bundle to a category code; None if no category applies. + Order matches the CATEGORIES dict: food > transit > park > culture > + attraction > shop > school > health > service > worship. + """ + amenity = tags.get("amenity") + tourism = tags.get("tourism") + leisure = tags.get("leisure") + railway = tags.get("railway") + pt = tags.get("public_transport") + aeroway = tags.get("aeroway") + shop = tags.get("shop") + historic = tags.get("historic") + + if amenity in {"restaurant", "cafe", "bar", "fast_food", "pub", + "food_court", "ice_cream", "biergarten"}: + return CATEGORIES["food"] + if (railway in {"station", "halt", "tram_stop"} + or pt == "station" + or amenity == "ferry_terminal" + or aeroway == "aerodrome"): + return CATEGORIES["transit"] + if leisure in {"park", "playground", "garden", "nature_reserve"}: + return CATEGORIES["park"] + if tourism in {"museum", "gallery"} or amenity in {"theatre", "cinema", "arts_centre", "library"}: + return CATEGORIES["culture"] + if tourism in {"attraction", "viewpoint", "zoo", "aquarium"} or historic: + return CATEGORIES["attraction"] + if shop in SHOP_ALLOWED: + return CATEGORIES["shop"] + if amenity in {"school", "university", "college"}: + return CATEGORIES["school"] + if amenity in {"hospital", "clinic", "pharmacy", "doctors"}: + return CATEGORIES["health"] + if amenity in {"post_office", "bank", "fuel", "police", "fire_station"}: + return CATEGORIES["service"] + if amenity == "place_of_worship": + return CATEGORIES["worship"] + return None + + +# Categories we KEEP even when name is missing (parks/playgrounds). +KEEP_UNNAMED_CATS = {CATEGORIES["park"]} + + +@dataclass +class _RawPOI: + lon: float + lat: float + category: int + name: str + + +class _WayCollector(osmium.SimpleHandler): + """Pass 0: find category-tagged ways; record node refs + tags.""" + + def __init__(self): + super().__init__() + self.way_tags: dict[int, dict[str, str]] = {} + self.way_refs: dict[int, list[int]] = {} + + def way(self, w): + tags = dict(w.tags) + cat = classify_tags(tags) + if cat is None: + return + name = tags.get("name", "") + if not name and cat not in KEEP_UNNAMED_CATS: + return + self.way_tags[w.id] = tags + self.way_refs[w.id] = [n.ref for n in w.nodes] + + +class _NodeCollector(osmium.SimpleHandler): + """Pass 1: collect standalone POIs (node-tagged) + coords for way nodes.""" + + def __init__(self, way_refs_flat: set[int]): + super().__init__() + self.way_refs_flat = way_refs_flat + self.standalone: list[_RawPOI] = [] + self.way_node_coords: dict[int, tuple[float, float]] = {} + + def node(self, n): + if n.id in self.way_refs_flat: + self.way_node_coords[n.id] = (n.location.lon, n.location.lat) + tags = dict(n.tags) + cat = classify_tags(tags) + if cat is None: + return + name = tags.get("name", "") + if not name and cat not in KEEP_UNNAMED_CATS: + return + self.standalone.append(_RawPOI(n.location.lon, n.location.lat, cat, name)) + + +def extract_pois( + pbf_path: Path, + bbox: tuple[float, float, float, float], + walk_graph: WalkGraphReader | None = None, +): + """Yield `POI` records from an OSM PBF. If `walk_graph` is None, + walk_node = 0 (used by extraction-only tests).""" + pbf_path = Path(pbf_path) + min_lon, min_lat, max_lon, max_lat = bbox + + # Pass 0: ways. + wc = _WayCollector() + wc.apply_file(str(pbf_path)) + + way_refs_flat: set[int] = set() + for refs in wc.way_refs.values(): + way_refs_flat.update(refs) + + # Pass 1: standalone POIs + way-node coords. + nc = _NodeCollector(way_refs_flat) + nc.apply_file(str(pbf_path)) + + # Compute centroids for ways. + way_pois: list[_RawPOI] = [] + for way_id, tags in wc.way_tags.items(): + coords = [nc.way_node_coords.get(n) for n in wc.way_refs[way_id]] + coords = [c for c in coords if c is not None] + if not coords: + continue + lons, lats = zip(*coords) + clon = (min(lons) + max(lons)) / 2 + clat = (min(lats) + max(lats)) / 2 + cat = classify_tags(tags) + way_pois.append(_RawPOI(clon, clat, cat, tags.get("name", ""))) + + all_raw = nc.standalone + way_pois + + for r in all_raw: + if not (min_lon <= r.lon <= max_lon and min_lat <= r.lat <= max_lat): + continue + if walk_graph is None: + walk_node = 0 + else: + node = walk_graph.snap(r.lon, r.lat) + if node is None or node not in walk_graph.lcc_nodes: + node = walk_graph.snap_in_lcc(r.lon, r.lat, max_m=100.0) + if node is None: + continue + walk_node = node + yield POI( + lon=r.lon, lat=r.lat, walk_node=walk_node, + category=r.category, name=r.name, + ) + + +def main(argv: list[str]) -> int: + if len(argv) != 4: + print("usage: pois.py ", + file=sys.stderr) + return 2 + pbf, wg, out = map(Path, argv[1:]) + + walk = WalkGraphReader(wg.read_bytes()) + # scripts/bbox.env BASEMAP_BBOX = -74.30,40.49,-71.85,41.20 + bbox = (-74.30, 40.49, -71.85, 41.20) + pois = list(extract_pois(pbf, bbox, walk)) + pois.sort(key=lambda p: (p.category, p.name)) + blob = write_poi_blob(pois, walk_graph_version=walk.version) + out.write_bytes(blob) + print(f"wrote {out} with {len(pois)} POIs ({len(blob)} bytes)") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/pipelines/tests/test_poi_emit.py b/pipelines/tests/test_poi_emit.py new file mode 100644 index 0000000..957c502 --- /dev/null +++ b/pipelines/tests/test_poi_emit.py @@ -0,0 +1,56 @@ +import struct + +import pytest + +from pipelines.poi_emit import POI, write_poi_blob, read_poi_header + + +def test_round_trip_three_pois(): + pois = [ + POI(lon=-74.0, lat=40.7, walk_node=0, category=1, name="Joe's Pizza"), + POI(lon=-73.99, lat=40.71, walk_node=5, category=3, name="Central Park"), + POI(lon=-73.95, lat=40.78, walk_node=9, category=4, name=""), # unnamed + ] + blob = write_poi_blob(pois, walk_graph_version=1) + assert blob[:4] == b"POI1" + header = read_poi_header(blob) + assert header["version"] == 1 + assert header["walk_graph_version"] == 1 + assert header["n_pois"] == 3 + + # Record 0 at offset 24. + rec0 = struct.unpack_from(" 100_000 + # Snap a street-adjacent point (14th & Park, NYC). + node = r.snap(-73.9879, 40.7363) + assert node is not None + assert 0 <= node < r.n_nodes + # Reachable from LCC. + assert node in r.lcc_nodes + # LCC should be the dominant component. + assert len(r.lcc_nodes) > r.n_nodes // 2 + + +def test_snap_in_lcc_returns_lcc_member(): + bin_path = REPO / "tiles" / "walk_graph.bin" + if not bin_path.exists(): + pytest.skip("walk_graph.bin missing") + r = WalkGraphReader(bin_path.read_bytes()) + node = r.snap_in_lcc(-73.9879, 40.7363, max_m=200.0) + assert node is not None + assert node in r.lcc_nodes + + +def test_snap_in_lcc_returns_none_when_too_far(): + bin_path = REPO / "tiles" / "walk_graph.bin" + if not bin_path.exists(): + pytest.skip("walk_graph.bin missing") + r = WalkGraphReader(bin_path.read_bytes()) + # Middle of the Atlantic — nothing within 100 m. + assert r.snap_in_lcc(-65.0, 35.0, max_m=100.0) is None diff --git a/pipelines/walk_graph_reader.py b/pipelines/walk_graph_reader.py new file mode 100644 index 0000000..56a1c9a --- /dev/null +++ b/pipelines/walk_graph_reader.py @@ -0,0 +1,195 @@ +""" +Decode a walk_graph.bin produced by pipelines/walk_graph.py. + +Binary layout (header + bincode body): + magic : 4 bytes = b"NWLK" + version : u32 LE + nodes : vec<(f64 lon, f64 lat)> + adj_offsets : vec # CSR; len = n_nodes + 1 + edges : vec<(u32 to, u32 seconds, u32 poly_start, u32 poly_end)> + polylines : vec<(f64 lon, f64 lat)> (skipped here) + snap_min_lon/lat : f64, f64 (skipped here) + snap_cell_deg : f64 (skipped here) + snap_cols/rows : u32, u32 (skipped here) + snap_cell_offsets : vec (skipped here) + snap_cell_nodes : vec (skipped here) + +We rebuild a simple snap dict and LCC ourselves so the reader is self- +contained — useful when offline tools (pois.py) need a stable view of +which nodes are routable. +""" +from __future__ import annotations + +import math +import struct + + +MAGIC = b"NWLK" +SNAP_CELL_DEG = 0.0025 # ~278 m at 40°N — local snap grid for our pre-snap + + +class _Cursor: + __slots__ = ("buf", "off") + + def __init__(self, buf: bytes): + self.buf = buf + self.off = 0 + + def take(self, fmt: str): + size = struct.calcsize(fmt) + vals = struct.unpack_from("<" + fmt, self.buf, self.off) + self.off += size + return vals + + def u32(self) -> int: + (v,) = self.take("I") + return v + + def u64(self) -> int: + (v,) = self.take("Q") + return v + + def f64(self) -> float: + (v,) = self.take("d") + return v + + def vec_lonlat(self) -> list[tuple[float, float]]: + n = self.u64() + # 2 f64 per item; unpack one big block for speed. + size = 16 * n + flat = struct.unpack_from(f"<{2*n}d", self.buf, self.off) + self.off += size + return list(zip(flat[0::2], flat[1::2])) + + def vec_u32(self) -> list[int]: + n = self.u64() + out = list(struct.unpack_from(f"<{n}I", self.buf, self.off)) + self.off += 4 * n + return out + + def vec_walk_edge(self): + n = self.u64() + # 4 u32 per edge. + flat = struct.unpack_from(f"<{4*n}I", self.buf, self.off) + self.off += 16 * n + return flat # interleaved (to, sec, ps, pe, ...) — we only use `to` + + def skip_bytes(self, n: int): + self.off += n + + +class WalkGraphReader: + """Read-only view of a walk_graph.bin. Builds LCC + local snap on demand.""" + + def __init__(self, blob: bytes): + if blob[:4] != MAGIC: + raise ValueError(f"bad magic: {blob[:4]!r} (expected {MAGIC!r})") + cur = _Cursor(blob) + cur.skip_bytes(4) # magic + self.version = cur.u32() + self.nodes: list[tuple[float, float]] = cur.vec_lonlat() + self.n_nodes = len(self.nodes) + # adj_offsets is CSR; length = n_nodes + 1. + self._adj_off: list[int] = cur.vec_u32() + edges_flat = cur.vec_walk_edge() + # Build a Python adjacency list keyed by source node. + # edges_flat is (to, sec, ps, pe) repeating; we only need `to`. + self._edge_to: list[int] = list(edges_flat[0::4]) + # (We ignore polylines + snap grid + everything after — pois.py + # builds its own snap.) + + self._lcc: frozenset[int] | None = None + self._snap_cells: dict[tuple[int, int], list[int]] | None = None + + # ---- adjacency (CSR access) ------------------------------------------- + + def neighbors(self, u: int) -> list[int]: + return self._edge_to[self._adj_off[u]:self._adj_off[u + 1]] + + # ---- largest connected component -------------------------------------- + + @property + def lcc_nodes(self) -> frozenset[int]: + if self._lcc is None: + self._lcc = self._compute_lcc() + return self._lcc + + def _compute_lcc(self) -> frozenset[int]: + visited = bytearray(self.n_nodes) + best: set[int] = set() + for start in range(self.n_nodes): + if visited[start]: + continue + stack = [start] + comp: set[int] = set() + while stack: + u = stack.pop() + if visited[u]: + continue + visited[u] = 1 + comp.add(u) + for v in self.neighbors(u): + if not visited[v]: + stack.append(v) + if len(comp) > len(best): + best = comp + return frozenset(best) + + # ---- local snap grid --------------------------------------------------- + + def _build_snap(self) -> None: + cells: dict[tuple[int, int], list[int]] = {} + for i, (lon, lat) in enumerate(self.nodes): + key = (int(lon / SNAP_CELL_DEG), int(lat / SNAP_CELL_DEG)) + cells.setdefault(key, []).append(i) + self._snap_cells = cells + + def snap(self, lon: float, lat: float) -> int | None: + if self._snap_cells is None: + self._build_snap() + cx = int(lon / SNAP_CELL_DEG) + cy = int(lat / SNAP_CELL_DEG) + best_d = math.inf + best_n: int | None = None + for dx in (-1, 0, 1): + for dy in (-1, 0, 1): + for n in self._snap_cells.get((cx + dx, cy + dy), ()): + nlon, nlat = self.nodes[n] + d = self._dist_m(lon, lat, nlon, nlat) + if d < best_d: + best_d = d + best_n = n + return best_n + + def snap_in_lcc( + self, lon: float, lat: float, max_m: float = 100.0, + ) -> int | None: + if self._snap_cells is None: + self._build_snap() + lcc = self.lcc_nodes + cx = int(lon / SNAP_CELL_DEG) + cy = int(lat / SNAP_CELL_DEG) + best_d = math.inf + best_n: int | None = None + # Widen the search radius a little — LCC misses can land further + # than the immediate 3x3 cell ring. + for dx in (-2, -1, 0, 1, 2): + for dy in (-2, -1, 0, 1, 2): + for n in self._snap_cells.get((cx + dx, cy + dy), ()): + if n not in lcc: + continue + nlon, nlat = self.nodes[n] + d = self._dist_m(lon, lat, nlon, nlat) + if d < best_d: + best_d = d + best_n = n + return best_n if best_d <= max_m else None + + # ---- helpers ---------------------------------------------------------- + + @staticmethod + def _dist_m(lon1: float, lat1: float, lon2: float, lat2: float) -> float: + # Equirectangular; fine for our distances. + dx = (lon2 - lon1) * 85_000 + dy = (lat2 - lat1) * 111_000 + return math.sqrt(dx * dx + dy * dy) diff --git a/scripts/assemble-pois.sh b/scripts/assemble-pois.sh new file mode 100755 index 0000000..3f6e510 --- /dev/null +++ b/scripts/assemble-pois.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Reassemble tiles/pois.bin from its committed chunks. +# Idempotent: only assembles when the full file is missing or older than any part. +# Mirrors scripts/assemble-walk-graph.sh. +set -euo pipefail +cd "$(dirname "$0")/.." + +OUT=tiles/pois.bin +PARTS=(tiles/pois.part-*) + +if [[ ${#PARTS[@]} -eq 0 || ! -e "${PARTS[0]}" ]]; then + echo "no chunks found at tiles/pois.part-* — run scripts/build-pois.sh first" >&2 + exit 1 +fi + +needs_rebuild=0 +if [[ ! -f "$OUT" ]]; then + needs_rebuild=1 +else + for p in "${PARTS[@]}"; do + if [[ "$p" -nt "$OUT" ]]; then needs_rebuild=1; break; fi + done +fi + +if [[ "$needs_rebuild" -eq 1 ]]; then + echo "assembling $OUT from ${#PARTS[@]} chunks" + cat "${PARTS[@]}" > "$OUT" + ls -lh "$OUT" +else + echo "$OUT is up to date" +fi diff --git a/scripts/build-pois.sh b/scripts/build-pois.sh new file mode 100755 index 0000000..55a2812 --- /dev/null +++ b/scripts/build-pois.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Build tiles/pois.bin from the cached OSM PBF and pre-snap each POI to +# walk_graph.bin's largest connected component. +# +# Outputs tiles/pois.bin and chunks it to tiles/pois.part-* for GitHub's +# 100 MB per-file cap. The assembled .bin is gitignored; only the chunks ship. +# +# Requires: scripts/build-walk-graph.sh has been run (PBF + walk_graph.bin +# must exist locally). + +set -euo pipefail +cd "$(dirname "$0")/.." + +EXTRACT=pipelines/cache/ny-metro.osm.pbf +WG=tiles/walk_graph.bin +OUT=tiles/pois.bin + +if [[ ! -f "$EXTRACT" ]]; then + echo "ERROR: $EXTRACT not found. Run scripts/build-walk-graph.sh first." >&2 + exit 1 +fi +if [[ ! -f "$WG" ]]; then + echo "ERROR: $WG not found. Assemble from chunks or run scripts/build-walk-graph.sh first." >&2 + exit 1 +fi + +mkdir -p tiles + +echo "extracting POIs..." +uv run --directory pipelines python pois.py \ + "$(pwd)/$EXTRACT" "$(pwd)/$WG" "$(pwd)/$OUT" + +SIZE=$(stat -c%s "$OUT" 2>/dev/null || stat -f%z "$OUT") +echo "pois.bin size: $SIZE bytes" +if (( SIZE < 1000000 )); then + echo "ERROR: pois.bin suspiciously small ($SIZE bytes)" >&2 + exit 1 +fi +if (( SIZE > 30000000 )); then + echo "ERROR: pois.bin suspiciously large ($SIZE bytes; allowlist may have leaked)" >&2 + exit 1 +fi + +echo "chunking..." +rm -f tiles/pois.part-* +split -b 50M -a 2 "$OUT" tiles/pois.part- +ls -lh tiles/pois.part-* diff --git a/tiles/pois.part-aa b/tiles/pois.part-aa new file mode 100644 index 0000000..7dd094b Binary files /dev/null and b/tiles/pois.part-aa differ