nordisk/korp-collectors

Cargo.toml

# The workspace. Until 2026-08-30 this manifest was a bare `[package]` and cargo
# treated it as a one-member workspace by implication; the members list is
# explicit now because there is a second member: `crates/korp-collectors-build-thing`,
# this repo's `impl BuildThing` (lane AG). Nothing korp-collectors ships links it
# — it exists so `.nornir/build-variants.toml` can be EMITTED from code instead
# of hand-written, and dwarves' forge finds it by package name.
[workspace]
resolver = "2"
members = [".", "crates/korp-collectors-build-thing"]

[package]
name = "korp-collectors"
# 0.1.8: DRIFTED from the published 0.1.7 — this tree carries the 2026-08-22/23 work
# (region pick-list keyed on the planet that produced it, the silenced-test audit
# landing RED here, and the OSM collector refusing to call a geography-free cut ONLINE).
# crates.io bytes are immutable, so consumers only see it as a new version.
version = "0.2.0"
edition = "2024"
license = "MIT OR Apache-2.0"
description = "korp-collectors — the standalone LEAF of korp's open-data collector layer: one object-safe `Collector` trait + per-access-METHOD import collectors (DCAT-AP-SE, PxWeb/JSON-stat2, ENTSO-E, generic REST/JSON, Trafikverket, OSM) parsing pure-Rust → Arrow, plus a `RegionSource` seam that FETCHES OSM extracts. Depends on neither korp nor facett so both can consume it (breaks the facett⇄korp cycle)."
repository = "https://git.gunnar.rs/nordisk/korp-collectors"
readme = "README.md"
# Development notes, agent instructions and machine-local tooling stay in the
# repository; they are not part of the published crate.
exclude = [
    ".nornir/",
    "CLAUDE.md",
    "bench_history.json",
    "brokkr.toml",
]

[features]
# DEFAULT = the object-safe collector trait/registry surface + the shared HTTP
# core + every LIGHT pure-Rust parser (DCAT-AP-SE RDF, PxWeb/JSON-stat2, ENTSO-E
# XML, generic REST/JSON, Trafikverket) → Arrow, plus the pure OSM admin-boundary
# tag-filter/trie. No heavy geo dep, no skade, no Iceberg — a consumer (e.g.
# facett-demo) that only wants the parsers + the memory BatchSink pulls just this.
default = ["parsers"]
parsers = [
    "dep:reqwest", "dep:rustls", "dep:webpki-root-certs",
    "dep:quick-xml",
    "dep:oxrdfxml",
    "dep:oxrdf",
    "dep:governor",
    "dep:async-trait",
    "dep:anyhow",
]
# `osm` — the OSM geometry connectors (osm.rs way→WKB + the OSM admin-relation
# read) via znippy-zoomies/osm-katana, AND the `RegionSource` fetch seam
# (region.rs: CachedLocalSource + HttpOsmSource). Heavier geo dep → opt-in.
osm = ["parsers", "dep:osm-katana"]
# `daemon` — the collector DAEMON architecture (`src/daemon.rs`): the object-safe
# Config + State + Collector trait trio, the `CollectorCtx` seams (S3 bucket / jera
# phase handle / log / BIOS status bus), the `Phase` state machine and the
# self-registering daemon registry. Per `.nornir/collectors-daemon-design.md`. Pure
# (`serde` + `async-trait` + `tokio`, all already under `parsers`) — the leaf pulls
# NOTHING new; korp wires the concrete aegir/jera backends behind the seams.
# `blake3` rides along because `acquire::Completeness::Blake3` is the only honest answer
# for a source that publishes a digest, and a completeness enum whose digest arm refuses
# to verify would be a marker that cannot fail. Same crate, same major, as znippy and
# skade — one hashing path in the house, not a second opinion. Measured cost of adding it
# to this step: +4 crates (blake3, arrayref, arrayvec, constant_time_eq).
daemon = ["parsers", "dep:redb", "dep:blake3"]
# `osm-daemon` — the `osm-cut` daemon (`src/osm_cut.rs`), the reference implementor of
# `daemon`: planet → region catalog (osm-katana, gatling) → the lettered a..g region
# cut → ONLINE. Pulls `osm` (osm-katana) + `daemon` + the `gatling` (rotaryengine)
# fork-join engine osm-katana's convert drives.
osm-daemon = ["daemon", "osm", "dep:rotaryengine", "dep:libc", "dep:blake3"]
# `osm-daemon-download` — turn on the in-collector librqbit planet TORRENT (design
# §9: "the download lives HERE"). Forwards osm-katana's `planet-fixture` (librqbit +
# tokio); OFF by default so the lean build/tests pull no BitTorrent client and rely
# on the on-disk seed. Mirrors korp's own `planet-download` gate.
osm-daemon-download = ["osm-daemon", "osm-katana/planet-fixture"]
# `kvm-guest` — build `korp-chain-init`, the a–e chain driver that runs as PID 1 inside
# a QEMU/KVM guest (see the `[[bin]]` block below and `tests/osm_chain_kvm.rs`). Pulls
# nothing new: `osm-daemon` already carries `libc` (the mount/reboot syscalls) and
# `serde_json` is unconditional. Kept a NAMED feature rather than folded into
# `osm-daemon` so the guest binary cannot be dragged into a substrate build by accident.
kvm-guest = ["osm-daemon"]
# `service` — **this build IS a service.** The nursery (`src/nursery.rs`): one process
# that supervises many collectors, restarts them with backoff, publishes each one's live
# phase+facet to the register so a separate process can read it, and takes its two runtime
# decisions — wait for the network, and what to do when the work dies — from
# `skidbladnir::readiness` rather than reimplementing either. That is why the feature
# carries the Skidbladnir dep: a *service* needs lifecycle, and a parsers-only consumer
# that only wants the collectors gets neither.
service = ["daemon", "dep:skidbladnir"]
# `systemd` — the THIRD runtime substrate: a collector as its own systemd unit, with
# its own cgroup, its own `Restart=` and its own service account. Every byte of systemd
# handling routes through **Skidbladnir** (which owns service lifecycle in this house):
# this crate contributes one `UnitSpec -> ComponentSpec` adapter and renders no unit
# text. Off by default so a parser-only consumer pulls no service framework.
#
# Implies `service`: installing a collector as a unit and having nothing to install are
# not two configurations worth supporting, and the unit's `ExecStart` is the nursery.
systemd = ["service"]
# `skade-sink` — land the parsed Arrow batches in Apache Iceberg through skade
# (the `BatchSink::iceberg` backend + the path-agnostic `run()` runner). Optional
# so a parser-only consumer needs no Iceberg engine.
skade-sink = ["parsers", "dep:skade", "dep:futures-util"]
# `csv` — the RFC-4180 (+ real-world dialects) CSV collector: delimiter sniff,
# BOM strip, Latin-1/UTF-8 transcode, decimal-comma, and a typed two-pass schema
# sniff → Arrow. Pulls only the light pure-Rust `csv` + `encoding_rs`.
csv = ["parsers", "dep:csv", "dep:encoding_rs"]
# `mainframe` — the EBCDIC + COBOL-copybook + COMP-3 fixed-width collector (and
# its ASCII fixed-width cousin). Fully hand-rolled (copybook PIC parser, IBM-037
# table, packed/zoned/binary decode) → zero new deps, keeping the leaf pure.
mainframe = ["parsers"]
# `gtfs` — the GTFS static transit feed collector: a **pure-Rust ZIP reader**
# (central-directory walk, STORED + DEFLATE via flate2's miniz_oxide backend, no
# `zip` crate) over the feed archive, then the `csv` reader per known GTFS file
# (`stops`/`routes`/`trips`) → typed Arrow. Pulls only `flate2` (pure-Rust) + the
# same light `csv` reader the CSV collector uses.
gtfs = ["parsers", "dep:flate2", "dep:csv"]
# `gribnc` — the NetCDF/GRIB weather-model collector: Copernicus CDS (retrieve
# API, key from env `CDSAPI_KEY`) + DWD ICON open data. Hand-rolled PURE-RUST
# GRIB2 + NetCDF-classic decoders (zero new deps, like the mainframe/gtfs-rt
# hand-rolled binary readers): a variable/time/bbox METADATA CATALOG AND the
# data-ARRAY unpack → Arrow. GRIB2 Section 5/6/7 decodes simple (5.0) + complex
# (5.2) + complex-with-spatial-differencing (5.3) packing with Section-6 bitmaps;
# NetCDF-classic reads variable data arrays (fixed + record dims, numeric types).
# `params.mode = "values"` emits per-point grid values; default = catalog. Only
# NetCDF-4/HDF5 (`\x89HDF`) stays DEFERRED behind a would-be OPTIONAL native
# `netcdf` (libnetcdf C) feature — documented, never default; not wired yet.
gribnc = ["parsers"]
# `znippy` — the collected-data CONVERTER: pack parsed Arrow batches (or raw byte
# blobs) into a **znippy** archive (compact, blake3-checksummed, OpenZL-compressed
# storage/transport) and read them back BYTE-IDENTICAL. Reuses the `znippy-common`
# pack format (no bespoke compression) + `arrow-ipc` for the batch serialization.
# Opt-in (pulls the OpenZL C codec via znippy-common), like `osm`/`skade-sink`.
znippy = ["parsers", "dep:znippy-common", "dep:arrow-ipc", "dep:tempfile"]
# `testmatrix` — turn the tests' `functional_status(component, check, ok, detail)`
# self-reports into real nornir test-matrix rows. Compiled OUT of the default
# build (no nornir dep by default); pulls nornir-testmatrix's own `testmatrix`
# mode so the emit is a row-append, not a no-op. Run: `cargo test --features
# testmatrix,osm,skade-sink`.
testmatrix = ["dep:nornir-testmatrix", "nornir-testmatrix/testmatrix"]

[dependencies]
arrow-array = "59"
arrow-schema = "59"
arrow-select = "59"
parquet = { version = "59", default-features = false, features = ["arrow"] }
bytes = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"

# ── parsers feature ─────────────────────────────────────────────────────────
# Shared HTTP core + pure-Rust parsers. `blocking` powers the synchronous
# `RegionSource::ensure` fetch (osm feature) while the async client serves the
# streaming collectors; both share one rustls stack.
#
# `gzip` is not a size optimisation, it is a HARD REQUIREMENT of a source korp
# reads: Fintraffic's Digitraffic gateway answers every request that does not
# carry `Accept-Encoding: gzip` with **HTTP 406** and the body
# `Use of gzip compression is required with Accept-Encoding: gzip header.`
# (measured live 2026-09-03 against `meri.digitraffic.fi/api/ais/v1/locations`).
# reqwest sends the header and transparently inflates only when this feature is
# on, so without it the AIS collector cannot fetch at all.
reqwest = { version = "0.13", default-features = false, features = ["rustls-no-provider", "json", "blocking", "gzip", "query", "form"], optional = true }
# reqwest 0.13 has no bundled roots and its `rustls` feature means aws-lc-rs (C) + the
# OS trust store; ring + the Mozilla bundle keep 0.12 `rustls-tls` behaviour (see the
# `tls_roots` helpers). 2026-10-02.
rustls = { version = "0.23", default-features = false, features = ["ring", "std"], optional = true }
webpki-root-certs = { version = "1", optional = true }
quick-xml = { version = "0.37", optional = true }

# ── csv feature ───────────────────────────────────────────────────────────────
# BurntSushi's canonical RFC-4180 reader + encoding_rs for BOM strip / Latin-1
# transcode. Both light, pure-Rust, feature-gated so parser-only consumers skip
# them.
csv = { version = "1.4", optional = true }
encoding_rs = { version = "0.8", optional = true }

# ── gtfs feature ──────────────────────────────────────────────────────────────
# Pure-Rust DEFLATE (miniz_oxide backend — flate2's default `rust_backend`, no C
# zlib) for the GTFS feed ZIP. The ZIP container itself is hand-rolled in
# `gtfs.rs` (central-directory walk), keeping the leaf free of the `zip` crate.
flate2 = { version = "1", optional = true }
oxrdfxml = { version = "0.2.3", optional = true }
oxrdf = { version = "0.3.3", optional = true }
governor = { version = "0.10", optional = true }
async-trait = { version = "0.1", optional = true }
anyhow = { version = "1", optional = true }
tokio = { version = "1", default-features = false, features = ["rt", "rt-multi-thread", "macros", "time", "sync", "signal"], optional = false }

# ── osm feature ─────────────────────────────────────────────────────────────
# OSM PBF/XML → WKB geometry (znippy-zoomies geo primitives; pure-Rust CPU).
osm-katana = { path = "../znippy-zoomies/osm-katana", version = "0.3.0", optional = true }

# ── osm-daemon feature ───────────────────────────────────────────────────────
# The `gatling` fork-join engine (crate `rotaryengine`, aliased `gatling`) — the
# EXACT primitive `osm_katana::tiles::tile_region` fans its per-tile encode over,
# so the all-cores tiling bencher drives it directly to prove the cut saturates
# every core through gatling (NOT rayon). Same crate/version osm-katana pins.
rotaryengine = { path = "../znippy-zoomies/gatling", version = "0.1.13", package = "rotaryengine", optional = true }

# `statvfs(2)` — the ONE thing the standard library cannot answer and the collector
# must know before starting a 31–35 minute planet convert: how many bytes are free on
# the filesystem it is about to write 6 GB of extract and 2 GB of pyramid into. On
# 2026-08-02 `/home` hit 100 % three times, killing a build (`mold: failed to write to
# an output file. Disk full?`) and a full re-cut (`No space left on device`).
# `libc` is bindings only — no runtime, no transitive tree — and is ALREADY in this
# crate's lock file transitively, so it costs nothing to compile and nothing to the
# leaf discipline (it is not a korp/facett/geo dep). Gated to `osm-daemon`; the
# parser-only consumer still pulls it not at all.
libc = { version = "0.2", optional = true }

# The planet's IDENTITY. The owner, 2026-08-02: an extract is stale when a NEW planet
# exists to fetch — not when a timer expires — so the source needs a digest, and the
# extract stamp carries it. blake3 is the fleet's hash already (znippy, skade); this
# is the SAME crate at the SAME major, not a second hashing path (LAW 5).
#
# Gated to `daemon` (not `osm-daemon`) because `acquire::Completeness::Blake3` needs it,
# and a large-artefact seam whose digest check is compiled out is a completeness marker
# that cannot say no.
blake3 = { version = "1", optional = true }

# ── znippy feature ──────────────────────────────────────────────────────────
# The znippy pack format (lean crate: create_archive_to_vec / ZnippyArchive) +
# Arrow-IPC for the batch serialization + tempfile to stage in-memory archive
# bytes for the path-based reader. Feature-gated so parser-only consumers pull
# none of the OpenZL C codec.
znippy-common = { path = "../znippy/znippy-common", version = "0.10.0", optional = true }
arrow-ipc = { version = "59", optional = true }
tempfile = { version = "3", optional = true }

# ── daemon feature: the collector STATE STORE (src/state.rs) ──────────────────
# The embedded ACID ledger where each step of the a–e chain records that it FINISHED.
# redb 2 is already the constellation's embedded store (nornir's funnel, edda's jera), so
# this is the same crate at the same major rather than a second embedded-KV opinion.
# Pure Rust, single file, one writer + lock-free readers (see the module's concurrency
# note — redb takes flock(LOCK_EX|LOCK_NB) and that trap is designed for, not discovered).
redb = { version = "2", optional = true }

# ── skade-sink feature ──────────────────────────────────────────────────────
skade = { version = "0.6.0", default-features = false, optional = true }
futures-util = { version = "0.3", optional = true }

# ── testmatrix feature ──────────────────────────────────────────────────────
# Path dep guarantees the `functional_status` emitter (nornir-testmatrix ≥0.2.0);
# off by default so consumers pull nothing.
nornir-testmatrix = { version = "0.2", default-features = false, optional = true }

# Service lifecycle for the `systemd` substrate — account, data directory, unit,
# enable/start, and THE uninstall. Skidbladnir owns all of it; this crate only adapts a
# `UnitSpec` into its `ComponentSpec`. Optional, `default-features = false` (no PKI, no
# kube), so the leaf's default build links nothing of it.
skidbladnir = { path = "../Skidbladnir", version = "0.2.0", default-features = false, optional = true }

[dev-dependencies]
arrow-array = "59"
arrow-schema = "59"
# `test-util` is here for ONE reason and it is a correctness one: `tests/not_startable.rs`
# proves a daemon with no key does not tick a second time, and the wait between ticks is
# 30 s growing to 3600 s. `tokio::time::pause()` makes that assertion take milliseconds
# instead of an hour, so the guard can live in a plain `cargo test` rather than being the
# kind of slow test somebody eventually marks `#[ignore]`.
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "time", "sync", "test-util"] }
tempfile = "3"
# Per-collector parse/normalize throughput benches (benches/collectors_parse.rs),
# fed from the offline-safe `sample::*_try_get_real_sample_data` source so they
# run with no network. Pure-Rust harness; matches the znippy-zoomies bench setup.
criterion = { version = "0.8", features = ["html_reports"] }
# The KVM boot harness (`tests/osm_chain_kvm.rs`) — tunnr's pure-Rust QEMU+OVMF+KVM
# primitive: `locate_kernel`/`locate_ovmf`, the cpio+gzip initramfs packer, and the
# KILLABLE `boot_test`/`BootHandle` whose `kill()` tears down the QEMU process group.
# That last one is the whole reason this is a dep and not 200 lines of local
# `Command::new("qemu-system-x86_64")` (LAW 5): a power cut mid-convert is the ONE
# interruption a cooperative host-side test cannot express, and this is the seam that
# expresses it. Default features only — libc + rotaryengine + flate2, all three already
# in this crate's graph under `osm-daemon`, so the dev graph gains no new weight.
tunnr-vm = { path = "../tunnr" }
# **The silenced-test guard, on a PLAIN `cargo test`.** Not the optional
# `testmatrix` feature above (that one is the functional-status EMITTER) — this is
# the audit that reads `.nornir/testmatrix-arms.json` and fails when a test target
# compiles to `running 0 tests` without a declared rescue arm. It is a dev-dep and
# NOT feature-gated on purpose: a guard against dark test files that is itself only
# built behind a feature is the joke writing itself. See
# `tests/this_repo_has_no_silenced_tests.rs`.
nornir-testmatrix = { version = "0.2.8", default-features = false }

# The parse/normalize throughput bench per collector. `harness = false` → the
# criterion runner owns `main`. Fed from the offline-safe sample source.
#
# Needs `parsers` — and only `parsers`. Its gtfs/osm benchmarks already carry
# their own `#[cfg(feature = ...)]`, so requiring those here would stop the bench
# building in the DEFAULT config for no reason.
#
# Declared as `required-features` rather than the in-file `#![cfg(...)]` self-gate
# the TEST targets use, because `harness = false` means this file owns `main` and
# cfg-ing the file away would delete the `main` the bench target must link.
[[bench]]
name = "collectors_parse"
harness = false
required-features = ["parsers"]

# ── the KVM guest PID 1 (feature `kvm-guest`) ────────────────────────────────
#
# `korp-chain-init` is the a–e chain driver that runs INSIDE a QEMU/KVM guest, built
# static and packed as `/init` in an initramfs by `tests/osm_chain_kvm.rs`. It is not a
# shipped product and no substrate execs it: it exists so the three properties the
# collector's design claims — survives a restart, keeps producing artefacts, nukes an
# incomplete one and reproduces from the last healthy step — are asserted on a machine
# that can actually be cut off at the power.
#
# `required-features` is what makes the name true: it cannot be produced without the
# osm-cut daemon it drives, so a file called `korp-chain-init` always carries the real
# chain and never a stub. OFF by default — a parser-only consumer builds no VM guest.
# The provider-fixture fetcher (part 3 of the owner's not-startable pattern, 2026-09-06).
# `daemon` for BLAKE3, the house hash the provenance manifest records — the fetcher does
# not otherwise need the daemon architecture. It is a dev tool: nothing in a test or a
# shipped path runs it, and it opens a socket, which is exactly why it is not a test.
[[bin]]
name = "collector-fixtures"
path = "src/bin/collector-fixtures.rs"
required-features = ["daemon"]

[[bin]]
name = "korp-chain-init"
path = "src/bin/korp-chain-init.rs"
required-features = ["kvm-guest"]

# 2026-10-05 (merge to main): no [patch.crates-io] any more — crates.io serves
# skade, znippy-zoomies (published 2026-10-04; the siblings carry the same code, at most test/doc edits since).
# znippy-zoomies, for the SAME reason korp patches it (korp/Cargo.toml:988-999) and
# for one more: the crates.io 0.1.15 that `skade` and `skade-katalog` resolve to
# declares rayon, and LAW 3 forbids it on a shipped edge. Measured 2026-08-03 at
# korp-collectors 1971f91: `cargo tree -e normal -i rayon --target all --all-features`
# printed `rayon <- znippy-zoomies 0.1.15 <- {skade, skade-katalog} <- korp-collectors`,
# and `--no-default-features --features skade-sink` printed the same. korp itself is
# clean under `--features collectors` — not because of different code, but because korp's
# own `[patch.crates-io]` already unifies every znippy-zoomies edge onto the local,
# rayon-free 0.1.21. This table did not, so korp-collectors built as its OWN root linked
# rayon while the identical crate inside korp did not. Same patch, same line, same reason.

# The owner, 2026-09-05: test artifacts carry line tables only. Full DWARF in every test
# binary was the bulk of a target dir on the day the disks filled (~/scratch/cargo held
# 27.9 GB for this crate alone, korp/target 852 GB); backtraces still show file:line.
[profile.test]
debug = "line-tables-only"