// build.rs — stamp the commit identity into the binary, and (for the
// `embed-tract` / `embed-ort` features) fetch the jina-v2-base-code model
// weights.
//
// The `nornir.v1` proto and its prost+tonic codegen used to live here; they
// moved to the `nornir-wire` edda leaf (`nornir_wire::pb`). nornir now consumes
// the generated stubs from that crate rather than compiling the proto itself.
use std::io::Read;
use std::path::{Path, PathBuf};
// The embedding-model registry, shared verbatim with the crate
// (`src/vector/embed_registry.rs`). `include!` keeps `build.rs` and the runtime
// in lock-step: the model fetched here is exactly the one the embedder loads.
// The file is `std`-only by design (no sibling-module `use`, no extern crates).
#[allow(dead_code)] // build.rs only uses `selected`; the rest is the crate's API.
mod embed_registry {
include!("src/vector/embed_registry.rs");
}
/// Stamp the commit identity into the binary — the short git SHA and the build
/// epoch. Emitted UNCONDITIONALLY (every build, all features), before any
/// feature-gated early return.
///
/// The SHA is read from the git metadata on disk with **pure `std::fs`** — NO
/// `Command::new("git")`, per nornir's pure-Rust / zero-shell law (build code is
/// production code; it must not shell out). Handles a symbolic or detached HEAD,
/// `packed-refs`, and linked worktrees (`.git` as a pointer file). Falls back to
/// `unknown` off a git checkout.
///
/// Emits two envs carrying the SAME short SHA:
/// - `NORNIR_GIT_HASH` — the canonical identity hash (`lib.rs` builds
/// `VERSION_LINE`/`GIT_HASH` from it: `--version`, `Health.Ping`, About).
/// - `NORNIR_BUILD_SHA` — historical alias the viz "is this a fresh client?"
/// check reads; kept for back-compat.
fn emit_build_stamp() {
let hash = git_short_hash().unwrap_or_else(|| "unknown".to_string());
println!("cargo:rustc-env=NORNIR_GIT_HASH={hash}");
println!("cargo:rustc-env=NORNIR_BUILD_SHA={hash}");
let epoch = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
println!("cargo:rustc-env=NORNIR_BUILD_EPOCH={epoch}");
}
/// Read the short (7-char) commit hash from git metadata on disk without
/// shelling out. Returns `None` off a git checkout.
fn git_short_hash() -> Option<String> {
let (gitdir, commondir) = resolve_git_dirs()?;
println!("cargo:rerun-if-changed={}", gitdir.join("HEAD").display());
let head = std::fs::read_to_string(gitdir.join("HEAD")).ok()?;
let head = head.trim();
let full = if let Some(ref_name) = head.strip_prefix("ref: ") {
// Symbolic HEAD → resolve the branch ref (loose file or packed-refs).
resolve_ref(ref_name.trim(), &gitdir, &commondir)?
} else {
// Detached HEAD: the file IS the object id.
head.to_string()
};
let full = full.trim();
if full.len() < 7 || !full.bytes().all(|b| b.is_ascii_hexdigit()) {
return None;
}
Some(full[..7].to_string())
}
/// Locate the real gitdir and the common dir. For a normal checkout both are
/// `.git`. For a linked worktree `.git` is a FILE (`gitdir: <path>`) and the
/// shared refs live in the common dir named by `<gitdir>/commondir`.
fn resolve_git_dirs() -> Option<(PathBuf, PathBuf)> {
let dot_git = PathBuf::from(".git");
let meta = std::fs::metadata(&dot_git).ok()?;
let gitdir = if meta.is_dir() {
dot_git
} else {
println!("cargo:rerun-if-changed=.git");
let contents = std::fs::read_to_string(&dot_git).ok()?;
PathBuf::from(contents.trim().strip_prefix("gitdir:")?.trim())
};
let commondir = match std::fs::read_to_string(gitdir.join("commondir")) {
Ok(rel) => gitdir.join(rel.trim()),
Err(_) => gitdir.clone(),
};
Some((gitdir, commondir))
}
/// Resolve a symbolic ref (e.g. `refs/heads/main`) to an object id: try the
/// loose ref file under the gitdir, then the common dir, then `packed-refs`.
fn resolve_ref(ref_name: &str, gitdir: &Path, commondir: &Path) -> Option<String> {
for base in [gitdir, commondir] {
let loose = base.join(ref_name);
println!("cargo:rerun-if-changed={}", loose.display());
if let Ok(s) = std::fs::read_to_string(&loose) {
let s = s.trim();
if !s.is_empty() && !s.starts_with("ref: ") {
return Some(s.to_string());
}
}
}
let packed = commondir.join("packed-refs");
println!("cargo:rerun-if-changed={}", packed.display());
let packed = std::fs::read_to_string(&packed).ok()?;
for line in packed.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') || line.starts_with('^') {
continue;
}
// Each entry is "<sha> <refname>".
if let Some((sha, name)) = line.split_once(' ') {
if name.trim() == ref_name {
return Some(sha.trim().to_string());
}
}
}
None
}
fn main() {
emit_build_stamp();
// The `nornir.v1` proto + its prost/tonic codegen moved to the `nornir-wire`
// edda leaf (`nornir_wire::pb`); nornir no longer compiles the proto here.
println!("cargo:rerun-if-changed=build.rs");
if std::env::var_os("CARGO_FEATURE_EMBED_TRACT").is_some()
|| std::env::var_os("CARGO_FEATURE_EMBED_ORT").is_some()
{
fetch_model();
}
// `manual-embed`: stage the embeddable `book.znippy` so the crate can
// `include_bytes!` it unconditionally.
if std::env::var_os("CARGO_FEATURE_MANUAL_EMBED").is_some() {
stage_manual_embed();
}
// The host triple, for `viz::preflight`'s shell-free rustup install. `rustup-init`
// is published per-triple at static.rust-lang.org, and the shell script at
// sh.rustup.rs exists mainly to detect this — cargo already knows it, so nornir
// does not need a shell to ask.
println!(
"cargo:rustc-env=NORNIR_HOST_TRIPLE={}",
std::env::var("TARGET").unwrap_or_else(|_| "x86_64-unknown-linux-gnu".into())
);
}
/// Stage the embeddable manual at `$OUT_DIR/book.znippy` so the crate can
/// `include_bytes!` it unconditionally (feature `manual-embed`). Copies the
/// repo's generated `docs/book.znippy` when present; otherwise writes a 0-byte
/// placeholder so a clean checkout (manual not generated yet) still compiles —
/// the runtime then embeds nothing and falls back to reading the manual off disk.
fn stage_manual_embed() {
println!("cargo:rerun-if-changed=docs/book.znippy");
let out = PathBuf::from(std::env::var_os("OUT_DIR").expect("OUT_DIR")).join("book.znippy");
let src = Path::new("docs/book.znippy");
if src.is_file() {
std::fs::copy(src, &out).expect("stage docs/book.znippy → OUT_DIR");
} else {
std::fs::write(&out, []).expect("stage empty book.znippy placeholder");
}
}
/// Download the **selected** embedding model (onnx model + tokenizer.json) into
/// a stable per-model local cache, then export its path + content hashes to the
/// crate via `rustc-env`. The model is chosen by `$NORNIR_EMBED_MODEL` from the
/// shared [`embed_registry`] (default `jina-v2-base-code`, 768-dim) — the exact
/// same selection the embedder makes at runtime, so the fetched weights and the
/// loaded model can never disagree. Files are fetched once and reused; set
/// `NORNIR_SKIP_MODEL_FETCH=1` to use whatever is already present.
fn fetch_model() {
// Re-run if the model selection changes.
println!("cargo:rerun-if-env-changed=NORNIR_EMBED_MODEL");
println!("cargo:rerun-if-env-changed=NORNIR_MODEL_CACHE");
let model = embed_registry::selected().unwrap_or_else(|e| panic!("{e}"));
// (remote path, local filename)
// The local filename is DATA now, not the literal "model.onnx" it used to be: a
// static-table model's weights are a `.safetensors`, and the loader looks for them
// under exactly the name this fetch writes. One authority (`weights_local_name`)
// for the writer here and the reader in `embed_static`.
let files: &[(&str, &str)] = &[
(model.weights_path, model.weights_local_name),
(model.tokenizer_path, "tokenizer.json"),
];
let cache = model_cache_dir(model.cache_subdir);
std::fs::create_dir_all(&cache).expect("create model cache dir");
// The registry's content pin, in the same order as `files`.
let pins = [model.weights_sha256, model.tokenizer_sha256];
let skip = std::env::var_os("NORNIR_SKIP_MODEL_FETCH").is_some();
let mut shas: Vec<String> = Vec::with_capacity(files.len());
for ((remote, local), pin) in files.iter().zip(pins) {
let dest = cache.join(local);
if !dest.exists() {
if skip {
panic!(
"NORNIR_SKIP_MODEL_FETCH set but {} is missing",
dest.display()
);
}
let url = format!(
"https://huggingface.co/{}/resolve/main/{remote}?download=true",
model.hf_repo
);
eprintln!(
"nornir: fetching {remote} for embed feature (model `{}`) …",
model.id
);
if let Err(e) = download(&url, &dest) {
// A SEALED build (the pinned toolchain container runs
// `--network none`) cannot fetch. The model is a RUNTIME input:
// `embed_support::model_dir` finds the files when the binary
// runs, and all the compile needs is its identity, which the pin
// is. Unpinned, there is no identity to compile in, so refuse.
let Some(pin) = pin else {
panic!(
"{e}\n`{}` has no content pin in embed_registry (weights_sha256 / \
tokenizer_sha256), so a build that cannot fetch it has no identity \
to compile in. Pin it, or put the files at {} first.",
model.id,
dest.display()
);
};
println!(
"cargo:warning=nornir: {local} for `{}` not fetched ({e}); compiling in its \
pinned identity {pin} and resolving the files at run time \
($NORNIR_MODEL_DIR, /opt/nornir/models, ~/.cache/nornir/models/{})",
model.id, model.cache_subdir
);
shas.push(pin.to_string());
continue;
}
}
let got = sha256_file(&dest);
if let Some(pin) = pin {
// `resolve/main` moves; the pin does not. Bytes that differ from the
// pin would silently change the model identity every vector is keyed by.
assert_eq!(
got,
pin,
"{} does not match the pin for `{}` in embed_registry — upstream moved or \
the cache is corrupt. Delete it to refetch, or re-pin deliberately.",
dest.display(),
model.id
);
}
shas.push(got);
}
let (weights_sha, tokenizer_sha) = (shas[0].clone(), shas[1].clone());
println!("cargo:rustc-env=NORNIR_MODEL_DIR={}", cache.display());
println!("cargo:rustc-env=NORNIR_MODEL_WEIGHTS_SHA={weights_sha}");
println!("cargo:rustc-env=NORNIR_MODEL_TOKENIZER_SHA={tokenizer_sha}");
}
fn model_cache_dir(subdir: &str) -> PathBuf {
// `NORNIR_MODEL_CACHE`, when set, is the model dir itself (back-compat: the
// operator points it straight at one model's files).
if let Some(dir) = std::env::var_os("NORNIR_MODEL_CACHE") {
return PathBuf::from(dir);
}
let base = std::env::var_os("HOME")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(std::env::var_os("OUT_DIR").expect("OUT_DIR")));
base.join(".cache/nornir/models").join(subdir)
}
/// Fetch `url` into `dest`. `Err` only when the request itself fails (no
/// network, DNS, an HTTP error) — the one case a sealed build recovers from
/// with a pin (see `fetch_model`). A failure after bytes started is a panic.
fn download(url: &str, dest: &Path) -> Result<(), String> {
let resp = ureq::Agent::config_builder().proxy(None).max_redirects(5).build().new_agent().get(url)
.call()
.map_err(|e| format!("download {url}: {e}"))?;
let mut reader = resp.into_body().into_reader();
let tmp = dest.with_extension("part");
let mut out =
std::fs::File::create(&tmp).unwrap_or_else(|e| panic!("create {}: {e}", tmp.display()));
let mut buf = [0u8; 1 << 16];
loop {
let n = reader
.read(&mut buf)
.unwrap_or_else(|e| panic!("read {url}: {e}"));
if n == 0 {
break;
}
use std::io::Write;
out.write_all(&buf[..n])
.unwrap_or_else(|e| panic!("write {}: {e}", tmp.display()));
}
drop(out);
std::fs::rename(&tmp, dest)
.unwrap_or_else(|e| panic!("rename {} -> {}: {e}", tmp.display(), dest.display()));
Ok(())
}
fn sha256_file(path: &Path) -> String {
nornir_hash::sha256_file(path).unwrap_or_else(|e| panic!("hash {}: {e}", path.display()))
}