mikronova/modgunn
benches/engine_ab.rs
//! HEAVY benchmark — **Engine A (linear) vs Engine B (inverted index)** head-to-head.
//!
//! Builds a large synthetic advisory db + component fleet and scans the whole fleet
//! through each engine at **full cores** (the SAME znippy-zoomies gatling fork-join
//! the production `AdvisoryEngine::fleet` / warehouse scan use — no rayon, one shared
//! engine), reporting build time, wall time, throughput (verdicts/sec), the winner,
//! and peak process RSS. It also *asserts A ≡ B* on the total finding count so the
//! benchmark doubles as a large-scale equivalence check.
//!
//! No smoke variant — sizes default to a heavy corpus and are only overridable
//! UPWARD in practice. Tunables (env):
//! MODGUNN_BENCH_ADVISORIES (default 25_000)
//! MODGUNN_BENCH_PACKAGES (default 3_000)
//! MODGUNN_BENCH_ARTIFACTS (default 1_000_000)
//! MODGUNN_BENCH_REPEATS (default 3 — timed passes per engine, best kept)
//!
//! Run: `CARGO_TARGET_DIR=/mnt/work4t/... cargo bench --features scan --bench engine_ab`
use std::time::Instant;
use znippy_zoomies::gatling_forkjoin::gatling_for_each;
use modgunn::core::{ArtifactRef, Severity};
use modgunn::engine::{AdvisoryEngine, IndexedEngine, LinearEngine};
use modgunn::scan::{
Advisory, AdvisoryDb, Artifact, LicensePolicy, Policy, Provenance, ProvenancePolicy,
};
use modgunn::versions::RangeEvent::{Fixed, Introduced};
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = self.0;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
fn below(&mut self, n: u64) -> u64 {
self.next() % n
}
}
const ECOS: [&str; 4] = ["cargo", "npm", "pypi", "maven"];
fn env_usize(key: &str, default: usize) -> usize {
std::env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
/// Peak resident set size (VmHWM) in MiB, read from /proc/self/status. Linux-only;
/// returns 0.0 elsewhere.
fn peak_rss_mib() -> f64 {
std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|s| {
s.lines().find(|l| l.starts_with("VmHWM:")).and_then(|l| {
l.split_whitespace()
.nth(1)
.and_then(|kb| kb.parse::<f64>().ok())
})
})
.map(|kb| kb / 1024.0)
.unwrap_or(0.0)
}
fn gen_db(rng: &mut Rng, n_adv: usize, n_pkg: usize) -> AdvisoryDb {
let sev = [
Severity::Low,
Severity::Medium,
Severity::High,
Severity::Critical,
];
let mut advisories = Vec::with_capacity(n_adv);
for i in 0..n_adv {
let eco = ECOS[rng.below(ECOS.len() as u64) as usize];
let pkg = format!("pkg{}", rng.below(n_pkg as u64));
let severity = sev[rng.below(sev.len() as u64) as usize];
let mut adv = match rng.below(3) {
0 => {
let v = format!("1.{}.0", rng.below(5));
Advisory::basic(format!("ADV-{i}"), eco, &pkg, vec![v], severity, "exact")
}
1 => {
let lo = rng.below(3);
let hi = lo + 1 + rng.below(3);
Advisory::basic(format!("ADV-{i}"), eco, &pkg, vec![], severity, "range")
.with_ranges(vec![
Introduced(format!("1.{lo}.0")),
Fixed(format!("1.{hi}.0")),
])
}
_ => Advisory::basic(format!("ADV-{i}"), eco, &pkg, vec![], severity, "all"),
};
if rng.below(8) == 0 {
adv.withdrawn = true;
}
if rng.below(16) == 0 {
adv.kev = true;
}
advisories.push(adv);
}
AdvisoryDb::new("db-rev-bench", advisories)
}
fn gen_artifacts(rng: &mut Rng, n: usize, n_pkg: usize) -> Vec<Artifact> {
let mut out = Vec::with_capacity(n);
for _ in 0..n {
let eco = ECOS[rng.below(ECOS.len() as u64) as usize];
let pkg = format!("pkg{}", rng.below((n_pkg + n_pkg / 4) as u64));
let version = format!("1.{}.{}", rng.below(6), rng.below(4));
let license = match rng.below(4) {
0 => None,
1 => Some("GPL-3.0".to_string()),
2 => Some("MIT".to_string()),
_ => Some("Apache-2.0".to_string()),
};
let prov = if rng.below(2) == 0 {
Provenance::signed_by("ci")
} else {
Provenance::unsigned()
};
out.push(Artifact::new(
ArtifactRef {
ecosystem: eco.into(),
name: pkg,
version,
blob_sha256: String::new(),
},
license,
prov,
));
}
out
}
/// One full-fleet scan across all cores on the gatling fork-join (one worker per
/// core, `n_workers == 0`); returns (wall_secs, total_findings). This is the exact
/// engine the shipped `AdvisoryEngine::fleet` / `warehouse::scan_components` run.
fn parallel_fleet(engine: &dyn AdvisoryEngine, policy: &Policy, arts: &[Artifact]) -> (f64, usize) {
let t = Instant::now();
let per: Vec<usize> = gatling_for_each(arts.len(), 0, |i| {
engine.verdict(policy, &arts[i]).findings.len()
});
let total: usize = per.into_iter().sum();
(t.elapsed().as_secs_f64(), total)
}
fn main() {
let n_adv = env_usize("MODGUNN_BENCH_ADVISORIES", 25_000);
let n_pkg = env_usize("MODGUNN_BENCH_PACKAGES", 3_000);
let n_art = env_usize("MODGUNN_BENCH_ARTIFACTS", 1_000_000);
let repeats = env_usize("MODGUNN_BENCH_REPEATS", 3);
let cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
eprintln!("== modgunn Engine A/B HEAVY bench ==");
eprintln!(
"cores={cores} advisories={n_adv} packages={n_pkg} artifacts={n_art} repeats={repeats}"
);
let mut rng = Rng(0xA11CE_5EED_u64 ^ 0x1234_5678);
let build_t = Instant::now();
let db = gen_db(&mut rng, n_adv, n_pkg);
let artifacts = gen_artifacts(&mut rng, n_art, n_pkg);
eprintln!(
"corpus generated in {:.2}s",
build_t.elapsed().as_secs_f64()
);
let policy = Policy {
version: "bench".into(),
license: LicensePolicy {
allow: vec!["MIT".into(), "Apache-2.0".into()],
deny: vec!["GPL-3.0".into()],
on_unknown: modgunn::core::Decision::Warn,
},
provenance: ProvenancePolicy::Warn,
epss_block_threshold: Some(0.5),
..Default::default()
};
// ── Engine A — linear ────────────────────────────────────────────────────
let ta = Instant::now();
let a = LinearEngine::new(&db);
let a_build = ta.elapsed().as_secs_f64();
let mut a_best = f64::INFINITY;
let mut a_find = 0usize;
for _ in 0..repeats {
let (w, f) = parallel_fleet(&a, &policy, &artifacts);
a_find = f;
a_best = a_best.min(w);
}
// ── Engine B — inverted index ────────────────────────────────────────────
let tb = Instant::now();
let b = IndexedEngine::new(&db);
let b_build = tb.elapsed().as_secs_f64();
let buckets = b.bucket_count();
let mut b_best = f64::INFINITY;
let mut b_find = 0usize;
for _ in 0..repeats {
let (w, f) = parallel_fleet(&b, &policy, &artifacts);
b_find = f;
b_best = b_best.min(w);
}
assert_eq!(
a_find, b_find,
"EQUIVALENCE VIOLATION: A={a_find} findings, B={b_find}"
);
let a_tput = n_art as f64 / a_best;
let b_tput = n_art as f64 / b_best;
let speedup = a_best / b_best;
let rss = peak_rss_mib();
// Human table on stderr…
eprintln!();
eprintln!(
"{:<10} {:>12} {:>14} {:>16} {:>14}",
"engine", "build(s)", "scan(s)", "verdicts/s", "µs/verdict"
);
eprintln!(
"{:<10} {:>12.4} {:>14.3} {:>16.0} {:>14.3}",
"A linear",
a_build,
a_best,
a_tput,
a_best / n_art as f64 * 1e6
);
eprintln!(
"{:<10} {:>12.4} {:>14.3} {:>16.0} {:>14.3}",
"B indexed",
b_build,
b_best,
b_tput,
b_best / n_art as f64 * 1e6
);
eprintln!();
eprintln!("total findings (A==B): {a_find}");
eprintln!("B index buckets: {buckets}");
eprintln!("peak RSS: {rss:.0} MiB");
if speedup >= 1.0 {
eprintln!("WINNER: Engine B is {speedup:.1}x faster on this fleet ({cores} cores)");
} else {
eprintln!(
"WINNER: Engine A is {:.1}x faster on this fleet ({cores} cores)",
1.0 / speedup
);
}
// …and a machine-readable one-liner on stdout for the results file.
println!(
"RESULT cores={cores} advisories={n_adv} packages={n_pkg} artifacts={n_art} \
a_build_s={a_build:.4} a_scan_s={a_best:.3} a_tput={a_tput:.0} \
b_build_s={b_build:.4} b_scan_s={b_best:.3} b_tput={b_tput:.0} \
buckets={buckets} findings={a_find} peak_rss_mib={rss:.0} speedup_b_over_a={speedup:.2}"
);
}