// SPDX-License-Identifier: AGPL-3.0-only
//! A znippy v0.7 archive reader over a byte slice — no file, no thread.
//!
//! The on-disk layout (znippy-common `index.rs`):
//!
//! ```text
//! [blobs ...][sub-index IPC streams ...][manifest IPC stream]["ZNPYMIDX"][u64 LE manifest_offset]
//! ```
//!
//! The manifest stream lists every sub-index (`pkg_type, repo, module_name,
//! index_offset, index_len, row_count`). Reserved modules (`__znippy_*__`,
//! `__gunnar_*__`) are skipped exactly as znippy's own reader skips them,
//! except the delta map, which would change what a chunk MEANS and is refused.
//! Each data sub-index holds one row per chunk: `relative_path, chunk_seq,
//! fdata_offset, compressed, uncompressed_size, blob_offset, blob_size,
//! checksum` (blake3 of the chunk's bytes).
//!
//! Stricter than znippy, on purpose — this is the code that reads hostile bytes:
//! * every region (manifest, sub-index, blob) must lie inside the file and every
//! blob before the manifest;
//! * the checksum column is REQUIRED (znippy skips verification without it);
//! * a file's chunks must tile `[0, size)` exactly — no gap, no overlap, no
//! duplicate `(path, chunk_seq)`;
//! * a stored chunk's `blob_size` must equal its `uncompressed_size`;
//! * a compressed chunk is decoded with OpenZL (feature `openzl`, [`crate::zl`])
//! only after its frame header declares exactly the size the index declares;
//! without the feature it is refused by name ([`Refusal::Codec`]);
//! * every chunk's blake3 (over the UNCOMPRESSED bytes, as znippy writes it) is
//! checked before the member is used.
use std::collections::BTreeMap;
use crate::ipc::{self, Batch, Column};
use crate::limits::{
EXPANSION_RATIO_FLOOR_BYTES, MAX_EXPANSION_RATIO, MAX_INDEX_ROWS, MAX_INDEX_SECTIONS,
MAX_MEMBERS, ZNIPPY_FORMAT_VERSION,
};
use crate::pathsafe::PathGuard;
use crate::refusal::{Refusal, Result};
pub const MULTI_INDEX_MAGIC: [u8; 8] = *b"ZNPYMIDX";
pub const DELTA_MODULE: &str = "__znippy_delta__";
const FORMAT_VERSION_KEY: &str = "znippy_format_version";
/// One chunk of one file.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Chunk {
pub chunk_seq: u32,
pub fdata_offset: u64,
pub compressed: bool,
pub uncompressed_size: u64,
pub blob_offset: u64,
pub blob_size: u64,
pub checksum: [u8; 32],
}
/// One file: its chunks, sorted by `fdata_offset`, tiling `[0, size)`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Entry {
pub size: u64,
pub chunks: Vec<Chunk>,
}
impl Entry {
pub fn is_compressed(&self) -> bool {
self.chunks.iter().any(|c| c.compressed)
}
}
/// A parsed archive: the index, validated, over the borrowed bytes.
#[derive(Debug)]
pub struct Archive<'a> {
bytes: &'a [u8],
entries: BTreeMap<String, Entry>,
}
/// One manifest row.
#[derive(Debug, Clone)]
struct Section {
module_name: String,
index_offset: u64,
index_len: u64,
}
fn bad(why: impl Into<String>) -> Refusal {
Refusal::Archive(why.into())
}
fn col<'b>(batch: &'b Batch, name: &str, what: &str) -> Result<&'b Column> {
batch
.columns
.get(name)
.ok_or_else(|| bad(format!("{what}: missing column {name}")))
}
macro_rules! typed {
($batch:expr, $name:expr, $what:expr, $variant:ident) => {
match col($batch, $name, $what)? {
Column::$variant(v) => v,
_ => {
return Err(bad(format!(
"{}: column {} has the wrong type",
$what, $name
)))
}
}
};
}
fn in_file(offset: u64, len: u64, limit: u64) -> bool {
offset.checked_add(len).is_some_and(|end| end <= limit)
}
impl<'a> Archive<'a> {
/// Parse and validate the whole index. Reads no blob.
pub fn parse(bytes: &'a [u8]) -> Result<Self> {
let file_len = bytes.len() as u64;
if bytes.len() < 16 {
return Err(bad("shorter than the 16-byte footer"));
}
let tail = &bytes[bytes.len() - 16..];
if tail[..8] != MULTI_INDEX_MAGIC {
return Err(bad("no ZNPYMIDX footer (not a v0.7 multi-index archive)"));
}
let manifest_offset = u64::from_le_bytes(tail[8..].try_into().expect("8 bytes"));
let manifest_end = file_len - 16;
if manifest_offset > manifest_end {
return Err(bad("manifest offset points past the footer"));
}
let manifest = &bytes[manifest_offset as usize..manifest_end as usize];
let sections = parse_manifest(manifest)?;
let mut rows: Vec<(String, Chunk)> = Vec::new();
let mut version_checked = false;
for s in §ions {
if !in_file(s.index_offset, s.index_len, manifest_offset) {
return Err(bad(format!(
"sub-index {:?} lies outside the region before the manifest",
s.module_name
)));
}
if s.module_name == DELTA_MODULE {
return Err(Refusal::Codec(
"the archive carries a delta map (__znippy_delta__); delta chunks are not \
reconstructed in the guest"
.into(),
));
}
if is_reserved_module(&s.module_name) {
continue;
}
let sub = &bytes[s.index_offset as usize..(s.index_offset + s.index_len) as usize];
let stream = ipc::decode(sub, "a data sub-index")?;
if !version_checked {
check_format_version(&stream.metadata)?;
version_checked = true;
}
let batches = stream.batches;
for batch in &batches {
read_rows(batch, manifest_offset, &mut rows)?;
}
}
let entries = build_entries(rows)?;
Ok(Archive { bytes, entries })
}
pub fn file_len(&self) -> u64 {
self.bytes.len() as u64
}
/// Every member path, sorted.
pub fn names(&self) -> impl Iterator<Item = &str> {
self.entries.keys().map(String::as_str)
}
pub fn len(&self) -> usize {
self.entries.len()
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
pub fn entry(&self, path: &str) -> Option<&Entry> {
self.entries.get(path)
}
/// The total size the index declares for every member.
pub fn declared_total(&self) -> u64 {
self.entries
.values()
.fold(0u64, |acc, e| acc.saturating_add(e.size))
}
/// Refuse the decompression-bomb SHAPE from the index alone: an archive whose
/// members claim more than [`MAX_EXPANSION_RATIO`] times its own size (once
/// past [`EXPANSION_RATIO_FLOOR_BYTES`]). Nothing is extracted to find out.
pub fn check_expansion_ratio(&self) -> Result<()> {
let total = self.declared_total();
if total > EXPANSION_RATIO_FLOOR_BYTES {
let allowed = self
.file_len()
.saturating_mul(MAX_EXPANSION_RATIO)
.max(EXPANSION_RATIO_FLOOR_BYTES);
if total > allowed {
return Err(Refusal::LimitExceeded(format!(
"archive declares {total} bytes from {} on disk, over the \
{MAX_EXPANSION_RATIO}:1 limit — refusing a decompression bomb",
self.file_len()
)));
}
}
Ok(())
}
/// Read one member, checking every chunk's blake3 before its bytes are kept.
/// `max_len` is checked against the DECLARED size before anything is
/// allocated; the declared size is then re-proven by the bytes themselves.
pub fn read(&self, path: &str, max_len: u64) -> Result<Vec<u8>> {
let entry = self
.entries
.get(path)
.ok_or_else(|| Refusal::MissingMember(path.to_string()))?;
if entry.size > max_len {
return Err(Refusal::LimitExceeded(format!(
"member {path:?} declares {} bytes, over {max_len}",
entry.size
)));
}
#[cfg(not(feature = "openzl"))]
if let Some(c) = entry.chunks.iter().find(|c| c.compressed) {
return Err(Refusal::Codec(format!(
"member {path:?} chunk {} is OpenZL-compressed and this guest was built \
without the `openzl` feature",
c.chunk_seq
)));
}
// Allocated once, at the DECLARED size (already under `max_len`, the
// archive's expansion ratio and the member caps); no chunk can grow it.
let size = usize::try_from(entry.size).map_err(|_| {
Refusal::LimitExceeded(format!(
"member {path:?} declares {} bytes, more than this guest can address",
entry.size
))
})?;
let mut out = Vec::new();
out.try_reserve_exact(size).map_err(|_| {
Refusal::LimitExceeded(format!(
"member {path:?} declares {size} bytes, more than the guest's memory"
))
})?;
out.resize(size, 0);
let mut at = 0usize;
for c in &entry.chunks {
// Bounds were proven at parse time; slice again defensively.
let start = c.blob_offset as usize;
let end = start
.checked_add(c.blob_size as usize)
.filter(|&e| e <= self.bytes.len())
.ok_or_else(|| bad(format!("chunk {} of {path:?} out of bounds", c.chunk_seq)))?;
let blob = &self.bytes[start..end];
let want = c.uncompressed_size as usize;
let dst = out
.get_mut(at..at + want)
.ok_or_else(|| bad(format!("chunk {} of {path:?} out of bounds", c.chunk_seq)))?;
if c.compressed {
decode_chunk(path, c, blob, dst)?;
} else {
dst.copy_from_slice(blob);
}
check_chunk(path, c, dst)?;
at += want;
}
if at != out.len() {
out.truncate(at);
}
if out.len() as u64 != entry.size {
return Err(Refusal::DigestMismatch {
path: path.to_string(),
which: "size",
});
}
Ok(out)
}
}
/// Decode one OpenZL chunk into `dst`, which is exactly the size the index
/// declares. The frame's own header must declare that same size BEFORE the
/// decoder runs (a frame that claims more is a size lie or a bomb, refused
/// without decoding), and the decoder must fill all of it.
#[cfg(feature = "openzl")]
fn decode_chunk(path: &str, c: &Chunk, blob: &[u8], dst: &mut [u8]) -> Result<()> {
let extract = |why: String| {
Refusal::Container(format!(
"cannot extract bundle member: chunk {} of {path:?}: {why}",
c.chunk_seq
))
};
let declared = crate::zl::frame_size(blob)
.map_err(|e| extract(format!("not an OpenZL frame: {}", e.0)))?;
if declared != dst.len() as u64 {
return Err(extract(format!(
"the OpenZL frame declares {declared} bytes but the index declares {}",
dst.len()
)));
}
let n = crate::zl::decode(blob, dst)
.map_err(|e| extract(format!("the OpenZL frame does not decode: {}", e.0)))?;
if n != dst.len() {
return Err(extract(format!(
"the OpenZL frame decoded to {n} bytes, not the declared {}",
dst.len()
)));
}
Ok(())
}
#[cfg(not(feature = "openzl"))]
fn decode_chunk(_: &str, _: &Chunk, _: &[u8], _: &mut [u8]) -> Result<()> {
unreachable!("compressed chunks are refused before decoding without `openzl`")
}
fn check_chunk(path: &str, c: &Chunk, bytes: &[u8]) -> Result<()> {
if c.checksum == [0u8; 32] && bytes.is_empty() {
return Ok(()); // an empty file's row carries no digest
}
if blake3::hash(bytes).as_bytes() != &c.checksum {
return Err(Refusal::Container(format!(
"chunk {} of {path:?} does not match its blake3 checksum",
c.chunk_seq
)));
}
Ok(())
}
/// znippy's reserved sub-index names (`index.rs` `RESERVED_MODULES`): derived
/// structures the data reader skips. The delta map is refused before this.
const RESERVED_MODULES: &[&str] = &[
"__znippy_lookup__",
"__znippy_trie__",
"__znippy_sign_artifacts__",
"__znippy_sign_archive__",
"__znippy_meta__",
"__gunnar_oid__",
"__gunnar_graph__",
"__gunnar_reach__",
"__gunnar_refs__",
"__gunnar_secrets__",
DELTA_MODULE,
"__znippy_fsmeta__",
];
fn is_reserved_module(name: &str) -> bool {
RESERVED_MODULES.contains(&name)
}
fn check_format_version(meta: &BTreeMap<String, String>) -> Result<()> {
if let Some(raw) = meta.get(FORMAT_VERSION_KEY) {
let v: u32 = raw
.parse()
.map_err(|_| bad(format!("invalid znippy format version {raw:?}")))?;
if v > ZNIPPY_FORMAT_VERSION {
return Err(bad(format!(
"znippy format v{v} is newer than this reader (max v{ZNIPPY_FORMAT_VERSION})"
)));
}
}
Ok(())
}
fn parse_manifest(bytes: &[u8]) -> Result<Vec<Section>> {
let what = "the archive manifest";
let stream = ipc::decode(bytes, what)?;
let mut out = Vec::new();
for batch in &stream.batches {
let _pkg = typed!(batch, "pkg_type", what, Int8);
let _repo = typed!(batch, "repo", what, Utf8);
let module = typed!(batch, "module_name", what, Utf8);
let off = typed!(batch, "index_offset", what, UInt64);
let len = typed!(batch, "index_len", what, UInt64);
let _rows = typed!(batch, "row_count", what, UInt64);
for i in 0..batch.rows {
if out.len() >= MAX_INDEX_SECTIONS {
return Err(Refusal::LimitExceeded(format!(
"more than {MAX_INDEX_SECTIONS} index sections"
)));
}
out.push(Section {
module_name: module[i].clone(),
index_offset: off[i],
index_len: len[i],
});
}
}
Ok(out)
}
fn read_rows(batch: &Batch, blob_limit: u64, rows: &mut Vec<(String, Chunk)>) -> Result<()> {
let what = "a data sub-index";
let paths = typed!(batch, "relative_path", what, Utf8);
let seq = typed!(batch, "chunk_seq", what, UInt32);
let fdata = typed!(batch, "fdata_offset", what, UInt64);
let compressed = typed!(batch, "compressed", what, Boolean);
let usize_ = typed!(batch, "uncompressed_size", what, UInt64);
let boff = typed!(batch, "blob_offset", what, UInt64);
let bsize = typed!(batch, "blob_size", what, UInt64);
let (width, checksum) = match col(batch, "checksum", what)? {
Column::FixedSizeBinary(w, v) => (*w, v),
_ => return Err(bad(format!("{what}: column checksum has the wrong type"))),
};
if width != 32 {
return Err(bad(format!(
"checksum column is {width} bytes wide, not 32"
)));
}
for i in 0..batch.rows {
if rows.len() >= MAX_INDEX_ROWS {
return Err(Refusal::LimitExceeded(format!(
"more than {MAX_INDEX_ROWS} index rows"
)));
}
let c = Chunk {
chunk_seq: seq[i],
fdata_offset: fdata[i],
compressed: compressed[i],
uncompressed_size: usize_[i],
blob_offset: boff[i],
blob_size: bsize[i],
checksum: checksum[i].as_slice().try_into().expect("width checked"),
};
let path = paths[i].as_str();
if !in_file(c.blob_offset, c.blob_size, blob_limit) {
return Err(bad(format!(
"chunk {} of {path:?} lies outside the blob region",
c.chunk_seq
)));
}
if !c.compressed && c.blob_size != c.uncompressed_size {
return Err(bad(format!(
"stored chunk {} of {path:?} has blob_size {} but uncompressed_size {}",
c.chunk_seq, c.blob_size, c.uncompressed_size
)));
}
rows.push((path.to_string(), c));
}
Ok(())
}
fn build_entries(rows: Vec<(String, Chunk)>) -> Result<BTreeMap<String, Entry>> {
let mut grouped: BTreeMap<String, Vec<Chunk>> = BTreeMap::new();
for (p, c) in rows {
grouped.entry(p).or_default().push(c);
}
if grouped.len() > MAX_MEMBERS {
return Err(Refusal::LimitExceeded(format!(
"member count {} exceeds {MAX_MEMBERS}",
grouped.len()
)));
}
let mut guard = PathGuard::new();
let mut out = BTreeMap::new();
for (path, mut chunks) in grouped {
guard.check(&path)?;
chunks.sort_by_key(|c| (c.fdata_offset, c.chunk_seq));
let mut seqs: Vec<u32> = chunks.iter().map(|c| c.chunk_seq).collect();
seqs.sort_unstable();
if seqs.windows(2).any(|w| w[0] == w[1]) {
return Err(Refusal::UnsafePath(format!(
"duplicate index entry (path, chunk_seq) for {path:?}"
)));
}
let mut cursor: u64 = 0;
for c in &chunks {
if c.fdata_offset != cursor {
return Err(bad(format!(
"the chunks of {path:?} do not tile the file (gap or overlap at {})",
c.fdata_offset
)));
}
cursor = cursor
.checked_add(c.uncompressed_size)
.ok_or_else(|| Refusal::LimitExceeded(format!("size of {path:?} overflows")))?;
}
out.insert(
path,
Entry {
size: cursor,
chunks,
},
);
}
Ok(out)
}