choir_store/
lib.rs

1//! L0 content-addressed store: BLAKE3 + FastCDC chunking (DECISIONS.md D6).
2//!
3//! One-way-door rules (DECISIONS.md) enforced here:
4//! - manifests carry `format_version` and record the exact chunking
5//!   parameters used, per object, so parameter changes never orphan data
6//! - all identifiers are self-describing [`ContentHash`] envelopes
7//!
8//! The chunk-store seam (D7) is [`ChunkStore`]; the production backend is an
9//! S3-compatible object store, so implementations must stay within plain
10//! put/get/has semantics.
11//!
12//! # Examples
13//!
14//! ```
15//! use choir_store::{ChunkerParams, MemStore, put_blob, get_blob};
16//!
17//! let mut store = MemStore::new();
18//! let data = vec![7u8; 100_000];
19//! let manifest_hash = put_blob(&mut store, &data, ChunkerParams::default()).unwrap();
20//! assert_eq!(get_blob(&store, &manifest_hash).unwrap(), data);
21//! ```
22//!
23//! # Where this sits
24//!
25//! `docs/architecture.md` is the map of the whole workspace.
26//! This crate is L0, the content-addressed store.
27//!
28//! It builds on [`choir_hash`].
29
30use choir_hash::ContentHash;
31use serde::{Deserialize, Serialize};
32
33/// Current manifest-format version. Bump on any incompatible change;
34/// additive changes keep the version (DECISIONS.md).
35pub const FORMAT_VERSION: u16 = 1;
36
37/// FastCDC parameters, recorded per object (one-way-door rule, D6): a blob
38/// is always rechunkable for verification because its manifest says exactly
39/// how it was cut.
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
41pub struct ChunkerParams {
42    /// Minimum chunk size in bytes.
43    pub min: u32,
44    /// Target average chunk size in bytes.
45    pub avg: u32,
46    /// Maximum chunk size in bytes.
47    pub max: u32,
48}
49
50impl Default for ChunkerParams {
51    /// 4 KiB / 16 KiB / 64 KiB, the plan's small-file-friendly starting point.
52    fn default() -> Self {
53        Self {
54            min: 4 * 1024,
55            avg: 16 * 1024,
56            max: 64 * 1024,
57        }
58    }
59}
60
61/// A blob's recipe: ordered chunk addresses plus the parameters that cut it.
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct Manifest {
64    /// Manifest wire-format version; see [`FORMAT_VERSION`].
65    pub format_version: u16,
66    /// Chunking parameters this blob was cut with.
67    pub params: ChunkerParams,
68    /// Total blob length in bytes.
69    pub len: u64,
70    /// Content addresses of the chunks, in order.
71    pub chunks: Vec<ContentHash>,
72}
73
74/// Failure modes of a [`ChunkStore`] or blob operation.
75#[derive(Debug)]
76pub enum StoreError {
77    /// Requested hash is not in the store.
78    NotFound(ContentHash),
79    /// Stored bytes do not hash to their address (corruption or tampering).
80    HashMismatch(ContentHash),
81    /// Stored manifest could not be decoded.
82    BadManifest(String),
83    /// Underlying storage I/O failure.
84    Io(std::io::Error),
85}
86
87/// The chunk-store seam (D7). Conformance suite: `tests/conformance.rs`.
88///
89/// `put` is content-addressed and idempotent; `get` must return exactly the
90/// bytes that hash to `hash` or an error, never silently wrong data.
91pub trait ChunkStore: Send {
92    /// Stores `data` and returns its content address. Idempotent.
93    fn put(&mut self, data: &[u8]) -> Result<ContentHash, StoreError>;
94
95    /// Retrieves the bytes for `hash`, verifying them against the address.
96    fn get(&self, hash: &ContentHash) -> Result<Vec<u8>, StoreError>;
97
98    /// Whether `hash` is present.
99    fn has(&self, hash: &ContentHash) -> bool;
100}
101
102/// Primary in-memory implementation (also the dev/test runtime).
103#[derive(Default)]
104pub struct MemStore {
105    chunks: std::collections::HashMap<ContentHash, Vec<u8>>,
106}
107
108impl MemStore {
109    /// Creates an empty in-memory store.
110    pub fn new() -> Self {
111        Self::default()
112    }
113
114    /// Number of distinct chunks held (used by dedup tests).
115    pub fn chunk_count(&self) -> usize {
116        self.chunks.len()
117    }
118}
119
120impl ChunkStore for MemStore {
121    fn put(&mut self, data: &[u8]) -> Result<ContentHash, StoreError> {
122        let hash = ContentHash::blake3(data);
123        self.chunks
124            .entry(hash.clone())
125            .or_insert_with(|| data.to_vec());
126        Ok(hash)
127    }
128
129    fn get(&self, hash: &ContentHash) -> Result<Vec<u8>, StoreError> {
130        let data = self
131            .chunks
132            .get(hash)
133            .ok_or_else(|| StoreError::NotFound(hash.clone()))?;
134        if &ContentHash::blake3(data) != hash {
135            return Err(StoreError::HashMismatch(hash.clone()));
136        }
137        Ok(data.clone())
138    }
139
140    fn has(&self, hash: &ContentHash) -> bool {
141        self.chunks.contains_key(hash)
142    }
143}
144
145/// Second implementation (seam rule): filesystem store, one file per chunk,
146/// sharded by the first digest byte.
147pub struct FsStore {
148    root: std::path::PathBuf,
149}
150
151impl FsStore {
152    /// Opens (creating if absent) a store rooted at `root`.
153    ///
154    /// # Errors
155    ///
156    /// Returns [`StoreError::Io`] when the root cannot be created.
157    pub fn open(root: &std::path::Path) -> Result<Self, StoreError> {
158        std::fs::create_dir_all(root).map_err(StoreError::Io)?;
159        Ok(Self {
160            root: root.to_path_buf(),
161        })
162    }
163
164    fn path_for(&self, hash: &ContentHash) -> std::path::PathBuf {
165        let hex = hash.to_hex();
166        // hex is "cc-dddd…"; shard on the first two digest nibbles.
167        self.root.join(&hex[3..5]).join(&hex)
168    }
169}
170
171impl ChunkStore for FsStore {
172    fn put(&mut self, data: &[u8]) -> Result<ContentHash, StoreError> {
173        let hash = ContentHash::blake3(data);
174        let path = self.path_for(&hash);
175        if !path.exists() {
176            std::fs::create_dir_all(path.parent().unwrap()).map_err(StoreError::Io)?;
177            // Content-addressed entries are immutable: write to a temp name,
178            // then atomically rename, so readers never see partial chunks.
179            let tmp = path.with_extension("tmp");
180            std::fs::write(&tmp, data).map_err(StoreError::Io)?;
181            std::fs::rename(&tmp, &path).map_err(StoreError::Io)?;
182        }
183        Ok(hash)
184    }
185
186    fn get(&self, hash: &ContentHash) -> Result<Vec<u8>, StoreError> {
187        let path = self.path_for(hash);
188        let data = std::fs::read(&path).map_err(|e| {
189            if e.kind() == std::io::ErrorKind::NotFound {
190                StoreError::NotFound(hash.clone())
191            } else {
192                StoreError::Io(e)
193            }
194        })?;
195        if &ContentHash::blake3(&data) != hash {
196            return Err(StoreError::HashMismatch(hash.clone()));
197        }
198        Ok(data)
199    }
200
201    fn has(&self, hash: &ContentHash) -> bool {
202        self.path_for(hash).exists()
203    }
204}
205
206/// Chunks `data` with FastCDC under `params`, stores every chunk and the
207/// manifest, and returns the manifest's content address.
208///
209/// # Errors
210///
211/// Propagates any [`StoreError`] from the underlying store.
212pub fn put_blob(
213    store: &mut dyn ChunkStore,
214    data: &[u8],
215    params: ChunkerParams,
216) -> Result<ContentHash, StoreError> {
217    let mut chunks = Vec::new();
218    for chunk in fastcdc::v2020::FastCDC::new(data, params.min, params.avg, params.max) {
219        chunks.push(store.put(&data[chunk.offset..chunk.offset + chunk.length])?);
220    }
221    let manifest = Manifest {
222        format_version: FORMAT_VERSION,
223        params,
224        len: data.len() as u64,
225        chunks,
226    };
227    let bytes = serde_json::to_vec(&manifest).expect("Manifest is always serializable");
228    store.put(&bytes)
229}
230
231/// Reassembles a blob from its manifest address, verifying every chunk.
232///
233/// # Errors
234///
235/// Returns [`StoreError::BadManifest`] when the manifest fails to decode,
236/// names an unsupported format version, or disagrees with the reassembled
237/// length, and propagates chunk lookup and verification failures.
238pub fn get_blob(
239    store: &dyn ChunkStore,
240    manifest_hash: &ContentHash,
241) -> Result<Vec<u8>, StoreError> {
242    let manifest: Manifest = serde_json::from_slice(&store.get(manifest_hash)?)
243        .map_err(|e| StoreError::BadManifest(e.to_string()))?;
244    if manifest.format_version != FORMAT_VERSION {
245        return Err(StoreError::BadManifest(format!(
246            "unsupported manifest format version {} (expected {FORMAT_VERSION})",
247            manifest.format_version
248        )));
249    }
250    let mut data = Vec::with_capacity(manifest.len as usize);
251    for chunk_hash in &manifest.chunks {
252        data.extend_from_slice(&store.get(chunk_hash)?);
253    }
254    if data.len() as u64 != manifest.len {
255        return Err(StoreError::BadManifest(format!(
256            "reassembled {} bytes, manifest says {}",
257            data.len(),
258            manifest.len
259        )));
260    }
261    Ok(data)
262}
choir_store/lib.rs

choir_store/
lib.rs
