Content-defined chunking and incremental synchronization primitives for MoonBit
moon check
moon test --target js
moon run cmd/main --target jslet input = "hello MoonChunk".to_bytes().to_array()
let cfg = @moonchunk.ChunkerConfig::new(min_size=16, average_size=32, max_size=64)
let chunks = @moonchunk.chunk_bytes(input, cfg~)
let manifest = @moonchunk.manifest_from_chunks("hello.txt", chunks)
let store = @moonchunk.ChunkStore::new()
@moonchunk.store_chunks(store, chunks)
let report = @moonchunk.verify_store(store, manifest)
assert_true(report.passed())MoonChunk demonstration
source bytes: 250
chunks: 9
root: 7019a4e06126793bfd0a35c6654fbc33
valid: 9 chunks, 250 bytes| File | Responsibility |
|---|---|
| types.mbt | Public data model and synchronization result types |
| digest.mbt | Deterministic digest and Merkle primitives |
| chunker.mbt | Batch and streaming content-defined chunking |
| manifest.mbt | Manifest construction and canonical encoding |
| sync.mbt | Diff plans, in-memory store, restore |
| verify.mbt | Integrity reports and proof checks |
| cmd/main | Runnable demonstration |
pub(all) struct BenchmarkResult {
name : String
input_bytes : Int
chunks : Int
digest_operations : Int
manifest_bytes : Int
restored_bytes : Int
reused_bytes : Int
uploaded_bytes : Int
checksum : String
} derive(Show)fn ChunkerConfig::new(min_size? : Int, average_size? : Int, max_size? : Int, seed? : UInt) -> ChunkerConfigpub(all) struct StorageReport {
logical_bytes : Int
physical_bytes : Int
chunk_count : Int
unique_chunks : Int
deduplicated_bytes : Int
manifest_bytes : Int
} derive(Show)pub(all) struct StreamingChunker {
config : ChunkerConfig
buffer : Array[Byte]
total_seen : Int
next_ordinal : Int
rolling : UInt
boundaries : Array[Int]
}pub(all) struct VerificationReport {
result : VerifyResult
checked_chunks : Int
checked_bytes : Int
warnings : Array[String]
} derive(Show)pub(all) struct VersionComparison {
old_bytes : Int
new_bytes : Int
reused_bytes : Int
uploaded_bytes : Int
removed_chunks : Int
reuse_ratio : Double
upload_ratio : Double
} derive(Show)fn apply_plan(old_store : ChunkStore, new_store : ChunkStore, old_manifest : Manifest, new_manifest : Manifest) -> Result[Array[Byte], String]fn run_comparison(old : BenchmarkInput, next : BenchmarkInput, config : ChunkerConfig) -> BenchmarkResultContent-defined chunking and incremental synchronization primitives for MoonBit