gsh-digital-services/Stratum
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented reed-solomon erasure coding (encoding and decoding) with dynamically set amount of data shards and parity shards via config

Browse Source
This commit is contained in:
Ramez Soliman
2026-03-18 23:45:59 +01:00
parent 644314f878
commit 330bd76fb3
13 changed files with 751 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

497
Cargo.lock generated
View File

@@ -2,6 +2,23 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "ahash"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
dependencies = [
"getrandom 0.2.17",
"once_cell",
"version_check",
]
[[package]]
name = "anyhow"
version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -60,12 +77,48 @@ dependencies = [
"tracing",
]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
[[package]]
name = "bumpalo"
version = "3.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
[[package]]
name = "bytes"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -108,6 +161,60 @@ dependencies = [
"slab",
]
[[package]]
name = "getrandom"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "getrandom"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasip2",
"wasip3",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash",
]
[[package]]
name = "hashbrown"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"foldhash",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "http"
version = "1.4.0"
@@ -189,24 +296,91 @@ dependencies = [
"tower-service",
]
[[package]]
name = "id-arena"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "indexmap"
version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
"serde",
"serde_core",
]
[[package]]
name = "instant"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
]
[[package]]
name = "itoa"
version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
[[package]]
name = "js-sys"
version = "0.3.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"
dependencies = [
"once_cell",
"wasm-bindgen",
]
[[package]]
name = "leb128fmt"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "libc"
version = "0.2.183"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
[[package]]
name = "libm"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lru"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"
dependencies = [
"hashbrown 0.12.3",
]
[[package]]
name = "matchit"
version = "0.8.4"
@@ -242,6 +416,31 @@ version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -260,6 +459,16 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
@@ -278,12 +487,58 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "reed-solomon-erasure"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f"
dependencies = [
"libm",
"lru",
"parking_lot",
"smallvec",
"spin",
]
[[package]]
name = "rustversion"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ryu"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "semver"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
[[package]]
name = "serde"
version = "1.0.228"
@@ -371,6 +626,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "stratum"
version = "0.1.0"
@@ -400,6 +661,12 @@ version = "0.1.0"
[[package]]
name = "stratum-storage"
version = "0.1.0"
dependencies = [
"anyhow",
"reed-solomon-erasure",
"tokio",
"uuid",
]
[[package]]
name = "stratum-tiering"
@@ -501,12 +768,154 @@ version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "uuid"
version = "1.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
dependencies = [
"getrandom 0.4.2",
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasip2"
version = "1.0.2+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasip3"
version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
dependencies = [
"unicode-ident",
]
[[package]]
name = "wasm-encoder"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
dependencies = [
"leb128fmt",
"wasmparser",
]
[[package]]
name = "wasm-metadata"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
dependencies = [
"anyhow",
"indexmap",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags 2.11.0",
"hashbrown 0.15.5",
"indexmap",
"semver",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-link"
version = "0.2.1"
@@ -522,6 +931,94 @@ dependencies = [
"windows-link",
]
[[package]]
name = "wit-bindgen"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
dependencies = [
"wit-bindgen-rust-macro",
]
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
dependencies = [
"anyhow",
"heck",
"wit-parser",
]
[[package]]
name = "wit-bindgen-rust"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
dependencies = [
"anyhow",
"heck",
"indexmap",
"prettyplease",
"syn",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
]
[[package]]
name = "wit-bindgen-rust-macro"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
dependencies = [
"anyhow",
"prettyplease",
"proc-macro2",
"quote",
"syn",
"wit-bindgen-core",
"wit-bindgen-rust",
]
[[package]]
name = "wit-component"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags 2.11.0",
"indexmap",
"log",
"serde",
"serde_derive",
"serde_json",
"wasm-encoder",
"wasm-metadata",
"wasmparser",
"wit-parser",
]
[[package]]
name = "wit-parser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
dependencies = [
"anyhow",
"id-arena",
"indexmap",
"log",
"semver",
"serde",
"serde_derive",
"serde_json",
"unicode-xid",
"wasmparser",
]
[[package]]
name = "zmij"
version = "1.0.21"

1
Cargo.toml
View File

@@ -28,3 +28,4 @@ tower = "0.5.3"
tracing = "0.1.44"
tracing-subscriber = "0.3.23"
uuid = { version = "1.22.0", features = ["v4"] }
anyhow = "1.0.102"

2
src/stratum-api-s3/src/errors/error.rs
View File

@@ -1,5 +1,3 @@
use std::fmt::write;
use axum::{
http::StatusCode,
response::{IntoResponse, Response},

2
src/stratum-api-s3/src/handlers/bucket.rs
View File

@@ -1,4 +1,3 @@
// src/stratum-api-s3/src/handlers/bucket.rs
use axum::http::StatusCode;
use axum::response::IntoResponse;
@@ -20,7 +19,6 @@ pub async fn head_bucket() -> impl IntoResponse {
#[cfg(test)]
mod tests {
use super::*;
use axum::{
body::Body,
http::{Request, StatusCode},

4
src/stratum-storage/Cargo.toml
View File

@@ -4,3 +4,7 @@ version = "0.1.0"
edition = "2024"
[dependencies]
reed-solomon-erasure.workspace = true
uuid.workspace = true
tokio.workspace = true
anyhow.workspace = true

9
src/stratum-storage/src/definitions/config.rs Normal file
View File

@@ -0,0 +1,9 @@
use std::path::PathBuf;
pub struct StorageConfig {
pub hot_path: PathBuf,
pub warm_path: PathBuf,
pub cold_endpoint: String,
pub data_shards: usize, // default 4
pub parity_shards: usize, // default 2
}

11
src/stratum-storage/src/definitions/location.rs Normal file
View File

@@ -0,0 +1,11 @@
use std::path::PathBuf;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Location {
Local(Vec<PathBuf>),
Remote {
endpoint: String,
bucket: String,
keys: Vec<String>
}
}

7
src/stratum-storage/src/definitions/mod.rs Normal file
View File

@@ -0,0 +1,7 @@
mod location;
mod storage_tier;
mod object_manifest;
mod config;
pub use object_manifest::ObjectManifest;
pub use config::StorageConfig;

13
src/stratum-storage/src/definitions/object_manifest.rs Normal file
View File

@@ -0,0 +1,13 @@
use crate::definitions::{location::Location, storage_tier::StorageTier};
#[derive(Debug, Clone)]
pub struct ObjectManifest {
pub id: String,
pub tier: StorageTier,
pub location : Location,
pub size : u64,
pub checksum : String,
pub created_at : u64,
pub last_accessed: u64,
pub access_count: u64
}

6
src/stratum-storage/src/definitions/storage_tier.rs Normal file
View File

@@ -0,0 +1,6 @@
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum StorageTier {
Hot = 1,
Warm = 2,
Cold = 3
}

16
src/stratum-storage/src/lib.rs
View File

@@ -1,14 +1,2 @@
pub fn add(left: u64, right: u64) -> u64 {
left + right
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn it_works() {
let result = add(2, 2);
assert_eq!(result, 4);
}
}
pub mod definitions;
pub mod shard;

198
src/stratum-storage/src/shard.rs Normal file
View File

@@ -0,0 +1,198 @@
use reed_solomon_erasure::galois_8::ReedSolomon;
use anyhow::{Ok, Result};
use crate::shard;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS;
pub struct ShardEncoder {
rs: ReedSolomon,
}
impl ShardEncoder {
pub fn new() -> Result<Self> {
Ok(Self {
rs: ReedSolomon::new(DATA_SHARDS, PARITY_SHARDS)?,
})
}
// Split bytes into 6 shards (4 data + 2 parity)
pub fn encode(&self, data: &[u8]) -> Result<Vec<Vec<u8>>> {
// calculate chunk size, padding to equal length
let chunk_size = (data.len() + DATA_SHARDS - 1) / DATA_SHARDS;
// split data into 4 equal shards
let mut shards : Vec<Vec<u8>> = (0..DATA_SHARDS)
.map(|i| {
let start = i * chunk_size;
let end = (start + chunk_size).min(data.len());
let mut chunk = data[start..end].to_vec();
chunk.resize(chunk_size, 0); // pad last chunk if needed
chunk
}).collect();
// add empty parity shards
for _ in 0..PARITY_SHARDS {
shards.push(vec![0u8; chunk_size]);
}
self.rs.encode(&mut shards)?;
Ok(shards)
}
pub fn decode(
&self,
shards: Vec<Option<Vec<u8>>>,
original_size: usize
) -> Result<Vec<u8>> {
let mut shards = shards;
self.rs.reconstruct(&mut shards)?;
// reconstruct missing shards
let mut result = Vec::with_capacity(original_size);
// reassemble data shards only (ignore parity)
for i in 0..DATA_SHARDS {
if let Some(shard) = &shards[i] {
result.extend_from_slice(shard);
}
}
// trim padding back to original size
result.truncate(original_size);
Ok(result)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn encoder() -> ShardEncoder {
ShardEncoder::new().unwrap()
}
#[test]
fn test_encode_produces_six_shards() {
let encoder = encoder();
let data = b"hello world this is test data!!";
let shards = encoder.encode(data).unwrap();
assert_eq!(shards.len(), TOTAL_SHARDS);
}
#[test]
fn test_encode_decode_roundtrip() {
let encoder = encoder();
let data = b"hello world this is test data!!";
let shards = encoder.encode(data).unwrap();
let wrapped: Vec<Option<Vec<u8>>> = shards
.into_iter()
.map(Some)
.collect();
let decoded = encoder.decode(wrapped, data.len()).unwrap();
assert_eq!(decoded, data);
}
#[test]
fn test_decode_with_one_missing_shard() {
let encoder = encoder();
let data = b"hello world this is test data!!";
let shards = encoder.encode(data).unwrap();
// lose shard 0
let mut wrapped: Vec<Option<Vec<u8>>> = shards
.into_iter()
.map(Some)
.collect();
wrapped[0] = None;
let decoded = encoder.decode(wrapped, data.len()).unwrap();
assert_eq!(decoded, data);
}
#[test]
fn test_decode_with_two_missing_shards() {
let encoder = encoder();
let data = b"hello world this is test data!!";
let shards = encoder.encode(data).unwrap();
// lose shard 0 and shard 2
let mut wrapped: Vec<Option<Vec<u8>>> = shards
.into_iter()
.map(Some)
.collect();
wrapped[0] = None;
wrapped[2] = None;
let decoded = encoder.decode(wrapped, data.len()).unwrap();
assert_eq!(decoded, data);
}
#[test]
fn test_decode_with_two_parity_shards_missing() {
let encoder = encoder();
let data = b"hello world this is test data!!";
let shards = encoder.encode(data).unwrap();
// lose both parity shards
let mut wrapped: Vec<Option<Vec<u8>>> = shards
.into_iter()
.map(Some)
.collect();
wrapped[4] = None;
wrapped[5] = None;
let decoded = encoder.decode(wrapped, data.len()).unwrap();
assert_eq!(decoded, data);
}
#[test]
fn test_shards_are_equal_size() {
let encoder = encoder();
let data = b"hello world this is test data!!";
let shards = encoder.encode(data).unwrap();
let first_size = shards[0].len();
for shard in &shards {
assert_eq!(shard.len(), first_size);
}
}
#[test]
fn test_large_object() {
let encoder = encoder();
let data = vec![42u8; 1024 * 1024]; // 1MB
let shards = encoder.encode(&data).unwrap();
let wrapped: Vec<Option<Vec<u8>>> = shards
.into_iter()
.map(Some)
.collect();
let decoded = encoder.decode(wrapped, data.len()).unwrap();
assert_eq!(decoded, data);
}
#[test]
fn test_three_missing_shards_fails() {
let encoder = encoder();
let data = b"hello world this is test data!!";
let shards = encoder.encode(data).unwrap();
// lose 3 shards - beyond recovery
let mut wrapped: Vec<Option<Vec<u8>>> = shards
.into_iter()
.map(Some)
.collect();
wrapped[0] = None;
wrapped[1] = None;
wrapped[2] = None;
let result = encoder.decode(wrapped, data.len());
assert!(result.is_err());
}
}

6
todo.md
View File

@@ -4,9 +4,9 @@
### 1. `stratum-storage` — Volume Layer
- [ ] `config.rs` — StorageConfig with hot/warm/cold paths
- [ ] `tier.rs` — StorageTier enum (Hot, Warm, Cold)
- [ ] `location.rs` — Location + ShardLocation enums (Local/Remote/Mixed)
- [ ] `volume.rs` — Volume struct with access tracking fields
- [x] `tier.rs` — StorageTier enum (Hot, Warm, Cold)
- [x] `location.rs` — Location + ShardLocation enums (Local/Remote/Mixed)
- [x] `manifest.rs` — Volume struct with access tracking fields
- [ ] `store.rs` — VolumeStore (in-memory HashMap for now)
- [ ] `shard.rs` — async read/write/delete shard files via tokio::fs
- [ ] Tests for all of the above