diff --git a/Cargo.lock b/Cargo.lock index 3d032dd..e081513 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,23 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + [[package]] name = "atomic-waker" version = "1.1.2" @@ -60,12 +77,48 @@ dependencies = [ "tracing", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -108,6 +161,60 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "http" version = "1.4.0" @@ -189,24 +296,91 @@ dependencies = [ "tower-service", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "itoa" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + [[package]] name = "matchit" version = "0.8.4" @@ -242,6 +416,31 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -260,6 +459,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -278,12 +487,58 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "reed-solomon-erasure" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f" +dependencies = [ + "libm", + "lru", + "parking_lot", + "smallvec", + "spin", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "serde" version = "1.0.228" @@ -371,6 +626,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "stratum" version = "0.1.0" @@ -400,6 +661,12 @@ version = "0.1.0" [[package]] name = "stratum-storage" version = "0.1.0" +dependencies = [ + "anyhow", + "reed-solomon-erasure", + "tokio", + "uuid", +] [[package]] name = "stratum-tiering" @@ -501,12 +768,154 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "uuid" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -522,6 +931,94 @@ dependencies = [ "windows-link", ] +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 41a0d94..9df451b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,3 +28,4 @@ tower = "0.5.3" tracing = "0.1.44" tracing-subscriber = "0.3.23" uuid = { version = "1.22.0", features = ["v4"] } +anyhow = "1.0.102" \ No newline at end of file diff --git a/src/stratum-api-s3/src/errors/error.rs b/src/stratum-api-s3/src/errors/error.rs index 3a68990..5a1e8ed 100644 --- a/src/stratum-api-s3/src/errors/error.rs +++ b/src/stratum-api-s3/src/errors/error.rs @@ -1,5 +1,3 @@ -use std::fmt::write; - use axum::{ http::StatusCode, response::{IntoResponse, Response}, diff --git a/src/stratum-api-s3/src/handlers/bucket.rs b/src/stratum-api-s3/src/handlers/bucket.rs index de00c2b..6abc16f 100644 --- a/src/stratum-api-s3/src/handlers/bucket.rs +++ b/src/stratum-api-s3/src/handlers/bucket.rs @@ -1,4 +1,3 @@ -// src/stratum-api-s3/src/handlers/bucket.rs use axum::http::StatusCode; use axum::response::IntoResponse; @@ -20,7 +19,6 @@ pub async fn head_bucket() -> impl IntoResponse { #[cfg(test)] mod tests { - use super::*; use axum::{ body::Body, http::{Request, StatusCode}, diff --git a/src/stratum-storage/Cargo.toml b/src/stratum-storage/Cargo.toml index d8177cb..be15cd6 100644 --- a/src/stratum-storage/Cargo.toml +++ b/src/stratum-storage/Cargo.toml @@ -4,3 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] +reed-solomon-erasure.workspace = true +uuid.workspace = true +tokio.workspace = true +anyhow.workspace = true diff --git a/src/stratum-storage/src/definitions/config.rs b/src/stratum-storage/src/definitions/config.rs new file mode 100644 index 0000000..b9f7854 --- /dev/null +++ b/src/stratum-storage/src/definitions/config.rs @@ -0,0 +1,9 @@ +use std::path::PathBuf; + +pub struct StorageConfig { + pub hot_path: PathBuf, + pub warm_path: PathBuf, + pub cold_endpoint: String, + pub data_shards: usize, // default 4 + pub parity_shards: usize, // default 2 +} \ No newline at end of file diff --git a/src/stratum-storage/src/definitions/location.rs b/src/stratum-storage/src/definitions/location.rs new file mode 100644 index 0000000..2e727c0 --- /dev/null +++ b/src/stratum-storage/src/definitions/location.rs @@ -0,0 +1,11 @@ +use std::path::PathBuf; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Location { + Local(Vec), + Remote { + endpoint: String, + bucket: String, + keys: Vec + } +} \ No newline at end of file diff --git a/src/stratum-storage/src/definitions/mod.rs b/src/stratum-storage/src/definitions/mod.rs new file mode 100644 index 0000000..1598b11 --- /dev/null +++ b/src/stratum-storage/src/definitions/mod.rs @@ -0,0 +1,7 @@ +mod location; +mod storage_tier; +mod object_manifest; +mod config; + +pub use object_manifest::ObjectManifest; +pub use config::StorageConfig; \ No newline at end of file diff --git a/src/stratum-storage/src/definitions/object_manifest.rs b/src/stratum-storage/src/definitions/object_manifest.rs new file mode 100644 index 0000000..6772e2d --- /dev/null +++ b/src/stratum-storage/src/definitions/object_manifest.rs @@ -0,0 +1,13 @@ +use crate::definitions::{location::Location, storage_tier::StorageTier}; + +#[derive(Debug, Clone)] +pub struct ObjectManifest { + pub id: String, + pub tier: StorageTier, + pub location : Location, + pub size : u64, + pub checksum : String, + pub created_at : u64, + pub last_accessed: u64, + pub access_count: u64 +} \ No newline at end of file diff --git a/src/stratum-storage/src/definitions/storage_tier.rs b/src/stratum-storage/src/definitions/storage_tier.rs new file mode 100644 index 0000000..9695168 --- /dev/null +++ b/src/stratum-storage/src/definitions/storage_tier.rs @@ -0,0 +1,6 @@ +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum StorageTier { + Hot = 1, + Warm = 2, + Cold = 3 +} \ No newline at end of file diff --git a/src/stratum-storage/src/lib.rs b/src/stratum-storage/src/lib.rs index b93cf3f..df306a0 100644 --- a/src/stratum-storage/src/lib.rs +++ b/src/stratum-storage/src/lib.rs @@ -1,14 +1,2 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +pub mod definitions; +pub mod shard; \ No newline at end of file diff --git a/src/stratum-storage/src/shard.rs b/src/stratum-storage/src/shard.rs new file mode 100644 index 0000000..5559975 --- /dev/null +++ b/src/stratum-storage/src/shard.rs @@ -0,0 +1,198 @@ +use reed_solomon_erasure::galois_8::ReedSolomon; +use anyhow::{Ok, Result}; + +use crate::shard; + +const DATA_SHARDS: usize = 4; +const PARITY_SHARDS: usize = 2; +const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS; + +pub struct ShardEncoder { + rs: ReedSolomon, +} + +impl ShardEncoder { + pub fn new() -> Result { + Ok(Self { + rs: ReedSolomon::new(DATA_SHARDS, PARITY_SHARDS)?, + }) + } + + // Split bytes into 6 shards (4 data + 2 parity) + pub fn encode(&self, data: &[u8]) -> Result>> { + // calculate chunk size, padding to equal length + let chunk_size = (data.len() + DATA_SHARDS - 1) / DATA_SHARDS; + + // split data into 4 equal shards + let mut shards : Vec> = (0..DATA_SHARDS) + .map(|i| { + let start = i * chunk_size; + let end = (start + chunk_size).min(data.len()); + let mut chunk = data[start..end].to_vec(); + chunk.resize(chunk_size, 0); // pad last chunk if needed + chunk + }).collect(); + + // add empty parity shards + for _ in 0..PARITY_SHARDS { + shards.push(vec![0u8; chunk_size]); + } + + self.rs.encode(&mut shards)?; + Ok(shards) + } + + pub fn decode( + &self, + shards: Vec>>, + original_size: usize + ) -> Result> { + let mut shards = shards; + self.rs.reconstruct(&mut shards)?; + + // reconstruct missing shards + let mut result = Vec::with_capacity(original_size); + + // reassemble data shards only (ignore parity) + for i in 0..DATA_SHARDS { + if let Some(shard) = &shards[i] { + result.extend_from_slice(shard); + } + } + + // trim padding back to original size + result.truncate(original_size); + Ok(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encoder() -> ShardEncoder { + ShardEncoder::new().unwrap() + } + + #[test] + fn test_encode_produces_six_shards() { + let encoder = encoder(); + let data = b"hello world this is test data!!"; + let shards = encoder.encode(data).unwrap(); + assert_eq!(shards.len(), TOTAL_SHARDS); + } + + #[test] + fn test_encode_decode_roundtrip() { + let encoder = encoder(); + let data = b"hello world this is test data!!"; + let shards = encoder.encode(data).unwrap(); + + let wrapped: Vec>> = shards + .into_iter() + .map(Some) + .collect(); + + let decoded = encoder.decode(wrapped, data.len()).unwrap(); + assert_eq!(decoded, data); + } + + #[test] + fn test_decode_with_one_missing_shard() { + let encoder = encoder(); + let data = b"hello world this is test data!!"; + let shards = encoder.encode(data).unwrap(); + + // lose shard 0 + let mut wrapped: Vec>> = shards + .into_iter() + .map(Some) + .collect(); + wrapped[0] = None; + + let decoded = encoder.decode(wrapped, data.len()).unwrap(); + assert_eq!(decoded, data); + } + + #[test] + fn test_decode_with_two_missing_shards() { + let encoder = encoder(); + let data = b"hello world this is test data!!"; + let shards = encoder.encode(data).unwrap(); + + // lose shard 0 and shard 2 + let mut wrapped: Vec>> = shards + .into_iter() + .map(Some) + .collect(); + wrapped[0] = None; + wrapped[2] = None; + + let decoded = encoder.decode(wrapped, data.len()).unwrap(); + assert_eq!(decoded, data); + } + + #[test] + fn test_decode_with_two_parity_shards_missing() { + let encoder = encoder(); + let data = b"hello world this is test data!!"; + let shards = encoder.encode(data).unwrap(); + + // lose both parity shards + let mut wrapped: Vec>> = shards + .into_iter() + .map(Some) + .collect(); + wrapped[4] = None; + wrapped[5] = None; + + let decoded = encoder.decode(wrapped, data.len()).unwrap(); + assert_eq!(decoded, data); + } + + #[test] + fn test_shards_are_equal_size() { + let encoder = encoder(); + let data = b"hello world this is test data!!"; + let shards = encoder.encode(data).unwrap(); + + let first_size = shards[0].len(); + for shard in &shards { + assert_eq!(shard.len(), first_size); + } + } + + #[test] + fn test_large_object() { + let encoder = encoder(); + let data = vec![42u8; 1024 * 1024]; // 1MB + let shards = encoder.encode(&data).unwrap(); + + let wrapped: Vec>> = shards + .into_iter() + .map(Some) + .collect(); + + let decoded = encoder.decode(wrapped, data.len()).unwrap(); + assert_eq!(decoded, data); + } + + #[test] + fn test_three_missing_shards_fails() { + let encoder = encoder(); + let data = b"hello world this is test data!!"; + let shards = encoder.encode(data).unwrap(); + + // lose 3 shards - beyond recovery + let mut wrapped: Vec>> = shards + .into_iter() + .map(Some) + .collect(); + wrapped[0] = None; + wrapped[1] = None; + wrapped[2] = None; + + let result = encoder.decode(wrapped, data.len()); + assert!(result.is_err()); + } +} diff --git a/todo.md b/todo.md index 0b870a5..c782942 100644 --- a/todo.md +++ b/todo.md @@ -4,9 +4,9 @@ ### 1. `stratum-storage` — Volume Layer - [ ] `config.rs` — StorageConfig with hot/warm/cold paths -- [ ] `tier.rs` — StorageTier enum (Hot, Warm, Cold) -- [ ] `location.rs` — Location + ShardLocation enums (Local/Remote/Mixed) -- [ ] `volume.rs` — Volume struct with access tracking fields +- [x] `tier.rs` — StorageTier enum (Hot, Warm, Cold) +- [x] `location.rs` — Location + ShardLocation enums (Local/Remote/Mixed) +- [x] `manifest.rs` — Volume struct with access tracking fields - [ ] `store.rs` — VolumeStore (in-memory HashMap for now) - [ ] `shard.rs` — async read/write/delete shard files via tokio::fs - [ ] Tests for all of the above