Fast and robust atproto CAR file processing in rust

type alias for rkey finally

+10 -7
+2 -2
readme.md
··· 62 62 63 63 more recent todo 64 64 - [ ] add a zero-copy rkyv process function example 65 - - [ ] repo car slices 66 - - [ ] lazy-value stream (rkey -> CID diffing for tap-like `#sync` handling) 65 + - [ ] car slices 66 + - [ ] lazy-value stream (for rkey -> CID diffing; tap-like `#sync` handling; save a fjall record `.get` when not needed) 67 67 - [x] get an *emtpy* car for the test suite 68 68 - [x] implement a max size on disk limit 69 69
+2
src/lib.rs
··· 90 90 91 91 pub type Bytes = Vec<u8>; 92 92 93 + pub type Rkey = String; 94 + 93 95 pub(crate) use hashbrown::HashMap; 94 96 95 97 #[doc = include_str!("../readme.md")]
+2 -1
src/mst.rs
··· 3 3 //! The primary aim is to work through the **tree** structure. Non-node blocks 4 4 //! are left as raw bytes, for upper levels to parse into DAG-CBOR or whatever. 5 5 6 + use crate::Rkey; 6 7 use cid::Cid; 7 8 use serde::Deserialize; 8 9 use sha2::{Digest, Sha256}; ··· 63 64 #[derive(Debug)] 64 65 pub(crate) enum ThingKind { 65 66 Tree, 66 - Value { rkey: String }, 67 + Value { rkey: Rkey }, 67 68 } 68 69 69 70 impl<'de> Deserialize<'de> for MstNode {
+4 -4
src/walk.rs
··· 1 1 //! Depth-first MST traversal 2 2 3 3 use crate::mst::{Depth, MstNode, NodeThing, ThingKind}; 4 - use crate::{Bytes, HashMap, disk::DiskStore, drive::MaybeProcessedBlock}; 4 + use crate::{Bytes, HashMap, Rkey, disk::DiskStore, drive::MaybeProcessedBlock}; 5 5 use cid::Cid; 6 6 use std::convert::Infallible; 7 7 ··· 30 30 #[error("MST depth underflow: depth-0 node with child trees")] 31 31 DepthUnderflow, 32 32 #[error("Encountered rkey {rkey:?} which cannot follow the previous: {prev:?}")] 33 - RkeyOutOfOrder { prev: String, rkey: String }, 33 + RkeyOutOfOrder { prev: Rkey, rkey: Rkey }, 34 34 } 35 35 36 36 /// Walker outputs 37 37 #[derive(Debug, PartialEq)] 38 38 pub struct Output { 39 - pub rkey: String, 39 + pub rkey: Rkey, 40 40 pub cid: Cid, 41 41 pub data: Bytes, 42 42 } ··· 46 46 /// Walks the tree from left-to-right in depth-first order 47 47 #[derive(Debug)] 48 48 pub struct Walker { 49 - prev_rkey: String, 49 + prev_rkey: Rkey, 50 50 root_depth: Depth, 51 51 todo: Vec<Vec<NodeThing>>, 52 52 }