microcosm.blue
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
make identity cache also configurable
authored by
+33
-16
+1
-1
slingshot/src/firehose_cache.rs
+1
-1
slingshot/src/firehose_cache.rs
···
15
15
.with_device_options(
16
16
DirectFsDeviceOptions::new(cache_dir)
17
17
.with_capacity(disk_gb * 2_usize.pow(30))
18
-
.with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records
18
+
.with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size (records should be max 1mb cbor, bit bigger json)
19
19
)
20
20
.build()
21
21
.await
+8
-4
slingshot/src/identity.rs
+8
-4
slingshot/src/identity.rs
···
161
161
}
162
162
163
163
impl Identity {
164
-
pub async fn new(cache_dir: impl AsRef<Path>) -> Result<Self, IdentityError> {
164
+
pub async fn new(
165
+
cache_dir: impl AsRef<Path>,
166
+
memory_mb: usize,
167
+
disk_gb: usize,
168
+
) -> Result<Self, IdentityError> {
165
169
let http_client = Arc::new(DefaultHttpClient::default());
166
170
let handle_resolver = AtprotoHandleResolver::new(AtprotoHandleResolverConfig {
167
171
dns_txt_resolver: HickoryDnsTxtResolver::new().unwrap(),
···
174
178
175
179
let cache = HybridCacheBuilder::new()
176
180
.with_name("identity")
177
-
.memory(16 * 2_usize.pow(20))
181
+
.memory(memory_mb * 2_usize.pow(20))
178
182
.with_weighter(|k, v| std::mem::size_of_val(k) + std::mem::size_of_val(v))
179
183
.storage(Engine::small())
180
184
.with_device_options(
181
185
DirectFsDeviceOptions::new(cache_dir)
182
-
.with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something)
183
-
.with_file_size(2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records
186
+
.with_capacity(disk_gb * 2_usize.pow(30)) // TODO: configurable (1GB to have something)
187
+
.with_file_size(2_usize.pow(20)), // note: this does limit the max cached item size!
184
188
)
185
189
.build()
186
190
.await?;
+24
-11
slingshot/src/main.rs
+24
-11
slingshot/src/main.rs
···
31
31
#[arg(long, env = "SLINGSHOT_BIND")]
32
32
#[clap(default_value = "0.0.0.0:8080")]
33
33
bind: std::net::SocketAddr,
34
-
/// memory cache size in megabytes
35
-
#[arg(long, env = "SLINGSHOT_CACHE_MEMORY_MB")]
34
+
/// memory cache size in megabytes for records
35
+
#[arg(long, env = "SLINGSHOT_RECORD_CACHE_MEMORY_MB")]
36
36
#[clap(default_value_t = 64)]
37
-
cache_memory_mb: usize,
38
-
/// disk cache size in gigabytes
39
-
#[arg(long, env = "SLINGHSOT_CACHE_DISK_DB")]
37
+
record_cache_memory_mb: usize,
38
+
/// disk cache size in gigabytes for records
39
+
#[arg(long, env = "SLINGSHOT_RECORD_CACHE_DISK_DB")]
40
40
#[clap(default_value_t = 1)]
41
-
cache_disk_gb: usize,
41
+
record_cache_disk_gb: usize,
42
+
/// memory cache size in megabytes for identities
43
+
#[arg(long, env = "SLINGSHOT_IDENTITY_CACHE_MEMORY_MB")]
44
+
#[clap(default_value_t = 64)]
45
+
identity_cache_memory_mb: usize,
46
+
/// disk cache size in gigabytes for identities
47
+
#[arg(long, env = "SLINGSHOT_IDENTITY_CACHE_DISK_DB")]
48
+
#[clap(default_value_t = 1)]
49
+
identity_cache_disk_gb: usize,
42
50
/// the domain pointing to this server
43
51
///
44
52
/// if present:
···
118
126
log::info!("setting up firehose cache...");
119
127
let cache = firehose_cache(
120
128
cache_dir.join("./firehose"),
121
-
args.cache_memory_mb,
122
-
args.cache_disk_gb,
129
+
args.record_cache_memory_mb,
130
+
args.record_cache_disk_gb,
123
131
)
124
132
.await?;
125
133
log::info!("firehose cache ready.");
···
127
135
let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new();
128
136
129
137
log::info!("starting identity service...");
130
-
let identity = Identity::new(cache_dir.join("./identity"))
131
-
.await
132
-
.map_err(|e| format!("identity setup failed: {e:?}"))?;
138
+
let identity = Identity::new(
139
+
cache_dir.join("./identity"),
140
+
args.identity_cache_memory_mb,
141
+
args.identity_cache_disk_gb,
142
+
)
143
+
.await
144
+
.map_err(|e| format!("identity setup failed: {e:?}"))?;
145
+
133
146
log::info!("identity service ready.");
134
147
let identity_refresher = identity.clone();
135
148
let identity_shutdown = shutdown.clone();