nautilus_persistence/catalog/
factory.rs1use std::sync::Arc;
19
20use ahash::AHashMap;
21use indexmap::IndexMap;
22use nautilus_core::Params;
23use serde_json::Value;
24
25use crate::{
26 catalog::traits::DataCatalog,
27 common::paths::file_protocol_uri,
28 config::{CatalogBackendType, CatalogCompression},
29};
30
31pub const PARQUET_CATALOG_FACTORY_NAME: &str = "Parquet";
33
34#[derive(Debug, Clone)]
36pub struct CatalogConnectConfig {
37 pub uri: String,
40 pub storage_options: Option<AHashMap<String, String>>,
42 pub batch_size: Option<usize>,
44 pub compression: Option<CatalogCompression>,
46 pub max_row_group_size: Option<usize>,
48 pub params: Option<Params>,
51}
52
53impl CatalogConnectConfig {
54 #[must_use]
56 pub fn new(uri: impl Into<String>, storage_options: Option<AHashMap<String, String>>) -> Self {
57 Self {
58 uri: uri.into(),
59 storage_options,
60 batch_size: None,
61 compression: None,
62 max_row_group_size: None,
63 params: None,
64 }
65 }
66
67 #[must_use]
73 pub fn from_path_and_protocol(
74 path: &str,
75 fs_protocol: Option<&str>,
76 storage_options: Option<AHashMap<String, String>>,
77 ) -> Self {
78 let uri = match fs_protocol {
79 _ if path.contains("://") => path.to_string(),
80 Some("file") => file_protocol_uri(path),
81 Some(protocol) => format!("{protocol}://{path}"),
82 None => path.to_string(),
83 };
84
85 Self::new(uri, storage_options)
86 }
87}
88
89pub type CatalogFactory =
91 Arc<dyn Fn(&CatalogConnectConfig) -> anyhow::Result<DataCatalog> + Send + Sync>;
92
93pub type CatalogFactoryRegistry = IndexMap<String, CatalogFactory>;
95
96pub fn create_catalog(
104 backend: &CatalogBackendType,
105 config: &CatalogConnectConfig,
106 factories: &CatalogFactoryRegistry,
107) -> anyhow::Result<DataCatalog> {
108 let name = backend.to_string();
109 factories
110 .get(&name)
111 .ok_or_else(|| anyhow::anyhow!("No catalog factory registered for '{name}'"))?(config)
112}
113
114#[derive(Debug, Clone, Copy, PartialEq, Eq)]
116pub enum CatalogParamKind {
117 Object,
119 Count,
121 Text,
123 List,
125}
126
127impl CatalogParamKind {
128 fn accepts(self, value: &Value) -> bool {
129 match self {
130 Self::Object => value.is_object(),
131 Self::Count => value.is_u64(),
132 Self::Text => value.is_string(),
133 Self::List => value.is_array(),
134 }
135 }
136
137 const fn expected(self) -> &'static str {
138 match self {
139 Self::Object => "an object",
140 Self::Count => "a non-negative integer",
141 Self::Text => "a string",
142 Self::List => "a list",
143 }
144 }
145}
146
147pub fn validate_catalog_params(
157 catalog: &str,
158 params: Option<&Params>,
159 accepted: &[(&str, CatalogParamKind)],
160) -> anyhow::Result<()> {
161 let Some(params) = params else {
162 return Ok(());
163 };
164
165 for (key, value) in params {
166 let Some((_, kind)) = accepted.iter().find(|(name, _)| name == key) else {
167 if accepted.is_empty() {
168 anyhow::bail!(
169 "Unknown {catalog} catalog param '{key}': this catalog takes no params"
170 );
171 }
172
173 let names = accepted
174 .iter()
175 .map(|(name, _)| *name)
176 .collect::<Vec<_>>()
177 .join(", ");
178 anyhow::bail!("Unknown {catalog} catalog param '{key}', expected one of {names}");
179 };
180
181 anyhow::ensure!(
182 value.is_null() || kind.accepts(value),
183 "Invalid {catalog} catalog param '{key}': expected {}",
184 kind.expected()
185 );
186 }
187
188 Ok(())
189}
190
191#[cfg(test)]
192mod tests {
193 use rstest::rstest;
194 use serde_json::json;
195
196 use super::*;
197
198 #[rstest]
199 fn from_path_and_protocol_joins_scheme() {
200 let cfg = CatalogConnectConfig::from_path_and_protocol("bucket/cat", Some("s3"), None);
201 assert_eq!(cfg.uri, "s3://bucket/cat");
202
203 let cfg = CatalogConnectConfig::from_path_and_protocol("/tmp/cat", None, None);
204 assert_eq!(cfg.uri, "/tmp/cat");
205
206 let cfg = CatalogConnectConfig::from_path_and_protocol(
207 "postgres://user:pass@localhost/catalog",
208 Some("file"),
209 None,
210 );
211 assert_eq!(cfg.uri, "postgres://user:pass@localhost/catalog");
212 }
213
214 #[rstest]
215 #[case(r"C:\data\catalog", "file:///C:/data/catalog")]
216 #[case("C:/data/catalog", "file:///C:/data/catalog")]
217 #[case(r"D:\", "file:///D:/")]
218 fn from_path_and_protocol_normalizes_windows_drive_paths(
219 #[case] path: &str,
220 #[case] expected: &str,
221 ) {
222 let cfg = CatalogConnectConfig::from_path_and_protocol(path, Some("file"), None);
223 assert_eq!(cfg.uri, expected);
224 }
225
226 #[rstest]
227 fn from_path_and_protocol_keeps_non_drive_file_joins() {
228 let cfg = CatalogConnectConfig::from_path_and_protocol("/tmp/cat", Some("file"), None);
229 assert_eq!(cfg.uri, "file:///tmp/cat");
230
231 let cfg = CatalogConnectConfig::from_path_and_protocol(
232 r"\\server\share\catalog",
233 Some("file"),
234 None,
235 );
236 assert_eq!(cfg.uri, r"file://\\server\share\catalog");
237 }
238
239 #[rstest]
240 fn create_catalog_rejects_unregistered_backend() {
241 let config = CatalogConnectConfig::new("/tmp/catalog", None);
242
243 let error = create_catalog(
244 &CatalogBackendType::External("Missing".to_string()),
245 &config,
246 &CatalogFactoryRegistry::new(),
247 )
248 .expect_err("an unregistered backend should fail");
249
250 assert_eq!(
251 error.to_string(),
252 "No catalog factory registered for 'Missing'"
253 );
254 }
255
256 const ACCEPTED: &[(&str, CatalogParamKind)] = &[
257 ("options", CatalogParamKind::Object),
258 ("rows", CatalogParamKind::Count),
259 ("codec", CatalogParamKind::Text),
260 ("rules", CatalogParamKind::List),
261 ];
262
263 fn params(entries: &[(&str, Value)]) -> Params {
264 Params::new(
265 entries
266 .iter()
267 .map(|(key, value)| ((*key).to_string(), value.clone()))
268 .collect(),
269 )
270 }
271
272 #[rstest]
273 #[case::absent(None)]
274 #[case::every_kind(Some(params(&[
275 ("options", json!({"region": "eu-west-1"})),
276 ("rows", json!(1024)),
277 ("codec", json!("zstd")),
278 ("rules", json!([])),
279 ])))]
280 #[case::null_counts_as_absent(Some(params(&[("rows", Value::Null)])))]
281 fn validate_catalog_params_accepts_declared_keys(#[case] params: Option<Params>) {
282 assert!(validate_catalog_params("Test", params.as_ref(), ACCEPTED).is_ok());
283 }
284
285 #[rstest]
286 #[case::unknown(
287 "no_such_param",
288 json!(1),
289 "Unknown Test catalog param 'no_such_param', expected one of options, rows, codec, rules"
290 )]
291 #[case::unknown_null(
292 "no_such_param",
293 Value::Null,
294 "Unknown Test catalog param 'no_such_param', expected one of options, rows, codec, rules"
295 )]
296 #[case::text_for_count(
297 "rows",
298 json!("big"),
299 "Invalid Test catalog param 'rows': expected a non-negative integer"
300 )]
301 #[case::negative_count(
302 "rows",
303 json!(-1),
304 "Invalid Test catalog param 'rows': expected a non-negative integer"
305 )]
306 #[case::text_for_object(
307 "options",
308 json!("region=eu-west-1"),
309 "Invalid Test catalog param 'options': expected an object"
310 )]
311 #[case::count_for_text(
312 "codec",
313 json!(6),
314 "Invalid Test catalog param 'codec': expected a string"
315 )]
316 #[case::object_for_list(
317 "rules",
318 json!({}),
319 "Invalid Test catalog param 'rules': expected a list"
320 )]
321 fn validate_catalog_params_rejects_unknown_or_mistyped_key(
322 #[case] key: &str,
323 #[case] value: Value,
324 #[case] expected: &str,
325 ) {
326 let error =
327 validate_catalog_params("Test", Some(¶ms(&[(key, value)])), ACCEPTED).unwrap_err();
328
329 assert_eq!(error.to_string(), expected);
330 }
331
332 #[rstest]
333 fn validate_catalog_params_rejects_every_key_when_none_accepted() {
334 let error =
335 validate_catalog_params("Test", Some(¶ms(&[("rows", json!(1))])), &[]).unwrap_err();
336
337 assert_eq!(
338 error.to_string(),
339 "Unknown Test catalog param 'rows': this catalog takes no params"
340 );
341 }
342}