Skip to main content

nautilus_persistence/catalog/
factory.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Catalog factory registry primitives.
17
18use std::sync::Arc;
19
20use ahash::AHashMap;
21use indexmap::IndexMap;
22use nautilus_core::Params;
23use serde_json::Value;
24
25use crate::{
26    catalog::traits::DataCatalog,
27    common::paths::file_protocol_uri,
28    config::{CatalogBackendType, CatalogCompression},
29};
30
31/// Conventional name of the Parquet catalog factory registration.
32pub const PARQUET_CATALOG_FACTORY_NAME: &str = "Parquet";
33
34/// Minimal connection-config supplied to [`CatalogFactory`].
35#[derive(Debug, Clone)]
36pub struct CatalogConnectConfig {
37    /// URI for the catalog backend (e.g. `file:///tmp/cat`, `s3://bucket/cat`), or a relative local
38    /// path for the backend to resolve (e.g. `./catalog`).
39    pub uri: String,
40    /// Optional storage-backend options (credentials, region, ...) passed to `object_store`.
41    pub storage_options: Option<AHashMap<String, String>>,
42    /// The number of rows per batch the catalog reads and writes.
43    pub batch_size: Option<usize>,
44    /// The compression codec of written data files.
45    pub compression: Option<CatalogCompression>,
46    /// The maximum number of rows per written row group.
47    pub max_row_group_size: Option<usize>,
48    /// Backend-specific catalog parameters, which a factory validates with
49    /// [`validate_catalog_params`].
50    pub params: Option<Params>,
51}
52
53impl CatalogConnectConfig {
54    /// Creates a new [`CatalogConnectConfig`].
55    #[must_use]
56    pub fn new(uri: impl Into<String>, storage_options: Option<AHashMap<String, String>>) -> Self {
57        Self {
58            uri: uri.into(),
59            storage_options,
60            batch_size: None,
61            compression: None,
62            max_row_group_size: None,
63            params: None,
64        }
65    }
66
67    /// Builds a [`CatalogConnectConfig`] from `path` + optional `fs_protocol`.
68    ///
69    /// A `file` protocol with a Windows drive path becomes `file:///C:/...`.
70    /// A `file` protocol leaves a relative path unchanged, for the backend to resolve.
71    /// A path that already contains `://` is left unchanged.
72    #[must_use]
73    pub fn from_path_and_protocol(
74        path: &str,
75        fs_protocol: Option<&str>,
76        storage_options: Option<AHashMap<String, String>>,
77    ) -> Self {
78        let uri = match fs_protocol {
79            _ if path.contains("://") => path.to_string(),
80            Some("file") => file_protocol_uri(path),
81            Some(protocol) => format!("{protocol}://{path}"),
82            None => path.to_string(),
83        };
84
85        Self::new(uri, storage_options)
86    }
87}
88
89/// Factory for a named catalog backend.
90pub type CatalogFactory =
91    Arc<dyn Fn(&CatalogConnectConfig) -> anyhow::Result<DataCatalog> + Send + Sync>;
92
93/// Ordered registry of catalog factories keyed by name.
94pub type CatalogFactoryRegistry = IndexMap<String, CatalogFactory>;
95
96/// Resolves a catalog backend through the factory registry and opens the catalog.
97///
98/// All variants resolve through the registry so built-ins and custom names share one code path.
99///
100/// # Errors
101///
102/// Returns an error if the backend's factory is not registered or opening the catalog fails.
103pub fn create_catalog(
104    backend: &CatalogBackendType,
105    config: &CatalogConnectConfig,
106    factories: &CatalogFactoryRegistry,
107) -> anyhow::Result<DataCatalog> {
108    let name = backend.to_string();
109    factories
110        .get(&name)
111        .ok_or_else(|| anyhow::anyhow!("No catalog factory registered for '{name}'"))?(config)
112}
113
114/// The JSON type a catalog param holds.
115#[derive(Debug, Clone, Copy, PartialEq, Eq)]
116pub enum CatalogParamKind {
117    /// An object of named values.
118    Object,
119    /// A non-negative integer.
120    Count,
121    /// A string.
122    Text,
123    /// An array.
124    List,
125}
126
127impl CatalogParamKind {
128    fn accepts(self, value: &Value) -> bool {
129        match self {
130            Self::Object => value.is_object(),
131            Self::Count => value.is_u64(),
132            Self::Text => value.is_string(),
133            Self::List => value.is_array(),
134        }
135    }
136
137    const fn expected(self) -> &'static str {
138        match self {
139            Self::Object => "an object",
140            Self::Count => "a non-negative integer",
141            Self::Text => "a string",
142            Self::List => "a list",
143        }
144    }
145}
146
147/// Checks that `params` holds only the `accepted` keys, each with its declared kind.
148///
149/// A catalog factory calls this with the keys it reads. A `null` value satisfies any declared
150/// kind, but an undeclared key fails whatever its value.
151///
152/// # Errors
153///
154/// Returns an error if a key is not accepted or its value has the wrong kind. The error names the
155/// catalog and the key.
156pub fn validate_catalog_params(
157    catalog: &str,
158    params: Option<&Params>,
159    accepted: &[(&str, CatalogParamKind)],
160) -> anyhow::Result<()> {
161    let Some(params) = params else {
162        return Ok(());
163    };
164
165    for (key, value) in params {
166        let Some((_, kind)) = accepted.iter().find(|(name, _)| name == key) else {
167            if accepted.is_empty() {
168                anyhow::bail!(
169                    "Unknown {catalog} catalog param '{key}': this catalog takes no params"
170                );
171            }
172
173            let names = accepted
174                .iter()
175                .map(|(name, _)| *name)
176                .collect::<Vec<_>>()
177                .join(", ");
178            anyhow::bail!("Unknown {catalog} catalog param '{key}', expected one of {names}");
179        };
180
181        anyhow::ensure!(
182            value.is_null() || kind.accepts(value),
183            "Invalid {catalog} catalog param '{key}': expected {}",
184            kind.expected()
185        );
186    }
187
188    Ok(())
189}
190
191#[cfg(test)]
192mod tests {
193    use rstest::rstest;
194    use serde_json::json;
195
196    use super::*;
197
198    #[rstest]
199    fn from_path_and_protocol_joins_scheme() {
200        let cfg = CatalogConnectConfig::from_path_and_protocol("bucket/cat", Some("s3"), None);
201        assert_eq!(cfg.uri, "s3://bucket/cat");
202
203        let cfg = CatalogConnectConfig::from_path_and_protocol("/tmp/cat", None, None);
204        assert_eq!(cfg.uri, "/tmp/cat");
205
206        let cfg = CatalogConnectConfig::from_path_and_protocol(
207            "postgres://user:pass@localhost/catalog",
208            Some("file"),
209            None,
210        );
211        assert_eq!(cfg.uri, "postgres://user:pass@localhost/catalog");
212    }
213
214    #[rstest]
215    #[case(r"C:\data\catalog", "file:///C:/data/catalog")]
216    #[case("C:/data/catalog", "file:///C:/data/catalog")]
217    #[case(r"D:\", "file:///D:/")]
218    fn from_path_and_protocol_normalizes_windows_drive_paths(
219        #[case] path: &str,
220        #[case] expected: &str,
221    ) {
222        let cfg = CatalogConnectConfig::from_path_and_protocol(path, Some("file"), None);
223        assert_eq!(cfg.uri, expected);
224    }
225
226    #[rstest]
227    fn from_path_and_protocol_keeps_non_drive_file_joins() {
228        let cfg = CatalogConnectConfig::from_path_and_protocol("/tmp/cat", Some("file"), None);
229        assert_eq!(cfg.uri, "file:///tmp/cat");
230
231        let cfg = CatalogConnectConfig::from_path_and_protocol(
232            r"\\server\share\catalog",
233            Some("file"),
234            None,
235        );
236        assert_eq!(cfg.uri, r"file://\\server\share\catalog");
237    }
238
239    #[rstest]
240    fn create_catalog_rejects_unregistered_backend() {
241        let config = CatalogConnectConfig::new("/tmp/catalog", None);
242
243        let error = create_catalog(
244            &CatalogBackendType::External("Missing".to_string()),
245            &config,
246            &CatalogFactoryRegistry::new(),
247        )
248        .expect_err("an unregistered backend should fail");
249
250        assert_eq!(
251            error.to_string(),
252            "No catalog factory registered for 'Missing'"
253        );
254    }
255
256    const ACCEPTED: &[(&str, CatalogParamKind)] = &[
257        ("options", CatalogParamKind::Object),
258        ("rows", CatalogParamKind::Count),
259        ("codec", CatalogParamKind::Text),
260        ("rules", CatalogParamKind::List),
261    ];
262
263    fn params(entries: &[(&str, Value)]) -> Params {
264        Params::new(
265            entries
266                .iter()
267                .map(|(key, value)| ((*key).to_string(), value.clone()))
268                .collect(),
269        )
270    }
271
272    #[rstest]
273    #[case::absent(None)]
274    #[case::every_kind(Some(params(&[
275        ("options", json!({"region": "eu-west-1"})),
276        ("rows", json!(1024)),
277        ("codec", json!("zstd")),
278        ("rules", json!([])),
279    ])))]
280    #[case::null_counts_as_absent(Some(params(&[("rows", Value::Null)])))]
281    fn validate_catalog_params_accepts_declared_keys(#[case] params: Option<Params>) {
282        assert!(validate_catalog_params("Test", params.as_ref(), ACCEPTED).is_ok());
283    }
284
285    #[rstest]
286    #[case::unknown(
287        "no_such_param",
288        json!(1),
289        "Unknown Test catalog param 'no_such_param', expected one of options, rows, codec, rules"
290    )]
291    #[case::unknown_null(
292        "no_such_param",
293        Value::Null,
294        "Unknown Test catalog param 'no_such_param', expected one of options, rows, codec, rules"
295    )]
296    #[case::text_for_count(
297        "rows",
298        json!("big"),
299        "Invalid Test catalog param 'rows': expected a non-negative integer"
300    )]
301    #[case::negative_count(
302        "rows",
303        json!(-1),
304        "Invalid Test catalog param 'rows': expected a non-negative integer"
305    )]
306    #[case::text_for_object(
307        "options",
308        json!("region=eu-west-1"),
309        "Invalid Test catalog param 'options': expected an object"
310    )]
311    #[case::count_for_text(
312        "codec",
313        json!(6),
314        "Invalid Test catalog param 'codec': expected a string"
315    )]
316    #[case::object_for_list(
317        "rules",
318        json!({}),
319        "Invalid Test catalog param 'rules': expected a list"
320    )]
321    fn validate_catalog_params_rejects_unknown_or_mistyped_key(
322        #[case] key: &str,
323        #[case] value: Value,
324        #[case] expected: &str,
325    ) {
326        let error =
327            validate_catalog_params("Test", Some(&params(&[(key, value)])), ACCEPTED).unwrap_err();
328
329        assert_eq!(error.to_string(), expected);
330    }
331
332    #[rstest]
333    fn validate_catalog_params_rejects_every_key_when_none_accepted() {
334        let error =
335            validate_catalog_params("Test", Some(&params(&[("rows", json!(1))])), &[]).unwrap_err();
336
337        assert_eq!(
338            error.to_string(),
339            "Unknown Test catalog param 'rows': this catalog takes no params"
340        );
341    }
342}