Skip to main content

nautilus_persistence/backend/parquet/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Parquet catalog backend.
17
18use std::sync::Arc;
19
20use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel};
21
22use crate::{
23    catalog::{factory as catalog_factory, traits as catalog_traits},
24    config::CatalogCompression,
25};
26
27pub mod catalog;
28pub mod consolidation;
29pub mod delete;
30pub mod feather_session;
31pub mod file_admin;
32pub mod intervals;
33pub mod io;
34pub mod migration;
35pub mod paths;
36pub mod writer;
37
38pub(crate) mod metadata;
39
40mod consolidation_state;
41
42/// Default number of rows in a Parquet row group.
43pub const DEFAULT_ROW_GROUP_SIZE: usize = 131_072;
44
45pub(crate) fn register_catalog_factory(registry: &mut catalog_factory::CatalogFactoryRegistry) {
46    registry.insert(
47        catalog_factory::PARQUET_CATALOG_FACTORY_NAME.to_string(),
48        Arc::new(|config: &catalog_factory::CatalogConnectConfig| {
49            Ok(Box::new(open_catalog(config)?) as catalog_traits::DataCatalog)
50        }),
51    );
52}
53
54// Streaming promotion opens its catalog here too, so both honor the catalog settings
55pub(crate) fn open_catalog(
56    config: &catalog_factory::CatalogConnectConfig,
57) -> anyhow::Result<catalog::ParquetDataCatalog> {
58    // Parquet reads only the typed settings, so every `params` key is unknown
59    catalog_factory::validate_catalog_params(
60        catalog_factory::PARQUET_CATALOG_FACTORY_NAME,
61        config.params.as_ref(),
62        &[],
63    )?;
64
65    catalog::ParquetDataCatalog::from_uri(
66        &config.uri,
67        config.storage_options.clone(),
68        config.batch_size,
69        config.compression.map(Compression::from),
70        config.max_row_group_size,
71    )
72}
73
74impl From<CatalogCompression> for Compression {
75    fn from(compression: CatalogCompression) -> Self {
76        match compression {
77            CatalogCompression::Uncompressed => Self::UNCOMPRESSED,
78            CatalogCompression::Snappy => Self::SNAPPY,
79            CatalogCompression::Gzip => Self::GZIP(GzipLevel::default()),
80            CatalogCompression::Brotli => Self::BROTLI(BrotliLevel::default()),
81            CatalogCompression::Lz4 => Self::LZ4_RAW,
82            CatalogCompression::Zstd => Self::ZSTD(ZstdLevel::default()),
83        }
84    }
85}
86
87#[cfg(test)]
88mod tests {
89    use std::fs::File;
90
91    use arrow::record_batch::RecordBatch;
92    use nautilus_core::{Params, UnixNanos};
93    use nautilus_model::data::{NautilusDataType, QuoteTick, stubs::quote_audusd};
94    use parquet::file::{
95        metadata::{ParquetMetaData, RowGroupMetaData},
96        reader::{FileReader, SerializedFileReader},
97    };
98    use rstest::rstest;
99    use serde_json::json;
100    use tempfile::TempDir;
101
102    use super::*;
103    use crate::config::DataCatalogConfig;
104
105    #[rstest]
106    fn open_catalog_writes_with_typed_settings() {
107        let directory = TempDir::new().unwrap();
108        let config = DataCatalogConfig::builder()
109            .path(directory.path().to_string_lossy().to_string())
110            .batch_size(2)
111            .compression(CatalogCompression::Uncompressed)
112            .max_row_group_size(3)
113            .build()
114            .unwrap();
115        let catalog = open_catalog(&config.connect_config()).unwrap();
116        let quotes = (1..=5).map(quote).collect::<Vec<_>>();
117
118        catalog.write_to_parquet(&quotes, None, None, None).unwrap();
119
120        let batches = catalog.data_to_record_batches(&quotes).unwrap();
121        let metadata = quote_file_metadata(&catalog, &directory);
122
123        assert_eq!(catalog.batch_size, 2);
124        assert_eq!(
125            batches
126                .iter()
127                .map(RecordBatch::num_rows)
128                .collect::<Vec<_>>(),
129            vec![2, 2, 1]
130        );
131        assert_eq!(
132            metadata
133                .row_groups()
134                .iter()
135                .map(RowGroupMetaData::num_rows)
136                .collect::<Vec<_>>(),
137            vec![3, 2]
138        );
139        assert!(
140            metadata
141                .row_groups()
142                .iter()
143                .flat_map(RowGroupMetaData::columns)
144                .all(|column| column.compression() == Compression::UNCOMPRESSED)
145        );
146    }
147
148    #[rstest]
149    #[case::uncompressed(CatalogCompression::Uncompressed, Compression::UNCOMPRESSED)]
150    #[case::snappy(CatalogCompression::Snappy, Compression::SNAPPY)]
151    #[case::gzip(CatalogCompression::Gzip, Compression::GZIP(GzipLevel::default()))]
152    #[case::brotli(
153        CatalogCompression::Brotli,
154        Compression::BROTLI(BrotliLevel::default())
155    )]
156    #[case::lz4(CatalogCompression::Lz4, Compression::LZ4_RAW)]
157    #[case::zstd(CatalogCompression::Zstd, Compression::ZSTD(ZstdLevel::default()))]
158    fn open_catalog_writes_each_compression(
159        #[case] compression: CatalogCompression,
160        #[case] expected: Compression,
161    ) {
162        let directory = TempDir::new().unwrap();
163        let mut config =
164            catalog_factory::CatalogConnectConfig::new(directory.path().to_string_lossy(), None);
165        config.compression = Some(compression);
166        let catalog = open_catalog(&config).unwrap();
167
168        catalog
169            .write_to_parquet(&[quote(1)], None, None, None)
170            .unwrap();
171
172        let metadata = quote_file_metadata(&catalog, &directory);
173
174        assert_eq!(catalog.compression, expected);
175        assert!(
176            metadata
177                .row_group(0)
178                .columns()
179                .iter()
180                .all(|column| column.compression() == expected)
181        );
182    }
183
184    #[rstest]
185    #[case::bare("")]
186    #[case::dotted("./")]
187    fn open_catalog_resolves_relative_path(#[case] prefix: &str) {
188        let directory = TempDir::new_in(".").unwrap();
189        let name = directory.path().file_name().unwrap().to_string_lossy();
190        let config = DataCatalogConfig::new(format!("{prefix}{name}"), None, None);
191        let mut catalog = open_catalog(&config.connect_config()).unwrap();
192
193        catalog
194            .write_to_parquet(&[quote(1)], None, None, None)
195            .unwrap();
196
197        let quotes = catalog
198            .query::<QuoteTick>(None, None, None, None, None, true)
199            .unwrap();
200
201        assert_eq!(quotes, vec![quote(1)]);
202        quote_file_metadata(&catalog, &directory);
203    }
204
205    #[rstest]
206    fn open_catalog_rejects_empty_path() {
207        let config = DataCatalogConfig::new(String::new(), None, None);
208
209        assert!(open_catalog(&config.connect_config()).is_err());
210    }
211
212    #[rstest]
213    #[case::moved_setting("batch_size")]
214    #[case::unknown("no_such_param")]
215    fn open_catalog_rejects_params_naming_key(#[case] key: &str) {
216        let directory = TempDir::new().unwrap();
217        let params = Params::new([(key.to_string(), json!(1024))].into());
218        let config = DataCatalogConfig::builder()
219            .path(directory.path().to_string_lossy().to_string())
220            .params(params)
221            .build()
222            .unwrap();
223
224        let error = open_catalog(&config.connect_config()).unwrap_err();
225
226        assert_eq!(
227            error.to_string(),
228            format!("Unknown Parquet catalog param '{key}': this catalog takes no params")
229        );
230    }
231
232    // A deserialized config skips `DataCatalogConfig::validate`, so the factory checks counts too
233    #[rstest]
234    #[case::batch_size(Some(0), None, "batch_size")]
235    #[case::max_row_group_size(None, Some(0), "max_row_group_size")]
236    fn open_catalog_rejects_zero_count(
237        #[case] batch_size: Option<usize>,
238        #[case] max_row_group_size: Option<usize>,
239        #[case] field: &str,
240    ) {
241        let directory = TempDir::new().unwrap();
242        let mut config =
243            catalog_factory::CatalogConnectConfig::new(directory.path().to_string_lossy(), None);
244        config.batch_size = batch_size;
245        config.max_row_group_size = max_row_group_size;
246
247        let error = open_catalog(&config).unwrap_err();
248
249        assert_eq!(
250            error.to_string(),
251            format!(
252                "invalid {field}: must be a positive number of rows; omit the field for the \
253                 backend default"
254            )
255        );
256    }
257
258    #[rstest]
259    #[case::batch_size(Some(0), None, "batch_size")]
260    #[case::max_row_group_size(None, Some(0), "max_row_group_size")]
261    fn from_uri_rejects_zero_count(
262        #[case] batch_size: Option<usize>,
263        #[case] max_row_group_size: Option<usize>,
264        #[case] field: &str,
265    ) {
266        let directory = TempDir::new().unwrap();
267
268        let error = catalog::ParquetDataCatalog::from_uri(
269            &directory.path().to_string_lossy(),
270            None,
271            batch_size,
272            None,
273            max_row_group_size,
274        )
275        .unwrap_err();
276
277        assert_eq!(
278            error.to_string(),
279            format!(
280                "invalid {field}: must be a positive number of rows; omit the field for the \
281                 backend default"
282            )
283        );
284    }
285
286    fn quote_file_metadata(
287        catalog: &catalog::ParquetDataCatalog,
288        directory: &TempDir,
289    ) -> ParquetMetaData {
290        let files = catalog
291            .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
292            .unwrap();
293        assert_eq!(files.len(), 1);
294
295        SerializedFileReader::new(File::open(directory.path().join(&files[0])).unwrap())
296            .unwrap()
297            .metadata()
298            .clone()
299    }
300
301    fn quote(ts: u64) -> QuoteTick {
302        QuoteTick {
303            ts_event: UnixNanos::from(ts),
304            ts_init: UnixNanos::from(ts),
305            ..quote_audusd()
306        }
307    }
308}