nautilus_persistence/backend/parquet/
mod.rs1use std::sync::Arc;
19
20use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel};
21
22use crate::{
23 catalog::{factory as catalog_factory, traits as catalog_traits},
24 config::CatalogCompression,
25};
26
27pub mod catalog;
28pub mod consolidation;
29pub mod delete;
30pub mod feather_session;
31pub mod file_admin;
32pub mod intervals;
33pub mod io;
34pub mod migration;
35pub mod paths;
36pub mod writer;
37
38pub(crate) mod metadata;
39
40mod consolidation_state;
41
42pub const DEFAULT_ROW_GROUP_SIZE: usize = 131_072;
44
45pub(crate) fn register_catalog_factory(registry: &mut catalog_factory::CatalogFactoryRegistry) {
46 registry.insert(
47 catalog_factory::PARQUET_CATALOG_FACTORY_NAME.to_string(),
48 Arc::new(|config: &catalog_factory::CatalogConnectConfig| {
49 Ok(Box::new(open_catalog(config)?) as catalog_traits::DataCatalog)
50 }),
51 );
52}
53
54pub(crate) fn open_catalog(
56 config: &catalog_factory::CatalogConnectConfig,
57) -> anyhow::Result<catalog::ParquetDataCatalog> {
58 catalog_factory::validate_catalog_params(
60 catalog_factory::PARQUET_CATALOG_FACTORY_NAME,
61 config.params.as_ref(),
62 &[],
63 )?;
64
65 catalog::ParquetDataCatalog::from_uri(
66 &config.uri,
67 config.storage_options.clone(),
68 config.batch_size,
69 config.compression.map(Compression::from),
70 config.max_row_group_size,
71 )
72}
73
74impl From<CatalogCompression> for Compression {
75 fn from(compression: CatalogCompression) -> Self {
76 match compression {
77 CatalogCompression::Uncompressed => Self::UNCOMPRESSED,
78 CatalogCompression::Snappy => Self::SNAPPY,
79 CatalogCompression::Gzip => Self::GZIP(GzipLevel::default()),
80 CatalogCompression::Brotli => Self::BROTLI(BrotliLevel::default()),
81 CatalogCompression::Lz4 => Self::LZ4_RAW,
82 CatalogCompression::Zstd => Self::ZSTD(ZstdLevel::default()),
83 }
84 }
85}
86
87#[cfg(test)]
88mod tests {
89 use std::fs::File;
90
91 use arrow::record_batch::RecordBatch;
92 use nautilus_core::{Params, UnixNanos};
93 use nautilus_model::data::{NautilusDataType, QuoteTick, stubs::quote_audusd};
94 use parquet::file::{
95 metadata::{ParquetMetaData, RowGroupMetaData},
96 reader::{FileReader, SerializedFileReader},
97 };
98 use rstest::rstest;
99 use serde_json::json;
100 use tempfile::TempDir;
101
102 use super::*;
103 use crate::config::DataCatalogConfig;
104
105 #[rstest]
106 fn open_catalog_writes_with_typed_settings() {
107 let directory = TempDir::new().unwrap();
108 let config = DataCatalogConfig::builder()
109 .path(directory.path().to_string_lossy().to_string())
110 .batch_size(2)
111 .compression(CatalogCompression::Uncompressed)
112 .max_row_group_size(3)
113 .build()
114 .unwrap();
115 let catalog = open_catalog(&config.connect_config()).unwrap();
116 let quotes = (1..=5).map(quote).collect::<Vec<_>>();
117
118 catalog.write_to_parquet("es, None, None, None).unwrap();
119
120 let batches = catalog.data_to_record_batches("es).unwrap();
121 let metadata = quote_file_metadata(&catalog, &directory);
122
123 assert_eq!(catalog.batch_size, 2);
124 assert_eq!(
125 batches
126 .iter()
127 .map(RecordBatch::num_rows)
128 .collect::<Vec<_>>(),
129 vec![2, 2, 1]
130 );
131 assert_eq!(
132 metadata
133 .row_groups()
134 .iter()
135 .map(RowGroupMetaData::num_rows)
136 .collect::<Vec<_>>(),
137 vec![3, 2]
138 );
139 assert!(
140 metadata
141 .row_groups()
142 .iter()
143 .flat_map(RowGroupMetaData::columns)
144 .all(|column| column.compression() == Compression::UNCOMPRESSED)
145 );
146 }
147
148 #[rstest]
149 #[case::uncompressed(CatalogCompression::Uncompressed, Compression::UNCOMPRESSED)]
150 #[case::snappy(CatalogCompression::Snappy, Compression::SNAPPY)]
151 #[case::gzip(CatalogCompression::Gzip, Compression::GZIP(GzipLevel::default()))]
152 #[case::brotli(
153 CatalogCompression::Brotli,
154 Compression::BROTLI(BrotliLevel::default())
155 )]
156 #[case::lz4(CatalogCompression::Lz4, Compression::LZ4_RAW)]
157 #[case::zstd(CatalogCompression::Zstd, Compression::ZSTD(ZstdLevel::default()))]
158 fn open_catalog_writes_each_compression(
159 #[case] compression: CatalogCompression,
160 #[case] expected: Compression,
161 ) {
162 let directory = TempDir::new().unwrap();
163 let mut config =
164 catalog_factory::CatalogConnectConfig::new(directory.path().to_string_lossy(), None);
165 config.compression = Some(compression);
166 let catalog = open_catalog(&config).unwrap();
167
168 catalog
169 .write_to_parquet(&[quote(1)], None, None, None)
170 .unwrap();
171
172 let metadata = quote_file_metadata(&catalog, &directory);
173
174 assert_eq!(catalog.compression, expected);
175 assert!(
176 metadata
177 .row_group(0)
178 .columns()
179 .iter()
180 .all(|column| column.compression() == expected)
181 );
182 }
183
184 #[rstest]
185 #[case::bare("")]
186 #[case::dotted("./")]
187 fn open_catalog_resolves_relative_path(#[case] prefix: &str) {
188 let directory = TempDir::new_in(".").unwrap();
189 let name = directory.path().file_name().unwrap().to_string_lossy();
190 let config = DataCatalogConfig::new(format!("{prefix}{name}"), None, None);
191 let mut catalog = open_catalog(&config.connect_config()).unwrap();
192
193 catalog
194 .write_to_parquet(&[quote(1)], None, None, None)
195 .unwrap();
196
197 let quotes = catalog
198 .query::<QuoteTick>(None, None, None, None, None, true)
199 .unwrap();
200
201 assert_eq!(quotes, vec![quote(1)]);
202 quote_file_metadata(&catalog, &directory);
203 }
204
205 #[rstest]
206 fn open_catalog_rejects_empty_path() {
207 let config = DataCatalogConfig::new(String::new(), None, None);
208
209 assert!(open_catalog(&config.connect_config()).is_err());
210 }
211
212 #[rstest]
213 #[case::moved_setting("batch_size")]
214 #[case::unknown("no_such_param")]
215 fn open_catalog_rejects_params_naming_key(#[case] key: &str) {
216 let directory = TempDir::new().unwrap();
217 let params = Params::new([(key.to_string(), json!(1024))].into());
218 let config = DataCatalogConfig::builder()
219 .path(directory.path().to_string_lossy().to_string())
220 .params(params)
221 .build()
222 .unwrap();
223
224 let error = open_catalog(&config.connect_config()).unwrap_err();
225
226 assert_eq!(
227 error.to_string(),
228 format!("Unknown Parquet catalog param '{key}': this catalog takes no params")
229 );
230 }
231
232 #[rstest]
234 #[case::batch_size(Some(0), None, "batch_size")]
235 #[case::max_row_group_size(None, Some(0), "max_row_group_size")]
236 fn open_catalog_rejects_zero_count(
237 #[case] batch_size: Option<usize>,
238 #[case] max_row_group_size: Option<usize>,
239 #[case] field: &str,
240 ) {
241 let directory = TempDir::new().unwrap();
242 let mut config =
243 catalog_factory::CatalogConnectConfig::new(directory.path().to_string_lossy(), None);
244 config.batch_size = batch_size;
245 config.max_row_group_size = max_row_group_size;
246
247 let error = open_catalog(&config).unwrap_err();
248
249 assert_eq!(
250 error.to_string(),
251 format!(
252 "invalid {field}: must be a positive number of rows; omit the field for the \
253 backend default"
254 )
255 );
256 }
257
258 #[rstest]
259 #[case::batch_size(Some(0), None, "batch_size")]
260 #[case::max_row_group_size(None, Some(0), "max_row_group_size")]
261 fn from_uri_rejects_zero_count(
262 #[case] batch_size: Option<usize>,
263 #[case] max_row_group_size: Option<usize>,
264 #[case] field: &str,
265 ) {
266 let directory = TempDir::new().unwrap();
267
268 let error = catalog::ParquetDataCatalog::from_uri(
269 &directory.path().to_string_lossy(),
270 None,
271 batch_size,
272 None,
273 max_row_group_size,
274 )
275 .unwrap_err();
276
277 assert_eq!(
278 error.to_string(),
279 format!(
280 "invalid {field}: must be a positive number of rows; omit the field for the \
281 backend default"
282 )
283 );
284 }
285
286 fn quote_file_metadata(
287 catalog: &catalog::ParquetDataCatalog,
288 directory: &TempDir,
289 ) -> ParquetMetaData {
290 let files = catalog
291 .query_files(&NautilusDataType::QuoteTick.into(), None, None, None)
292 .unwrap();
293 assert_eq!(files.len(), 1);
294
295 SerializedFileReader::new(File::open(directory.path().join(&files[0])).unwrap())
296 .unwrap()
297 .metadata()
298 .clone()
299 }
300
301 fn quote(ts: u64) -> QuoteTick {
302 QuoteTick {
303 ts_event: UnixNanos::from(ts),
304 ts_init: UnixNanos::from(ts),
305 ..quote_audusd()
306 }
307 }
308}