Skip to main content

nautilus_tardis/csv/
convert.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{path::PathBuf, time::Duration};
17
18use ahash::AHashMap;
19use anyhow::Context;
20use csv::StringRecord;
21use nautilus_core::UnixNanos;
22use nautilus_model::{
23    data::{Data, HasTsInit},
24    enums::OptionKind,
25    identifiers::{InstrumentId, Symbol},
26    instruments::{CryptoOption, InstrumentAny},
27    types::{Currency, Price, Quantity, fixed::FIXED_PRECISION},
28};
29use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
30use rust_decimal::Decimal;
31
32use crate::{
33    common::{
34        enums::{TardisExchange, TardisOptionType},
35        parse::{option_terms, parse_instrument_id, parse_option_kind, parse_timestamp},
36    },
37    csv::{
38        create_csv_reader, infer_precision, load::OptionsChainPrecision, matches_underlying_filter,
39        normalize_underlying_filters, parse_options_chain_record,
40        parse_options_chain_record_as_quote, record::TardisOptionsChainRecord,
41    },
42};
43
44const DATA_FLUSH_ROWS: usize = 100_000;
45
46/// Configuration for converting Tardis `options_chain` CSV files into a Nautilus catalog.
47#[derive(Debug, Clone, bon::Builder)]
48pub struct TardisOptionsChainCSVConverterConfig {
49    /// Tardis daily `options_chain` CSV file paths, processed in order.
50    pub filepaths: Vec<PathBuf>,
51    /// Nautilus catalog path.
52    pub catalog_path: PathBuf,
53    /// Optional underlying prefixes, such as `BTC-`.
54    pub underlyings: Option<Vec<String>>,
55    /// Optional thinning interval. Keeps the last row per instrument per bucket.
56    pub snapshot_interval: Option<Duration>,
57    /// Whether to emit quotes from best bid/offer fields.
58    #[builder(default = true)]
59    pub extract_bbo_as_quotes: bool,
60    /// Whether to derive and write instrument definitions from the CSV rows.
61    #[builder(default = true)]
62    pub write_instruments: bool,
63    /// Optional explicit price precision.
64    pub price_precision: Option<u8>,
65    /// Optional explicit size precision.
66    pub size_precision: Option<u8>,
67}
68
69/// Converts Tardis `options_chain` CSV files into `QuoteTick`, `OptionGreeks`, and instruments.
70///
71/// # Errors
72///
73/// Returns an error if the catalog cannot be opened, a CSV file cannot be read, a row cannot be
74/// parsed, a complete best bid/offer row contains invalid values, instrument derivation fails, or
75/// catalog writes fail.
76pub fn convert_options_chain_csv(
77    config: &TardisOptionsChainCSVConverterConfig,
78) -> anyhow::Result<()> {
79    let underlyings = normalize_underlying_filters(config.underlyings.clone());
80    let catalog = ParquetDataCatalog::from_uri(
81        &config.catalog_path.to_string_lossy(),
82        None,
83        None,
84        None,
85        None,
86    )?;
87    let mut precision_by_instrument: AHashMap<InstrumentId, OptionsChainPrecision> =
88        AHashMap::new();
89    let mut instrument_states: AHashMap<InstrumentId, InstrumentBuildState> = AHashMap::new();
90    let mut data_buffers: AHashMap<InstrumentId, DataBuffer> = AHashMap::new();
91    let mut pending_records: AHashMap<(InstrumentId, u64), TardisOptionsChainRecord> =
92        AHashMap::new();
93    let mut current_bucket = None;
94
95    for filepath in &config.filepaths {
96        let mut reader = create_csv_reader(filepath)
97            .with_context(|| format!("failed to open CSV file {}", filepath.display()))?;
98        let mut csv_record = StringRecord::new();
99
100        while reader
101            .read_record(&mut csv_record)
102            .with_context(|| format!("failed to read CSV file {}", filepath.display()))?
103        {
104            if let Some(underlyings) = underlyings.as_deref() {
105                let Some(symbol) = csv_record.get(1) else {
106                    continue;
107                };
108                let symbol = symbol.to_uppercase();
109                if !matches_underlying_filter(&symbol, Some(underlyings)) {
110                    continue;
111                }
112            }
113
114            let record: TardisOptionsChainRecord = csv_record
115                .deserialize(None)
116                .with_context(|| format!("failed to parse CSV file {}", filepath.display()))?;
117            let instrument_id = parse_instrument_id(&record.exchange, record.symbol);
118            precision_by_instrument
119                .entry(instrument_id)
120                .or_insert_with(|| {
121                    OptionsChainPrecision::new(config.price_precision, config.size_precision)
122                })
123                .update(&record, config.price_precision, config.size_precision);
124            instrument_states
125                .entry(instrument_id)
126                .and_modify(|state| state.update_activation(record.local_timestamp))
127                .or_insert_with(|| InstrumentBuildState::new(record.clone()));
128
129            if let Some(interval) = config.snapshot_interval {
130                let interval_us = u64::try_from(interval.as_micros())
131                    .context("snapshot interval exceeds u64 microseconds")?;
132                anyhow::ensure!(interval_us > 0, "snapshot interval must be positive");
133                let bucket = record.local_timestamp / interval_us;
134
135                if let Some(current_bucket) = current_bucket {
136                    anyhow::ensure!(
137                        bucket >= current_bucket,
138                        "options_chain CSV rows must be ordered by local_timestamp when thinning"
139                    );
140                }
141
142                if current_bucket.is_none_or(|current| bucket > current) {
143                    flush_pending_records_before(
144                        &catalog,
145                        &mut pending_records,
146                        &mut data_buffers,
147                        &precision_by_instrument,
148                        bucket,
149                        config.extract_bbo_as_quotes,
150                    )?;
151                    current_bucket = Some(bucket);
152                }
153                pending_records
154                    .entry((instrument_id, bucket))
155                    .and_modify(|pending| {
156                        if record.local_timestamp >= pending.local_timestamp {
157                            *pending = record.clone();
158                        }
159                    })
160                    .or_insert(record);
161            } else {
162                flush_data_buffer_if_ready(
163                    &catalog,
164                    &mut data_buffers,
165                    instrument_id,
166                    parse_timestamp(record.local_timestamp),
167                )?;
168                let data = options_chain_record_to_data(
169                    &record,
170                    &precision_by_instrument,
171                    config.extract_bbo_as_quotes,
172                )?;
173                data_buffers
174                    .entry(instrument_id)
175                    .or_default()
176                    .extend(data, parse_timestamp(record.local_timestamp));
177            }
178        }
179
180        if config.snapshot_interval.is_some() {
181            flush_pending_records_before(
182                &catalog,
183                &mut pending_records,
184                &mut data_buffers,
185                &precision_by_instrument,
186                u64::MAX,
187                config.extract_bbo_as_quotes,
188            )?;
189            flush_data_buffers(&catalog, &mut data_buffers)?;
190            current_bucket = None;
191        }
192    }
193
194    flush_data_buffers(&catalog, &mut data_buffers)?;
195
196    if config.write_instruments {
197        let instruments = build_instruments(instrument_states, &precision_by_instrument)?;
198        catalog.write_instruments(instruments)?;
199    }
200
201    Ok(())
202}
203
204#[derive(Debug, Clone)]
205struct InstrumentBuildState {
206    record: TardisOptionsChainRecord,
207    activation: UnixNanos,
208}
209
210impl InstrumentBuildState {
211    fn new(record: TardisOptionsChainRecord) -> Self {
212        Self {
213            activation: parse_timestamp(record.local_timestamp),
214            record,
215        }
216    }
217
218    fn update_activation(&mut self, local_timestamp: u64) {
219        self.activation = self.activation.min(parse_timestamp(local_timestamp));
220    }
221}
222
223#[derive(Debug, Default)]
224struct DataBuffer {
225    data: Vec<Data>,
226    last_ts_init: Option<UnixNanos>,
227}
228
229impl DataBuffer {
230    fn extend(&mut self, mut data: Vec<Data>, ts_init: UnixNanos) {
231        self.data.append(&mut data);
232        self.last_ts_init = Some(ts_init);
233    }
234}
235
236fn flush_data_buffer_if_ready(
237    catalog: &ParquetDataCatalog,
238    data_buffers: &mut AHashMap<InstrumentId, DataBuffer>,
239    instrument_id: InstrumentId,
240    ts_init: UnixNanos,
241) -> anyhow::Result<()> {
242    let Some(buffer) = data_buffers.get_mut(&instrument_id) else {
243        return Ok(());
244    };
245
246    if buffer.data.len() >= DATA_FLUSH_ROWS
247        && buffer
248            .last_ts_init
249            .is_some_and(|last_ts_init| last_ts_init < ts_init)
250    {
251        write_data_buffer(catalog, &mut buffer.data)?;
252    }
253
254    Ok(())
255}
256
257fn flush_data_buffers(
258    catalog: &ParquetDataCatalog,
259    data_buffers: &mut AHashMap<InstrumentId, DataBuffer>,
260) -> anyhow::Result<()> {
261    for buffer in data_buffers.values_mut() {
262        write_data_buffer(catalog, &mut buffer.data)?;
263    }
264    data_buffers.clear();
265    Ok(())
266}
267
268fn flush_pending_records_before(
269    catalog: &ParquetDataCatalog,
270    pending_records: &mut AHashMap<(InstrumentId, u64), TardisOptionsChainRecord>,
271    data_buffers: &mut AHashMap<InstrumentId, DataBuffer>,
272    precision_by_instrument: &AHashMap<InstrumentId, OptionsChainPrecision>,
273    next_bucket: u64,
274    extract_bbo_as_quotes: bool,
275) -> anyhow::Result<()> {
276    let mut ready = Vec::new();
277    pending_records.retain(|(_, bucket), record| {
278        if *bucket < next_bucket {
279            ready.push(record.clone());
280            false
281        } else {
282            true
283        }
284    });
285    ready.sort_by_key(|record| (record.local_timestamp, record.symbol));
286
287    for record in ready {
288        let instrument_id = parse_instrument_id(&record.exchange, record.symbol);
289        let ts_init = parse_timestamp(record.local_timestamp);
290        flush_data_buffer_if_ready(catalog, data_buffers, instrument_id, ts_init)?;
291        let data =
292            options_chain_record_to_data(&record, precision_by_instrument, extract_bbo_as_quotes)?;
293        data_buffers
294            .entry(instrument_id)
295            .or_default()
296            .extend(data, ts_init);
297    }
298    Ok(())
299}
300
301fn options_chain_record_to_data(
302    record: &TardisOptionsChainRecord,
303    precision_by_instrument: &AHashMap<InstrumentId, OptionsChainPrecision>,
304    extract_bbo_as_quotes: bool,
305) -> anyhow::Result<Vec<Data>> {
306    let instrument_id = parse_instrument_id(&record.exchange, record.symbol);
307    let precision = precision_by_instrument
308        .get(&instrument_id)
309        .copied()
310        .unwrap_or_else(|| OptionsChainPrecision::new(None, None));
311    let mut data = Vec::with_capacity(2);
312
313    if extract_bbo_as_quotes
314        && let Some(quote) = parse_options_chain_record_as_quote(
315            record,
316            precision.price,
317            precision.size,
318            instrument_id,
319        )?
320    {
321        data.push(Data::Quote(quote));
322    }
323
324    data.push(Data::OptionGreeks(parse_options_chain_record(
325        record,
326        instrument_id,
327    )));
328    Ok(data)
329}
330
331fn write_data_buffer(catalog: &ParquetDataCatalog, data: &mut Vec<Data>) -> anyhow::Result<()> {
332    if data.is_empty() {
333        return Ok(());
334    }
335
336    let mut data_by_instrument: AHashMap<InstrumentId, Vec<Data>> = AHashMap::new();
337    for item in data.drain(..) {
338        data_by_instrument
339            .entry(item.instrument_id())
340            .or_default()
341            .push(item);
342    }
343
344    for instrument_data in data_by_instrument.values_mut() {
345        instrument_data.sort_by_key(HasTsInit::ts_init);
346        catalog.write_data_enum(instrument_data, None, None, None)?;
347    }
348
349    data.clear();
350    Ok(())
351}
352
353fn build_instruments(
354    instrument_states: AHashMap<InstrumentId, InstrumentBuildState>,
355    precision_by_instrument: &AHashMap<InstrumentId, OptionsChainPrecision>,
356) -> anyhow::Result<Vec<InstrumentAny>> {
357    let mut states = instrument_states.into_iter().collect::<Vec<_>>();
358    states.sort_by_key(|(instrument_id, _)| instrument_id.to_string());
359
360    states
361        .into_iter()
362        .map(|(instrument_id, state)| {
363            let precision = precision_by_instrument
364                .get(&instrument_id)
365                .copied()
366                .unwrap_or_else(|| OptionsChainPrecision::new(None, None));
367            create_crypto_option_from_options_chain_record(
368                &state.record,
369                instrument_id,
370                state.activation,
371                precision,
372            )
373        })
374        .collect()
375}
376
377fn create_crypto_option_from_options_chain_record(
378    record: &TardisOptionsChainRecord,
379    instrument_id: InstrumentId,
380    activation: UnixNanos,
381    precision: OptionsChainPrecision,
382) -> anyhow::Result<InstrumentAny> {
383    let underlying = record
384        .symbol
385        .as_str()
386        .split_once('-')
387        .map(|(underlying, _)| underlying)
388        .context("options_chain symbol missing underlying prefix")?;
389    let underlying_currency = Currency::get_or_create_crypto(underlying);
390    let (quote_currency, settlement_currency, is_inverse) =
391        option_currency_mapping(record.exchange, underlying_currency)?;
392    let instrument_price_precision = precision
393        .price
394        .max(infer_precision(record.strike_price).min(FIXED_PRECISION));
395    let price_increment = decimal_increment_price(instrument_price_precision)?;
396    let size_increment = decimal_increment_quantity(precision.size)?;
397    let strike_price = Price::from_decimal_dp(
398        Decimal::try_from(record.strike_price)?,
399        instrument_price_precision,
400    )?;
401    let expiration = parse_timestamp(record.expiration);
402    let option_kind = option_kind(record.option_type);
403    let (exercise_style, settlement_type) = option_terms(record.exchange);
404
405    Ok(InstrumentAny::CryptoOption(
406        CryptoOption::builder()
407            .instrument_id(instrument_id)
408            .raw_symbol(Symbol::from_ustr_unchecked(record.symbol))
409            .underlying(underlying_currency)
410            .quote_currency(quote_currency)
411            .settlement_currency(settlement_currency)
412            .is_inverse(is_inverse)
413            .option_kind(option_kind)
414            .strike_price(strike_price)
415            .activation_ns(activation)
416            .expiration_ns(expiration)
417            .price_precision(instrument_price_precision)
418            .size_precision(precision.size)
419            .price_increment(price_increment)
420            .size_increment(size_increment)
421            .lot_size(size_increment)
422            .min_quantity(size_increment)
423            .maybe_exercise_style(exercise_style)
424            .maybe_settlement_type(settlement_type)
425            .ts_event(parse_timestamp(record.timestamp))
426            .ts_init(parse_timestamp(record.local_timestamp))
427            .build()?,
428    ))
429}
430
431fn option_currency_mapping(
432    exchange: TardisExchange,
433    underlying_currency: Currency,
434) -> anyhow::Result<(Currency, Currency, bool)> {
435    match exchange {
436        TardisExchange::Deribit => Ok((underlying_currency, underlying_currency, true)),
437        exchange => anyhow::bail!(
438            "options_chain instrument derivation supports Deribit only, received {exchange}"
439        ),
440    }
441}
442
443fn decimal_increment_price(precision: u8) -> anyhow::Result<Price> {
444    Ok(Price::from_decimal_dp(
445        Decimal::new(1, u32::from(precision)),
446        precision,
447    )?)
448}
449
450fn decimal_increment_quantity(precision: u8) -> anyhow::Result<Quantity> {
451    Ok(Quantity::from_decimal_dp(
452        Decimal::new(1, u32::from(precision)),
453        precision,
454    )?)
455}
456
457const fn option_kind(value: TardisOptionType) -> OptionKind {
458    parse_option_kind(value)
459}
460
461#[cfg(test)]
462mod tests {
463    use std::{
464        fs,
465        path::{Path, PathBuf},
466        time::Duration,
467    };
468
469    use nautilus_model::{
470        data::{OptionGreeks, QuoteTick},
471        enums::{ExerciseStyle, OptionKind},
472        instruments::Instrument,
473    };
474    use nautilus_persistence::backend::parquet::catalog::ParquetDataCatalog;
475    use rstest::rstest;
476    use tempfile::TempDir;
477    use ustr::Ustr;
478
479    use super::*;
480    use crate::common::testing::get_test_data_path;
481
482    #[rstest]
483    fn test_options_chain_converter_config_defaults_extract_bbo_as_quotes() {
484        let config = TardisOptionsChainCSVConverterConfig::builder()
485            .filepaths(Vec::<PathBuf>::new())
486            .catalog_path(PathBuf::from("/tmp/options-chain-catalog"))
487            .build();
488
489        assert!(config.extract_bbo_as_quotes);
490        assert!(config.write_instruments);
491    }
492
493    #[rstest]
494    fn test_convert_options_chain_csv_returns_error_for_missing_catalog_directory() {
495        let temp_dir = TempDir::new().unwrap();
496        let catalog_path = temp_dir.path().join("missing");
497        let config = TardisOptionsChainCSVConverterConfig::builder()
498            .filepaths(vec![get_test_data_path("options_chain.csv")])
499            .catalog_path(catalog_path.clone())
500            .build();
501
502        let error = convert_options_chain_csv(&config).unwrap_err();
503
504        assert_eq!(
505            error.to_string(),
506            format!(
507                "failed to open local storage directory '{}'; create it if it does not exist \
508                 and check access permissions",
509                catalog_path.display()
510            )
511        );
512        assert!(!catalog_path.exists());
513    }
514
515    #[rstest]
516    fn test_convert_options_chain_csv_thins_and_round_trips_catalog() {
517        let temp_dir = TempDir::new().unwrap();
518        let filepath = get_test_data_path("options_chain.csv");
519        let config = TardisOptionsChainCSVConverterConfig {
520            filepaths: vec![filepath],
521            catalog_path: temp_dir.path().to_path_buf(),
522            underlyings: Some(vec!["BTC-9JUN20-9875".to_string()]),
523            snapshot_interval: Some(Duration::from_secs(60)),
524            extract_bbo_as_quotes: true,
525            write_instruments: true,
526            price_precision: None,
527            size_precision: None,
528        };
529
530        convert_options_chain_csv(&config).unwrap();
531
532        let mut catalog = ParquetDataCatalog::new(temp_dir.path(), None, None, None, None);
533        let instrument_id = "BTC-9JUN20-9875-P.DERIBIT".to_string();
534        let quotes = catalog
535            .query::<QuoteTick>(
536                Some(vec![instrument_id.clone()]),
537                None,
538                None,
539                None,
540                None,
541                true,
542            )
543            .unwrap();
544        let greeks = catalog
545            .query::<OptionGreeks>(
546                Some(vec![instrument_id.clone()]),
547                None,
548                None,
549                None,
550                None,
551                true,
552            )
553            .unwrap();
554        let instruments = catalog
555            .query_instruments(Some(std::slice::from_ref(&instrument_id)))
556            .unwrap();
557
558        assert_eq!(quotes.len(), 2);
559        assert_eq!(greeks.len(), 2);
560        assert_eq!(instruments.len(), 1);
561
562        assert_eq!(quotes[0].bid_price, Price::from("0.0206"));
563        assert_eq!(quotes[0].ask_price, Price::from("0.0236"));
564        assert_eq!(quotes[0].ts_init.as_u64(), 1_591_574_400_473_112_000);
565        assert_eq!(quotes[1].bid_price, Price::from("0.0207"));
566        assert_eq!(quotes[1].ts_init.as_u64(), 1_591_574_460_473_112_000);
567
568        let InstrumentAny::CryptoOption(option) = &instruments[0] else {
569            panic!("Expected CryptoOption");
570        };
571        assert_eq!(option.id().to_string(), instrument_id);
572        assert_eq!(option.raw_symbol(), Symbol::from("BTC-9JUN20-9875-P"));
573        assert_eq!(option.option_kind, OptionKind::Put);
574        assert_eq!(option.exercise_style, Some(ExerciseStyle::European));
575        assert_eq!(option.settlement_type, None);
576        assert_eq!(option.strike_price, Price::from("9875.0000"));
577        assert_eq!(
578            option.expiration_ns,
579            UnixNanos::from(1_591_689_600_000_000_000)
580        );
581        assert_eq!(
582            option.activation_ns,
583            UnixNanos::from(1_591_574_400_196_008_000)
584        );
585        assert_eq!(option.price_increment, Price::from("0.0001"));
586        assert_eq!(option.size_increment, Quantity::from("0.1"));
587        assert!(option.is_inverse);
588        assert_eq!(option.quote_currency, Currency::from("BTC"));
589        assert_eq!(option.settlement_currency, Currency::from("BTC"));
590    }
591
592    #[rstest]
593    fn test_convert_options_chain_csv_thinned_multi_instrument_writes_separate_catalogs() {
594        let temp_dir = TempDir::new().unwrap();
595        let filepath = get_test_data_path("options_chain.csv");
596        let config = TardisOptionsChainCSVConverterConfig {
597            filepaths: vec![filepath],
598            catalog_path: temp_dir.path().to_path_buf(),
599            underlyings: Some(vec!["BTC-".to_string()]),
600            snapshot_interval: Some(Duration::from_secs(60)),
601            extract_bbo_as_quotes: true,
602            write_instruments: false,
603            price_precision: None,
604            size_precision: None,
605        };
606
607        convert_options_chain_csv(&config).unwrap();
608
609        let mut catalog = ParquetDataCatalog::new(temp_dir.path(), None, None, None, None);
610        let call_id = "BTC-9JUN20-10000-C.DERIBIT".to_string();
611        let next_expiry_id = "BTC-10JUN20-10000-C.DERIBIT".to_string();
612        let call_quotes = catalog
613            .query::<QuoteTick>(Some(vec![call_id.clone()]), None, None, None, None, true)
614            .unwrap();
615        let next_expiry_greeks = catalog
616            .query::<OptionGreeks>(
617                Some(vec![next_expiry_id.clone()]),
618                None,
619                None,
620                None,
621                None,
622                true,
623            )
624            .unwrap();
625
626        assert_eq!(call_quotes.len(), 1);
627        assert_eq!(call_quotes[0].instrument_id.to_string(), call_id);
628        assert_eq!(call_quotes[0].bid_price, Price::from("0.0305"));
629        assert_eq!(next_expiry_greeks.len(), 1);
630        assert_eq!(
631            next_expiry_greeks[0].instrument_id.to_string(),
632            next_expiry_id
633        );
634        assert_eq!(next_expiry_greeks[0].greeks.delta, 0.0);
635    }
636
637    #[rstest]
638    fn test_convert_options_chain_csv_unthinned_round_trips_catalog() {
639        let temp_dir = TempDir::new().unwrap();
640        let filepath = get_test_data_path("options_chain.csv");
641        let config = TardisOptionsChainCSVConverterConfig {
642            filepaths: vec![filepath],
643            catalog_path: temp_dir.path().to_path_buf(),
644            underlyings: Some(vec!["BTC-9JUN20-9875".to_string()]),
645            snapshot_interval: None,
646            extract_bbo_as_quotes: true,
647            write_instruments: false,
648            price_precision: None,
649            size_precision: None,
650        };
651
652        convert_options_chain_csv(&config).unwrap();
653
654        let mut catalog = ParquetDataCatalog::new(temp_dir.path(), None, None, None, None);
655        let instrument_id = "BTC-9JUN20-9875-P.DERIBIT".to_string();
656        let quotes = catalog
657            .query::<QuoteTick>(
658                Some(vec![instrument_id.clone()]),
659                None,
660                None,
661                None,
662                None,
663                true,
664            )
665            .unwrap();
666        let greeks = catalog
667            .query::<OptionGreeks>(
668                Some(vec![instrument_id.clone()]),
669                None,
670                None,
671                None,
672                None,
673                true,
674            )
675            .unwrap();
676
677        assert_eq!(quotes.len(), 3);
678        assert_eq!(greeks.len(), 3);
679        assert_eq!(quotes[0].bid_price, Price::from("0.0205"));
680        assert_eq!(quotes[1].bid_price, Price::from("0.0206"));
681        assert_eq!(quotes[2].bid_price, Price::from("0.0207"));
682        assert_eq!(quotes[0].ts_init.as_u64(), 1_591_574_400_196_008_000);
683        assert_eq!(quotes[2].ts_init.as_u64(), 1_591_574_460_473_112_000);
684        assert!(
685            greeks
686                .iter()
687                .all(|greek| greek.instrument_id.to_string() == instrument_id)
688        );
689    }
690
691    #[rstest]
692    fn test_convert_options_chain_csv_can_suppress_bbo_quotes() {
693        let temp_dir = TempDir::new().unwrap();
694        let filepath = get_test_data_path("options_chain.csv");
695        let config = TardisOptionsChainCSVConverterConfig {
696            filepaths: vec![filepath],
697            catalog_path: temp_dir.path().to_path_buf(),
698            underlyings: Some(vec!["BTC-9JUN20-9875".to_string()]),
699            snapshot_interval: None,
700            extract_bbo_as_quotes: false,
701            write_instruments: false,
702            price_precision: None,
703            size_precision: None,
704        };
705
706        convert_options_chain_csv(&config).unwrap();
707
708        let mut catalog = ParquetDataCatalog::new(temp_dir.path(), None, None, None, None);
709        let instrument_id = "BTC-9JUN20-9875-P.DERIBIT".to_string();
710        let quotes = catalog
711            .query::<QuoteTick>(
712                Some(vec![instrument_id.clone()]),
713                None,
714                None,
715                None,
716                None,
717                true,
718            )
719            .unwrap();
720        let greeks = catalog
721            .query::<OptionGreeks>(
722                Some(vec![instrument_id.clone()]),
723                None,
724                None,
725                None,
726                None,
727                true,
728            )
729            .unwrap();
730
731        assert!(quotes.is_empty());
732        assert_eq!(greeks.len(), 3);
733        assert!(
734            greeks
735                .iter()
736                .all(|greek| greek.instrument_id.to_string() == instrument_id)
737        );
738    }
739
740    #[rstest]
741    fn test_convert_options_chain_csv_thinned_multi_file_resets_bucket_state() {
742        let temp_dir = TempDir::new().unwrap();
743        let fixture = fs::read_to_string(get_test_data_path("options_chain.csv")).unwrap();
744        let lines = fixture.lines().collect::<Vec<_>>();
745        let first_filepath = temp_dir.path().join("late_btc_options_chain.csv");
746        let second_filepath = temp_dir.path().join("early_eth_options_chain.csv");
747        let catalog_path = temp_dir.path().join("catalog");
748        fs::create_dir(&catalog_path).unwrap();
749        write_options_chain_rows(&first_filepath, lines[0], &[lines[7]]);
750        write_options_chain_rows(&second_filepath, lines[0], &[lines[5]]);
751
752        let config = TardisOptionsChainCSVConverterConfig {
753            filepaths: vec![first_filepath, second_filepath],
754            catalog_path: catalog_path.clone(),
755            underlyings: None,
756            snapshot_interval: Some(Duration::from_secs(60)),
757            extract_bbo_as_quotes: true,
758            write_instruments: false,
759            price_precision: None,
760            size_precision: None,
761        };
762
763        convert_options_chain_csv(&config).unwrap();
764
765        let mut catalog = ParquetDataCatalog::new(&catalog_path, None, None, None, None);
766        let btc_id = "BTC-9JUN20-9875-P.DERIBIT".to_string();
767        let eth_id = "ETH-9JUN20-250-P.DERIBIT".to_string();
768        let btc_quotes = catalog
769            .query::<QuoteTick>(Some(vec![btc_id.clone()]), None, None, None, None, true)
770            .unwrap();
771        let eth_quotes = catalog
772            .query::<QuoteTick>(Some(vec![eth_id.clone()]), None, None, None, None, true)
773            .unwrap();
774        let eth_greeks = catalog
775            .query::<OptionGreeks>(Some(vec![eth_id.clone()]), None, None, None, None, true)
776            .unwrap();
777
778        assert_eq!(btc_quotes.len(), 1);
779        assert_eq!(btc_quotes[0].instrument_id.to_string(), btc_id);
780        assert_eq!(btc_quotes[0].bid_price, Price::from("0.0207"));
781        assert_eq!(eth_quotes.len(), 1);
782        assert_eq!(eth_quotes[0].instrument_id.to_string(), eth_id);
783        assert_eq!(eth_quotes[0].bid_price, Price::from("0.12345"));
784        assert_eq!(eth_greeks.len(), 1);
785        assert_eq!(eth_greeks[0].instrument_id.to_string(), eth_id);
786    }
787
788    #[rstest]
789    fn test_create_crypto_option_preserves_fractional_strike_and_new_underlying() {
790        let record = TardisOptionsChainRecord {
791            exchange: TardisExchange::Deribit,
792            symbol: Ustr::from("NEW-9JUN20-2.05-C"),
793            timestamp: 1,
794            local_timestamp: 2,
795            option_type: TardisOptionType::Call,
796            strike_price: 2.05,
797            expiration: 1_591_689_600_000_000,
798            open_interest: None,
799            last_price: Some(0.1),
800            bid_price: Some(0.1),
801            bid_amount: Some(1.0),
802            bid_iv: None,
803            ask_price: Some(0.2),
804            ask_amount: Some(1.0),
805            ask_iv: None,
806            mark_price: None,
807            mark_iv: None,
808            underlying_index: "SYN.NEW-9JUN20".to_string(),
809            underlying_price: Some(2.0),
810            delta: None,
811            gamma: None,
812            vega: None,
813            theta: None,
814            rho: None,
815        };
816        let instrument_id = InstrumentId::from("NEW-9JUN20-2.05-C.DERIBIT");
817        let instrument = create_crypto_option_from_options_chain_record(
818            &record,
819            instrument_id,
820            UnixNanos::from(2_000),
821            OptionsChainPrecision { price: 1, size: 0 },
822        )
823        .unwrap();
824
825        let InstrumentAny::CryptoOption(option) = instrument else {
826            panic!("Expected CryptoOption");
827        };
828
829        assert_eq!(option.underlying, Currency::get_or_create_crypto("NEW"));
830        assert_eq!(option.strike_price, Price::from("2.05"));
831        assert_eq!(option.price_precision, 2);
832        assert_eq!(option.price_increment, Price::from("0.01"));
833    }
834
835    fn write_options_chain_rows(path: &Path, header: &str, rows: &[&str]) {
836        let mut contents = String::from(header);
837        contents.push('\n');
838        for row in rows {
839            contents.push_str(row);
840            contents.push('\n');
841        }
842        fs::write(path, contents).unwrap();
843    }
844}