Skip to main content

nautilus_coinapi/python/
loaders.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Python access to standalone CoinAPI ingestion.
17
18use std::{collections::HashMap, num::NonZeroUsize, path::PathBuf, sync::Arc};
19
20use jiff::civil::Date;
21use nautilus_core::{
22    UnixNanos,
23    python::{to_pyruntime_err, to_pyvalue_err},
24    string::secret::SecretString,
25};
26use nautilus_model::{
27    data::{Bar, BarType, OrderBookDelta, QuoteTick, TradeTick},
28    identifiers::InstrumentId,
29    python::instruments::{instrument_any_to_pyobject, pyobject_to_instrument_any},
30};
31use nautilus_network::retry::RetryConfig;
32use pyo3::{IntoPyObjectExt, prelude::*};
33use ustr::Ustr;
34
35use super::csv::{
36    CoinApiDeltaStreamIterator, CoinApiQuoteStreamIterator, CoinApiTradeStreamIterator,
37};
38use crate::{
39    config::CoinApiInstrumentSelection, http::client::CoinApiHttpConfig, loaders::CoinApiDataLoader,
40};
41
42/// Standalone ingestion with reusable instrument definitions and lazy credentials.
43///
44/// Register or request definitions before loading market data. Construction sends no requests.
45/// History bounds are integer UNIX nanoseconds; limits must be positive.
46/// Local decoding failures raise `ValueError`; downloads and REST failures raise `RuntimeError`.
47/// Requested chunk sizes must be in `[1, 1_000_000]`.
48#[derive(Debug)]
49#[pyclass(
50    name = "CoinApiDataLoader",
51    module = "nautilus_trader.adapters.coinapi",
52    frozen
53)]
54#[pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.coinapi")]
55pub struct PyCoinApiDataLoader {
56    inner: Arc<CoinApiDataLoader>,
57}
58
59#[pymethods]
60#[pyo3_stub_gen::derive::gen_stub_pymethods]
61impl PyCoinApiDataLoader {
62    /// Creates a loader with optional credentials and HTTP controls.
63    ///
64    /// Missing credentials fall back to `COINAPI_API_KEY` on the first network request.
65    /// Local files require no key. Construction sends no requests.
66    #[new]
67    #[pyo3(signature = (api_key=None, base_url=None, proxy_url=None, timeout_secs=None, max_concurrent_requests=None, credit_budget=None, max_retries=None, retry_delay_initial_ms=None, retry_delay_max_ms=None, retry_max_elapsed_ms=None))]
68    #[expect(clippy::too_many_arguments)]
69    fn py_new(
70        api_key: Option<String>,
71        base_url: Option<String>,
72        proxy_url: Option<String>,
73        timeout_secs: Option<u64>,
74        max_concurrent_requests: Option<usize>,
75        credit_budget: Option<u64>,
76        max_retries: Option<u32>,
77        retry_delay_initial_ms: Option<u64>,
78        retry_delay_max_ms: Option<u64>,
79        retry_max_elapsed_ms: Option<u64>,
80    ) -> Self {
81        let defaults = CoinApiHttpConfig::default();
82
83        let http_config = CoinApiHttpConfig {
84            base_url: base_url.unwrap_or(defaults.base_url),
85            proxy_url: proxy_url.map(SecretString::from),
86            max_concurrent_requests: max_concurrent_requests
87                .unwrap_or(defaults.max_concurrent_requests),
88            credit_budget: credit_budget.unwrap_or(defaults.credit_budget),
89            retry: RetryConfig {
90                operation_timeout_ms: timeout_secs
91                    .map(|value| value.saturating_mul(1_000))
92                    .or(defaults.retry.operation_timeout_ms),
93                max_retries: max_retries.unwrap_or(defaults.retry.max_retries),
94                initial_delay_ms: retry_delay_initial_ms.unwrap_or(defaults.retry.initial_delay_ms),
95                max_delay_ms: retry_delay_max_ms.unwrap_or(defaults.retry.max_delay_ms),
96                max_elapsed_ms: retry_max_elapsed_ms.or(defaults.retry.max_elapsed_ms),
97                ..defaults.retry
98            },
99        };
100
101        let loader =
102            CoinApiDataLoader::with_http_config(api_key.map(SecretString::from), http_config);
103
104        Self {
105            inner: Arc::new(loader),
106        }
107    }
108
109    /// Registers an instrument and optional REST and numeric file-symbol mappings.
110    #[pyo3(signature = (instrument, symbol_id=None, symbol_id_int=None))]
111    fn add_instrument(
112        &self,
113        py: Python<'_>,
114        instrument: Py<PyAny>,
115        symbol_id: Option<&str>,
116        symbol_id_int: Option<u64>,
117    ) -> PyResult<()> {
118        let instrument = pyobject_to_instrument_any(py, instrument)?;
119        self.inner
120            .add_instrument(instrument, symbol_id.map(Ustr::from), symbol_id_int)
121            .map_err(to_pyvalue_err)
122    }
123
124    /// Returns a registered instrument definition.
125    fn instrument(&self, py: Python<'_>, instrument_id: InstrumentId) -> PyResult<Py<PyAny>> {
126        let instrument = self
127            .inner
128            .instrument(instrument_id)
129            .map_err(to_pyvalue_err)?;
130        instrument_any_to_pyobject(py, instrument)
131    }
132
133    /// Retrieves the supplied selections and retains their definitions and mappings.
134    #[pyo3(signature = (instruments, price_precisions=None))]
135    #[gen_stub(override_return_type(type_repr = "typing.Awaitable[list[typing.Any]]", imports = ("typing",)))]
136    fn request_instruments<'py>(
137        &self,
138        py: Python<'py>,
139        instruments: Vec<CoinApiInstrumentSelection>,
140        price_precisions: Option<HashMap<InstrumentId, u8>>,
141    ) -> PyResult<Bound<'py, PyAny>> {
142        let loader = Arc::clone(&self.inner);
143        let price_precisions = price_precisions.unwrap_or_default();
144        pyo3_async_runtimes::tokio::future_into_py(py, async move {
145            let instruments = loader
146                .request_instruments(&instruments, &price_precisions)
147                .await
148                .map_err(to_pyruntime_err)?;
149            Python::attach(|py| {
150                instruments
151                    .into_iter()
152                    .map(|instrument| instrument_any_to_pyobject(py, instrument))
153                    .collect::<PyResult<Vec<_>>>()
154            })
155        })
156    }
157
158    /// Downloads a Flat Files object without requiring instrument definitions.
159    ///
160    /// Creates parent directories and replaces the destination only after a complete download.
161    /// Uses the configured credentials, proxy, retry, timeout, and concurrency controls.
162    /// `timeout_secs` bounds each attempt, including admission and the complete body; defaults to 30.
163    /// `retry_max_elapsed_ms` bounds the whole call; defaults to 120_000. Raise both for large files.
164    /// Each retry restarts the download and can incur additional billed transfer.
165    /// Flat Files usage does not consume the local REST credit allowance. Failures raise `RuntimeError`.
166    #[gen_stub(override_return_type(type_repr = "typing.Awaitable[None]", imports = ("typing",)))]
167    fn download_flat_file<'py>(
168        &self,
169        py: Python<'py>,
170        bucket: String,
171        object_key: String,
172        filepath: PathBuf,
173    ) -> PyResult<Bound<'py, PyAny>> {
174        let loader = Arc::clone(&self.inner);
175        pyo3_async_runtimes::tokio::future_into_py(py, async move {
176            loader
177                .download_flat_file(&bucket, &object_key, &filepath)
178                .await
179                .map_err(to_pyruntime_err)
180        })
181    }
182
183    /// Loads local book deltas using the registered instrument precision.
184    ///
185    /// Supply `date` or use a path with a `D-` partition; otherwise loading raises `ValueError`.
186    #[pyo3(signature = (filepath, instrument_id, date=None, limit=None))]
187    fn load_book_deltas(
188        &self,
189        filepath: PathBuf,
190        instrument_id: InstrumentId,
191        date: Option<Date>,
192        limit: Option<usize>,
193    ) -> PyResult<Vec<OrderBookDelta>> {
194        self.inner
195            .load_book_deltas(filepath, instrument_id, date, limit)
196            .map_err(to_pyvalue_err)
197    }
198
199    /// Streams local book deltas using the registered instrument precision.
200    ///
201    /// Chunks end at event boundaries and can exceed `chunk_size`.
202    ///
203    /// Supply `date` or use a path with a `D-` partition; otherwise loading raises `ValueError`.
204    #[pyo3(signature = (filepath, instrument_id, date=None, chunk_size=100_000, limit=None))]
205    fn stream_book_deltas(
206        &self,
207        filepath: PathBuf,
208        instrument_id: InstrumentId,
209        date: Option<Date>,
210        chunk_size: usize,
211        limit: Option<usize>,
212    ) -> PyResult<CoinApiDeltaStreamIterator> {
213        let stream = self
214            .inner
215            .stream_book_deltas(filepath, instrument_id, date, chunk_size, limit)
216            .map_err(to_pyvalue_err)?;
217        Ok(CoinApiDeltaStreamIterator {
218            stream: Box::new(stream),
219        })
220    }
221
222    /// Loads local quotes using the registered instrument precision.
223    #[pyo3(signature = (filepath, instrument_id, limit=None))]
224    fn load_quotes(
225        &self,
226        filepath: PathBuf,
227        instrument_id: InstrumentId,
228        limit: Option<usize>,
229    ) -> PyResult<Vec<QuoteTick>> {
230        self.inner
231            .load_quotes(filepath, instrument_id, limit)
232            .map_err(to_pyvalue_err)
233    }
234
235    /// Streams local quotes using the registered instrument precision.
236    #[pyo3(signature = (filepath, instrument_id, chunk_size=100_000, limit=None))]
237    fn stream_quotes(
238        &self,
239        filepath: PathBuf,
240        instrument_id: InstrumentId,
241        chunk_size: usize,
242        limit: Option<usize>,
243    ) -> PyResult<CoinApiQuoteStreamIterator> {
244        let stream = self
245            .inner
246            .stream_quotes(filepath, instrument_id, chunk_size, limit)
247            .map_err(to_pyvalue_err)?;
248        Ok(CoinApiQuoteStreamIterator {
249            stream: Box::new(stream),
250        })
251    }
252
253    /// Loads local trades using the registered instrument precision.
254    #[pyo3(signature = (filepath, instrument_id, limit=None))]
255    fn load_trades(
256        &self,
257        filepath: PathBuf,
258        instrument_id: InstrumentId,
259        limit: Option<usize>,
260    ) -> PyResult<Vec<TradeTick>> {
261        self.inner
262            .load_trades(filepath, instrument_id, limit)
263            .map_err(to_pyvalue_err)
264    }
265
266    /// Streams local trades using the registered instrument precision.
267    #[pyo3(signature = (filepath, instrument_id, chunk_size=100_000, limit=None))]
268    fn stream_trades(
269        &self,
270        filepath: PathBuf,
271        instrument_id: InstrumentId,
272        chunk_size: usize,
273        limit: Option<usize>,
274    ) -> PyResult<CoinApiTradeStreamIterator> {
275        let stream = self
276            .inner
277            .stream_trades(filepath, instrument_id, chunk_size, limit)
278            .map_err(to_pyvalue_err)?;
279        Ok(CoinApiTradeStreamIterator {
280            stream: Box::new(stream),
281        })
282    }
283
284    /// Loads consolidated OHLCV bars for registered numeric symbol IDs.
285    ///
286    /// Reads the period from `period_id` or the `TP-` file name; unmapped symbols are skipped.
287    #[pyo3(signature = (filepath, period_id=None, limit=None))]
288    fn load_bars(
289        &self,
290        filepath: PathBuf,
291        period_id: Option<&str>,
292        limit: Option<usize>,
293    ) -> PyResult<Vec<Bar>> {
294        self.inner
295            .load_bars(filepath, period_id, limit)
296            .map_err(to_pyvalue_err)
297    }
298
299    /// Retrieves a current L2 book using a registered REST symbol mapping.
300    #[pyo3(signature = (instrument_id, depth=None))]
301    #[gen_stub(override_return_type(type_repr = "typing.Awaitable[nautilus_trader.model.OrderBook]", imports = ("typing", "nautilus_trader.model")))]
302    fn request_book_snapshot<'py>(
303        &self,
304        py: Python<'py>,
305        instrument_id: InstrumentId,
306        depth: Option<u32>,
307    ) -> PyResult<Bound<'py, PyAny>> {
308        let loader = Arc::clone(&self.inner);
309        pyo3_async_runtimes::tokio::future_into_py(py, async move {
310            let book = loader
311                .request_book_snapshot(instrument_id, depth)
312                .await
313                .map_err(to_pyruntime_err)?;
314            Python::attach(|py| book.into_py_any(py))
315        })
316    }
317
318    /// Retrieves historical quotes using retained metadata.
319    ///
320    /// Bounds are UNIX nanoseconds in `[start, end)`; conversion or retrieval failures raise
321    /// without partial results.
322    #[pyo3(signature = (instrument_id, start, end, limit=None))]
323    #[gen_stub(override_return_type(type_repr = "typing.Awaitable[list[nautilus_trader.model.QuoteTick]]", imports = ("typing", "nautilus_trader.model")))]
324    fn request_quotes<'py>(
325        &self,
326        py: Python<'py>,
327        instrument_id: InstrumentId,
328        start: u64,
329        end: u64,
330        limit: Option<usize>,
331    ) -> PyResult<Bound<'py, PyAny>> {
332        let limit = limit
333            .map(NonZeroUsize::try_from)
334            .transpose()
335            .map_err(to_pyvalue_err)?;
336        let loader = Arc::clone(&self.inner);
337        pyo3_async_runtimes::tokio::future_into_py(py, async move {
338            let records = loader
339                .request_quotes(
340                    instrument_id,
341                    UnixNanos::from(start),
342                    UnixNanos::from(end),
343                    limit,
344                )
345                .await
346                .map_err(to_pyruntime_err)?;
347            Python::attach(|py| records.into_py_any(py))
348        })
349    }
350
351    /// Retrieves historical trades using retained metadata.
352    ///
353    /// Bounds are UNIX nanoseconds in `[start, end)`; conversion or retrieval failures raise
354    /// without partial results.
355    #[pyo3(signature = (instrument_id, start, end, limit=None))]
356    #[gen_stub(override_return_type(type_repr = "typing.Awaitable[list[nautilus_trader.model.TradeTick]]", imports = ("typing", "nautilus_trader.model")))]
357    fn request_trades<'py>(
358        &self,
359        py: Python<'py>,
360        instrument_id: InstrumentId,
361        start: u64,
362        end: u64,
363        limit: Option<usize>,
364    ) -> PyResult<Bound<'py, PyAny>> {
365        let limit = limit
366            .map(NonZeroUsize::try_from)
367            .transpose()
368            .map_err(to_pyvalue_err)?;
369        let loader = Arc::clone(&self.inner);
370        pyo3_async_runtimes::tokio::future_into_py(py, async move {
371            let records = loader
372                .request_trades(
373                    instrument_id,
374                    UnixNanos::from(start),
375                    UnixNanos::from(end),
376                    limit,
377                )
378                .await
379                .map_err(to_pyruntime_err)?;
380            Python::attach(|py| records.into_py_any(py))
381        })
382    }
383
384    /// Retrieves historical bars using retained metadata.
385    ///
386    /// Bounds are UNIX nanoseconds in `[start, end)`; conversion or retrieval failures raise
387    /// without partial results.
388    #[pyo3(signature = (bar_type, start, end, limit=None))]
389    #[gen_stub(override_return_type(type_repr = "typing.Awaitable[list[nautilus_trader.model.Bar]]", imports = ("typing", "nautilus_trader.model")))]
390    fn request_bars<'py>(
391        &self,
392        py: Python<'py>,
393        bar_type: BarType,
394        start: u64,
395        end: u64,
396        limit: Option<usize>,
397    ) -> PyResult<Bound<'py, PyAny>> {
398        let limit = limit
399            .map(NonZeroUsize::try_from)
400            .transpose()
401            .map_err(to_pyvalue_err)?;
402        let loader = Arc::clone(&self.inner);
403        pyo3_async_runtimes::tokio::future_into_py(py, async move {
404            let records = loader
405                .request_bars(
406                    bar_type,
407                    UnixNanos::from(start),
408                    UnixNanos::from(end),
409                    limit,
410                )
411                .await
412                .map_err(to_pyruntime_err)?;
413            Python::attach(|py| records.into_py_any(py))
414        })
415    }
416}