Loading External Data

Load CSV market data into the Parquet data catalog, then run a backtest with BacktestNode. This is a common workflow when you have historical data from an external vendor that is not directly supported by a NautilusTrader adapter.

View source on GitHub.

import os
import shutil
from pathlib import Path

from nautilus_trader.backtest import BacktestNode
from nautilus_trader.config import BacktestDataConfig
from nautilus_trader.config import BacktestEngineConfig
from nautilus_trader.config import BacktestRunConfig
from nautilus_trader.config import BacktestVenueConfig
from nautilus_trader.model import Quantity
from nautilus_trader.persistence import ParquetDataCatalog
from nautilus_trader.testkit.providers import TestDataProvider
from nautilus_trader.testkit.providers import TestInstrumentProvider
from nautilus_trader.trading import EmaCrossConfig

Load and wrangle the data

Place CSV tick files (e.g. from histdata.com) into ~/Downloads/Data/HISTDATA/. Set the NAUTILUS_DATA_DIR environment variable to the parent directory if your data lives elsewhere. TestDataProvider.quotes_from_histdata_csv converts the rows into Nautilus QuoteTick objects.

DATA_DIR = Path(os.environ.get("NAUTILUS_DATA_DIR", "~/Downloads/Data")).expanduser() / "HISTDATA"

path = DATA_DIR
raw_files = [
    f for f in path.iterdir() if f.is_file() and (f.suffix == ".csv" or f.name.endswith(".csv.gz"))
]
assert raw_files, f"Unable to find any data files in directory {path}"
raw_files

EURUSD = TestInstrumentProvider.default_fx_ccy("EUR/USD")
ticks = TestDataProvider.quotes_from_histdata_csv(EURUSD, raw_files[0])

Write to the data catalog

Create a ParquetDataCatalog and write the instrument definition and tick data. The catalog stores data in Parquet format for efficient querying across backtest runs.

CATALOG_PATH = Path.cwd() / "catalog"

# Clear if it already exists, then create fresh
if CATALOG_PATH.exists():
    shutil.rmtree(CATALOG_PATH)
CATALOG_PATH.mkdir(parents=True)

catalog = ParquetDataCatalog(str(CATALOG_PATH))

catalog.write_instruments([EURUSD])
catalog.write_quote_ticks(ticks)

# Verify instruments written to catalog
catalog.instruments()

start = ticks[0].ts_event
end = ticks[-1].ts_event + 1

ticks = catalog.query_quote_ticks(identifiers=[EURUSD.id.value], start=start, end=end)
ticks[:10]

Configure and run the backtest

Set up venue and data configs, build the node, then register the built-in EmaCross strategy. The same node and strategy pattern carries forward to live trading with LiveNode.

instrument = catalog.instruments()[0]

venue_configs = [
    BacktestVenueConfig(
        name="SIM",
        oms_type="HEDGING",
        account_type="MARGIN",
        base_currency="USD",
        starting_balances=["1000000 USD"],
    ),
]

data_configs = [
    BacktestDataConfig(
        catalog_path=str(CATALOG_PATH),
        data_type="QuoteTick",
        instrument_id=instrument.id,
        start_time=start,
        end_time=end,
    ),
]

config = BacktestRunConfig(
    engine=BacktestEngineConfig(),
    data=data_configs,
    venues=venue_configs,
)

node = BacktestNode(configs=[config])
node.build()
node.add_builtin_strategy(
    config.id,
    "EmaCross",
    EmaCrossConfig(
        instrument_id=instrument.id,
        trade_size=Quantity.from_int(1_000_000),
        fast_period=10,
        slow_period=20,
    ),
)

[result] = node.run()

result

Loading External Data

Load and wrangle the data

Write to the data catalog

Configure and run the backtest

On this page