1use std::{collections::HashMap, str::FromStr};
17
18use ahash::AHashMap;
19use bytes::Bytes;
20use futures::future::join_all;
21#[cfg(feature = "defi")]
22use nautilus_common::{cache::CacheSnapshotRef, defi::cache::InstrumentMigration};
23use nautilus_common::{
24 cache::database::{CacheMap, register_loaded_currencies},
25 enums::SerializationEncoding,
26};
27use nautilus_core::serialization::timestamps;
28use nautilus_model::{
29 accounts::AccountAny,
30 data::{CustomData, DataType, HasTsInit, InstrumentClose},
31 enums::CurrencyType,
32 events::{AccountState, OrderEventAny, OrderFilled, PositionSnapshot},
33 identifiers::{AccountId, ClientId, ClientOrderId, InstrumentId, PositionId},
34 instruments::{InstrumentAny, SyntheticInstrument},
35 orders::{Order, OrderAny},
36 position::Position,
37 types::Currency,
38};
39#[cfg(feature = "defi")]
40use nautilus_model::{
41 defi::currency::{self, TokenCurrency},
42 events::OrderSnapshot,
43 instruments::Instrument,
44};
45use redis::{AsyncCommands, aio::ConnectionManager};
46use serde::{Deserialize, Serialize, de::DeserializeOwned};
47use serde_json::Value;
48use ustr::Ustr;
49
50use super::get_index_key;
51#[cfg(feature = "defi")]
52use crate::defi::{contract_conflict, integrity_error, redis_read_error, validate_marker};
53
54#[derive(Debug, Clone, Serialize, Deserialize)]
60pub(crate) struct CurrencyRecord {
61 pub code: String,
62 pub precision: u8,
63 pub iso4217: u16,
64 pub name: String,
65 pub currency_type: CurrencyType,
66 #[cfg(feature = "defi")]
67 #[serde(default, skip_serializing_if = "Option::is_none")]
68 pub defi: Option<TokenCurrency>,
69 #[cfg(not(feature = "defi"))]
70 #[serde(default, skip_serializing_if = "Option::is_none")]
71 pub defi: Option<Value>,
72}
73
74impl From<&Currency> for CurrencyRecord {
75 fn from(currency: &Currency) -> Self {
76 Self {
77 code: currency.code.to_string(),
78 precision: currency.precision,
79 iso4217: currency.iso4217,
80 name: currency.name.to_string(),
81 currency_type: currency.currency_type,
82 #[cfg(feature = "defi")]
83 defi: currency::lookup(currency.code).map(|(_, metadata)| metadata),
84 #[cfg(not(feature = "defi"))]
85 defi: None,
86 }
87 }
88}
89
90impl TryFrom<CurrencyRecord> for Currency {
91 type Error = anyhow::Error;
92
93 fn try_from(record: CurrencyRecord) -> Result<Self, Self::Error> {
94 let currency = Self::new_checked(
95 record.code.as_str(),
96 record.precision,
97 record.iso4217,
98 record.name.as_str(),
99 record.currency_type,
100 )
101 .map_err(|e| anyhow::anyhow!("Invalid currency record: {e}"))?;
102
103 #[cfg(feature = "defi")]
104 if let Some(metadata) = record.defi {
105 return currency::register(currency, metadata).map_err(integrity_error);
106 } else if currency::lookup(currency.code).is_some() {
107 return Err(integrity_error(format!(
108 "Missing token metadata for {}",
109 currency.code
110 )));
111 }
112
113 #[cfg(not(feature = "defi"))]
114 if record.defi.is_some() {
115 anyhow::bail!("DeFi currency records require the defi feature");
116 }
117
118 Ok(currency)
119 }
120}
121
122enum CurrencyDecode {
124 Currency(Currency),
125 InvalidRecord(anyhow::Error),
127 Undecodable(anyhow::Error),
129}
130
131const INDEX: &str = "index";
133const GENERAL: &str = "general";
134const CURRENCIES: &str = "currencies";
135const INSTRUMENTS: &str = "instruments";
136const INSTRUMENT_CLOSES: &str = "instrument_closes";
137const SYNTHETICS: &str = "synthetics";
138const ACCOUNTS: &str = "accounts";
139const ORDERS: &str = "orders";
140const POSITIONS: &str = "positions";
141const SNAPSHOTS: &str = "snapshots";
142const ACTORS: &str = "actors";
143const STRATEGIES: &str = "strategies";
144const CUSTOM: &str = "custom";
145const REDIS_DELIMITER: char = ':';
146
147const INDEX_ORDER_IDS: &str = "index:order_ids";
149const INDEX_ORDER_POSITION: &str = "index:order_position";
150const INDEX_ORDER_CLIENT: &str = "index:order_client";
151const INDEX_ORDERS: &str = "index:orders";
152const INDEX_ORDERS_OPEN: &str = "index:orders_open";
153const INDEX_ORDERS_CLOSED: &str = "index:orders_closed";
154const INDEX_ORDERS_EMULATED: &str = "index:orders_emulated";
155const INDEX_ORDERS_INFLIGHT: &str = "index:orders_inflight";
156const INDEX_POSITIONS: &str = "index:positions";
157const INDEX_POSITIONS_OPEN: &str = "index:positions_open";
158const INDEX_POSITIONS_CLOSED: &str = "index:positions_closed";
159
160#[derive(Debug)]
161pub struct DatabaseQueries;
162
163impl DatabaseQueries {
164 pub fn serialize_payload<T: Serialize>(
170 encoding: SerializationEncoding,
171 payload: &T,
172 ) -> anyhow::Result<Vec<u8>> {
173 Self::serialize_payload_with_timestamps(encoding, payload, true)
174 }
175
176 pub(crate) fn serialize_payload_with_timestamps<T: Serialize>(
177 encoding: SerializationEncoding,
178 payload: &T,
179 timestamps_as_iso8601: bool,
180 ) -> anyhow::Result<Vec<u8>> {
181 let mut value = serde_json::to_value(payload)?;
182 timestamps::serialize(&mut value, timestamps_as_iso8601)?;
183 match encoding {
184 SerializationEncoding::MsgPack => rmp_serde::to_vec(&value)
185 .map_err(|e| anyhow::anyhow!("Failed to serialize msgpack `payload`: {e}")),
186 SerializationEncoding::Json => serde_json::to_vec(&value)
187 .map_err(|e| anyhow::anyhow!("Failed to serialize json `payload`: {e}")),
188 SerializationEncoding::Sbe => {
189 anyhow::bail!("SBE encoding is not supported for Redis cache payloads")
190 }
191 }
192 }
193
194 pub fn deserialize_payload<T: DeserializeOwned>(
202 encoding: SerializationEncoding,
203 payload: &[u8],
204 ) -> anyhow::Result<T> {
205 let mut value = match encoding {
206 SerializationEncoding::MsgPack => rmp_serde::from_slice(payload)
207 .map_err(|e| anyhow::anyhow!("Failed to deserialize msgpack `payload`: {e}"))?,
208 SerializationEncoding::Json => serde_json::from_slice(payload)
209 .map_err(|e| anyhow::anyhow!("Failed to deserialize json `payload`: {e}"))?,
210 SerializationEncoding::Sbe => {
211 anyhow::bail!("SBE encoding is not supported for Redis cache payloads")
212 }
213 };
214
215 timestamps::deserialize(&mut value)?;
216
217 deserialize_payload_value(value)
218 }
219
220 #[cfg(feature = "defi")]
221 #[allow(
222 clippy::too_many_arguments,
223 reason = "the atomic write checks token definitions and all instrument currency roles"
224 )]
225 pub(crate) async fn write_defi(
226 con: &mut ConnectionManager,
227 trader_key: &str,
228 encoding: SerializationEncoding,
229 key: &str,
230 codes: &[String],
231 payloads: &[Bytes],
232 references: &[String],
233 migration: Option<&InstrumentMigration>,
234 ) -> anyhow::Result<()> {
235 let instrument = key.starts_with("instruments:");
236 if payloads.len() != codes.len() + usize::from(instrument) {
237 return Err(integrity_error("Invalid DeFi write payload count"));
238 }
239
240 let mut keys = codes
241 .iter()
242 .map(|code| format!("{trader_key}:currencies:{code}"))
243 .collect::<Vec<_>>();
244
245 if instrument {
246 keys.push(format!("{trader_key}:{key}"));
247 }
248
249 let contracts = Self::contract_bindings(trader_key, encoding, &payloads[..codes.len()])?;
250 let contract_offset = keys.len();
251 keys.extend(contracts.iter().map(|(key, _)| key.clone()));
252 let write_count = keys.len();
253 let references = references
254 .iter()
255 .filter(|code| !codes.contains(code))
256 .collect::<Vec<_>>();
257 let reference_offset = keys.len();
258 keys.extend(
259 references
260 .iter()
261 .map(|code| format!("{trader_key}:currencies:{code}")),
262 );
263
264 for _ in 0..4 {
265 let previous = Self::read_bulk(con, &keys)
266 .await
267 .map_err(redis_read_error)?;
268 let mut replacement =
269 Self::check_currency_records(encoding, codes, payloads, &previous)?;
270
271 Self::check_contract_bindings(
272 con,
273 trader_key,
274 encoding,
275 &contracts,
276 &previous[contract_offset..reference_offset],
277 )
278 .await?;
279
280 Self::check_currency_references(encoding, &previous[reference_offset..])?;
281
282 if instrument {
283 Self::check_instrument_replacement(
284 con,
285 trader_key,
286 encoding,
287 previous[codes.len()].as_ref(),
288 &payloads[codes.len()],
289 codes,
290 migration,
291 )
292 .await?;
293 replacement.push(payloads[codes.len()].clone());
294 }
295
296 replacement.extend(
297 contracts
298 .iter()
299 .map(|(_, code)| Bytes::copy_from_slice(code.as_bytes())),
300 );
301
302 replacement.resize(keys.len(), Bytes::new());
303 let mut command = redis::cmd("EVAL");
305 command
306 .arg(
307 "for i = 1, #KEYS do
308 -- Keep command error codes intact on Redis 6.x
309 local value = redis.pcall('GET', KEYS[i])
310 if type(value) == 'table' and value.err then return value end
311 local offset = 1 + (i - 1) * 3
312 if ARGV[offset + 1] == '0' then
313 if value then return 0 end
314 elseif value ~= ARGV[offset + 2] then
315 return 0
316 end
317 end
318 local values = {}
319 for i = 1, tonumber(ARGV[1]) do
320 values[#values + 1] = KEYS[i]
321 values[#values + 1] = ARGV[(i - 1) * 3 + 4]
322 end
323 redis.call('MSET', unpack(values))
324 return 1
325 ",
326 )
327 .arg(keys.len())
328 .arg(&keys)
329 .arg(write_count);
330
331 for (old, new) in previous.iter().zip(&replacement) {
332 command
333 .arg(if old.is_some() { "1" } else { "0" })
334 .arg(old.as_deref().unwrap_or_default())
335 .arg(new.as_ref());
336 }
337
338 let written: i32 = command.query_async(con).await.map_err(redis_read_error)?;
339 if written == 1 {
340 return Ok(());
341 }
342 }
343
344 anyhow::bail!("Token dependencies changed concurrently; retry the write")
345 }
346
347 #[cfg(feature = "defi")]
348 fn check_currency_references(
349 encoding: SerializationEncoding,
350 previous: &[Option<Bytes>],
351 ) -> anyhow::Result<()> {
352 for bytes in previous.iter().flatten() {
353 let value: Value =
354 Self::deserialize_payload(encoding, bytes).map_err(integrity_error)?;
355
356 if value.get("defi").is_some_and(|value| !value.is_null()) {
357 return Err(integrity_error(
358 "Instrument token dependencies differ from persisted currency definitions",
359 ));
360 }
361 }
362
363 Ok(())
364 }
365
366 #[cfg(feature = "defi")]
367 fn contract_bindings(
368 trader_key: &str,
369 encoding: SerializationEncoding,
370 payloads: &[Bytes],
371 ) -> anyhow::Result<Vec<(String, String)>> {
372 let mut bindings = Vec::new();
373
374 for payload in payloads {
375 let record: CurrencyRecord =
376 Self::deserialize_payload(encoding, payload).map_err(integrity_error)?;
377
378 if let Some(metadata) = record.defi {
379 bindings.push((Self::contract_key(trader_key, &metadata), record.code));
380 }
381 }
382
383 Ok(bindings)
384 }
385
386 #[cfg(feature = "defi")]
387 fn contract_key(trader_key: &str, metadata: &TokenCurrency) -> String {
388 format!(
389 "{trader_key}:defi:contracts:{}:{}",
390 metadata.chain_id, metadata.address
391 )
392 }
393
394 #[cfg(feature = "defi")]
395 async fn check_contract_bindings(
396 con: &mut ConnectionManager,
397 trader_key: &str,
398 encoding: SerializationEncoding,
399 bindings: &[(String, String)],
400 previous: &[Option<Bytes>],
401 ) -> anyhow::Result<()> {
402 let mut missing = AHashMap::new();
403
404 for ((key, code), old) in bindings.iter().zip(previous) {
405 if let Some(old) = old {
406 let stored = std::str::from_utf8(old).map_err(integrity_error)?;
407 if stored != code {
408 return Err(contract_conflict(stored));
409 }
410 } else {
411 missing.insert(key.as_str(), code.as_str());
412 }
413 }
414
415 if missing.is_empty() {
416 return Ok(());
417 }
418
419 Self::check_unindexed_contracts(con, trader_key, encoding, &missing).await
420 }
421
422 #[cfg(feature = "defi")]
423 async fn check_unindexed_contracts(
424 con: &mut ConnectionManager,
425 trader_key: &str,
426 encoding: SerializationEncoding,
427 missing: &AHashMap<&str, &str>,
428 ) -> anyhow::Result<()> {
429 let keys = Self::scan_keys(con, format!("{trader_key}:currencies:*"))
430 .await
431 .map_err(redis_read_error)?;
432
433 for bytes in Self::read_bulk(con, &keys)
434 .await
435 .map_err(redis_read_error)?
436 .into_iter()
437 .flatten()
438 {
439 let value: Value =
440 Self::deserialize_payload(encoding, &bytes).map_err(integrity_error)?;
441
442 if value.get("defi").is_none_or(Value::is_null) {
443 continue;
444 }
445
446 let record: CurrencyRecord = serde_json::from_value(value).map_err(integrity_error)?;
447 let metadata = record
448 .defi
449 .ok_or_else(|| integrity_error("Missing token metadata"))?;
450 let key = Self::contract_key(trader_key, &metadata);
451 if let Some(code) = missing.get(key.as_str())
452 && *code != record.code
453 {
454 return Err(contract_conflict(&record.code));
455 }
456 }
457
458 Ok(())
459 }
460
461 #[cfg(feature = "defi")]
462 fn check_currency_records(
463 encoding: SerializationEncoding,
464 codes: &[String],
465 payloads: &[Bytes],
466 previous: &[Option<Bytes>],
467 ) -> anyhow::Result<Vec<Bytes>> {
468 let mut replacement = payloads[..codes.len()].to_vec();
469 for (i, code) in codes.iter().enumerate() {
470 let incoming: CurrencyRecord =
471 Self::deserialize_payload(encoding, &payloads[i]).map_err(integrity_error)?;
472
473 if incoming.code != *code {
474 return Err(integrity_error(format!(
475 "Invalid currency record for {code}"
476 )));
477 }
478
479 let token = incoming.defi.is_some();
480 Currency::try_from(incoming).map_err(integrity_error)?;
481
482 let Some(bytes) = &previous[i] else {
483 continue;
484 };
485
486 let value: Value =
487 Self::deserialize_payload(encoding, bytes).map_err(integrity_error)?;
488
489 if !token && value.as_str() == Some(code.as_str()) {
490 continue;
491 }
492
493 let stored: CurrencyRecord = serde_json::from_value(value).map_err(integrity_error)?;
494
495 if stored.code != *code || stored.defi.is_some() != token {
496 return Err(integrity_error(format!(
497 "Currency provenance conflicts at token code {code}"
498 )));
499 }
500
501 Currency::try_from(stored).map_err(integrity_error)?;
502
503 if token {
504 replacement[i] = bytes.clone();
505 }
506 }
507
508 Ok(replacement)
509 }
510
511 #[cfg(feature = "defi")]
512 async fn check_instrument_replacement(
513 con: &mut ConnectionManager,
514 trader_key: &str,
515 encoding: SerializationEncoding,
516 previous: Option<&Bytes>,
517 incoming: &Bytes,
518 codes: &[String],
519 migration: Option<&InstrumentMigration>,
520 ) -> anyhow::Result<()> {
521 let Some(bytes) = previous else {
522 return Ok(());
523 };
524
525 let value: Value = Self::deserialize_payload(encoding, bytes).map_err(integrity_error)?;
526 if value.get("instrument").is_some() || value.get("defi_currency_codes").is_some() {
527 let record: crate::defi::InstrumentRecord =
528 serde_json::from_value(value).map_err(integrity_error)?;
529
530 if record.defi_currency_codes != codes {
531 return Err(integrity_error(
532 "Cannot change a persisted token instrument's currency identities",
533 ));
534 }
535
536 let previous: InstrumentAny =
537 serde_json::from_value(record.instrument).map_err(integrity_error)?;
538 let record: crate::defi::InstrumentRecord =
539 Self::deserialize_payload(encoding, incoming).map_err(integrity_error)?;
540 let incoming: InstrumentAny =
541 serde_json::from_value(record.instrument).map_err(integrity_error)?;
542
543 if previous.base_currency() != incoming.base_currency()
544 || previous.quote_currency() != incoming.quote_currency()
545 || previous.settlement_currency() != incoming.settlement_currency()
546 {
547 return Err(integrity_error(
548 "Cannot change a persisted token instrument's currency identities",
549 ));
550 }
551
552 return Ok(());
553 }
554
555 if codes.is_empty() {
556 return Ok(());
557 }
558
559 let legacy: InstrumentAny = serde_json::from_value(value).map_err(integrity_error)?;
560 migration
561 .ok_or_else(|| {
562 integrity_error("Historical replacement requires the cache migration check")
563 })?
564 .validate()
565 .map_err(integrity_error)?;
566
567 Self::check_legacy_replacement(con, trader_key, encoding, legacy.id()).await
568 }
569
570 #[cfg(feature = "defi")]
571 async fn check_legacy_replacement(
572 con: &mut ConnectionManager,
573 trader_key: &str,
574 encoding: SerializationEncoding,
575 instrument_id: InstrumentId,
576 ) -> anyhow::Result<()> {
577 Self::check_position_history(con, trader_key, encoding, instrument_id).await?;
578
579 let prefix = format!("{trader_key}:orders:");
580
581 for key in Self::scan_keys(con, format!("{prefix}*"))
582 .await
583 .map_err(redis_read_error)?
584 {
585 let code = key
586 .strip_prefix(&prefix)
587 .ok_or_else(|| integrity_error("Invalid order key"))?;
588 let id = ClientOrderId::new_checked(code).map_err(integrity_error)?;
589 let order = Self::load_order(con, trader_key, &id, encoding)
590 .await
591 .map_err(redis_read_error)?
592 .ok_or_else(|| integrity_error(format!("Missing order state at {key}")))?;
593
594 if order.instrument_id() == instrument_id && !order.is_closed() {
595 return Err(integrity_error(format!(
596 "Instrument {instrument_id} has nonterminal orders"
597 )));
598 }
599 }
600
601 let prefix = format!("{trader_key}:snapshots:orders:");
602
603 for key in Self::scan_keys(con, format!("{prefix}*"))
604 .await
605 .map_err(redis_read_error)?
606 {
607 let values: Vec<Vec<u8>> = con.lrange(&key, 0, -1).await.map_err(redis_read_error)?;
608 for bytes in values {
609 let snapshot: OrderSnapshot =
610 Self::deserialize_payload(encoding, &bytes).map_err(integrity_error)?;
611
612 if snapshot.instrument_id == instrument_id && !snapshot.status.is_closed() {
613 return Err(integrity_error(format!(
614 "Instrument {instrument_id} has nonterminal order snapshots"
615 )));
616 }
617 }
618 }
619
620 Ok(())
621 }
622
623 #[cfg(feature = "defi")]
624 async fn check_position_history(
625 con: &mut ConnectionManager,
626 trader_key: &str,
627 encoding: SerializationEncoding,
628 instrument_id: InstrumentId,
629 ) -> anyhow::Result<()> {
630 let expected_id = instrument_id.to_string();
631
632 for collection in ["positions", "snapshots:positions"] {
633 let prefix = format!("{trader_key}:{collection}:");
634
635 for key in Self::scan_keys(con, format!("{prefix}*"))
636 .await
637 .map_err(redis_read_error)?
638 {
639 let values: Vec<Vec<u8>> =
640 con.lrange(&key, 0, -1).await.map_err(redis_read_error)?;
641
642 if values.is_empty() {
643 return Err(integrity_error(format!(
644 "Unreadable position history at {key}"
645 )));
646 }
647
648 for bytes in values {
649 let value: Value =
650 Self::deserialize_payload(encoding, &bytes).map_err(integrity_error)?;
651
652 let id = value
653 .get("instrument_id")
654 .and_then(Value::as_str)
655 .ok_or_else(|| {
656 integrity_error(format!(
657 "Missing instrument ID in position history at {key}"
658 ))
659 })?;
660
661 if id == expected_id {
662 return Err(integrity_error(format!(
663 "Instrument {instrument_id} has position history; retain its legacy currencies"
664 )));
665 }
666 }
667 }
668 }
669
670 Self::check_position_archives(con, trader_key, instrument_id).await
671 }
672
673 #[cfg(feature = "defi")]
674 async fn check_position_archives(
675 con: &mut ConnectionManager,
676 trader_key: &str,
677 instrument_id: InstrumentId,
678 ) -> anyhow::Result<()> {
679 let prefix = format!("{trader_key}:general:{}", CacheSnapshotRef::POSITION_PREFIX);
680
681 for key in Self::scan_keys(con, format!("{prefix}*"))
682 .await
683 .map_err(redis_read_error)?
684 {
685 let blob: Vec<u8> = con.get(&key).await.map_err(redis_read_error)?;
686 if CacheSnapshotRef::position_instrument_id(&blob).map_err(integrity_error)?
687 == instrument_id
688 {
689 return Err(integrity_error(format!(
690 "Instrument {instrument_id} has position history; retain its legacy currencies"
691 )));
692 }
693 }
694
695 Ok(())
696 }
697
698 pub async fn scan_keys(
704 con: &mut ConnectionManager,
705 pattern: String,
706 ) -> anyhow::Result<Vec<String>> {
707 let mut result = Vec::new();
708 let mut cursor = 0u64;
709
710 loop {
711 let scan_result: (u64, Vec<String>) = redis::cmd("SCAN")
712 .arg(cursor)
713 .arg("MATCH")
714 .arg(&pattern)
715 .arg("COUNT")
716 .arg(5000)
717 .query_async(con)
718 .await?;
719
720 let (new_cursor, keys) = scan_result;
721 result.extend(keys);
722
723 if new_cursor == 0 {
725 break;
726 }
727
728 cursor = new_cursor;
729 }
730
731 Ok(result)
732 }
733
734 pub async fn read_bulk(
740 con: &ConnectionManager,
741 keys: &[String],
742 ) -> anyhow::Result<Vec<Option<Bytes>>> {
743 if keys.is_empty() {
744 return Ok(vec![]);
745 }
746
747 let mut con = con.clone();
748
749 let results: Vec<Option<Vec<u8>>> =
751 redis::cmd("MGET").arg(keys).query_async(&mut con).await?;
752
753 let bytes_results: Vec<Option<Bytes>> = results
755 .into_iter()
756 .map(|opt| opt.map(Bytes::from))
757 .collect();
758
759 Ok(bytes_results)
760 }
761
762 pub async fn read_bulk_batched(
771 con: &ConnectionManager,
772 keys: &[String],
773 batch_size: usize,
774 ) -> anyhow::Result<Vec<Option<Bytes>>> {
775 if batch_size == 0 {
776 anyhow::bail!("`batch_size` must be greater than zero");
777 }
778
779 if keys.is_empty() {
780 return Ok(vec![]);
781 }
782
783 let mut all_results: Vec<Option<Bytes>> = Vec::with_capacity(keys.len());
784
785 for chunk in keys.chunks(batch_size) {
786 let mut con = con.clone();
787
788 let results: Vec<Option<Vec<u8>>> =
789 redis::cmd("MGET").arg(chunk).query_async(&mut con).await?;
790
791 all_results.extend(results.into_iter().map(|opt| opt.map(Bytes::from)));
792 }
793
794 Ok(all_results)
795 }
796
797 pub async fn read(
803 con: &ConnectionManager,
804 trader_key: &str,
805 key: &str,
806 ) -> anyhow::Result<Vec<Bytes>> {
807 let collection = Self::get_collection_key(key)?;
808 let full_key = format!("{trader_key}{REDIS_DELIMITER}{key}");
809
810 let mut con = con.clone();
811
812 match collection {
813 INDEX => Self::read_index(&mut con, &full_key).await,
814 GENERAL | CURRENCIES | INSTRUMENTS | INSTRUMENT_CLOSES | SYNTHETICS | ACTORS
815 | STRATEGIES => Self::read_string(&mut con, &full_key).await,
816 ACCOUNTS | ORDERS | POSITIONS | SNAPSHOTS => Self::read_list(&mut con, &full_key).await,
817 _ => anyhow::bail!("Unsupported operation: `read` for collection '{collection}'"),
818 }
819 }
820
821 pub async fn load_all(
829 con: &ConnectionManager,
830 encoding: SerializationEncoding,
831 trader_key: &str,
832 ) -> anyhow::Result<CacheMap> {
833 Self::load_all_batched(con, encoding, trader_key, usize::MAX).await
834 }
835
836 pub(crate) async fn load_all_batched(
837 con: &ConnectionManager,
838 encoding: SerializationEncoding,
839 trader_key: &str,
840 batch_size: usize,
841 ) -> anyhow::Result<CacheMap> {
842 let mut currencies =
845 Self::load_currencies_batched(con, trader_key, encoding, batch_size).await?;
846 register_loaded_currencies(&mut currencies)?;
847
848 let (instruments, instrument_closes, synthetics, accounts, orders, positions) =
849 tokio::try_join!(
850 Self::load_instruments(con, trader_key, encoding),
851 Self::load_instrument_closes_batched(con, trader_key, encoding, batch_size),
852 Self::load_synthetics(con, trader_key, encoding),
853 Self::load_accounts(con, trader_key, encoding),
854 Self::load_orders(con, trader_key, encoding),
855 Self::load_positions(con, trader_key, encoding)
856 )
857 .map_err(|e| anyhow::anyhow!("Error loading cache data: {e}"))?;
858
859 Ok(CacheMap {
860 currencies,
861 instruments,
862 instrument_closes,
863 synthetics,
864 accounts,
865 orders,
866 positions,
867 greeks: AHashMap::new(),
868 yield_curves: AHashMap::new(),
869 })
870 }
871
872 fn decode_currency(
880 encoding: SerializationEncoding,
881 value_bytes: &Bytes,
882 code: Ustr,
883 ) -> CurrencyDecode {
884 let record =
885 Self::deserialize_payload::<Value>(encoding, value_bytes).and_then(|mut value| {
886 if let Some(object) = value.as_object_mut()
887 && !object.contains_key("code")
888 {
889 object.insert("code".into(), Value::String(code.to_string()));
890 }
891
892 serde_json::from_value::<CurrencyRecord>(value).map_err(Into::into)
893 });
894
895 let record_err = match record {
896 Ok(record) => {
897 return match Currency::try_from(record) {
898 Ok(currency) => CurrencyDecode::Currency(currency),
899 Err(e) => CurrencyDecode::InvalidRecord(e),
900 };
901 }
902 Err(e) => e,
903 };
904
905 match Self::deserialize_payload::<String>(encoding, value_bytes) {
906 Ok(code) => match Currency::try_from_str(&code) {
907 Some(currency) => {
908 #[cfg(feature = "defi")]
909 if currency::lookup(currency.code).is_some() {
910 return CurrencyDecode::InvalidRecord(integrity_error(format!(
911 "Missing token metadata for {}",
912 currency.code,
913 )));
914 }
915
916 CurrencyDecode::Currency(currency)
917 }
918 None => CurrencyDecode::Undecodable(anyhow::anyhow!(
919 "bare currency code {code} is not registered"
920 )),
921 },
922 Err(_) => CurrencyDecode::InvalidRecord(anyhow::anyhow!(
923 "payload is not a bare code and does not decode as a currency record: {record_err}"
924 )),
925 }
926 }
927
928 pub async fn load_currencies(
935 con: &ConnectionManager,
936 trader_key: &str,
937 encoding: SerializationEncoding,
938 ) -> anyhow::Result<AHashMap<Ustr, Currency>> {
939 Self::load_currencies_batched(con, trader_key, encoding, usize::MAX).await
940 }
941
942 pub(crate) async fn load_currencies_batched(
943 con: &ConnectionManager,
944 trader_key: &str,
945 encoding: SerializationEncoding,
946 batch_size: usize,
947 ) -> anyhow::Result<AHashMap<Ustr, Currency>> {
948 let mut currencies = AHashMap::new();
949 let prefix = format!("{trader_key}{REDIS_DELIMITER}{CURRENCIES}{REDIS_DELIMITER}");
950 let pattern = format!("{prefix}*");
951 log::debug!("Loading {pattern}");
952
953 let mut con = con.clone();
954 let keys = Self::scan_keys(&mut con, pattern).await?;
955
956 if keys.is_empty() {
957 return Ok(currencies);
958 }
959
960 let bulk_values = Self::read_bulk_batched(&con, &keys, batch_size).await?;
962
963 for (key, value_opt) in keys.iter().zip(bulk_values.iter()) {
965 let currency_code = match key.as_str().strip_prefix(prefix.as_str()) {
968 Some(code) if !code.is_empty() => Ustr::from(code),
969 _ => {
970 log::error!("Invalid key format: {key}");
971 continue;
972 }
973 };
974
975 if let Some(value_bytes) = value_opt {
976 match Self::decode_currency(encoding, value_bytes, currency_code) {
977 CurrencyDecode::Currency(currency) => {
978 if currency.code != currency_code {
981 anyhow::bail!(
982 "Currency record under key {currency_code} holds code {}",
983 currency.code
984 );
985 }
986 currencies.insert(currency_code, currency);
987 }
988 CurrencyDecode::InvalidRecord(e) => {
989 anyhow::bail!("Invalid currency record for {currency_code}: {e}");
990 }
991 CurrencyDecode::Undecodable(e) => {
992 log::error!(
993 "Skipping currency {currency_code}: {e}; payloads denominated in it will not decode"
994 );
995 }
996 }
997 } else {
998 log::error!("Currency not found in Redis: {currency_code}");
999 }
1000 }
1001
1002 log::debug!("Loaded {} currencies(s)", currencies.len());
1003
1004 Ok(currencies)
1005 }
1006
1007 pub async fn load_instruments(
1018 con: &ConnectionManager,
1019 trader_key: &str,
1020 encoding: SerializationEncoding,
1021 ) -> anyhow::Result<AHashMap<InstrumentId, InstrumentAny>> {
1022 let mut instruments = AHashMap::new();
1023 let prefix = format!("{trader_key}{REDIS_DELIMITER}{INSTRUMENTS}{REDIS_DELIMITER}");
1024 let pattern = format!("{prefix}*");
1025 log::debug!("Loading {pattern}");
1026
1027 let mut con = con.clone();
1028 let keys = Self::scan_keys(&mut con, pattern).await?;
1029
1030 let futures: Vec<_> = keys
1031 .iter()
1032 .map(|key| {
1033 let con = con.clone();
1034 let prefix = &prefix;
1035 async move {
1036 let instrument_id = parse_instrument_key(key, prefix);
1037
1038 let Ok(instrument_id) = instrument_id else {
1039 log::error!("Failed to parse InstrumentId from Redis key: {key}");
1040 return Ok::<_, anyhow::Error>(None);
1041 };
1042
1043 match Self::load_instrument(&con, trader_key, &instrument_id, encoding).await {
1044 Ok(Some(instrument)) => Ok(Some((instrument_id, instrument))),
1045 Ok(None) => {
1046 log::error!("Instrument not found: {instrument_id}");
1047 Ok(None)
1048 }
1049 Err(e) => {
1050 #[cfg(feature = "defi")]
1051 if e.is::<crate::defi::IntegrityError>() {
1052 return Err(e);
1053 }
1054
1055 log::error!("Failed to load instrument {instrument_id}: {e}");
1056 Ok(None)
1057 }
1058 }
1059 }
1060 })
1061 .collect();
1062
1063 for result in join_all(futures).await {
1065 instruments.extend(result?);
1066 }
1067
1068 log::debug!("Loaded {} instruments(s)", instruments.len());
1069
1070 Ok(instruments)
1071 }
1072
1073 pub async fn load_instrument_closes(
1081 con: &ConnectionManager,
1082 trader_key: &str,
1083 encoding: SerializationEncoding,
1084 ) -> anyhow::Result<AHashMap<InstrumentId, InstrumentClose>> {
1085 Self::load_instrument_closes_batched(con, trader_key, encoding, usize::MAX).await
1086 }
1087
1088 pub(crate) async fn load_instrument_closes_batched(
1089 con: &ConnectionManager,
1090 trader_key: &str,
1091 encoding: SerializationEncoding,
1092 batch_size: usize,
1093 ) -> anyhow::Result<AHashMap<InstrumentId, InstrumentClose>> {
1094 let prefix = format!("{trader_key}{REDIS_DELIMITER}{INSTRUMENT_CLOSES}{REDIS_DELIMITER}");
1095 let pattern = format!("{prefix}*");
1096 log::debug!("Loading {pattern}");
1097
1098 let mut con = con.clone();
1099 let keys = Self::scan_keys(&mut con, pattern).await?;
1100 let values = Self::read_bulk_batched(&con, &keys, batch_size).await?;
1101 let mut closes = AHashMap::with_capacity(keys.len());
1102
1103 for (key, value) in keys.into_iter().zip(values) {
1104 let instrument_id = parse_instrument_key(&key, &prefix)?;
1105 let value = value
1106 .ok_or_else(|| anyhow::anyhow!("Instrument close not found in Redis: {key}"))?;
1107 let close: InstrumentClose = Self::deserialize_payload(encoding, &value)?;
1108 anyhow::ensure!(
1109 close.instrument_id == instrument_id,
1110 "Instrument close key ID {instrument_id} did not match payload ID {}",
1111 close.instrument_id,
1112 );
1113 closes.insert(instrument_id, close);
1114 }
1115
1116 log::debug!("Loaded {} instrument close(s)", closes.len());
1117 Ok(closes)
1118 }
1119
1120 pub async fn load_synthetics(
1126 con: &ConnectionManager,
1127 trader_key: &str,
1128 encoding: SerializationEncoding,
1129 ) -> anyhow::Result<AHashMap<InstrumentId, SyntheticInstrument>> {
1130 let mut synthetics = AHashMap::new();
1131 let pattern = format!("{trader_key}{REDIS_DELIMITER}{SYNTHETICS}*");
1132 log::debug!("Loading {pattern}");
1133
1134 let mut con = con.clone();
1135 let keys = Self::scan_keys(&mut con, pattern).await?;
1136
1137 let futures: Vec<_> = keys
1138 .iter()
1139 .map(|key| {
1140 let con = con.clone();
1141 async move {
1142 let instrument_id = key
1143 .as_str()
1144 .rsplit(':')
1145 .next()
1146 .ok_or_else(|| {
1147 log::error!("Invalid key format: {key}");
1148 "Invalid key format"
1149 })
1150 .and_then(|code| {
1151 InstrumentId::from_str(code).map_err(|e| {
1152 log::error!("Failed to parse InstrumentId for {key}: {e}");
1153 "Invalid instrument ID"
1154 })
1155 });
1156
1157 let Ok(instrument_id) = instrument_id else {
1158 return None;
1159 };
1160
1161 match Self::load_synthetic(&con, trader_key, &instrument_id, encoding).await {
1162 Ok(Some(synthetic)) => Some((instrument_id, synthetic)),
1163 Ok(None) => {
1164 log::error!("Synthetic not found: {instrument_id}");
1165 None
1166 }
1167 Err(e) => {
1168 log::error!("Failed to load synthetic {instrument_id}: {e}");
1169 None
1170 }
1171 }
1172 }
1173 })
1174 .collect();
1175
1176 synthetics.extend(join_all(futures).await.into_iter().flatten());
1178 log::debug!("Loaded {} synthetics(s)", synthetics.len());
1179
1180 Ok(synthetics)
1181 }
1182
1183 pub async fn load_accounts(
1194 con: &ConnectionManager,
1195 trader_key: &str,
1196 encoding: SerializationEncoding,
1197 ) -> anyhow::Result<AHashMap<AccountId, AccountAny>> {
1198 let mut accounts = AHashMap::new();
1199 let pattern = format!("{trader_key}{REDIS_DELIMITER}{ACCOUNTS}*");
1200 log::debug!("Loading {pattern}");
1201
1202 let mut con = con.clone();
1203 let keys = Self::scan_keys(&mut con, pattern).await?;
1204
1205 let futures: Vec<_> = keys
1206 .iter()
1207 .map(|key| {
1208 let con = con.clone();
1209 async move {
1210 let account_id = if let Some(code) = key.as_str().rsplit(':').next() {
1211 AccountId::from(code)
1212 } else {
1213 log::error!("Invalid key format: {key}");
1214 return None;
1215 };
1216
1217 match Self::load_account(&con, trader_key, &account_id, encoding).await {
1218 Ok(Some(account)) => Some((account_id, account)),
1219 Ok(None) => {
1220 log::error!("Account not found: {account_id}");
1221 None
1222 }
1223 Err(e) => {
1224 log::error!("Failed to load account {account_id}: {e}");
1225 None
1226 }
1227 }
1228 }
1229 })
1230 .collect();
1231
1232 accounts.extend(join_all(futures).await.into_iter().flatten());
1234 log::debug!("Loaded {} accounts(s)", accounts.len());
1235
1236 Ok(accounts)
1237 }
1238
1239 pub async fn load_orders(
1250 con: &ConnectionManager,
1251 trader_key: &str,
1252 encoding: SerializationEncoding,
1253 ) -> anyhow::Result<AHashMap<ClientOrderId, OrderAny>> {
1254 let mut orders = AHashMap::new();
1255 let pattern = format!("{trader_key}{REDIS_DELIMITER}{ORDERS}*");
1256 log::debug!("Loading {pattern}");
1257
1258 let mut con = con.clone();
1259 let keys = Self::scan_keys(&mut con, pattern).await?;
1260
1261 let futures: Vec<_> = keys
1262 .iter()
1263 .map(|key| {
1264 let con = con.clone();
1265 async move {
1266 let client_order_id = if let Some(code) = key.as_str().rsplit(':').next() {
1267 ClientOrderId::from(code)
1268 } else {
1269 log::error!("Invalid key format: {key}");
1270 return None;
1271 };
1272
1273 match Self::load_order(&con, trader_key, &client_order_id, encoding).await {
1274 Ok(Some(order)) => Some((client_order_id, order)),
1275 Ok(None) => {
1276 log::error!("Order not found: {client_order_id}");
1277 None
1278 }
1279 Err(e) => {
1280 log::error!("Failed to load order {client_order_id}: {e}");
1281 None
1282 }
1283 }
1284 }
1285 })
1286 .collect();
1287
1288 orders.extend(join_all(futures).await.into_iter().flatten());
1290 log::debug!("Loaded {} order(s)", orders.len());
1291
1292 Ok(orders)
1293 }
1294
1295 pub async fn load_positions(
1306 con: &ConnectionManager,
1307 trader_key: &str,
1308 encoding: SerializationEncoding,
1309 ) -> anyhow::Result<AHashMap<PositionId, Position>> {
1310 let mut positions = AHashMap::new();
1311 let pattern = format!("{trader_key}{REDIS_DELIMITER}{POSITIONS}*");
1312 log::debug!("Loading {pattern}");
1313
1314 let mut con = con.clone();
1315 let keys = Self::scan_keys(&mut con, pattern).await?;
1316
1317 let futures: Vec<_> = keys
1318 .iter()
1319 .map(|key| {
1320 let con = con.clone();
1321 async move {
1322 let position_id = if let Some(code) = key.as_str().rsplit(':').next() {
1323 PositionId::from(code)
1324 } else {
1325 log::error!("Invalid key format: {key}");
1326 return None;
1327 };
1328
1329 match Self::load_position(&con, trader_key, &position_id, encoding).await {
1330 Ok(Some(position)) => Some((position_id, position)),
1331 Ok(None) => {
1332 log::error!("Position not found: {position_id}");
1333 None
1334 }
1335 Err(e) => {
1336 log::error!("Failed to load position {position_id}: {e}");
1337 None
1338 }
1339 }
1340 }
1341 })
1342 .collect();
1343
1344 positions.extend(join_all(futures).await.into_iter().flatten());
1346 log::debug!("Loaded {} position(s)", positions.len());
1347
1348 Ok(positions)
1349 }
1350
1351 pub async fn load_index_order_position(
1357 con: &ConnectionManager,
1358 trader_key: &str,
1359 ) -> anyhow::Result<AHashMap<ClientOrderId, PositionId>> {
1360 let index = Self::read_index_hash(con, trader_key, INDEX_ORDER_POSITION).await?;
1361 Ok(index
1362 .into_iter()
1363 .map(|(k, v)| {
1364 (
1365 ClientOrderId::from(k.as_str()),
1366 PositionId::from(v.as_str()),
1367 )
1368 })
1369 .collect())
1370 }
1371
1372 pub async fn load_index_order_client(
1378 con: &ConnectionManager,
1379 trader_key: &str,
1380 ) -> anyhow::Result<AHashMap<ClientOrderId, ClientId>> {
1381 let index = Self::read_index_hash(con, trader_key, INDEX_ORDER_CLIENT).await?;
1382 Ok(index
1383 .into_iter()
1384 .map(|(k, v)| (ClientOrderId::from(k.as_str()), ClientId::from(v.as_str())))
1385 .collect())
1386 }
1387
1388 async fn read_index_hash(
1389 con: &ConnectionManager,
1390 trader_key: &str,
1391 key: &str,
1392 ) -> anyhow::Result<HashMap<String, String>> {
1393 let result = Self::read(con, trader_key, key).await?;
1394 if result.is_empty() {
1395 return Ok(HashMap::new());
1396 }
1397
1398 serde_json::from_slice(&result[0])
1399 .map_err(|e| anyhow::anyhow!("Failed to parse index hash '{key}': {e}"))
1400 }
1401
1402 pub async fn load_custom_data(
1412 con: &ConnectionManager,
1413 trader_key: &str,
1414 data_type: &DataType,
1415 ) -> anyhow::Result<Vec<CustomData>> {
1416 Self::load_custom_data_batched(con, trader_key, data_type, usize::MAX).await
1417 }
1418
1419 pub(crate) async fn load_custom_data_batched(
1420 con: &ConnectionManager,
1421 trader_key: &str,
1422 data_type: &DataType,
1423 batch_size: usize,
1424 ) -> anyhow::Result<Vec<CustomData>> {
1425 let pattern = format!("{trader_key}{REDIS_DELIMITER}{CUSTOM}*");
1426 log::debug!("Loading custom data {pattern}");
1427
1428 let mut con = con.clone();
1429 let keys = Self::scan_keys(&mut con, pattern).await?;
1430
1431 if keys.is_empty() {
1432 return Ok(Vec::new());
1433 }
1434
1435 let values = Self::read_bulk_batched(&con, &keys, batch_size).await?;
1436 let request_type_name = data_type.type_name();
1437 let request_short = request_type_name
1438 .rsplit([':', '.'])
1439 .next()
1440 .unwrap_or(request_type_name);
1441 let request_identifier = data_type.identifier().unwrap_or("");
1442
1443 let mut results = Vec::new();
1444
1445 for value_opt in values {
1446 let Some(value_bytes) = value_opt else {
1447 continue;
1448 };
1449 let custom = match CustomData::from_json_bytes(value_bytes.as_ref()) {
1450 Ok(c) => c,
1451 Err(e) => {
1452 log::warn!("Failed to deserialize custom data from Redis: {e}");
1453 continue;
1454 }
1455 };
1456 let stored_type_name = custom.data_type.type_name();
1457 let type_match =
1458 stored_type_name == request_type_name || stored_type_name == request_short;
1459 let identifier_match =
1460 custom.data_type.identifier().unwrap_or("") == request_identifier;
1461 let metadata_match = match (data_type.metadata(), custom.data_type.metadata()) {
1462 (None, None) => true,
1463 (Some(a), Some(b)) => serde_json::to_value(a).ok() == serde_json::to_value(b).ok(),
1464 _ => false,
1465 };
1466
1467 if type_match && identifier_match && metadata_match {
1468 results.push(custom);
1469 }
1470 }
1471
1472 results.sort_by_key(HasTsInit::ts_init);
1473 log::debug!("Loaded {} custom data item(s)", results.len());
1474 Ok(results)
1475 }
1476
1477 pub async fn load_currency(
1483 con: &ConnectionManager,
1484 trader_key: &str,
1485 code: &Ustr,
1486 encoding: SerializationEncoding,
1487 ) -> anyhow::Result<Option<Currency>> {
1488 let key = format!("{CURRENCIES}{REDIS_DELIMITER}{code}");
1489 let result = Self::read(con, trader_key, &key).await?;
1490
1491 if result.is_empty() {
1492 return Ok(None);
1493 }
1494
1495 match Self::decode_currency(encoding, &result[0], *code) {
1496 CurrencyDecode::Currency(currency) => Ok(Some(currency)),
1497 CurrencyDecode::InvalidRecord(e) | CurrencyDecode::Undecodable(e) => {
1498 Err(e.context(format!("Failed to load currency {code}")))
1499 }
1500 }
1501 }
1502
1503 pub async fn load_instrument(
1509 con: &ConnectionManager,
1510 trader_key: &str,
1511 instrument_id: &InstrumentId,
1512 encoding: SerializationEncoding,
1513 ) -> anyhow::Result<Option<InstrumentAny>> {
1514 let key = format!("{INSTRUMENTS}{REDIS_DELIMITER}{instrument_id}");
1515 let result = Self::read(con, trader_key, &key).await?;
1516 if result.is_empty() {
1517 return Ok(None);
1518 }
1519
1520 #[cfg(feature = "defi")]
1521 {
1522 let value: Value = Self::deserialize_payload(encoding, &result[0])?;
1523 if value.get("defi_currency_codes").is_some() || value.get("instrument").is_some() {
1524 let record: crate::defi::InstrumentRecord =
1525 serde_json::from_value(value).map_err(integrity_error)?;
1526
1527 if record.defi_currency_codes.is_empty() {
1528 return Err(integrity_error("Empty DeFi instrument dependency marker"));
1529 }
1530
1531 for code in &record.defi_currency_codes {
1532 let currency =
1533 Self::load_currency(con, trader_key, &Ustr::from(code), encoding)
1534 .await
1535 .map_err(integrity_error)?
1536 .ok_or_else(|| {
1537 integrity_error(format!("Missing token currency {code}"))
1538 })?;
1539
1540 if currency::lookup(currency.code).is_none() {
1541 return Err(integrity_error(format!(
1542 "Missing token metadata for {code}"
1543 )));
1544 }
1545 }
1546
1547 let instrument: InstrumentAny =
1548 serde_json::from_value(record.instrument).map_err(integrity_error)?;
1549 validate_marker(&instrument, &record.defi_currency_codes)?;
1550 if instrument.id() != *instrument_id {
1551 return Err(integrity_error("DeFi instrument key and payload ID differ"));
1552 }
1553
1554 return Ok(Some(instrument));
1555 }
1556 }
1557
1558 let instrument: InstrumentAny = Self::deserialize_payload(encoding, &result[0])?;
1559 #[cfg(feature = "defi")]
1560 validate_marker(&instrument, &[])?;
1561 Ok(Some(instrument))
1562 }
1563
1564 pub async fn load_synthetic(
1570 con: &ConnectionManager,
1571 trader_key: &str,
1572 instrument_id: &InstrumentId,
1573 encoding: SerializationEncoding,
1574 ) -> anyhow::Result<Option<SyntheticInstrument>> {
1575 let key = format!("{SYNTHETICS}{REDIS_DELIMITER}{instrument_id}");
1576 let result = Self::read(con, trader_key, &key).await?;
1577 if result.is_empty() {
1578 return Ok(None);
1579 }
1580
1581 let synthetic: SyntheticInstrument = Self::deserialize_payload(encoding, &result[0])?;
1582 Ok(Some(synthetic))
1583 }
1584
1585 pub async fn load_account(
1591 con: &ConnectionManager,
1592 trader_key: &str,
1593 account_id: &AccountId,
1594 encoding: SerializationEncoding,
1595 ) -> anyhow::Result<Option<AccountAny>> {
1596 let key = format!("{ACCOUNTS}{REDIS_DELIMITER}{account_id}");
1597 let result = Self::read(con, trader_key, &key).await?;
1598 if result.is_empty() {
1599 return Ok(None);
1600 }
1601
1602 let events: Vec<AccountState> = result
1603 .iter()
1604 .map(|payload| Self::deserialize_payload(encoding, payload))
1605 .collect::<anyhow::Result<_>>()?;
1606 let account = AccountAny::from_events(&events)?;
1607 Ok(Some(account))
1608 }
1609
1610 pub async fn load_order(
1616 con: &ConnectionManager,
1617 trader_key: &str,
1618 client_order_id: &ClientOrderId,
1619 encoding: SerializationEncoding,
1620 ) -> anyhow::Result<Option<OrderAny>> {
1621 let key = format!("{ORDERS}{REDIS_DELIMITER}{client_order_id}");
1622 let result = Self::read(con, trader_key, &key).await?;
1623 if result.is_empty() {
1624 return Ok(None);
1625 }
1626
1627 let order = Self::deserialize_order(encoding, &result)?;
1628 Ok(Some(order))
1629 }
1630
1631 pub(crate) fn deserialize_order(
1632 encoding: SerializationEncoding,
1633 payloads: &[Bytes],
1634 ) -> anyhow::Result<OrderAny> {
1635 let (first, remaining) = payloads
1636 .split_first()
1637 .ok_or_else(|| anyhow::anyhow!("Cannot restore an order without initialization"))?;
1638 let initial = Self::deserialize_payload(encoding, first)?;
1639 let mut order = OrderAny::from_events(vec![initial])?;
1640
1641 for payload in remaining {
1642 let value: Value = Self::deserialize_payload(encoding, payload)?;
1643 let legacy_init = value.get("type").and_then(Value::as_str) == Some("OrderInitialized");
1644 let event = deserialize_payload_value(value)?;
1645 if legacy_init {
1646 order = transform_order(&order, event)?;
1647 } else {
1648 order.apply(event)?;
1649 }
1650 }
1651
1652 Ok(order)
1653 }
1654
1655 pub async fn load_position(
1661 con: &ConnectionManager,
1662 trader_key: &str,
1663 position_id: &PositionId,
1664 encoding: SerializationEncoding,
1665 ) -> anyhow::Result<Option<Position>> {
1666 let snapshot_key =
1667 format!("{SNAPSHOTS}{REDIS_DELIMITER}{POSITIONS}{REDIS_DELIMITER}{position_id}");
1668 let snapshots = Self::read(con, trader_key, &snapshot_key).await?;
1669 for payload in snapshots.iter().rev() {
1670 let snapshot: PositionSnapshot = Self::deserialize_payload(encoding, payload)?;
1671 if let Some(replay_state) = snapshot.replay_state {
1672 let position: Position = serde_json::from_value(replay_state)
1673 .map_err(|e| anyhow::anyhow!("Failed to decode position replay state: {e}"))?;
1674 position.check_accounting()?;
1675 return Ok(Some(position));
1676 }
1677 }
1678
1679 let key = format!("{POSITIONS}{REDIS_DELIMITER}{position_id}");
1680 let result = Self::read(con, trader_key, &key).await?;
1681 if result.is_empty() {
1682 return Ok(None);
1683 }
1684
1685 let fills: Vec<OrderFilled> = result
1686 .iter()
1687 .map(|payload| Self::deserialize_payload(encoding, payload))
1688 .collect::<anyhow::Result<_>>()?;
1689 let Some((first_fill, remaining_fills)) = fills.split_first() else {
1690 return Ok(None);
1691 };
1692 let Some(instrument) =
1693 Self::load_instrument(con, trader_key, &first_fill.instrument_id, encoding).await?
1694 else {
1695 log::error!(
1696 "Instrument not found for position {position_id}: {}",
1697 first_fill.instrument_id
1698 );
1699 return Ok(None);
1700 };
1701
1702 let mut position = Position::new_checked(&instrument, first_fill.clone())?;
1703 for fill in remaining_fills {
1704 if position.trade_ids.contains(&fill.trade_id) {
1705 anyhow::bail!(
1706 "Duplicate fill event for position {position_id}: {}",
1707 fill.trade_id
1708 );
1709 }
1710
1711 position.try_apply(fill)?;
1712 }
1713
1714 Ok(Some(position))
1715 }
1716
1717 fn get_collection_key(key: &str) -> anyhow::Result<&str> {
1718 key.split_once(REDIS_DELIMITER)
1719 .map(|(collection, _)| collection)
1720 .ok_or_else(|| {
1721 anyhow::anyhow!("Invalid `key`, missing a '{REDIS_DELIMITER}' delimiter, was {key}")
1722 })
1723 }
1724
1725 async fn read_index(conn: &mut ConnectionManager, key: &str) -> anyhow::Result<Vec<Bytes>> {
1726 let index_key = get_index_key(key)?;
1727 match index_key {
1728 INDEX_ORDER_IDS
1729 | INDEX_ORDERS
1730 | INDEX_ORDERS_OPEN
1731 | INDEX_ORDERS_CLOSED
1732 | INDEX_ORDERS_EMULATED
1733 | INDEX_ORDERS_INFLIGHT
1734 | INDEX_POSITIONS
1735 | INDEX_POSITIONS_OPEN
1736 | INDEX_POSITIONS_CLOSED => Self::read_set(conn, key).await,
1737 INDEX_ORDER_POSITION | INDEX_ORDER_CLIENT => Self::read_hset(conn, key).await,
1738 _ => anyhow::bail!("Index unknown '{index_key}' on read"),
1739 }
1740 }
1741
1742 async fn read_string(conn: &mut ConnectionManager, key: &str) -> anyhow::Result<Vec<Bytes>> {
1743 let result: Vec<u8> = conn.get(key).await?;
1744
1745 if result.is_empty() {
1746 Ok(vec![])
1747 } else {
1748 Ok(vec![Bytes::from(result)])
1749 }
1750 }
1751
1752 async fn read_set(conn: &mut ConnectionManager, key: &str) -> anyhow::Result<Vec<Bytes>> {
1753 let result: Vec<Bytes> = conn.smembers(key).await?;
1754 Ok(result)
1755 }
1756
1757 async fn read_hset(conn: &mut ConnectionManager, key: &str) -> anyhow::Result<Vec<Bytes>> {
1758 let result: HashMap<String, String> = conn.hgetall(key).await?;
1759 let json = serde_json::to_string(&result)?;
1760 Ok(vec![Bytes::from(json.into_bytes())])
1761 }
1762
1763 async fn read_list(conn: &mut ConnectionManager, key: &str) -> anyhow::Result<Vec<Bytes>> {
1764 let result: Vec<Bytes> = conn.lrange(key, 0, -1).await?;
1765 Ok(result)
1766 }
1767}
1768
1769fn transform_order(order: &OrderAny, event: OrderEventAny) -> anyhow::Result<OrderAny> {
1770 let history = order.events();
1771 anyhow::ensure!(
1772 !history.contains(&&event),
1773 "Duplicate initialization in legacy order history"
1774 );
1775 let mut transformed = OrderAny::from_events(vec![event])?;
1776 anyhow::ensure!(
1777 (
1778 transformed.trader_id(),
1779 transformed.strategy_id(),
1780 transformed.instrument_id(),
1781 transformed.client_order_id()
1782 ) == (
1783 order.trader_id(),
1784 order.strategy_id(),
1785 order.instrument_id(),
1786 order.client_order_id()
1787 ),
1788 "Legacy order transformation changes the order identity"
1789 );
1790 let history = history.into_iter().cloned();
1791
1792 match &mut transformed {
1793 OrderAny::Market(target) => {
1794 target.ts_init = order.ts_init();
1795 target.prepend_events(history);
1796 }
1797 OrderAny::Limit(target) => {
1798 target.ts_init = order.ts_init();
1799 target.liquidity_side = order.liquidity_side();
1800 target.prepend_events(history);
1801 }
1802 _ => anyhow::bail!("Legacy order transformation must produce a market or limit order"),
1803 }
1804
1805 Ok(transformed)
1806}
1807
1808fn deserialize_payload_value<T: DeserializeOwned>(mut value: Value) -> anyhow::Result<T> {
1809 normalize_account_state(&mut value);
1810 if let Some(kind) = value
1811 .get("type")
1812 .and_then(Value::as_str)
1813 .and_then(legacy_variant)
1814 {
1815 let kind = kind.to_string();
1816 if kind == "Initialized" {
1817 normalize_order_options(&mut value);
1818 }
1819
1820 if let Some(object) = value.as_object_mut()
1821 && let Some(tick_scheme) = object.remove("tick_scheme_name")
1822 {
1823 object.entry("tick_scheme").or_insert(tick_scheme);
1824 }
1825
1826 if let Ok(payload) = T::deserialize(&value) {
1827 return Ok(payload);
1828 }
1829
1830 value = Value::Object(serde_json::Map::from_iter([(kind, value)]));
1831 }
1832
1833 serde_json::from_value(value)
1834 .map_err(|e| anyhow::anyhow!("Failed to convert value to target type: {e}"))
1835}
1836
1837fn normalize_account_state(value: &mut Value) {
1838 if value.get("type").and_then(Value::as_str) != Some("AccountState") {
1839 return;
1840 }
1841
1842 let Some(object) = value.as_object_mut() else {
1843 return;
1844 };
1845
1846 let Some(reported) = object.remove("reported") else {
1847 return;
1848 };
1849
1850 object.entry("is_reported").or_insert(reported);
1851 for field in ["balances", "margins"] {
1852 if let Some(Value::Array(balances)) = object.get_mut(field) {
1853 for balance in balances {
1854 normalize_balance(balance);
1855 }
1856 }
1857 }
1858}
1859
1860fn normalize_balance(value: &mut Value) {
1861 let Some(balance) = value.as_object_mut() else {
1862 return;
1863 };
1864
1865 let Some(code) = balance
1866 .get("currency")
1867 .and_then(Value::as_str)
1868 .map(str::to_string)
1869 else {
1870 return;
1871 };
1872
1873 for field in ["total", "locked", "free", "initial", "maintenance"] {
1874 if let Some(Value::String(value)) = balance.get_mut(field)
1875 && !value.contains(' ')
1876 {
1877 value.push(' ');
1878 value.push_str(&code);
1879 }
1880 }
1881}
1882
1883fn normalize_order_options(value: &mut Value) {
1884 let Some(object) = value.as_object_mut() else {
1885 return;
1886 };
1887
1888 let Some(Value::Object(options)) = object.remove("options") else {
1889 return;
1890 };
1891
1892 for (key, mut value) in options {
1893 let key = if key == "expire_time_ns" {
1894 if value.as_u64() == Some(0) {
1895 value = Value::Null;
1896 }
1897
1898 "expire_time".into()
1899 } else {
1900 key
1901 };
1902
1903 object.entry(key).or_insert(value);
1904 }
1905}
1906
1907fn legacy_variant(kind: &str) -> Option<&str> {
1908 match kind {
1909 "BettingInstrument" => Some("Betting"),
1910 "BinaryOption"
1911 | "Cfd"
1912 | "Commodity"
1913 | "CryptoFuture"
1914 | "CryptoFuturesSpread"
1915 | "CryptoOption"
1916 | "CryptoOptionSpread"
1917 | "CryptoPerpetual"
1918 | "CurrencyPair"
1919 | "Equity"
1920 | "FuturesContract"
1921 | "FuturesSpread"
1922 | "IndexInstrument"
1923 | "OptionContract"
1924 | "OptionSpread"
1925 | "PerpetualContract" => Some(kind),
1926 _ => kind.strip_prefix("Order"),
1927 }
1928}
1929
1930fn parse_instrument_key(key: &str, prefix: &str) -> anyhow::Result<InstrumentId> {
1931 let value = key
1932 .strip_prefix(prefix)
1933 .filter(|value| !value.is_empty())
1934 .ok_or_else(|| anyhow::anyhow!("Invalid instrument key '{key}'"))?;
1935 InstrumentId::from_str(value)
1936 .map_err(|e| anyhow::anyhow!("Failed to parse instrument ID from key '{key}': {e}"))
1937}
1938
1939#[cfg(test)]
1940mod tests {
1941 use std::{hint::black_box, str::FromStr, time::Instant};
1942
1943 use bytes::Bytes;
1944 use nautilus_common::enums::SerializationEncoding;
1945 use nautilus_core::{UUID4, UnixNanos};
1946 use nautilus_model::{
1947 data::{Data, stubs::stub_custom_data},
1948 enums::{AccountType, CurrencyType, OrderType},
1949 events::{
1950 AccountState, OrderEventAny, OrderInitialized,
1951 order::spec::{OrderFillVoidedSpec, OrderFilledSpec},
1952 },
1953 identifiers::{AccountId, InstrumentId, PositionId, TradeId},
1954 instruments::{Instrument, InstrumentAny, stubs::currency_pair_btcusdt},
1955 orders::{Order, OrderError},
1956 position::Position,
1957 types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
1958 };
1959 use rstest::rstest;
1960 use serde::{Deserialize, Serialize, de::DeserializeOwned};
1961 use ustr::Ustr;
1962
1963 use super::{CurrencyDecode, CurrencyRecord, DatabaseQueries, parse_instrument_key};
1964
1965 #[rstest]
1966 #[case(SerializationEncoding::Json)]
1967 #[case(SerializationEncoding::MsgPack)]
1968 fn test_legacy_margin_account_preserves_amounts(#[case] encoding: SerializationEncoding) {
1969 let fixture: serde_json::Value =
1970 serde_json::from_str(include_str!("../../test_data/redis_cache_v1.json")).unwrap();
1971
1972 let payload = match encoding {
1973 SerializationEncoding::Json => serde_json::to_vec(&fixture["margin_account"]).unwrap(),
1974 SerializationEncoding::MsgPack => {
1975 rmp_serde::to_vec_named(&fixture["margin_account"]).unwrap()
1976 }
1977 SerializationEncoding::Sbe => unreachable!(),
1978 };
1979
1980 let expected = AccountState::new(
1981 AccountId::from("BINANCE-MARGIN-001"),
1982 AccountType::Margin,
1983 vec![AccountBalance::new(
1984 Money::from("999 USDT"),
1985 Money::from("100 USDT"),
1986 Money::from("899 USDT"),
1987 )],
1988 vec![MarginBalance::new(
1989 Money::from("12.34 USDT"),
1990 Money::from("5.67 USDT"),
1991 Some(InstrumentId::from("BTCUSDT.BINANCE")),
1992 )],
1993 false,
1994 UUID4::from_str("77777777-7777-4777-8777-777777777777").unwrap(),
1995 UnixNanos::from(700),
1996 UnixNanos::from(701),
1997 Some(Currency::USDT()),
1998 );
1999
2000 let restored: AccountState =
2001 DatabaseQueries::deserialize_payload(encoding, &payload).unwrap();
2002
2003 assert_eq!(
2004 serde_json::to_value(restored).unwrap(),
2005 serde_json::to_value(expected).unwrap()
2006 );
2007 }
2008
2009 #[rstest]
2010 fn test_legacy_order_transformation_rejects_stop_limit() {
2011 let fixture: serde_json::Value =
2012 serde_json::from_str(include_str!("../../test_data/redis_cache_v1.json")).unwrap();
2013 let original = fixture["order_initialized"].clone();
2014 let mut transformed = original.clone();
2015 transformed["order_type"] = "STOP_LIMIT".into();
2016 transformed["options"]["trigger_price"] = "12340.00".into();
2017 transformed["options"]["trigger_type"] = "LAST_PRICE".into();
2018 let payloads =
2019 [original, transformed].map(|value| Bytes::from(serde_json::to_vec(&value).unwrap()));
2020
2021 let error =
2022 DatabaseQueries::deserialize_order(SerializationEncoding::Json, &payloads).unwrap_err();
2023
2024 assert_eq!(
2025 error.to_string(),
2026 "Legacy order transformation must produce a market or limit order"
2027 );
2028 }
2029
2030 #[rstest]
2031 #[case(SerializationEncoding::Json)]
2032 #[case(SerializationEncoding::MsgPack)]
2033 fn test_native_order_timestamp_parameters_round_trip(
2034 #[case] encoding: SerializationEncoding,
2035 #[values(false, true)] iso8601: bool,
2036 ) {
2037 let fixture: serde_json::Value =
2038 serde_json::from_str(include_str!("../../test_data/redis_cache_v1.json")).unwrap();
2039 let mut event: OrderInitialized = DatabaseQueries::deserialize_payload(
2040 SerializationEncoding::Json,
2041 &serde_json::to_vec(&fixture["order_initialized"]).unwrap(),
2042 )
2043 .unwrap();
2044 event.exec_algorithm_params = Some(
2045 [
2046 (Ustr::from("ts_start"), Ustr::from("9007199254740993")),
2047 (
2048 Ustr::from("ts_end"),
2049 Ustr::from("1970-01-01T00:00:00.000000987Z"),
2050 ),
2051 ]
2052 .into_iter()
2053 .collect(),
2054 );
2055 let event = OrderEventAny::Initialized(event);
2056 let payload =
2057 DatabaseQueries::serialize_payload_with_timestamps(encoding, &event, iso8601).unwrap();
2058
2059 let restored: OrderEventAny =
2060 DatabaseQueries::deserialize_payload(encoding, &payload).unwrap();
2061 let order = DatabaseQueries::deserialize_order(encoding, &[Bytes::from(payload)]).unwrap();
2062
2063 assert_eq!(restored, event);
2064 assert_eq!(order.events(), vec![&event]);
2065 }
2066
2067 #[rstest]
2068 #[case(SerializationEncoding::Json)]
2069 #[case(SerializationEncoding::MsgPack)]
2070 fn test_native_instrument_legacy_info_timestamps(#[case] encoding: SerializationEncoding) {
2071 let fixture: serde_json::Value =
2072 serde_json::from_str(include_str!("../../test_data/redis_cache_v1.json")).unwrap();
2073 let mut instrument = currency_pair_btcusdt();
2074 instrument.info = Some(
2075 serde_json::from_value(serde_json::json!({
2076 "ts_source": 456,
2077 "label": "legacy-source"
2078 }))
2079 .unwrap(),
2080 );
2081 let expected = InstrumentAny::CurrencyPair(instrument);
2082 let mut value = serde_json::to_value(&expected).unwrap();
2083 value["CurrencyPair"]["info"] = fixture["native_info"].clone();
2084 let payload = match encoding {
2085 SerializationEncoding::Json => serde_json::to_vec(&value).unwrap(),
2086 SerializationEncoding::MsgPack => rmp_serde::to_vec_named(&value).unwrap(),
2087 SerializationEncoding::Sbe => unreachable!(),
2088 };
2089
2090 let restored: InstrumentAny =
2091 DatabaseQueries::deserialize_payload(encoding, &payload).unwrap();
2092
2093 assert_eq!(
2094 serde_json::to_value(restored).unwrap(),
2095 serde_json::to_value(expected).unwrap()
2096 );
2097 }
2098
2099 #[rstest]
2100 #[case(SerializationEncoding::Json)]
2101 #[case(SerializationEncoding::MsgPack)]
2102 fn test_native_fill_timestamp_info_round_trip(
2103 #[case] encoding: SerializationEncoding,
2104 #[values(false, true)] iso8601: bool,
2105 #[values(false, true)] voided: bool,
2106 ) {
2107 let info = [
2108 ("ts_source", "123"),
2109 ("expire_time_ns", "456"),
2110 ("ts_original", "1970-01-01T00:00:00.000000789Z"),
2111 ("label", "fill-source"),
2112 ]
2113 .map(|(key, value)| (Ustr::from(key), Ustr::from(value)))
2114 .into_iter()
2115 .collect();
2116
2117 let event = if voided {
2118 OrderEventAny::FillVoided(OrderFillVoidedSpec::builder().info(info).build())
2119 } else {
2120 OrderEventAny::Filled(OrderFilledSpec::builder().info(info).build())
2121 };
2122
2123 let payload =
2124 DatabaseQueries::serialize_payload_with_timestamps(encoding, &event, iso8601).unwrap();
2125
2126 let restored: OrderEventAny =
2127 DatabaseQueries::deserialize_payload(encoding, &payload).unwrap();
2128
2129 assert_eq!(restored, event);
2130 }
2131
2132 #[rstest]
2133 #[case(SerializationEncoding::Json)]
2134 #[case(SerializationEncoding::MsgPack)]
2135 fn test_legacy_order_transformation_preserves_history(
2136 #[case] encoding: SerializationEncoding,
2137 #[values("MARKET", "LIMIT")] order_type: &str,
2138 ) {
2139 let fixture: serde_json::Value =
2140 serde_json::from_str(include_str!("../../test_data/redis_cache_v1.json")).unwrap();
2141 let original = fixture["order_initialized"].clone();
2142 let mut transformed = original.clone();
2143 transformed["order_type"] = order_type.into();
2144 transformed["event_id"] = "33333333-3333-4333-8333-333333333333".into();
2145 transformed["ts_event"] = "600".into();
2146 transformed["ts_init"] = "600".into();
2147 transformed["time_in_force"] = "GTC".into();
2148 transformed["options"]["expire_time_ns"] = 0.into();
2149 if order_type == "MARKET" {
2150 transformed["post_only"] = false.into();
2151 transformed["options"] = serde_json::Value::Object(serde_json::Map::new());
2152 }
2153
2154 let payloads = [original, transformed].map(|value| {
2155 Bytes::from(match encoding {
2156 SerializationEncoding::Json => serde_json::to_vec(&value).unwrap(),
2157 SerializationEncoding::MsgPack => rmp_serde::to_vec_named(&value).unwrap(),
2158 SerializationEncoding::Sbe => unreachable!(),
2159 })
2160 });
2161
2162 let expected_initial: OrderEventAny =
2163 DatabaseQueries::deserialize_payload(encoding, &payloads[0]).unwrap();
2164 let expected_transformed: OrderEventAny =
2165 DatabaseQueries::deserialize_payload(encoding, &payloads[1]).unwrap();
2166 let concrete: OrderInitialized =
2167 DatabaseQueries::deserialize_payload(encoding, &payloads[0]).unwrap();
2168
2169 let order = DatabaseQueries::deserialize_order(encoding, &payloads).unwrap();
2170
2171 assert_eq!(
2172 order.order_type(),
2173 if order_type == "MARKET" {
2174 OrderType::Market
2175 } else {
2176 OrderType::Limit
2177 }
2178 );
2179 assert_eq!(order.ts_init(), UnixNanos::from(100));
2180 assert_eq!(order.ts_last(), UnixNanos::from(600));
2181 assert_eq!(
2182 order.events(),
2183 vec![&expected_initial, &expected_transformed]
2184 );
2185 assert_eq!(expected_initial, OrderEventAny::Initialized(concrete));
2186 }
2187
2188 #[rstest]
2189 #[case::duplicate(false, "Duplicate initialization")]
2190 #[case::identity(true, "changes the order identity")]
2191 fn test_legacy_order_transformation_rejects_invalid_history(
2192 #[case] change_identity: bool,
2193 #[case] expected: &str,
2194 ) {
2195 let fixture: serde_json::Value =
2196 serde_json::from_str(include_str!("../../test_data/redis_cache_v1.json")).unwrap();
2197 let original = fixture["order_initialized"].clone();
2198 let mut transformed = original.clone();
2199 if change_identity {
2200 transformed["client_order_id"] = "O-OTHER".into();
2201 }
2202
2203 let payloads =
2204 [original, transformed].map(|value| Bytes::from(serde_json::to_vec(&value).unwrap()));
2205
2206 let error =
2207 DatabaseQueries::deserialize_order(SerializationEncoding::Json, &payloads).unwrap_err();
2208
2209 assert!(error.to_string().contains(expected), "{error}");
2210 }
2211
2212 #[rstest]
2213 fn test_native_order_history_still_rejects_duplicate_initialization() {
2214 let fixture: serde_json::Value =
2215 serde_json::from_str(include_str!("../../test_data/redis_cache_v1.json")).unwrap();
2216 let event: OrderEventAny = DatabaseQueries::deserialize_payload(
2217 SerializationEncoding::Json,
2218 &serde_json::to_vec(&fixture["order_initialized"]).unwrap(),
2219 )
2220 .unwrap();
2221 let payload = Bytes::from(serde_json::to_vec(&event).unwrap());
2222
2223 let error = DatabaseQueries::deserialize_order(
2224 SerializationEncoding::Json,
2225 &[payload.clone(), payload],
2226 )
2227 .unwrap_err();
2228
2229 assert!(matches!(
2230 error.downcast_ref::<OrderError>(),
2231 Some(OrderError::AlreadyInitialized)
2232 ));
2233 }
2234
2235 #[rstest]
2240 #[case(SerializationEncoding::Json)]
2241 #[case(SerializationEncoding::MsgPack)]
2242 fn test_currency_record_round_trips_an_unregistered_code(
2243 #[case] encoding: SerializationEncoding,
2244 ) {
2245 let code = match encoding {
2247 SerializationEncoding::MsgPack => "ZZREDISM",
2248 _ => "ZZREDISJ",
2249 };
2250 assert!(
2251 Currency::try_from_str(code).is_none(),
2252 "the fixture code must be unregistered for this test to mean anything"
2253 );
2254
2255 let currency = Currency::new(code, 3, 0, "Redis Fixture", CurrencyType::Crypto);
2256 let payload =
2257 DatabaseQueries::serialize_payload(encoding, &CurrencyRecord::from(¤cy)).unwrap();
2258
2259 let CurrencyDecode::Currency(restored) =
2260 DatabaseQueries::decode_currency(encoding, &Bytes::from(payload), currency.code)
2261 else {
2262 panic!("a valid record decodes to a currency");
2263 };
2264
2265 assert_eq!(restored.code.as_str(), code);
2266 assert_eq!(restored.precision, 3);
2267 assert_eq!(restored.iso4217, 0);
2268 assert_eq!(restored.name.as_str(), "Redis Fixture");
2269 assert_eq!(restored.currency_type, CurrencyType::Crypto);
2270 }
2271
2272 #[rstest]
2274 #[case(SerializationEncoding::Json)]
2275 #[case(SerializationEncoding::MsgPack)]
2276 fn test_legacy_bare_code_still_loads(#[case] encoding: SerializationEncoding) {
2277 let payload = DatabaseQueries::serialize_payload(encoding, &Currency::USD()).unwrap();
2278
2279 let CurrencyDecode::Currency(restored) =
2280 DatabaseQueries::decode_currency(encoding, &Bytes::from(payload), Currency::USD().code)
2281 else {
2282 panic!("a registered bare code decodes to a currency");
2283 };
2284
2285 assert_eq!(restored, Currency::USD());
2286 }
2287
2288 #[rstest]
2290 #[case(SerializationEncoding::Json)]
2291 #[case(SerializationEncoding::MsgPack)]
2292 fn test_invalid_record_is_classified_as_invalid(#[case] encoding: SerializationEncoding) {
2293 let record = CurrencyRecord {
2294 code: "ZZQ1".to_string(),
2295 precision: 20,
2296 iso4217: 0,
2297 name: "ZZQ1".to_string(),
2298 currency_type: CurrencyType::Crypto,
2299 defi: None,
2300 };
2301 let bytes = Bytes::from(DatabaseQueries::serialize_payload(encoding, &record).unwrap());
2302
2303 assert!(matches!(
2304 DatabaseQueries::decode_currency(encoding, &bytes, Ustr::from("ZZQ1")),
2305 CurrencyDecode::InvalidRecord(_)
2306 ));
2307 }
2308
2309 #[rstest]
2314 #[case(SerializationEncoding::Json)]
2315 #[case(SerializationEncoding::MsgPack)]
2316 fn test_record_with_an_unknown_currency_type_is_invalid(
2317 #[case] encoding: SerializationEncoding,
2318 ) {
2319 let record = serde_json::json!({
2320 "code": "ZZQ3",
2321 "precision": 6,
2322 "iso4217": 0,
2323 "name": "ZZQ3",
2324 "currency_type": "STABLECOIN",
2325 });
2326 let bytes = Bytes::from(DatabaseQueries::serialize_payload(encoding, &record).unwrap());
2327
2328 assert!(matches!(
2329 DatabaseQueries::decode_currency(encoding, &bytes, Ustr::from("ZZQ3")),
2330 CurrencyDecode::InvalidRecord(_)
2331 ));
2332 }
2333
2334 #[rstest]
2336 #[case(SerializationEncoding::Json)]
2337 #[case(SerializationEncoding::MsgPack)]
2338 fn test_unknown_bare_code_is_classified_as_undecodable(
2339 #[case] encoding: SerializationEncoding,
2340 ) {
2341 let bytes = Bytes::from(DatabaseQueries::serialize_payload(encoding, &"ZZQ2").unwrap());
2342
2343 let CurrencyDecode::Undecodable(error) =
2344 DatabaseQueries::decode_currency(encoding, &bytes, Ustr::from("ZZQ2"))
2345 else {
2346 panic!("an unregistered bare code is undecodable");
2347 };
2348 assert!(
2349 error.to_string().contains("ZZQ2"),
2350 "the error names the code so the operator can register it: {error}"
2351 );
2352 }
2353
2354 #[derive(Debug, Deserialize, PartialEq, Eq)]
2355 struct TimestampPayload {
2356 ts_event: UnixNanos,
2357 ts_init: UnixNanos,
2358 }
2359
2360 #[rstest]
2361 #[case(SerializationEncoding::Json)]
2362 #[case(SerializationEncoding::MsgPack)]
2363 fn test_deserialize_chrono_timestamp_payload(#[case] encoding: SerializationEncoding) {
2364 let json = include_bytes!("../../test_data/redis_cache_timestamp_chrono.json");
2365 let payload = match encoding {
2366 SerializationEncoding::Json => json.to_vec(),
2367 SerializationEncoding::MsgPack => {
2368 let value = serde_json::from_slice::<serde_json::Value>(json).unwrap();
2369 rmp_serde::to_vec(&value).unwrap()
2370 }
2371 SerializationEncoding::Sbe => unreachable!(),
2372 };
2373
2374 let result =
2375 DatabaseQueries::deserialize_payload::<TimestampPayload>(encoding, &payload).unwrap();
2376
2377 assert_eq!(
2378 result,
2379 TimestampPayload {
2380 ts_event: UnixNanos::from(1_123_456_789),
2381 ts_init: UnixNanos::from(2_987_654_321),
2382 }
2383 );
2384 }
2385
2386 #[rstest]
2387 #[case(SerializationEncoding::Json)]
2388 #[case(SerializationEncoding::MsgPack)]
2389 fn test_wallet_account_state_round_trips_unregistered_currency(
2390 #[case] encoding: SerializationEncoding,
2391 ) {
2392 let currency = Currency::new(
2393 "ENG729C",
2394 6,
2395 0,
2396 "Cache round-trip token",
2397 CurrencyType::Crypto,
2398 );
2399 let total = Money::from_mantissa_exponent(123_456_789, -6, currency);
2400 let state = AccountState::new(
2401 AccountId::new("WALLET-CACHE-001"),
2402 AccountType::Wallet,
2403 vec![AccountBalance::new(total, Money::zero(currency), total)],
2404 vec![],
2405 true,
2406 nautilus_core::UUID4::new(),
2407 UnixNanos::from(1),
2408 UnixNanos::from(2),
2409 None,
2410 );
2411 assert!(Currency::try_from_str("ENG729C").is_none());
2412
2413 let payload = DatabaseQueries::serialize_payload(encoding, &state).unwrap();
2414 let restored: AccountState =
2415 DatabaseQueries::deserialize_payload(encoding, &payload).unwrap();
2416 let restored = restored.balances[0];
2417
2418 assert_eq!(restored.total.raw(), total.raw());
2419 assert_eq!(restored.locked.raw(), 0);
2420 assert_eq!(restored.free.raw(), total.raw());
2421 assert_eq!(restored.currency.code, currency.code);
2422 assert_eq!(restored.currency.precision, currency.precision);
2423 assert_eq!(restored.currency.iso4217, currency.iso4217);
2424 assert_eq!(restored.currency.name, currency.name);
2425 assert_eq!(restored.currency.currency_type, currency.currency_type);
2426 assert!(Currency::try_from_str("ENG729C").is_none());
2427 }
2428
2429 #[rstest]
2430 #[ignore = "optimized Redis cache payload measurements without a Redis connection"]
2431 fn redis_payload_measurements() {
2432 let samples = std::env::var("NAUTILUS_BENCH_SAMPLES")
2433 .map_or(10_000, |value| value.parse::<usize>().unwrap());
2434 let trials = std::env::var("NAUTILUS_BENCH_TRIALS")
2435 .map_or(3, |value| value.parse::<usize>().unwrap());
2436
2437 for trial in 0..trials {
2438 let formats = [SerializationEncoding::Json, SerializationEncoding::MsgPack];
2439 for offset in 0..formats.len() {
2440 let encoding = formats[(offset + trial) % formats.len()];
2441
2442 let format = match encoding {
2443 SerializationEncoding::Json => "json",
2444 SerializationEncoding::MsgPack => "msgpack",
2445 SerializationEncoding::Sbe => unreachable!(),
2446 };
2447
2448 if std::env::var("NAUTILUS_BENCH_FORMAT").is_ok_and(|selected| selected != format) {
2449 continue;
2450 }
2451
2452 cache_cases(encoding, trial, samples);
2453 }
2454 }
2455 }
2456
2457 fn cache_cases(encoding: SerializationEncoding, trial: usize, samples: usize) {
2458 let mut state = AccountState::new(
2459 AccountId::from("SIM-1"),
2460 AccountType::Cash,
2461 vec![AccountBalance::new(
2462 Money::from("1000.75 USD"),
2463 Money::from("250.25 USD"),
2464 Money::from("750.50 USD"),
2465 )],
2466 vec![],
2467 true,
2468 UUID4::new(),
2469 UnixNanos::from(1_700_000_000_000_000_123),
2470 UnixNanos::from(1_700_000_000_000_000_456),
2471 Some(Currency::USD()),
2472 );
2473 let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
2474 let custom = Data::Custom(stub_custom_data(77, -88, None, None));
2475 let mut fill = OrderFilledSpec::builder()
2476 .instrument_id(instrument.id())
2477 .position_id(PositionId::from("POSITION-1"))
2478 .last_qty(Quantity::from("2.750"))
2479 .last_px(Price::from("16384.00"))
2480 .currency(Currency::USDT())
2481 .commission(Money::from("0.73 USDT"))
2482 .ts_event(UnixNanos::from(1_700_000_000_000_000_789))
2483 .ts_init(UnixNanos::from(1_700_000_000_000_000_987))
2484 .build();
2485
2486 for populated in [false, true] {
2487 state.info = populated.then(|| {
2488 serde_json::from_str(include_str!(
2489 "../../../common/test_data/message_params.json"
2490 ))
2491 .unwrap()
2492 });
2493
2494 let name = if populated {
2495 "account_state_nested"
2496 } else {
2497 "account_state_plain"
2498 };
2499
2500 measure_cache_payload(name, &state, encoding, trial, samples);
2501 }
2502
2503 measure_cache_payload("instrument", &instrument, encoding, trial, samples);
2504 measure_cache_payload("custom", &custom, encoding, trial, samples);
2505 fill.trade_id = TradeId::from("FILL-0");
2506 let mut position = Position::new(&instrument, fill.clone());
2507 for count in [1, 32, 256] {
2508 for index in position.events.len()..count {
2509 fill.event_id = UUID4::new();
2510 fill.trade_id = TradeId::new(format!("FILL-{index}"));
2511 position.apply(&fill);
2512 }
2513
2514 assert_eq!(position.events.len(), count);
2515 measure_cache_payload(
2516 &format!("position_{count}"),
2517 &position,
2518 encoding,
2519 trial,
2520 samples,
2521 );
2522 }
2523 }
2524
2525 #[allow(
2526 clippy::cast_possible_truncation,
2527 clippy::cast_precision_loss,
2528 reason = "nanosecond samples fit in u64 and timing means are approximate"
2529 )]
2530 fn measure_cache_payload<T: Serialize + DeserializeOwned>(
2531 name: &str,
2532 value: &T,
2533 encoding: SerializationEncoding,
2534 trial: usize,
2535 samples: usize,
2536 ) {
2537 if std::env::var("NAUTILUS_BENCH_CASE").is_ok_and(|selected| selected != name) {
2538 return;
2539 }
2540
2541 assert!(samples > 0);
2542 let payload = DatabaseQueries::serialize_payload(encoding, value).unwrap();
2543 let decoded: T = DatabaseQueries::deserialize_payload(encoding, &payload).unwrap();
2544 assert_eq!(
2545 serde_json::to_value(&decoded).unwrap(),
2546 serde_json::to_value(value).unwrap()
2547 );
2548
2549 let warmup = std::env::var("NAUTILUS_BENCH_WARMUP")
2550 .map_or(1_000, |value| value.parse::<usize>().unwrap());
2551
2552 for _ in 0..warmup {
2553 black_box(DatabaseQueries::serialize_payload(encoding, black_box(value)).unwrap());
2554 black_box(
2555 DatabaseQueries::deserialize_payload::<T>(encoding, black_box(&payload)).unwrap(),
2556 );
2557 }
2558
2559 if let Ok(operation) = std::env::var("NAUTILUS_BENCH_PROFILE") {
2560 for _ in 0..samples {
2561 match operation.as_str() {
2562 "encode" => {
2563 black_box(cache_encode(encoding, black_box(value)));
2564 }
2565 "decode" => {
2566 black_box(cache_decode::<T>(encoding, black_box(&payload)));
2567 }
2568 _ => panic!("profile must be encode or decode"),
2569 }
2570 }
2571
2572 println!(
2573 "{}",
2574 serde_json::json!({"case": name, "operation": operation,
2575 "profile_calls": samples, "bytes": payload.len()})
2576 );
2577 return;
2578 }
2579
2580 let mut encoded = Vec::with_capacity(samples);
2581 let mut decoded = Vec::with_capacity(samples);
2582 for _ in 0..samples {
2583 let started = Instant::now();
2584 let bytes = cache_encode(encoding, black_box(value));
2585 encoded.push(started.elapsed().as_nanos() as u64);
2586 black_box(bytes);
2587 let started = Instant::now();
2588 let value = cache_decode::<T>(encoding, black_box(&payload));
2589 decoded.push(started.elapsed().as_nanos() as u64);
2590 black_box(value);
2591 }
2592
2593 for (operation, mut times) in [("encode", encoded), ("decode", decoded)] {
2594 let mean = times.iter().map(|value| *value as f64).sum::<f64>() / samples as f64;
2595 times.sort_unstable();
2596 let percentile = |n| times[(samples - 1) * n / 1_000];
2597 println!(
2598 "{}",
2599 serde_json::json!({
2600 "case": name, "boundary": "redis_payload", "format": encoding,
2601 "trial": trial, "operation": operation, "samples": samples, "bytes": payload.len(),
2602 "mean_ns": mean, "p50_ns": percentile(500), "p95_ns": percentile(950),
2603 "p99_ns": percentile(990), "p999_ns": percentile(999), "max_ns": times.last().unwrap(),
2604 })
2605 );
2606 }
2607 }
2608
2609 #[inline(never)]
2610 fn cache_encode<T: Serialize>(encoding: SerializationEncoding, value: &T) -> Vec<u8> {
2611 DatabaseQueries::serialize_payload(encoding, value).unwrap()
2612 }
2613
2614 #[inline(never)]
2615 fn cache_decode<T: DeserializeOwned>(encoding: SerializationEncoding, payload: &[u8]) -> T {
2616 DatabaseQueries::deserialize_payload(encoding, payload).unwrap()
2617 }
2618
2619 #[rstest]
2620 #[case("0xC31E54c7a869B9FcBEcc14363CF510d1c41fa443.Arbitrum:UniswapV3")]
2621 #[case(concat!(
2622 "0xc9bc8043294146424a4e4607d8ad837d",
2623 "6a659142822bbaaabc83bb57e7447461.Arbitrum:UniswapV4",
2624 ))]
2625 fn test_parse_instrument_key_preserves_colons_in_venue(#[case] value: &str) {
2626 let prefix = "TRADER-001:instruments:";
2627 let key = format!("{prefix}{value}");
2628
2629 let result = parse_instrument_key(&key, prefix).unwrap();
2630
2631 assert_eq!(result, InstrumentId::from_str(value).unwrap());
2632 }
2633}