988 lines
33 KiB
Python
988 lines
33 KiB
Python
import copy
|
|
import json
|
|
import math
|
|
import re
|
|
import tempfile
|
|
import unittest
|
|
from datetime import date, datetime
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
|
|
from quant60.broker import SimBroker
|
|
from quant60.config import ExecutionConfig, FeeConfig
|
|
from quant60.contracts import (
|
|
ContractValidationError as RuntimeContractValidationError,
|
|
validate_broker_snapshot,
|
|
)
|
|
from quant60.domain import Bar, Portfolio, Side
|
|
from quant60.ledger import (
|
|
DuplicateEventError,
|
|
HashChainLedger,
|
|
LedgerIntegrityError,
|
|
)
|
|
from quant60.reconcile import reconcile_snapshot
|
|
from quant60.replay import (
|
|
ConflictingDuplicateError,
|
|
LedgerProjector,
|
|
LedgerReplayError,
|
|
OutOfOrderError,
|
|
OverfillError,
|
|
UnknownOrderError,
|
|
project_ledger,
|
|
)
|
|
from quant60.snapshot_pipeline import first_executable_window
|
|
|
|
|
|
class ContractValidationError(AssertionError):
|
|
pass
|
|
|
|
|
|
def validate_json_schema(instance, schema, *, root=None, path="$"):
|
|
"""Validate the JSON-Schema subset used by the replay contract.
|
|
|
|
This deliberately small validator keeps the contract test dependency-free;
|
|
it is not exposed as a general JSON-Schema implementation.
|
|
"""
|
|
|
|
root = schema if root is None else root
|
|
if "$ref" in schema:
|
|
reference = schema["$ref"]
|
|
if not reference.startswith("#/"):
|
|
raise ContractValidationError(
|
|
f"{path}: only local schema references are supported"
|
|
)
|
|
target = root
|
|
for component in reference[2:].split("/"):
|
|
target = target[component.replace("~1", "/").replace("~0", "~")]
|
|
validate_json_schema(instance, target, root=root, path=path)
|
|
return
|
|
|
|
expected_type = schema.get("type")
|
|
if expected_type is not None:
|
|
expected = (
|
|
expected_type
|
|
if isinstance(expected_type, list)
|
|
else [expected_type]
|
|
)
|
|
|
|
def matches_type(name):
|
|
if name == "null":
|
|
return instance is None
|
|
if name == "object":
|
|
return isinstance(instance, dict)
|
|
if name == "array":
|
|
return isinstance(instance, list)
|
|
if name == "string":
|
|
return isinstance(instance, str)
|
|
if name == "boolean":
|
|
return isinstance(instance, bool)
|
|
if name == "integer":
|
|
return isinstance(instance, int) and not isinstance(instance, bool)
|
|
if name == "number":
|
|
return (
|
|
isinstance(instance, (int, float))
|
|
and not isinstance(instance, bool)
|
|
and math.isfinite(float(instance))
|
|
)
|
|
raise ContractValidationError(
|
|
f"{path}: unsupported schema type {name}"
|
|
)
|
|
|
|
if not any(matches_type(name) for name in expected):
|
|
raise ContractValidationError(
|
|
f"{path}: expected type {expected}, got {type(instance).__name__}"
|
|
)
|
|
|
|
if "const" in schema and instance != schema["const"]:
|
|
raise ContractValidationError(
|
|
f"{path}: expected constant {schema['const']!r}"
|
|
)
|
|
if "enum" in schema and instance not in schema["enum"]:
|
|
raise ContractValidationError(f"{path}: value is outside enum")
|
|
if "minLength" in schema and len(instance) < schema["minLength"]:
|
|
raise ContractValidationError(f"{path}: string is too short")
|
|
if "pattern" in schema and re.search(schema["pattern"], instance) is None:
|
|
raise ContractValidationError(f"{path}: string does not match pattern")
|
|
if "minimum" in schema and instance < schema["minimum"]:
|
|
raise ContractValidationError(f"{path}: value is below minimum")
|
|
if (
|
|
"exclusiveMinimum" in schema
|
|
and instance <= schema["exclusiveMinimum"]
|
|
):
|
|
raise ContractValidationError(
|
|
f"{path}: value is not above exclusive minimum"
|
|
)
|
|
|
|
if schema.get("format") == "date":
|
|
try:
|
|
date.fromisoformat(instance)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ContractValidationError(
|
|
f"{path}: value is not an ISO date"
|
|
) from exc
|
|
if schema.get("format") == "date-time":
|
|
try:
|
|
parsed = datetime.fromisoformat(instance.replace("Z", "+00:00"))
|
|
except (AttributeError, ValueError) as exc:
|
|
raise ContractValidationError(
|
|
f"{path}: value is not an ISO date-time"
|
|
) from exc
|
|
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
|
raise ContractValidationError(
|
|
f"{path}: date-time has no timezone offset"
|
|
)
|
|
|
|
if isinstance(instance, dict):
|
|
required = schema.get("required", [])
|
|
missing = [key for key in required if key not in instance]
|
|
if missing:
|
|
raise ContractValidationError(
|
|
f"{path}: missing required fields {missing}"
|
|
)
|
|
properties = schema.get("properties", {})
|
|
if schema.get("additionalProperties") is False:
|
|
unknown = sorted(set(instance) - set(properties))
|
|
if unknown:
|
|
raise ContractValidationError(
|
|
f"{path}: unsupported fields {unknown}"
|
|
)
|
|
for key, subschema in properties.items():
|
|
if key in instance:
|
|
validate_json_schema(
|
|
instance[key],
|
|
subschema,
|
|
root=root,
|
|
path=f"{path}.{key}",
|
|
)
|
|
for trigger, dependencies in schema.get(
|
|
"dependentRequired",
|
|
{},
|
|
).items():
|
|
if trigger in instance:
|
|
absent = [key for key in dependencies if key not in instance]
|
|
if absent:
|
|
raise ContractValidationError(
|
|
f"{path}: {trigger} requires {absent}"
|
|
)
|
|
|
|
if "oneOf" in schema:
|
|
matches = 0
|
|
failures = []
|
|
for candidate in schema["oneOf"]:
|
|
try:
|
|
validate_json_schema(
|
|
instance,
|
|
candidate,
|
|
root=root,
|
|
path=path,
|
|
)
|
|
matches += 1
|
|
except ContractValidationError as exc:
|
|
failures.append(str(exc))
|
|
if matches != 1:
|
|
raise ContractValidationError(
|
|
f"{path}: expected exactly one schema branch, got {matches}; "
|
|
f"{failures}"
|
|
)
|
|
|
|
|
|
def order_payload(
|
|
order_id,
|
|
side,
|
|
quantity,
|
|
status,
|
|
*,
|
|
filled=0,
|
|
average="0",
|
|
):
|
|
return {
|
|
"order_id": order_id,
|
|
"symbol": "600000.XSHG",
|
|
"side": side,
|
|
"quantity": quantity,
|
|
"filled_quantity": filled,
|
|
"average_fill_price": average,
|
|
"status": status,
|
|
"submitted_at": "2024-01-02",
|
|
"metadata": {},
|
|
"reason": None,
|
|
}
|
|
|
|
|
|
def fill_payload(
|
|
fill_id,
|
|
order_id,
|
|
side,
|
|
quantity,
|
|
price,
|
|
*,
|
|
commission="5",
|
|
stamp_duty="0",
|
|
transfer_fee="0",
|
|
cash_delta=None,
|
|
):
|
|
payload = {
|
|
"fill_id": fill_id,
|
|
"order_id": order_id,
|
|
"trading_date": "2024-01-03",
|
|
"symbol": "600000.XSHG",
|
|
"side": side,
|
|
"quantity": quantity,
|
|
"price": price,
|
|
"commission": commission,
|
|
"stamp_duty": stamp_duty,
|
|
"transfer_fee": transfer_fee,
|
|
}
|
|
if cash_delta is not None:
|
|
payload["cash_delta"] = cash_delta
|
|
return payload
|
|
|
|
|
|
def golden_ledger():
|
|
ledger = HashChainLedger()
|
|
ledger.append(
|
|
"ORDER_STATUS",
|
|
order_payload("O1", "BUY", 200, "ACCEPTED"),
|
|
timestamp="2024-01-03T09:30:00+08:00",
|
|
event_id="order:O1:accepted",
|
|
)
|
|
ledger.append(
|
|
"FILL",
|
|
fill_payload("F1", "O1", "BUY", 100, "10", cash_delta="-1005"),
|
|
timestamp="2024-01-03T09:31:00+08:00",
|
|
event_id="fill:F1",
|
|
)
|
|
ledger.append(
|
|
"ORDER_STATUS",
|
|
order_payload(
|
|
"O1",
|
|
"BUY",
|
|
200,
|
|
"PARTIALLY_FILLED",
|
|
filled=100,
|
|
average="10",
|
|
),
|
|
timestamp="2024-01-03T09:31:00+08:00",
|
|
event_id="order:O1:partial:100",
|
|
)
|
|
ledger.append(
|
|
"FILL",
|
|
fill_payload("F2", "O1", "BUY", 100, "11", cash_delta="-1105"),
|
|
timestamp="2024-01-03T09:32:00+08:00",
|
|
event_id="fill:F2",
|
|
)
|
|
ledger.append(
|
|
"ORDER_STATUS",
|
|
order_payload(
|
|
"O1",
|
|
"BUY",
|
|
200,
|
|
"FILLED",
|
|
filled=200,
|
|
average="10.5",
|
|
),
|
|
timestamp="2024-01-03T09:32:00+08:00",
|
|
event_id="order:O1:filled",
|
|
)
|
|
ledger.append(
|
|
"ORDER_STATUS",
|
|
order_payload("O2", "SELL", 100, "ACCEPTED"),
|
|
timestamp="2024-01-03T09:33:00+08:00",
|
|
event_id="order:O2:accepted",
|
|
)
|
|
ledger.append(
|
|
"FILL",
|
|
fill_payload(
|
|
"F3",
|
|
"O2",
|
|
"SELL",
|
|
100,
|
|
"12",
|
|
commission="5",
|
|
stamp_duty="0.6",
|
|
transfer_fee="0.01",
|
|
cash_delta="1194.39",
|
|
),
|
|
timestamp="2024-01-03T09:34:00+08:00",
|
|
event_id="fill:F3",
|
|
)
|
|
ledger.append(
|
|
"ORDER_STATUS",
|
|
order_payload(
|
|
"O2",
|
|
"SELL",
|
|
100,
|
|
"FILLED",
|
|
filled=100,
|
|
average="12",
|
|
),
|
|
timestamp="2024-01-03T09:34:00+08:00",
|
|
event_id="order:O2:filled",
|
|
)
|
|
return ledger
|
|
|
|
|
|
class LedgerReconcileTests(unittest.TestCase):
|
|
def test_broker_snapshot_schema_requires_fresh_source_and_order_intent(self):
|
|
snapshot = {
|
|
"schema_version": "1.0",
|
|
"snapshot_id": "snapshot-1",
|
|
"signal_as_of": "2024-01-02T15:00:00+08:00",
|
|
"next_trading_session": "2024-01-03",
|
|
"first_executable_window": first_executable_window(
|
|
date(2024, 1, 3)
|
|
),
|
|
"observation_as_of": "2024-01-03T09:10:00+08:00",
|
|
"as_of": "2024-01-03T09:10:00+08:00",
|
|
"source_time": "2024-01-03T09:09:59+08:00",
|
|
"broker": "fake",
|
|
"account_hash": "account-hash",
|
|
"cash": 1000.0,
|
|
"total_asset": 2000.0,
|
|
"positions": [
|
|
{
|
|
"symbol": "600000.XSHG",
|
|
"quantity": 100,
|
|
"sellable_quantity": 100,
|
|
}
|
|
],
|
|
"open_orders": [
|
|
{
|
|
"order_id": "C1",
|
|
"broker_order_id": "B1",
|
|
"symbol": "600000.XSHG",
|
|
"side": "BUY",
|
|
"quantity": 100,
|
|
"filled_quantity": 0,
|
|
"status": "ACCEPTED",
|
|
"limit_price": 10.0,
|
|
"order_type": "LIMIT",
|
|
"time_in_force": "DAY",
|
|
}
|
|
],
|
|
}
|
|
validate_broker_snapshot(snapshot)
|
|
|
|
no_source = copy.deepcopy(snapshot)
|
|
del no_source["source_time"]
|
|
with self.assertRaises(RuntimeContractValidationError):
|
|
validate_broker_snapshot(no_source)
|
|
no_price = copy.deepcopy(snapshot)
|
|
del no_price["open_orders"][0]["limit_price"]
|
|
with self.assertRaises(RuntimeContractValidationError):
|
|
validate_broker_snapshot(no_price)
|
|
|
|
def test_schema_validates_actual_sim_broker_replay_envelopes(self):
|
|
ledger = HashChainLedger()
|
|
broker = SimBroker(
|
|
Portfolio(Decimal("100000")),
|
|
ExecutionConfig(
|
|
lot_size=100,
|
|
participation_rate=0.1,
|
|
slippage_bps=0,
|
|
),
|
|
FeeConfig(
|
|
commission_rate=0,
|
|
minimum_commission=0,
|
|
stamp_duty_rate=0,
|
|
transfer_fee_rate=0,
|
|
),
|
|
ledger,
|
|
)
|
|
broker.submit(
|
|
"600000.SH",
|
|
Side.BUY,
|
|
100,
|
|
date(2024, 1, 2),
|
|
)
|
|
value = Decimal("10")
|
|
broker.execute_session(
|
|
{
|
|
"600000.XSHG": Bar(
|
|
trading_date=date(2024, 1, 2),
|
|
symbol="600000.XSHG",
|
|
open=value,
|
|
high=value,
|
|
low=value,
|
|
close=value,
|
|
volume=1000,
|
|
)
|
|
}
|
|
)
|
|
broker.execute_session(
|
|
{
|
|
"600000.XSHG": Bar(
|
|
trading_date=date(2024, 1, 3),
|
|
symbol="600000.XSHG",
|
|
open=value,
|
|
high=value,
|
|
low=value,
|
|
close=value,
|
|
volume=1000,
|
|
)
|
|
}
|
|
)
|
|
envelopes = [
|
|
record.as_dict()
|
|
for record in ledger.records
|
|
if record.event_type in {"ORDER_STATUS", "FILL"}
|
|
]
|
|
self.assertEqual(
|
|
[record["event_type"] for record in envelopes],
|
|
["ORDER_STATUS", "FILL", "ORDER_STATUS"],
|
|
)
|
|
schema_path = (
|
|
Path(__file__).resolve().parents[1]
|
|
/ "schemas"
|
|
/ "order_event.schema.json"
|
|
)
|
|
schema = json.loads(schema_path.read_text(encoding="utf-8"))
|
|
for envelope in envelopes:
|
|
with self.subTest(event_id=envelope["event_id"]):
|
|
validate_json_schema(envelope, schema)
|
|
|
|
projection = project_ledger(envelopes)
|
|
self.assertEqual(projection.cash_delta, Decimal("-1000.000000"))
|
|
self.assertEqual(
|
|
projection.position_deltas,
|
|
{"600000.XSHG": 100},
|
|
)
|
|
self.assertEqual(projection.orders["O00000001"].status, "FILLED")
|
|
|
|
def test_schema_discriminates_payloads_and_projector_enforces_envelope(self):
|
|
schema_path = (
|
|
Path(__file__).resolve().parents[1]
|
|
/ "schemas"
|
|
/ "order_event.schema.json"
|
|
)
|
|
schema = json.loads(schema_path.read_text(encoding="utf-8"))
|
|
accepted = golden_ledger().records[0].as_dict()
|
|
|
|
flat_legacy = {
|
|
"schema_version": "1.0",
|
|
"event_id": accepted["event_id"],
|
|
"event_time": accepted["timestamp"],
|
|
**accepted["payload"],
|
|
}
|
|
with self.assertRaises(ContractValidationError):
|
|
validate_json_schema(flat_legacy, schema)
|
|
with self.assertRaises(LedgerReplayError):
|
|
project_ledger([flat_legacy])
|
|
|
|
crossed = copy.deepcopy(accepted)
|
|
crossed["event_type"] = "FILL"
|
|
with self.assertRaises(ContractValidationError):
|
|
validate_json_schema(crossed, schema)
|
|
|
|
unexpected = copy.deepcopy(accepted)
|
|
unexpected["payload"]["undocumented"] = True
|
|
with self.assertRaises(ContractValidationError):
|
|
validate_json_schema(unexpected, schema)
|
|
with self.assertRaises(LedgerReplayError):
|
|
project_ledger([unexpected])
|
|
|
|
unpaired_hash = copy.deepcopy(accepted)
|
|
del unpaired_hash["hash"]
|
|
with self.assertRaises(ContractValidationError):
|
|
validate_json_schema(unpaired_hash, schema)
|
|
with self.assertRaises(LedgerReplayError):
|
|
project_ledger([unpaired_hash])
|
|
|
|
no_sequence = copy.deepcopy(accepted)
|
|
del no_sequence["sequence"]
|
|
with self.assertRaises(ContractValidationError):
|
|
validate_json_schema(no_sequence, schema)
|
|
with self.assertRaises(LedgerReplayError):
|
|
project_ledger([no_sequence])
|
|
|
|
alias_fixture = copy.deepcopy(accepted)
|
|
del alias_fixture["previous_hash"]
|
|
del alias_fixture["hash"]
|
|
alias_fixture["payload"]["order_quantity"] = (
|
|
alias_fixture["payload"].pop("quantity")
|
|
)
|
|
alias_fixture["payload"]["cumulative_filled_quantity"] = (
|
|
alias_fixture["payload"].pop("filled_quantity")
|
|
)
|
|
with self.assertRaises(ContractValidationError):
|
|
validate_json_schema(alias_fixture, schema)
|
|
self.assertEqual(
|
|
project_ledger([alias_fixture]).orders["O1"].quantity,
|
|
200,
|
|
)
|
|
conflicting_alias = copy.deepcopy(accepted)
|
|
del conflicting_alias["previous_hash"]
|
|
del conflicting_alias["hash"]
|
|
conflicting_alias["payload"]["order_quantity"] = 300
|
|
with self.assertRaises(ConflictingDuplicateError):
|
|
project_ledger([conflicting_alias])
|
|
|
|
def test_hash_chain_duplicate_and_tamper_detection(self):
|
|
ledger = HashChainLedger()
|
|
first = ledger.append(
|
|
"TEST",
|
|
{"value": 1},
|
|
timestamp="2024-01-02T15:00:00+08:00",
|
|
event_id="event-1",
|
|
)
|
|
self.assertIs(
|
|
ledger.append(
|
|
"TEST",
|
|
{"value": 1},
|
|
timestamp="2024-01-02T15:00:00+08:00",
|
|
event_id="event-1",
|
|
),
|
|
first,
|
|
)
|
|
with self.assertRaises(DuplicateEventError):
|
|
ledger.append(
|
|
"TEST",
|
|
{"value": 2},
|
|
timestamp="2024-01-02T15:00:00+08:00",
|
|
event_id="event-1",
|
|
)
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = ledger.write_jsonl(Path(directory) / "events.jsonl")
|
|
self.assertTrue(HashChainLedger.read_jsonl(path).verify())
|
|
line = json.loads(path.read_text(encoding="utf-8").splitlines()[0])
|
|
line["payload"]["value"] = 99
|
|
path.write_text(json.dumps(line) + "\n", encoding="utf-8")
|
|
with self.assertRaises(LedgerIntegrityError):
|
|
HashChainLedger.read_jsonl(path)
|
|
|
|
def test_legacy_reconciliation_api_remains_compatible(self):
|
|
okay = reconcile_snapshot(
|
|
local_cash="100",
|
|
local_positions={"600000.SH": 100},
|
|
broker_cash="100.001",
|
|
broker_positions={"SH600000": 100},
|
|
cash_tolerance="0.01",
|
|
)
|
|
self.assertTrue(okay.balanced)
|
|
self.assertFalse(okay.complete)
|
|
self.assertFalse(okay.safe_to_open)
|
|
bad = reconcile_snapshot(
|
|
local_cash="100",
|
|
local_positions={"600000.SH": 100},
|
|
broker_cash="99",
|
|
broker_positions={"600000.SH": 0},
|
|
)
|
|
self.assertFalse(bad.safe_to_open)
|
|
self.assertEqual(
|
|
{item.kind for item in bad.differences},
|
|
{"CASH", "POSITION"},
|
|
)
|
|
|
|
def test_identical_duplicate_open_order_identity_fails_closed(self):
|
|
order = {
|
|
"order_id": "DUPLICATE",
|
|
"broker_order_id": "B-DUPLICATE",
|
|
"symbol": "600000.XSHG",
|
|
"side": "BUY",
|
|
"quantity": 100,
|
|
"filled_quantity": 0,
|
|
"status": "ACCEPTED",
|
|
"limit_price": 10.0,
|
|
"order_type": "LIMIT",
|
|
"time_in_force": "DAY",
|
|
}
|
|
local = {
|
|
"cash": "100",
|
|
"positions": [],
|
|
"open_orders": [order, copy.deepcopy(order)],
|
|
"as_of": "2024-01-03T12:00:00+08:00",
|
|
}
|
|
broker = {
|
|
**local,
|
|
"open_orders": [copy.deepcopy(order), copy.deepcopy(order)],
|
|
"source_time": "2024-01-03T12:00:00+08:00",
|
|
}
|
|
report = reconcile_snapshot(
|
|
local_snapshot=local,
|
|
broker_snapshot=broker,
|
|
now="2024-01-03T12:00:00+08:00",
|
|
)
|
|
self.assertTrue(report.complete)
|
|
self.assertFalse(report.balanced)
|
|
self.assertFalse(report.safe_to_open)
|
|
self.assertIn(
|
|
("DUPLICATE_OPEN_ORDER", "DUPLICATE"),
|
|
{(item.kind, item.key) for item in report.differences},
|
|
)
|
|
|
|
def test_complete_snapshot_checks_sellable_orders_and_freshness(self):
|
|
local = {
|
|
"cash": "1000",
|
|
"positions": [
|
|
{
|
|
"symbol": "600000.XSHG",
|
|
"quantity": 200,
|
|
"sellable_quantity": 100,
|
|
}
|
|
],
|
|
"open_orders": [
|
|
{
|
|
"order_id": "C1",
|
|
"broker_order_id": "B-C1",
|
|
"symbol": "600000.XSHG",
|
|
"side": "SELL",
|
|
"quantity": 100,
|
|
"filled_quantity": 0,
|
|
"status": "ACCEPTED",
|
|
"limit_price": 10.0,
|
|
"order_type": "LIMIT",
|
|
"time_in_force": "DAY",
|
|
}
|
|
],
|
|
"as_of": "2024-01-03T12:00:09+08:00",
|
|
}
|
|
broker = {
|
|
"cash": "1000.001",
|
|
"positions": [
|
|
{
|
|
"symbol": "600000.SH",
|
|
"quantity": 200,
|
|
"sellable_quantity": 100,
|
|
}
|
|
],
|
|
"open_orders": [
|
|
{
|
|
"order_id": "C1",
|
|
"broker_order_id": "B-C1",
|
|
"symbol": "SH600000",
|
|
"side": "SELL",
|
|
"quantity": 100,
|
|
"filled_quantity": 0,
|
|
"status": "ACCEPTED",
|
|
"limit_price": 10.0,
|
|
"order_type": "LIMIT",
|
|
"time_in_force": "DAY",
|
|
}
|
|
],
|
|
"as_of": "2024-01-03T12:00:08+08:00",
|
|
"source_time": "2024-01-03T12:00:05+08:00",
|
|
}
|
|
report = reconcile_snapshot(
|
|
local_snapshot=local,
|
|
broker_snapshot=broker,
|
|
now="2024-01-03T12:00:10+08:00",
|
|
max_snapshot_age_seconds=15,
|
|
max_source_age_seconds=15,
|
|
)
|
|
self.assertTrue(report.safe_to_open)
|
|
|
|
wrong_price = copy.deepcopy(broker)
|
|
wrong_price["open_orders"][0]["limit_price"] = 10.01
|
|
price_report = reconcile_snapshot(
|
|
local_snapshot=local,
|
|
broker_snapshot=wrong_price,
|
|
now="2024-01-03T12:00:10+08:00",
|
|
max_snapshot_age_seconds=15,
|
|
max_source_age_seconds=15,
|
|
)
|
|
self.assertFalse(price_report.safe_to_open)
|
|
self.assertIn(
|
|
("OPEN_ORDER", "C1"),
|
|
{
|
|
(item.kind, item.key)
|
|
for item in price_report.differences
|
|
},
|
|
)
|
|
|
|
unsafe_broker = copy.deepcopy(broker)
|
|
unsafe_broker["positions"][0]["sellable_quantity"] = 50
|
|
unsafe_broker["open_orders"][0]["status"] = "UNKNOWN"
|
|
unsafe_broker["source_time"] = "2024-01-03T11:55:00+08:00"
|
|
unsafe = reconcile_snapshot(
|
|
local_snapshot=local,
|
|
broker_snapshot=unsafe_broker,
|
|
now="2024-01-03T12:00:10+08:00",
|
|
max_snapshot_age_seconds=15,
|
|
max_source_age_seconds=15,
|
|
)
|
|
self.assertFalse(unsafe.safe_to_open)
|
|
self.assertEqual(
|
|
{item.kind for item in unsafe.differences},
|
|
{"OPEN_ORDER", "SELLABLE", "STALE_SOURCE", "UNKNOWN_ORDER"},
|
|
)
|
|
|
|
def test_complete_snapshot_missing_or_stale_as_of_fails_closed(self):
|
|
local = {
|
|
"cash": "100",
|
|
"positions": [],
|
|
"open_orders": [],
|
|
"as_of": "2024-01-03T11:00:00+08:00",
|
|
}
|
|
broker = {
|
|
"cash": "100",
|
|
"positions": [],
|
|
"open_orders": [],
|
|
"as_of": "2024-01-03T11:00:00+08:00",
|
|
"source_time": "2024-01-03T11:00:00+08:00",
|
|
}
|
|
stale = reconcile_snapshot(
|
|
local_snapshot=local,
|
|
broker_snapshot=broker,
|
|
now="2024-01-03T12:00:00+08:00",
|
|
)
|
|
self.assertFalse(stale.safe_to_open)
|
|
self.assertIn(
|
|
"STALE_SNAPSHOT",
|
|
{item.kind for item in stale.differences},
|
|
)
|
|
missing = copy.deepcopy(broker)
|
|
del missing["open_orders"]
|
|
report = reconcile_snapshot(
|
|
local_snapshot={
|
|
**local,
|
|
"as_of": "2024-01-03T12:00:00+08:00",
|
|
},
|
|
broker_snapshot={
|
|
**missing,
|
|
"as_of": "2024-01-03T12:00:00+08:00",
|
|
"source_time": "2024-01-03T12:00:00+08:00",
|
|
},
|
|
now="2024-01-03T12:00:00+08:00",
|
|
)
|
|
self.assertFalse(report.safe_to_open)
|
|
self.assertIn(
|
|
("MISSING_FIELD", "broker.open_orders"),
|
|
{(item.kind, item.key) for item in report.differences},
|
|
)
|
|
|
|
def test_freshness_tolerances_reject_nonfinite_and_negative_values(self):
|
|
snapshot = {
|
|
"cash": "100",
|
|
"positions": [],
|
|
"open_orders": [],
|
|
"as_of": "2024-01-03T12:00:00+08:00",
|
|
"source_time": "2024-01-03T12:00:00+08:00",
|
|
}
|
|
base = {
|
|
"local_snapshot": {
|
|
key: value
|
|
for key, value in snapshot.items()
|
|
if key != "source_time"
|
|
},
|
|
"broker_snapshot": snapshot,
|
|
"now": "2024-01-03T12:00:00+08:00",
|
|
}
|
|
tolerance_names = (
|
|
"max_snapshot_age_seconds",
|
|
"max_source_age_seconds",
|
|
"max_clock_skew_seconds",
|
|
"max_age_seconds",
|
|
)
|
|
invalid_values = (
|
|
float("nan"),
|
|
float("inf"),
|
|
float("-inf"),
|
|
-0.001,
|
|
)
|
|
for name in tolerance_names:
|
|
for value in invalid_values:
|
|
with self.subTest(tolerance=name, value=value):
|
|
with self.assertRaisesRegex(
|
|
ValueError,
|
|
rf"{name} must be a finite non-negative number",
|
|
):
|
|
reconcile_snapshot(**base, **{name: value})
|
|
|
|
def test_invalid_specific_tolerance_is_not_hidden_by_age_alias(self):
|
|
legacy = {
|
|
"local_cash": "100",
|
|
"local_positions": {},
|
|
"broker_cash": "100",
|
|
"broker_positions": {},
|
|
"max_age_seconds": 10,
|
|
}
|
|
for name in (
|
|
"max_snapshot_age_seconds",
|
|
"max_source_age_seconds",
|
|
):
|
|
with self.subTest(tolerance=name):
|
|
with self.assertRaisesRegex(
|
|
ValueError,
|
|
rf"{name} must be a finite non-negative number",
|
|
):
|
|
reconcile_snapshot(**legacy, **{name: float("nan")})
|
|
|
|
def test_replay_frozen_golden_digest_and_accounting(self):
|
|
projection = project_ledger(golden_ledger())
|
|
self.assertEqual(projection.cash_delta, Decimal("-915.610000"))
|
|
self.assertEqual(
|
|
projection.position_deltas,
|
|
{"600000.XSHG": 100},
|
|
)
|
|
self.assertEqual(projection.orders["O1"].filled_quantity, 200)
|
|
self.assertEqual(
|
|
projection.orders["O1"].average_fill_price,
|
|
Decimal("10.5000"),
|
|
)
|
|
self.assertEqual(projection.orders["O2"].position_delta, -100)
|
|
self.assertTrue(projection.safe_to_reconcile)
|
|
self.assertEqual(
|
|
projection.digest,
|
|
"b9b48e5f11f43262e435318f6b6a39be74d3090506a3e5952bef4af9944545e5",
|
|
)
|
|
|
|
def test_replay_identical_duplicates_are_idempotent(self):
|
|
ledger = golden_ledger()
|
|
baseline = project_ledger(ledger)
|
|
records = [record.as_dict() for record in ledger.records]
|
|
duplicate_fill = copy.deepcopy(records[6])
|
|
duplicate_fill["sequence"] = 9
|
|
duplicate_fill["event_id"] = "fill:F3:duplicate-delivery"
|
|
duplicate_status = copy.deepcopy(records[7])
|
|
duplicate_status["sequence"] = 10
|
|
duplicate_status["event_id"] = "order:O2:filled:duplicate-delivery"
|
|
replayed = project_ledger(records + [duplicate_fill, duplicate_status])
|
|
self.assertEqual(replayed.digest, baseline.digest)
|
|
self.assertEqual(replayed.cash_delta, baseline.cash_delta)
|
|
self.assertEqual(replayed.fill_ids, baseline.fill_ids)
|
|
|
|
def test_replay_duplicate_id_cannot_hide_sequence_jump(self):
|
|
first = {
|
|
"sequence": 1,
|
|
"event_id": "order:e1",
|
|
"event_type": "ORDER_STATUS",
|
|
"timestamp": "2024-01-03T09:30:00+08:00",
|
|
"payload": order_payload(
|
|
"E1",
|
|
"BUY",
|
|
100,
|
|
"ACCEPTED",
|
|
),
|
|
}
|
|
exact_retry = copy.deepcopy(first)
|
|
second = copy.deepcopy(first)
|
|
second["sequence"] = 2
|
|
second["event_id"] = "order:e2"
|
|
|
|
legitimate = LedgerProjector()
|
|
legitimate.process(first)
|
|
legitimate.process(exact_retry)
|
|
legitimate.process(second)
|
|
projection = legitimate.projection()
|
|
self.assertEqual(projection.source_last_sequence, 2)
|
|
self.assertEqual(projection.relevant_event_count, 1)
|
|
|
|
renumbered_retry = copy.deepcopy(first)
|
|
renumbered_retry["sequence"] = 2
|
|
wrong_sequence = LedgerProjector()
|
|
wrong_sequence.process(first)
|
|
with self.assertRaisesRegex(
|
|
ConflictingDuplicateError,
|
|
"first observed at sequence 1, not 2",
|
|
):
|
|
wrong_sequence.process(renumbered_retry)
|
|
|
|
changed_timestamp = copy.deepcopy(first)
|
|
changed_timestamp["timestamp"] = "2024-01-03T09:30:01+08:00"
|
|
wrong_timestamp = LedgerProjector()
|
|
wrong_timestamp.process(first)
|
|
with self.assertRaises(ConflictingDuplicateError):
|
|
wrong_timestamp.process(changed_timestamp)
|
|
|
|
jumped_retry = copy.deepcopy(first)
|
|
jumped_retry["sequence"] = 100
|
|
poisoned = LedgerProjector()
|
|
poisoned.process(first)
|
|
with self.assertRaisesRegex(OutOfOrderError, "sequence gap"):
|
|
poisoned.process(jumped_retry)
|
|
with self.assertRaisesRegex(
|
|
LedgerReplayError,
|
|
"fail-closed after a prior replay error",
|
|
):
|
|
poisoned.process(second)
|
|
|
|
late_retry = LedgerProjector()
|
|
late_retry.process(first)
|
|
late_retry.process(second)
|
|
with self.assertRaises(OutOfOrderError):
|
|
late_retry.process(exact_retry)
|
|
|
|
def test_replay_rejects_conflicting_duplicates(self):
|
|
ledger = golden_ledger()
|
|
records = [record.as_dict() for record in ledger.records]
|
|
conflict = copy.deepcopy(records[6])
|
|
conflict["sequence"] = 9
|
|
conflict["event_id"] = "fill:F3:conflicting-delivery"
|
|
conflict["payload"]["price"] = "13"
|
|
with self.assertRaises(ConflictingDuplicateError):
|
|
project_ledger(records + [conflict])
|
|
|
|
event_conflict = copy.deepcopy(records[0])
|
|
event_conflict["payload"]["quantity"] = 300
|
|
with self.assertRaises(ConflictingDuplicateError):
|
|
project_ledger([records[0], event_conflict])
|
|
|
|
def test_replay_rejects_unknown_order_overfill_and_out_of_order(self):
|
|
unknown_fill = {
|
|
"sequence": 1,
|
|
"event_id": "fill:unknown",
|
|
"event_type": "FILL",
|
|
"timestamp": "2024-01-03T09:31:00+08:00",
|
|
"payload": fill_payload("FU", "MISSING", "BUY", 100, "10"),
|
|
}
|
|
with self.assertRaises(UnknownOrderError):
|
|
project_ledger([unknown_fill])
|
|
|
|
accepted = {
|
|
"sequence": 1,
|
|
"event_id": "order:small",
|
|
"event_type": "ORDER_STATUS",
|
|
"timestamp": "2024-01-03T09:30:00+08:00",
|
|
"payload": order_payload("SMALL", "BUY", 100, "ACCEPTED"),
|
|
}
|
|
overfill = {
|
|
"sequence": 2,
|
|
"event_id": "fill:too-much",
|
|
"event_type": "FILL",
|
|
"timestamp": "2024-01-03T09:31:00+08:00",
|
|
"payload": fill_payload("F-OVER", "SMALL", "BUY", 200, "10"),
|
|
}
|
|
with self.assertRaises(OverfillError):
|
|
project_ledger([accepted, overfill])
|
|
projector = LedgerProjector()
|
|
projector.process(accepted)
|
|
with self.assertRaises(OverfillError):
|
|
projector.process(overfill)
|
|
with self.assertRaises(LedgerReplayError):
|
|
projector.projection()
|
|
with self.assertRaises(LedgerReplayError):
|
|
projector.process(
|
|
{
|
|
**accepted,
|
|
"sequence": 3,
|
|
"event_id": "order:after-failure",
|
|
}
|
|
)
|
|
|
|
second = copy.deepcopy(accepted)
|
|
second["sequence"] = 2
|
|
second["event_id"] = "order:second"
|
|
first_late = copy.deepcopy(accepted)
|
|
first_late["event_id"] = "order:first-late"
|
|
with self.assertRaises(OutOfOrderError):
|
|
project_ledger([second, first_late])
|
|
|
|
regression = copy.deepcopy(accepted)
|
|
regression["sequence"] = 2
|
|
regression["event_id"] = "order:regression"
|
|
regression["payload"]["status"] = "NEW"
|
|
with self.assertRaises(OutOfOrderError):
|
|
project_ledger([accepted, regression])
|
|
|
|
time_regression = copy.deepcopy(accepted)
|
|
time_regression["sequence"] = 2
|
|
time_regression["event_id"] = "order:time-regression"
|
|
time_regression["timestamp"] = "2024-01-03T09:29:59+08:00"
|
|
time_regression["payload"]["status"] = "UNKNOWN"
|
|
with self.assertRaises(OutOfOrderError):
|
|
project_ledger([accepted, time_regression])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|