feat: add Quant OS A-share baseline
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
"""Quant60: a dependency-free, auditable A-share research baseline."""
|
||||
|
||||
from .backtest import BacktestEngine, BacktestResult
|
||||
from .config import BacktestConfig, load_config
|
||||
from .domain import Bar, Fill, Order, OrderStatus, Portfolio, Position, Side
|
||||
from .synthetic import generate_synthetic_bars
|
||||
|
||||
__all__ = [
|
||||
"BacktestConfig",
|
||||
"BacktestEngine",
|
||||
"BacktestResult",
|
||||
"Bar",
|
||||
"Fill",
|
||||
"Order",
|
||||
"OrderStatus",
|
||||
"Portfolio",
|
||||
"Position",
|
||||
"Side",
|
||||
"generate_synthetic_bars",
|
||||
"load_config",
|
||||
]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,5 @@
|
||||
from .cli import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,672 @@
|
||||
"""Deterministic event-driven local backtest and evidence artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .broker import SimBroker
|
||||
from .config import BacktestConfig
|
||||
from .contracts import schema_directory, validate_records
|
||||
from .domain import Bar, OrderStatus, Portfolio, Side, money
|
||||
from .ledger import HashChainLedger, canonical_json, jsonable
|
||||
from .portable_core import build_rebalance_plan
|
||||
from .risk import check_target_weights, weights_from_positions
|
||||
from .synthetic import group_bars_by_date
|
||||
|
||||
|
||||
def _sha256(value: Any) -> str:
|
||||
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _atomic_write_bytes(path: Path, payload: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
dir=str(path.parent),
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_name, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temporary_name)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _engine_source_manifest() -> tuple[str, dict[str, str]]:
|
||||
package = Path(__file__).resolve().parent
|
||||
sources = {
|
||||
path.name: hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in sorted(package.glob("*.py"), key=lambda item: item.name)
|
||||
}
|
||||
return _sha256(sources), sources
|
||||
|
||||
|
||||
def _contract_manifest() -> tuple[str, dict[str, str]]:
|
||||
directory = schema_directory()
|
||||
names = (
|
||||
"signal.schema.json",
|
||||
"target.schema.json",
|
||||
"order_event.schema.json",
|
||||
"broker_snapshot.schema.json",
|
||||
)
|
||||
contracts = {
|
||||
name: hashlib.sha256((directory / name).read_bytes()).hexdigest()
|
||||
for name in names
|
||||
}
|
||||
return _sha256(contracts), contracts
|
||||
|
||||
|
||||
def _strict_json_file(path: Path) -> Any:
|
||||
def reject_constant(value: str) -> None:
|
||||
raise ValueError(f"non-finite JSON constant {value}")
|
||||
|
||||
return json.loads(
|
||||
path.read_text(encoding="utf-8"),
|
||||
parse_constant=reject_constant,
|
||||
)
|
||||
|
||||
|
||||
def verify_artifact_manifest(
|
||||
manifest_path: str | Path,
|
||||
*,
|
||||
require_current_source: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Verify a completed local artifact set and its current source identity."""
|
||||
path = Path(manifest_path)
|
||||
root = path.parent
|
||||
marker = root / ".quant60-incomplete"
|
||||
if marker.exists():
|
||||
raise ValueError(f"artifact publication is incomplete: {marker}")
|
||||
manifest = _strict_json_file(path)
|
||||
if not isinstance(manifest, dict):
|
||||
raise ValueError("manifest root must be an object")
|
||||
expected = manifest.get("artifact_sha256")
|
||||
if not isinstance(expected, dict):
|
||||
raise ValueError("manifest.artifact_sha256 must be an object")
|
||||
engine_sources = manifest.get("engine_sources")
|
||||
if not isinstance(engine_sources, dict) or not engine_sources:
|
||||
raise ValueError("manifest.engine_sources must be a non-empty object")
|
||||
if _sha256(engine_sources) != manifest.get("engine_source_sha256"):
|
||||
raise ValueError("engine_sources aggregate hash mismatch")
|
||||
if (
|
||||
engine_sources.get("portable_core.py")
|
||||
!= manifest.get("portable_core_sha256")
|
||||
):
|
||||
raise ValueError("portable_core source hash mismatch")
|
||||
contract_schemas = manifest.get("contract_schemas")
|
||||
if not isinstance(contract_schemas, dict) or not contract_schemas:
|
||||
raise ValueError(
|
||||
"manifest.contract_schemas must be a non-empty object"
|
||||
)
|
||||
if _sha256(contract_schemas) != manifest.get("contracts_sha256"):
|
||||
raise ValueError("contract_schemas aggregate hash mismatch")
|
||||
filenames = {
|
||||
"signal": "signal.json",
|
||||
"target": "target.json",
|
||||
"equity": "equity.json",
|
||||
"report": "report.json",
|
||||
"events": "events.jsonl",
|
||||
}
|
||||
if set(expected) != set(filenames):
|
||||
raise ValueError("manifest artifact set is incomplete or unknown")
|
||||
allowed_entries = set(filenames.values()) | {path.name}
|
||||
actual_entries = {entry.name for entry in root.iterdir()}
|
||||
unexpected_entries = sorted(actual_entries - allowed_entries)
|
||||
missing_entries = sorted(allowed_entries - actual_entries)
|
||||
if unexpected_entries or missing_entries:
|
||||
raise ValueError(
|
||||
"artifact directory is not an exact completed set: "
|
||||
f"unexpected={unexpected_entries}, missing={missing_entries}"
|
||||
)
|
||||
verified: dict[str, str] = {}
|
||||
for name, filename in filenames.items():
|
||||
artifact = root / filename
|
||||
actual = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
if actual != expected[name]:
|
||||
raise ValueError(f"{name} artifact hash mismatch")
|
||||
verified[name] = actual
|
||||
|
||||
ledger = HashChainLedger.read_jsonl(root / filenames["events"])
|
||||
ledger.verify()
|
||||
if ledger.head_hash != manifest.get("ledger_head_hash"):
|
||||
raise ValueError("ledger head does not match manifest")
|
||||
signals = _strict_json_file(root / filenames["signal"])
|
||||
targets = _strict_json_file(root / filenames["target"])
|
||||
if not isinstance(signals, list) or not isinstance(targets, list):
|
||||
raise ValueError("signal and target artifacts must be arrays")
|
||||
validate_records(signals, "signal.schema.json")
|
||||
validate_records(targets, "target.schema.json")
|
||||
validate_records(
|
||||
(
|
||||
record.as_dict()
|
||||
for record in ledger.records
|
||||
if record.event_type in {"ORDER_STATUS", "FILL"}
|
||||
),
|
||||
"order_event.schema.json",
|
||||
)
|
||||
|
||||
report = _strict_json_file(root / filenames["report"])
|
||||
for field in (
|
||||
"run_id",
|
||||
"config_hash",
|
||||
"data_version",
|
||||
"engine_source_sha256",
|
||||
"rules_sha256",
|
||||
"strategy_sha256",
|
||||
"contracts_sha256",
|
||||
"ledger_head_hash",
|
||||
):
|
||||
if report.get(field) != manifest.get(field):
|
||||
raise ValueError(f"report/manifest {field} mismatch")
|
||||
|
||||
current_source_sha256, unused_sources = _engine_source_manifest()
|
||||
del unused_sources
|
||||
source_matches = (
|
||||
current_source_sha256 == manifest.get("engine_source_sha256")
|
||||
)
|
||||
if require_current_source and not source_matches:
|
||||
raise ValueError("current quant60 source does not match run manifest")
|
||||
current_contracts_sha256, unused_contracts = _contract_manifest()
|
||||
del unused_contracts
|
||||
contracts_match = (
|
||||
current_contracts_sha256 == manifest.get("contracts_sha256")
|
||||
)
|
||||
if require_current_source and not contracts_match:
|
||||
raise ValueError("current quant60 schemas do not match run manifest")
|
||||
return {
|
||||
"ok": True,
|
||||
"run_id": manifest.get("run_id"),
|
||||
"artifacts": verified,
|
||||
"ledger_records": len(ledger.records),
|
||||
"ledger_head_hash": ledger.head_hash,
|
||||
"current_source_matches": source_matches,
|
||||
"current_contracts_match": contracts_match,
|
||||
}
|
||||
|
||||
|
||||
def _bar_payload(bar: Bar) -> dict[str, Any]:
|
||||
return {
|
||||
"date": bar.trading_date.isoformat(),
|
||||
"symbol": bar.symbol,
|
||||
"open": format(bar.open, "f"),
|
||||
"high": format(bar.high, "f"),
|
||||
"low": format(bar.low, "f"),
|
||||
"close": format(bar.close, "f"),
|
||||
"volume": bar.volume,
|
||||
"previous_close": (
|
||||
format(bar.previous_close, "f")
|
||||
if bar.previous_close is not None
|
||||
else None
|
||||
),
|
||||
"limit_up": format(bar.limit_up, "f") if bar.limit_up is not None else None,
|
||||
"limit_down": (
|
||||
format(bar.limit_down, "f") if bar.limit_down is not None else None
|
||||
),
|
||||
"suspended": bar.suspended,
|
||||
}
|
||||
|
||||
|
||||
def _metrics(
|
||||
equity_curve: list[dict[str, Any]],
|
||||
broker: SimBroker,
|
||||
initial_cash: float,
|
||||
) -> dict[str, Any]:
|
||||
equities = [float(item["equity"]) for item in equity_curve]
|
||||
returns = [
|
||||
equities[index] / equities[index - 1] - 1.0
|
||||
for index in range(1, len(equities))
|
||||
if equities[index - 1] > 0
|
||||
]
|
||||
total_return = equities[-1] / initial_cash - 1.0 if equities else 0.0
|
||||
annualized_return = (
|
||||
(1.0 + total_return) ** (252.0 / max(1, len(returns))) - 1.0
|
||||
if 1.0 + total_return > 0 and returns
|
||||
else total_return
|
||||
)
|
||||
daily_volatility = statistics.stdev(returns) if len(returns) > 1 else 0.0
|
||||
annualized_volatility = daily_volatility * math.sqrt(252.0)
|
||||
mean_return = statistics.fmean(returns) if returns else 0.0
|
||||
sharpe = (
|
||||
mean_return / daily_volatility * math.sqrt(252.0)
|
||||
if daily_volatility > 0
|
||||
else 0.0
|
||||
)
|
||||
peak = 0.0
|
||||
max_drawdown = 0.0
|
||||
for value in equities:
|
||||
peak = max(peak, value)
|
||||
if peak > 0:
|
||||
max_drawdown = min(max_drawdown, value / peak - 1.0)
|
||||
|
||||
filled_quantity = sum(fill.quantity for fill in broker.fills)
|
||||
submitted_quantity = sum(order.quantity for order in broker.orders.values())
|
||||
traded_notional = sum(float(fill.notional) for fill in broker.fills)
|
||||
total_fees = sum(float(fill.total_fees) for fill in broker.fills)
|
||||
average_equity = statistics.fmean(equities) if equities else initial_cash
|
||||
blocked: dict[str, int] = {}
|
||||
for record in broker.ledger.records:
|
||||
if record.event_type == "ORDER_BLOCKED":
|
||||
reason = str(record.payload.get("reason", "UNKNOWN"))
|
||||
blocked[reason] = blocked.get(reason, 0) + 1
|
||||
statuses: dict[str, int] = {}
|
||||
for order in broker.orders.values():
|
||||
statuses[order.status.value] = statuses.get(order.status.value, 0) + 1
|
||||
|
||||
return {
|
||||
"trading_days": len(equity_curve),
|
||||
"initial_cash": initial_cash,
|
||||
"final_equity": equities[-1] if equities else initial_cash,
|
||||
"total_return": total_return,
|
||||
"annualized_return": annualized_return,
|
||||
"annualized_volatility": annualized_volatility,
|
||||
"sharpe_zero_rate": sharpe,
|
||||
"max_drawdown": max_drawdown,
|
||||
"order_count": len(broker.orders),
|
||||
"fill_count": len(broker.fills),
|
||||
"submitted_quantity": submitted_quantity,
|
||||
"filled_quantity": filled_quantity,
|
||||
"fill_rate": (
|
||||
filled_quantity / submitted_quantity if submitted_quantity else 0.0
|
||||
),
|
||||
"turnover_on_average_equity": (
|
||||
traded_notional / average_equity if average_equity > 0 else 0.0
|
||||
),
|
||||
"total_fees": total_fees,
|
||||
"blocked_events": dict(sorted(blocked.items())),
|
||||
"terminal_order_statuses": dict(sorted(statuses.items())),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BacktestResult:
|
||||
run_id: str
|
||||
config_hash: str
|
||||
data_version: str
|
||||
signals: list[dict[str, Any]]
|
||||
targets: list[dict[str, Any]]
|
||||
equity_curve: list[dict[str, Any]]
|
||||
report: dict[str, Any]
|
||||
manifest: dict[str, Any]
|
||||
ledger: HashChainLedger
|
||||
|
||||
def write_artifacts(self, output_dir: str | Path) -> dict[str, Path]:
|
||||
root = Path(output_dir)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
validate_records(self.signals, "signal.schema.json")
|
||||
validate_records(self.targets, "target.schema.json")
|
||||
validate_records(
|
||||
(
|
||||
record.as_dict()
|
||||
for record in self.ledger.records
|
||||
if record.event_type in {"ORDER_STATUS", "FILL"}
|
||||
),
|
||||
"order_event.schema.json",
|
||||
)
|
||||
marker = root / ".quant60-incomplete"
|
||||
_atomic_write_bytes(marker, (self.run_id + "\n").encode("ascii"))
|
||||
payloads = {
|
||||
"signal": self.signals,
|
||||
"target": self.targets,
|
||||
"equity": self.equity_curve,
|
||||
"report": self.report,
|
||||
}
|
||||
paths: dict[str, Path] = {}
|
||||
for name, payload in payloads.items():
|
||||
path = root / f"{name}.json"
|
||||
encoded = (
|
||||
json.dumps(
|
||||
jsonable(payload),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
).encode("utf-8")
|
||||
_atomic_write_bytes(path, encoded)
|
||||
paths[name] = path
|
||||
paths["events"] = self.ledger.write_jsonl(root / "events.jsonl")
|
||||
|
||||
manifest = dict(self.manifest)
|
||||
artifact_hashes: dict[str, str] = {}
|
||||
for name, path in sorted(paths.items()):
|
||||
artifact_hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
manifest["artifact_sha256"] = artifact_hashes
|
||||
manifest["ledger_head_hash"] = self.ledger.head_hash
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest_encoded = (
|
||||
json.dumps(
|
||||
jsonable(manifest),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
).encode("utf-8")
|
||||
_atomic_write_bytes(manifest_path, manifest_encoded)
|
||||
marker.unlink()
|
||||
paths["manifest"] = manifest_path
|
||||
return paths
|
||||
|
||||
|
||||
class BacktestEngine:
|
||||
"""Reference clock: T completed close decides, T+1 open attempts fills."""
|
||||
|
||||
def __init__(self, config: BacktestConfig):
|
||||
config.validate()
|
||||
self.config = config
|
||||
|
||||
def run(self, bars: Iterable[Bar]) -> BacktestResult:
|
||||
all_bars = list(bars)
|
||||
sessions = group_bars_by_date(all_bars)
|
||||
if not sessions:
|
||||
raise ValueError("backtest requires at least one bar")
|
||||
expected_symbols = set(self.config.symbols)
|
||||
seen_symbols = {bar.symbol for bar in all_bars}
|
||||
missing = expected_symbols - seen_symbols
|
||||
if missing:
|
||||
raise ValueError(f"bars are missing configured symbols: {sorted(missing)}")
|
||||
for trading_date, session_bars in sessions.items():
|
||||
missing_for_session = expected_symbols - set(session_bars)
|
||||
if missing_for_session:
|
||||
raise ValueError(
|
||||
"incomplete market-data session "
|
||||
f"{trading_date.isoformat()}: {sorted(missing_for_session)}"
|
||||
)
|
||||
|
||||
config_payload = self.config.as_dict()
|
||||
config_hash = _sha256(config_payload)
|
||||
data_version = _sha256([_bar_payload(bar) for bar in all_bars])
|
||||
engine_source_sha256, engine_sources = _engine_source_manifest()
|
||||
contracts_sha256, contract_schemas = _contract_manifest()
|
||||
portable_path = Path(__file__).with_name("portable_core.py")
|
||||
portable_core_sha256 = hashlib.sha256(
|
||||
portable_path.read_bytes()
|
||||
).hexdigest()
|
||||
strategy_sha256 = _sha256(
|
||||
{
|
||||
"version": "portable-momentum-v1",
|
||||
"parameters": config_payload["strategy"],
|
||||
"portable_core_sha256": portable_core_sha256,
|
||||
}
|
||||
)
|
||||
rules_sha256 = _sha256(
|
||||
{
|
||||
"version": "synthetic-cn-a-v2",
|
||||
"fees": config_payload["fees"],
|
||||
"execution": config_payload["execution"],
|
||||
"clock": "weekly_first_close_to_next_open",
|
||||
}
|
||||
)
|
||||
run_id = (
|
||||
"q60-"
|
||||
+ _sha256(
|
||||
{
|
||||
"config": config_hash,
|
||||
"data": data_version,
|
||||
"engine_source": engine_source_sha256,
|
||||
"strategy": strategy_sha256,
|
||||
"rules": rules_sha256,
|
||||
"contracts": contracts_sha256,
|
||||
}
|
||||
)[:16]
|
||||
)
|
||||
portfolio = Portfolio(Decimal(str(self.config.initial_cash)))
|
||||
ledger = HashChainLedger()
|
||||
broker = SimBroker(
|
||||
portfolio,
|
||||
self.config.execution,
|
||||
self.config.fees,
|
||||
ledger,
|
||||
)
|
||||
histories: dict[str, list[float]] = {
|
||||
symbol: [] for symbol in self.config.symbols
|
||||
}
|
||||
signals: list[dict[str, Any]] = []
|
||||
targets: list[dict[str, Any]] = []
|
||||
equity_curve: list[dict[str, Any]] = []
|
||||
strategy = self.config.strategy
|
||||
required_history = strategy.lookback + strategy.skip + 1
|
||||
|
||||
session_items = list(sessions.items())
|
||||
for session_index, (trading_date, session_bars) in enumerate(session_items):
|
||||
# Purchases filled yesterday become sellable before today's open.
|
||||
portfolio.start_session()
|
||||
broker.execute_session(session_bars)
|
||||
if self.config.execution.cancel_open_orders_at_end:
|
||||
# The reference execution window is one daily bar. Any
|
||||
# unfilled residual from yesterday is explicitly cancelled
|
||||
# before a new decision can be published.
|
||||
broker.cancel_open_orders(
|
||||
trading_date,
|
||||
"DAILY_WINDOW_END",
|
||||
phase="post_open",
|
||||
)
|
||||
|
||||
portfolio.mark(session_bars.values())
|
||||
for symbol in self.config.symbols:
|
||||
bar = session_bars.get(symbol)
|
||||
if bar is not None:
|
||||
histories[symbol].append(float(bar.close))
|
||||
|
||||
equity_curve.append(
|
||||
{
|
||||
"date": trading_date.isoformat(),
|
||||
"cash": float(portfolio.cash),
|
||||
"market_value": float(portfolio.market_value),
|
||||
"equity": float(portfolio.equity),
|
||||
"positions": portfolio.quantities(),
|
||||
}
|
||||
)
|
||||
ledger.append(
|
||||
"PORTFOLIO_SNAPSHOT",
|
||||
equity_curve[-1],
|
||||
timestamp=f"{trading_date.isoformat()}T15:00:00+08:00",
|
||||
event_id=f"portfolio:{trading_date.isoformat()}",
|
||||
)
|
||||
|
||||
ready = all(len(values) >= required_history for values in histories.values())
|
||||
# Canonical cross-engine clock:
|
||||
# first trading session of each ISO week, after its close;
|
||||
# execution is the next available session's open.
|
||||
# Hosted adapters implement the same relation with a daily
|
||||
# state machine; no fixed weekday is assumed.
|
||||
first_session_of_week = (
|
||||
session_index == 0
|
||||
or session_items[session_index - 1][0].isocalendar()[:2]
|
||||
!= trading_date.isocalendar()[:2]
|
||||
)
|
||||
rebalance = (
|
||||
ready
|
||||
and first_session_of_week
|
||||
and session_index < len(session_items) - 1
|
||||
)
|
||||
if not rebalance:
|
||||
continue
|
||||
|
||||
prices = {
|
||||
symbol: float(session_bars[symbol].close)
|
||||
for symbol in self.config.symbols
|
||||
if symbol in session_bars
|
||||
}
|
||||
current_weights = weights_from_positions(
|
||||
portfolio.quantities(),
|
||||
prices,
|
||||
float(portfolio.equity),
|
||||
)
|
||||
plan = build_rebalance_plan(
|
||||
price_history=histories,
|
||||
current=portfolio.quantities(),
|
||||
sellable=portfolio.sellable_quantities(),
|
||||
equity=float(portfolio.equity),
|
||||
lookback=strategy.lookback,
|
||||
skip=strategy.skip,
|
||||
top_n=strategy.top_n,
|
||||
max_weight=strategy.max_weight,
|
||||
gross_target=strategy.gross_target,
|
||||
cash_buffer=strategy.cash_buffer,
|
||||
lot_size=self.config.execution.lot_size,
|
||||
)
|
||||
risk = check_target_weights(
|
||||
plan["weights"],
|
||||
current_weights,
|
||||
max_single_weight=strategy.max_weight,
|
||||
max_gross_weight=min(
|
||||
strategy.gross_target, 1.0 - strategy.cash_buffer
|
||||
),
|
||||
# The smoke permits a full initial deployment. Production
|
||||
# turnover is a separate, account-specific authorization.
|
||||
max_one_way_turnover=1.0,
|
||||
)
|
||||
risk.require_approved()
|
||||
decision_id = f"D-{trading_date.strftime('%Y%m%d')}"
|
||||
as_of = f"{trading_date.isoformat()}T15:00:00+08:00"
|
||||
|
||||
for symbol in sorted(plan["scores"]):
|
||||
score = plan["scores"][symbol]
|
||||
if score is None or not math.isfinite(float(score)):
|
||||
continue
|
||||
signals.append(
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"run_id": run_id,
|
||||
"signal_id": f"S-{trading_date.strftime('%Y%m%d')}-{symbol}",
|
||||
"as_of": as_of,
|
||||
"symbol": symbol,
|
||||
"horizon_trading_days": strategy.rebalance_every,
|
||||
"score": float(score),
|
||||
"expected_excess_return": None,
|
||||
"confidence": None,
|
||||
"model_version": "portable-momentum-v1",
|
||||
"data_version": data_version,
|
||||
"feature_version": (
|
||||
f"close_momentum_{strategy.lookback}_skip_{strategy.skip}"
|
||||
),
|
||||
"metadata": {"selected": plan["weights"].get(symbol, 0.0) > 0},
|
||||
}
|
||||
)
|
||||
for symbol in sorted(plan["targets"]):
|
||||
targets.append(
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"run_id": run_id,
|
||||
"decision_id": decision_id,
|
||||
"as_of": as_of,
|
||||
"symbol": symbol,
|
||||
"target_weight": float(plan["weights"].get(symbol, 0.0)),
|
||||
"target_quantity": int(plan["targets"][symbol]),
|
||||
"lot_size": self.config.execution.lot_size,
|
||||
"signal_id": f"S-{trading_date.strftime('%Y%m%d')}-{symbol}",
|
||||
"constraint_state": "FEASIBLE",
|
||||
"config_hash": config_hash,
|
||||
"metadata": {
|
||||
"gross_weight": risk.gross_weight,
|
||||
"one_way_turnover": risk.one_way_turnover,
|
||||
},
|
||||
}
|
||||
)
|
||||
ledger.append(
|
||||
"DECISION",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"decision_id": decision_id,
|
||||
"as_of": as_of,
|
||||
"weights": plan["weights"],
|
||||
"targets": plan["targets"],
|
||||
"orders": plan["orders"],
|
||||
"risk_approved": risk.approved,
|
||||
},
|
||||
timestamp=as_of,
|
||||
event_id=f"decision:{decision_id}",
|
||||
)
|
||||
for symbol, delta in sorted(
|
||||
plan["orders"].items(), key=lambda item: (item[1] > 0, item[0])
|
||||
):
|
||||
if not delta:
|
||||
continue
|
||||
broker.submit(
|
||||
symbol,
|
||||
Side.BUY if delta > 0 else Side.SELL,
|
||||
abs(int(delta)),
|
||||
trading_date,
|
||||
metadata={"run_id": run_id, "decision_id": decision_id},
|
||||
)
|
||||
|
||||
last_date = session_items[-1][0]
|
||||
broker.cancel_open_orders(last_date, "BACKTEST_END")
|
||||
ledger.verify()
|
||||
report = _metrics(
|
||||
equity_curve,
|
||||
broker,
|
||||
float(self.config.initial_cash),
|
||||
)
|
||||
report.update(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"config_hash": config_hash,
|
||||
"data_version": data_version,
|
||||
"engine_source_sha256": engine_source_sha256,
|
||||
"rules_sha256": rules_sha256,
|
||||
"strategy_sha256": strategy_sha256,
|
||||
"contracts_sha256": contracts_sha256,
|
||||
"final_cash": float(portfolio.cash),
|
||||
"final_positions": portfolio.quantities(),
|
||||
"ledger_records": len(ledger.records),
|
||||
"ledger_head_hash": ledger.head_hash,
|
||||
"evidence_class": "synthetic-local",
|
||||
"investment_value_claim": False,
|
||||
}
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": "1.0",
|
||||
"run_id": run_id,
|
||||
"engine": "local",
|
||||
"mode": "fixture",
|
||||
"evidence_class": "synthetic",
|
||||
"config_hash": config_hash,
|
||||
"data_version": data_version,
|
||||
"engine_source_sha256": engine_source_sha256,
|
||||
"engine_sources": engine_sources,
|
||||
"portable_core_sha256": portable_core_sha256,
|
||||
"rules_sha256": rules_sha256,
|
||||
"strategy_sha256": strategy_sha256,
|
||||
"contracts_sha256": contracts_sha256,
|
||||
"contract_schemas": contract_schemas,
|
||||
"strategy_version": "portable-momentum-v1",
|
||||
"rules_version": "synthetic-cn-a-v2",
|
||||
"timezone": "Asia/Shanghai",
|
||||
"signal_clock": "T_CLOSE_COMPLETE",
|
||||
"first_executable_clock": "T_PLUS_1_OPEN",
|
||||
"price_adjustment": "synthetic_raw",
|
||||
"fill_model": "prior_day_volume_capped_open_slippage_v2",
|
||||
"deterministic": True,
|
||||
}
|
||||
return BacktestResult(
|
||||
run_id=run_id,
|
||||
config_hash=config_hash,
|
||||
data_version=data_version,
|
||||
signals=signals,
|
||||
targets=targets,
|
||||
equity_curve=equity_curve,
|
||||
report=report,
|
||||
manifest=manifest,
|
||||
ledger=ledger,
|
||||
)
|
||||
@@ -0,0 +1,368 @@
|
||||
"""Deterministic daily-bar execution simulator for A-share constraints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
|
||||
from .config import ExecutionConfig, FeeConfig
|
||||
from .domain import (
|
||||
PRICE_QUANT,
|
||||
Bar,
|
||||
Fill,
|
||||
Order,
|
||||
OrderStatus,
|
||||
Portfolio,
|
||||
Side,
|
||||
decimal,
|
||||
money,
|
||||
)
|
||||
from .ledger import HashChainLedger
|
||||
|
||||
|
||||
class SimBroker:
|
||||
def __init__(
|
||||
self,
|
||||
portfolio: Portfolio,
|
||||
execution: ExecutionConfig,
|
||||
fees: FeeConfig,
|
||||
ledger: HashChainLedger,
|
||||
) -> None:
|
||||
self.portfolio = portfolio
|
||||
self.execution = execution
|
||||
self.fees = fees
|
||||
self.ledger = ledger
|
||||
self.orders: dict[str, Order] = {}
|
||||
self.fills: list[Fill] = []
|
||||
self._order_sequence = 0
|
||||
self._fill_sequence = 0
|
||||
# A T+1 open fill may only use liquidity observed by T close.
|
||||
# Current-day full-session volume/high/low are future information at
|
||||
# 09:30 and are deliberately excluded from the execution decision.
|
||||
self._prior_session_volume: dict[str, int] = {}
|
||||
|
||||
def _timestamp(self, trading_date: date, phase: str) -> str:
|
||||
times = {
|
||||
"open": "09:30:00",
|
||||
"post_open": "09:31:00",
|
||||
"close": "15:00:00",
|
||||
"cancel": "15:01:00",
|
||||
}
|
||||
return f"{trading_date.isoformat()}T{times[phase]}+08:00"
|
||||
|
||||
def submit(
|
||||
self,
|
||||
symbol: str,
|
||||
side: Side,
|
||||
quantity: int,
|
||||
submitted_at: date,
|
||||
metadata: dict[str, object] | None = None,
|
||||
) -> Order:
|
||||
self._order_sequence += 1
|
||||
order_id = f"O{self._order_sequence:08d}"
|
||||
order = Order(
|
||||
order_id=order_id,
|
||||
symbol=symbol,
|
||||
side=side,
|
||||
quantity=int(quantity),
|
||||
submitted_at=submitted_at,
|
||||
metadata=dict(metadata or {}),
|
||||
)
|
||||
self.orders[order_id] = order
|
||||
position = self.portfolio.position(order.symbol)
|
||||
full_balance_sell = (
|
||||
side == Side.SELL
|
||||
and int(quantity) == position.quantity
|
||||
and int(quantity) == position.sellable_quantity
|
||||
)
|
||||
code = order.symbol.split(".", 1)[0]
|
||||
is_star_market = (
|
||||
order.symbol.endswith(".XSHG")
|
||||
and code.startswith(("688", "689"))
|
||||
)
|
||||
if is_star_market and int(quantity) < 200 and not full_balance_sell:
|
||||
order.transition(
|
||||
OrderStatus.REJECTED,
|
||||
"BELOW_MARKET_MINIMUM",
|
||||
)
|
||||
elif quantity % self.execution.lot_size and not full_balance_sell:
|
||||
order.transition(OrderStatus.REJECTED, "NOT_BOARD_LOT")
|
||||
else:
|
||||
order.transition(OrderStatus.ACCEPTED)
|
||||
self.ledger.append(
|
||||
"ORDER_STATUS",
|
||||
self._order_payload(order),
|
||||
timestamp=self._timestamp(submitted_at, "close"),
|
||||
event_id=f"order:{order_id}:{order.status.value}",
|
||||
)
|
||||
return order
|
||||
|
||||
@staticmethod
|
||||
def _order_payload(order: Order) -> dict[str, object]:
|
||||
return {
|
||||
"order_id": order.order_id,
|
||||
"symbol": order.symbol,
|
||||
"side": order.side.value,
|
||||
"quantity": order.quantity,
|
||||
"filled_quantity": order.filled_quantity,
|
||||
"average_fill_price": format(order.average_fill_price, "f"),
|
||||
"status": order.status.value,
|
||||
"reason": order.status_reason,
|
||||
"submitted_at": order.submitted_at.isoformat(),
|
||||
"metadata": order.metadata,
|
||||
}
|
||||
|
||||
def _fees(
|
||||
self, symbol: str, side: Side, quantity: int, price: Decimal
|
||||
) -> tuple[Decimal, Decimal, Decimal]:
|
||||
notional = price * quantity
|
||||
commission = max(
|
||||
decimal(self.fees.minimum_commission),
|
||||
notional * decimal(self.fees.commission_rate),
|
||||
)
|
||||
stamp = (
|
||||
notional * decimal(self.fees.stamp_duty_rate)
|
||||
if side == Side.SELL
|
||||
else Decimal("0")
|
||||
)
|
||||
transfer = notional * decimal(self.fees.transfer_fee_rate)
|
||||
return money(commission), money(stamp), money(transfer)
|
||||
|
||||
def _fill_price(self, order: Order, bar: Bar) -> Decimal:
|
||||
slip = decimal(self.execution.slippage_bps) / Decimal("10000")
|
||||
if order.side == Side.BUY:
|
||||
price = bar.open * (Decimal("1") + slip)
|
||||
if bar.limit_up is not None:
|
||||
price = min(price, bar.limit_up)
|
||||
else:
|
||||
price = bar.open * (Decimal("1") - slip)
|
||||
if bar.limit_down is not None:
|
||||
price = max(price, bar.limit_down)
|
||||
return price.quantize(PRICE_QUANT, rounding=ROUND_HALF_UP)
|
||||
|
||||
def _affordable_buy_quantity(
|
||||
self, order: Order, desired: int, price: Decimal
|
||||
) -> int:
|
||||
lot = self.execution.lot_size
|
||||
approximate = int(self.portfolio.cash / price) // lot * lot
|
||||
quantity = min(desired, approximate)
|
||||
while quantity > 0:
|
||||
commission, stamp, transfer = self._fees(
|
||||
order.symbol, order.side, quantity, price
|
||||
)
|
||||
required = price * quantity + commission + stamp + transfer
|
||||
if required <= self.portfolio.cash:
|
||||
return quantity
|
||||
quantity -= lot
|
||||
return 0
|
||||
|
||||
def _blocked_reason(self, order: Order, bar: Bar) -> str | None:
|
||||
if bar.suspended:
|
||||
return "SUSPENDED"
|
||||
# At the opening boundary only the open and today's published price
|
||||
# limits are known. A buy at limit-up / sell at limit-down is
|
||||
# conservatively rejected without consulting the future daily range.
|
||||
if (
|
||||
order.side == Side.BUY
|
||||
and bar.limit_up is not None
|
||||
and bar.open >= bar.limit_up
|
||||
):
|
||||
return "LOCKED_LIMIT_UP"
|
||||
if (
|
||||
order.side == Side.SELL
|
||||
and bar.limit_down is not None
|
||||
and bar.open <= bar.limit_down
|
||||
):
|
||||
return "LOCKED_LIMIT_DOWN"
|
||||
return None
|
||||
|
||||
def _record_block(
|
||||
self, order: Order, bar: Bar, reason: str
|
||||
) -> None:
|
||||
self.ledger.append(
|
||||
"ORDER_BLOCKED",
|
||||
{
|
||||
"order_id": order.order_id,
|
||||
"symbol": order.symbol,
|
||||
"reason": reason,
|
||||
"remaining_quantity": order.remaining_quantity,
|
||||
},
|
||||
timestamp=self._timestamp(bar.trading_date, "open"),
|
||||
event_id=(
|
||||
f"block:{order.order_id}:{bar.trading_date.isoformat()}:{reason}"
|
||||
),
|
||||
)
|
||||
|
||||
def execute_session(self, bars: dict[str, Bar]) -> list[Fill]:
|
||||
if not bars:
|
||||
return []
|
||||
dates = {bar.trading_date for bar in bars.values()}
|
||||
if len(dates) != 1:
|
||||
raise ValueError("all bars in a session must have the same date")
|
||||
trading_date = next(iter(dates))
|
||||
volume_left = {
|
||||
symbol: (
|
||||
int(
|
||||
self._prior_session_volume.get(symbol, 0)
|
||||
* self.execution.participation_rate
|
||||
)
|
||||
// self.execution.lot_size
|
||||
* self.execution.lot_size
|
||||
)
|
||||
for symbol in bars
|
||||
}
|
||||
session_fills: list[Fill] = []
|
||||
active = [
|
||||
order
|
||||
for order in self.orders.values()
|
||||
if order.status
|
||||
in {OrderStatus.ACCEPTED, OrderStatus.PARTIALLY_FILLED}
|
||||
and order.submitted_at < trading_date
|
||||
]
|
||||
active.sort(
|
||||
key=lambda order: (
|
||||
0 if order.side == Side.SELL else 1,
|
||||
order.order_id,
|
||||
)
|
||||
)
|
||||
for order in active:
|
||||
bar = bars.get(order.symbol)
|
||||
if bar is None:
|
||||
self._record_block(
|
||||
order,
|
||||
Bar(
|
||||
trading_date=trading_date,
|
||||
symbol=order.symbol,
|
||||
open=Decimal("1"),
|
||||
high=Decimal("1"),
|
||||
low=Decimal("1"),
|
||||
close=Decimal("1"),
|
||||
volume=0,
|
||||
suspended=True,
|
||||
),
|
||||
"NO_MARKET_DATA",
|
||||
)
|
||||
continue
|
||||
reason = self._blocked_reason(order, bar)
|
||||
if reason is not None:
|
||||
self._record_block(order, bar, reason)
|
||||
continue
|
||||
capacity = volume_left.get(order.symbol, 0)
|
||||
desired = min(order.remaining_quantity, capacity)
|
||||
if order.side == Side.SELL:
|
||||
position = self.portfolio.position(order.symbol)
|
||||
sellable = position.sellable_quantity
|
||||
desired = min(desired, sellable)
|
||||
full_balance_sell = (
|
||||
desired == order.remaining_quantity
|
||||
and desired == position.quantity
|
||||
and desired == sellable
|
||||
)
|
||||
if not full_balance_sell:
|
||||
desired = (
|
||||
desired
|
||||
// self.execution.lot_size
|
||||
* self.execution.lot_size
|
||||
)
|
||||
if desired <= 0:
|
||||
self._record_block(order, bar, "T_PLUS_ONE")
|
||||
continue
|
||||
else:
|
||||
desired = (
|
||||
desired
|
||||
// self.execution.lot_size
|
||||
* self.execution.lot_size
|
||||
)
|
||||
price = self._fill_price(order, bar)
|
||||
if order.side == Side.BUY:
|
||||
desired = self._affordable_buy_quantity(
|
||||
order, desired, price
|
||||
)
|
||||
if desired <= 0:
|
||||
self._record_block(order, bar, "INSUFFICIENT_CASH")
|
||||
continue
|
||||
if desired <= 0:
|
||||
self._record_block(order, bar, "PARTICIPATION_LIMIT")
|
||||
continue
|
||||
commission, stamp, transfer = self._fees(
|
||||
order.symbol, order.side, desired, price
|
||||
)
|
||||
self._fill_sequence += 1
|
||||
fill = Fill(
|
||||
fill_id=f"F{self._fill_sequence:08d}",
|
||||
order_id=order.order_id,
|
||||
trading_date=trading_date,
|
||||
symbol=order.symbol,
|
||||
side=order.side,
|
||||
quantity=desired,
|
||||
price=price,
|
||||
commission=commission,
|
||||
stamp_duty=stamp,
|
||||
transfer_fee=transfer,
|
||||
)
|
||||
self.portfolio.apply_fill(fill)
|
||||
order.apply_fill(desired, price)
|
||||
volume_left[order.symbol] -= desired
|
||||
self.fills.append(fill)
|
||||
session_fills.append(fill)
|
||||
self.ledger.append(
|
||||
"FILL",
|
||||
{
|
||||
"fill_id": fill.fill_id,
|
||||
"order_id": fill.order_id,
|
||||
"trading_date": fill.trading_date.isoformat(),
|
||||
"symbol": fill.symbol,
|
||||
"side": fill.side.value,
|
||||
"quantity": fill.quantity,
|
||||
"price": format(fill.price, "f"),
|
||||
"commission": format(fill.commission, "f"),
|
||||
"stamp_duty": format(fill.stamp_duty, "f"),
|
||||
"transfer_fee": format(fill.transfer_fee, "f"),
|
||||
"cash_delta": format(fill.cash_delta, "f"),
|
||||
},
|
||||
timestamp=self._timestamp(trading_date, "open"),
|
||||
event_id=f"fill:{fill.fill_id}",
|
||||
)
|
||||
self.ledger.append(
|
||||
"ORDER_STATUS",
|
||||
self._order_payload(order),
|
||||
timestamp=self._timestamp(trading_date, "open"),
|
||||
event_id=(
|
||||
f"order:{order.order_id}:{order.status.value}:"
|
||||
f"{order.filled_quantity}"
|
||||
),
|
||||
)
|
||||
self._prior_session_volume = {
|
||||
symbol: int(bar.volume) for symbol, bar in bars.items()
|
||||
}
|
||||
return session_fills
|
||||
|
||||
def cancel_open_orders(
|
||||
self,
|
||||
trading_date: date,
|
||||
reason: str = "EXECUTION_WINDOW_END",
|
||||
*,
|
||||
phase: str = "cancel",
|
||||
) -> None:
|
||||
if phase not in {"post_open", "cancel"}:
|
||||
raise ValueError("cancel phase must be post_open or cancel")
|
||||
for order in sorted(self.orders.values(), key=lambda item: item.order_id):
|
||||
if order.status in {
|
||||
OrderStatus.ACCEPTED,
|
||||
OrderStatus.PARTIALLY_FILLED,
|
||||
OrderStatus.UNKNOWN,
|
||||
}:
|
||||
order.transition(OrderStatus.CANCEL_PENDING, reason)
|
||||
self.ledger.append(
|
||||
"ORDER_STATUS",
|
||||
self._order_payload(order),
|
||||
timestamp=self._timestamp(trading_date, phase),
|
||||
event_id=f"order:{order.order_id}:CANCEL_PENDING",
|
||||
)
|
||||
order.transition(OrderStatus.CANCELLED, reason)
|
||||
self.ledger.append(
|
||||
"ORDER_STATUS",
|
||||
self._order_payload(order),
|
||||
timestamp=self._timestamp(trading_date, phase),
|
||||
event_id=f"order:{order.order_id}:CANCELLED",
|
||||
)
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Command-line entry point for the dependency-free local baseline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import date
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
from . import __version__
|
||||
from .backtest import BacktestEngine, verify_artifact_manifest
|
||||
from .config import BacktestConfig, load_config
|
||||
from .experiment import (
|
||||
run_research_smoke,
|
||||
verify_research_manifest,
|
||||
write_research_artifacts,
|
||||
)
|
||||
from .ledger import HashChainLedger, jsonable
|
||||
from .snapshot_pipeline import (
|
||||
build_snapshot_decision,
|
||||
verify_snapshot_decision,
|
||||
write_snapshot_decision,
|
||||
)
|
||||
from .snapshot_backtest import run_snapshot_backtest
|
||||
from .synthetic import generate_synthetic_bars
|
||||
|
||||
|
||||
def _project_root() -> Path:
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _default_config_path() -> Path:
|
||||
return _project_root() / "configs" / "baseline.json"
|
||||
|
||||
|
||||
def _load_or_default(path: str | None) -> BacktestConfig:
|
||||
return load_config(path) if path else BacktestConfig()
|
||||
|
||||
|
||||
def _run_backtest(config: BacktestConfig, output: str | Path) -> dict:
|
||||
bars = generate_synthetic_bars(
|
||||
config.symbols,
|
||||
start_date=config.synthetic.start_date,
|
||||
trading_days_count=config.synthetic.trading_days,
|
||||
seed=config.synthetic.seed,
|
||||
base_volume=config.synthetic.base_volume,
|
||||
)
|
||||
result = BacktestEngine(config).run(bars)
|
||||
paths = result.write_artifacts(output)
|
||||
return {
|
||||
"ok": True,
|
||||
"run_id": result.run_id,
|
||||
"report": result.report,
|
||||
"artifacts": {name: str(path.resolve()) for name, path in paths.items()},
|
||||
}
|
||||
|
||||
|
||||
def _preflight() -> dict:
|
||||
project = _project_root()
|
||||
return {
|
||||
"local": {
|
||||
"available": True,
|
||||
"python": sys.version.split()[0],
|
||||
},
|
||||
"qlib": {
|
||||
"available": importlib.util.find_spec("qlib") is not None,
|
||||
"required_runtime": "CPython 3.12 + pyqlib==0.9.7",
|
||||
},
|
||||
"jqdatasdk": {
|
||||
"available": importlib.util.find_spec("jqdatasdk") is not None,
|
||||
"required_runtime": "authorized JQData account + jqdatasdk==1.9.8",
|
||||
},
|
||||
"cvxpy": {
|
||||
"available": importlib.util.find_spec("cvxpy") is not None,
|
||||
"fallback": "dependency-free deterministic optimizer",
|
||||
},
|
||||
"xtquant": {
|
||||
"available": importlib.util.find_spec("xtquant") is not None,
|
||||
"required_runtime": "licensed QMT/MiniQMT environment",
|
||||
},
|
||||
"joinquant_bundle": {
|
||||
"available": (project / "dist" / "joinquant_strategy.py").is_file(),
|
||||
},
|
||||
"qmt_bundle": {
|
||||
"available": (project / "dist" / "qmt_builtin_strategy.py").is_file(),
|
||||
},
|
||||
"colab_notebook": {
|
||||
"available": (
|
||||
project / "notebooks" / "quant_os_colab.ipynb"
|
||||
).is_file(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="quant60", description=__doc__)
|
||||
parser.add_argument("--version", action="version", version=__version__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
validate = subparsers.add_parser(
|
||||
"validate-config", help="validate a baseline JSON configuration"
|
||||
)
|
||||
validate.add_argument(
|
||||
"--config", default=str(_default_config_path()), help="configuration path"
|
||||
)
|
||||
|
||||
smoke = subparsers.add_parser(
|
||||
"smoke", help="run the deterministic synthetic local backtest"
|
||||
)
|
||||
smoke.add_argument(
|
||||
"--config", default=str(_default_config_path()), help="configuration path"
|
||||
)
|
||||
smoke.add_argument("--output", required=True, help="artifact directory")
|
||||
|
||||
backtest = subparsers.add_parser(
|
||||
"backtest",
|
||||
help="alias of smoke: run the deterministic synthetic local backtest",
|
||||
)
|
||||
backtest.add_argument(
|
||||
"--config", default=str(_default_config_path()), help="configuration path"
|
||||
)
|
||||
backtest.add_argument("--output", required=True, help="artifact directory")
|
||||
|
||||
verify = subparsers.add_parser(
|
||||
"verify-ledger", help="verify an events.jsonl hash chain"
|
||||
)
|
||||
verify.add_argument("path")
|
||||
|
||||
verify_manifest = subparsers.add_parser(
|
||||
"verify-manifest",
|
||||
help="verify a completed local artifact manifest and current source",
|
||||
)
|
||||
verify_manifest.add_argument("path")
|
||||
|
||||
research = subparsers.add_parser(
|
||||
"research-smoke",
|
||||
help="run the deterministic research-to-target walk-forward experiment",
|
||||
)
|
||||
research.add_argument("--output", required=True, help="artifact directory")
|
||||
research.add_argument("--trading-days", type=int, default=320)
|
||||
research.add_argument("--seed", type=int, default=60)
|
||||
|
||||
verify_research = subparsers.add_parser(
|
||||
"verify-research-manifest",
|
||||
help="verify a completed research experiment artifact set",
|
||||
)
|
||||
verify_research.add_argument("path")
|
||||
|
||||
snapshot_decision = subparsers.add_parser(
|
||||
"snapshot-decision",
|
||||
help=(
|
||||
"build a PIT portable signal/target from a verified provider "
|
||||
"snapshot"
|
||||
),
|
||||
)
|
||||
snapshot_decision.add_argument("--snapshot", required=True)
|
||||
snapshot_decision.add_argument(
|
||||
"--config",
|
||||
default=str(_default_config_path()),
|
||||
help="strategy/execution configuration path",
|
||||
)
|
||||
snapshot_decision.add_argument(
|
||||
"--as-of",
|
||||
required=True,
|
||||
type=date.fromisoformat,
|
||||
)
|
||||
funding = snapshot_decision.add_mutually_exclusive_group(required=True)
|
||||
funding.add_argument("--equity", type=float)
|
||||
funding.add_argument(
|
||||
"--broker-snapshot",
|
||||
help="canonical broker snapshot JSON; open orders must be empty",
|
||||
)
|
||||
snapshot_decision.add_argument("--output", required=True)
|
||||
|
||||
snapshot_backtest = subparsers.add_parser(
|
||||
"snapshot-backtest",
|
||||
help=(
|
||||
"run the local event backtest with PIT membership, raw prices, "
|
||||
"adjustment factors and provider daily trading fields"
|
||||
),
|
||||
)
|
||||
snapshot_backtest.add_argument("--snapshot", required=True)
|
||||
snapshot_backtest.add_argument(
|
||||
"--config",
|
||||
default=str(_default_config_path()),
|
||||
)
|
||||
snapshot_backtest.add_argument("--output", required=True)
|
||||
|
||||
verify_snapshot = subparsers.add_parser(
|
||||
"verify-snapshot-decision",
|
||||
help="verify a provider-snapshot decision artifact set",
|
||||
)
|
||||
verify_snapshot.add_argument("path")
|
||||
|
||||
subparsers.add_parser(
|
||||
"preflight", help="report local and optional platform capabilities"
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.command == "validate-config":
|
||||
config = load_config(args.config)
|
||||
payload = {"ok": True, "config": config.as_dict()}
|
||||
elif args.command in {"smoke", "backtest"}:
|
||||
payload = _run_backtest(load_config(args.config), args.output)
|
||||
elif args.command == "verify-ledger":
|
||||
ledger = HashChainLedger.read_jsonl(args.path)
|
||||
payload = {
|
||||
"ok": ledger.verify(),
|
||||
"records": len(ledger.records),
|
||||
"head_hash": ledger.head_hash,
|
||||
}
|
||||
elif args.command == "verify-manifest":
|
||||
payload = verify_artifact_manifest(args.path)
|
||||
elif args.command == "research-smoke":
|
||||
result = run_research_smoke(
|
||||
trading_days=args.trading_days,
|
||||
seed=args.seed,
|
||||
)
|
||||
paths = write_research_artifacts(result, args.output)
|
||||
payload = {
|
||||
"ok": True,
|
||||
"run_id": result["manifest"]["run_id"],
|
||||
"report": result["report"],
|
||||
"artifacts": paths,
|
||||
}
|
||||
elif args.command == "verify-research-manifest":
|
||||
payload = verify_research_manifest(args.path)
|
||||
elif args.command == "snapshot-decision":
|
||||
broker_snapshot = None
|
||||
if args.broker_snapshot:
|
||||
broker_snapshot = json.loads(
|
||||
Path(args.broker_snapshot).read_text(encoding="utf-8")
|
||||
)
|
||||
result = build_snapshot_decision(
|
||||
snapshot_manifest=args.snapshot,
|
||||
config=load_config(args.config),
|
||||
as_of=args.as_of,
|
||||
equity=args.equity,
|
||||
broker_snapshot=broker_snapshot,
|
||||
)
|
||||
paths = write_snapshot_decision(result, args.output)
|
||||
payload = {
|
||||
"ok": True,
|
||||
"run_id": result["manifest"]["run_id"],
|
||||
"decision": result["decision"],
|
||||
"artifacts": paths,
|
||||
}
|
||||
elif args.command == "verify-snapshot-decision":
|
||||
payload = verify_snapshot_decision(args.path)
|
||||
elif args.command == "snapshot-backtest":
|
||||
result = run_snapshot_backtest(
|
||||
snapshot_manifest=args.snapshot,
|
||||
config=load_config(args.config),
|
||||
)
|
||||
paths = result.write_artifacts(args.output)
|
||||
payload = {
|
||||
"ok": True,
|
||||
"run_id": result.run_id,
|
||||
"report": result.report,
|
||||
"artifacts": {
|
||||
name: str(path.resolve())
|
||||
for name, path in paths.items()
|
||||
},
|
||||
}
|
||||
elif args.command == "preflight":
|
||||
payload = {"ok": True, "capabilities": _preflight()}
|
||||
else:
|
||||
raise ValueError(f"unsupported command {args.command}")
|
||||
except Exception as exc:
|
||||
print(
|
||||
json.dumps(
|
||||
{"ok": False, "error": type(exc).__name__, "message": str(exc)},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
print(
|
||||
json.dumps(
|
||||
jsonable(payload),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Validated JSON configuration for local and platform-neutral runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
def _finite_number(value: Any, name: str) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{name} must be a finite number")
|
||||
result = float(value)
|
||||
if not math.isfinite(result):
|
||||
raise ValueError(f"{name} must be a finite number")
|
||||
return result
|
||||
|
||||
|
||||
def _exact_int(value: Any, name: str) -> int:
|
||||
if type(value) is not int:
|
||||
raise ValueError(f"{name} must be an integer")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FeeConfig:
|
||||
commission_rate: float = 0.0002
|
||||
minimum_commission: float = 5.0
|
||||
stamp_duty_rate: float = 0.0005
|
||||
transfer_fee_rate: float = 0.00001
|
||||
|
||||
def validate(self) -> None:
|
||||
values = {
|
||||
"commission_rate": self.commission_rate,
|
||||
"minimum_commission": self.minimum_commission,
|
||||
"stamp_duty_rate": self.stamp_duty_rate,
|
||||
"transfer_fee_rate": self.transfer_fee_rate,
|
||||
}
|
||||
for name, value in values.items():
|
||||
numeric = _finite_number(value, name)
|
||||
if numeric < 0:
|
||||
raise ValueError(f"{name} must be finite and non-negative")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExecutionConfig:
|
||||
lot_size: int = 100
|
||||
participation_rate: float = 0.10
|
||||
slippage_bps: float = 2.0
|
||||
cancel_open_orders_at_end: bool = True
|
||||
|
||||
def validate(self) -> None:
|
||||
if _exact_int(self.lot_size, "lot_size") <= 0:
|
||||
raise ValueError("lot_size must be positive")
|
||||
participation_rate = _finite_number(
|
||||
self.participation_rate,
|
||||
"participation_rate",
|
||||
)
|
||||
if not (0 < participation_rate <= 1):
|
||||
raise ValueError("participation_rate must lie in (0, 1]")
|
||||
slippage_bps = _finite_number(self.slippage_bps, "slippage_bps")
|
||||
if not (0 <= slippage_bps < 10_000):
|
||||
raise ValueError("slippage_bps must lie in [0, 10000)")
|
||||
if type(self.cancel_open_orders_at_end) is not bool:
|
||||
raise ValueError("cancel_open_orders_at_end must be a bool")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UniverseConfig:
|
||||
mode: str = "pit_index"
|
||||
index_symbol: str = "000905.XSHG"
|
||||
qmt_sector_name: str = "中证500"
|
||||
dedicated_account_required: bool = True
|
||||
exclude_st: bool = True
|
||||
|
||||
def validate(self) -> None:
|
||||
if self.mode not in {"pit_index", "fixed"}:
|
||||
raise ValueError("universe mode must be pit_index or fixed")
|
||||
normalize_symbol(self.index_symbol)
|
||||
if not str(self.qmt_sector_name).strip():
|
||||
raise ValueError("qmt_sector_name must be non-empty")
|
||||
if type(self.dedicated_account_required) is not bool:
|
||||
raise ValueError("dedicated_account_required must be a bool")
|
||||
if type(self.exclude_st) is not bool:
|
||||
raise ValueError("exclude_st must be a bool")
|
||||
if self.mode == "pit_index" and not self.dedicated_account_required:
|
||||
raise ValueError(
|
||||
"pit_index mode requires a dedicated strategy account"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StrategyConfig:
|
||||
lookback: int = 20
|
||||
skip: int = 0
|
||||
top_n: int = 5
|
||||
max_weight: float = 0.20
|
||||
gross_target: float = 0.95
|
||||
cash_buffer: float = 0.02
|
||||
rebalance_every: int = 5
|
||||
rebalance_schedule: str = "weekly_first_close"
|
||||
|
||||
def validate(self) -> None:
|
||||
lookback = _exact_int(self.lookback, "lookback")
|
||||
skip = _exact_int(self.skip, "skip")
|
||||
top_n = _exact_int(self.top_n, "top_n")
|
||||
if lookback <= 0 or skip < 0 or top_n <= 0:
|
||||
raise ValueError("invalid momentum window/top_n")
|
||||
max_weight = _finite_number(self.max_weight, "max_weight")
|
||||
gross_target = _finite_number(self.gross_target, "gross_target")
|
||||
cash_buffer = _finite_number(self.cash_buffer, "cash_buffer")
|
||||
if not (0 < max_weight <= 1):
|
||||
raise ValueError("max_weight must lie in (0, 1]")
|
||||
if not (0 <= gross_target <= 1):
|
||||
raise ValueError("gross_target must lie in [0, 1]")
|
||||
if not (0 <= cash_buffer < 1):
|
||||
raise ValueError("cash_buffer must lie in [0, 1)")
|
||||
if _exact_int(self.rebalance_every, "rebalance_every") <= 0:
|
||||
raise ValueError("rebalance_every must be positive")
|
||||
if (
|
||||
type(self.rebalance_schedule) is not str
|
||||
or self.rebalance_schedule != "weekly_first_close"
|
||||
):
|
||||
raise ValueError(
|
||||
"rebalance_schedule must be weekly_first_close"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SyntheticConfig:
|
||||
start_date: str = "2024-01-02"
|
||||
trading_days: int = 90
|
||||
seed: int = 60
|
||||
base_volume: int = 100_000
|
||||
|
||||
def validate(self) -> None:
|
||||
if type(self.start_date) is not str:
|
||||
raise ValueError("start_date must be an ISO date string")
|
||||
date.fromisoformat(self.start_date)
|
||||
trading_days = _exact_int(self.trading_days, "trading_days")
|
||||
_exact_int(self.seed, "seed")
|
||||
base_volume = _exact_int(self.base_volume, "base_volume")
|
||||
if trading_days <= 1 or base_volume < 100:
|
||||
raise ValueError("synthetic data needs >1 day and volume >=100")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BacktestConfig:
|
||||
initial_cash: float = 1_000_000.0
|
||||
symbols: tuple[str, ...] = (
|
||||
"600000.XSHG",
|
||||
"000001.XSHE",
|
||||
"300750.XSHE",
|
||||
"830799.XBSE",
|
||||
)
|
||||
universe: UniverseConfig = field(default_factory=UniverseConfig)
|
||||
fees: FeeConfig = field(default_factory=FeeConfig)
|
||||
execution: ExecutionConfig = field(default_factory=ExecutionConfig)
|
||||
strategy: StrategyConfig = field(default_factory=StrategyConfig)
|
||||
synthetic: SyntheticConfig = field(default_factory=SyntheticConfig)
|
||||
|
||||
def validate(self) -> None:
|
||||
initial_cash = _finite_number(self.initial_cash, "initial_cash")
|
||||
if initial_cash <= 0:
|
||||
raise ValueError("initial_cash must be finite and positive")
|
||||
normalized = tuple(normalize_symbol(symbol) for symbol in self.symbols)
|
||||
if not normalized or len(normalized) != len(set(normalized)):
|
||||
raise ValueError("symbols must be non-empty and unique")
|
||||
self.universe.validate()
|
||||
self.fees.validate()
|
||||
self.execution.validate()
|
||||
self.strategy.validate()
|
||||
self.synthetic.validate()
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
result = asdict(self)
|
||||
result["symbols"] = list(self.symbols)
|
||||
return result
|
||||
|
||||
|
||||
def _only_known(mapping: dict[str, Any], allowed: set[str], name: str) -> None:
|
||||
unknown = set(mapping) - allowed
|
||||
if unknown:
|
||||
raise ValueError(f"unknown {name} keys: {sorted(unknown)}")
|
||||
|
||||
|
||||
def config_from_dict(raw: dict[str, Any]) -> BacktestConfig:
|
||||
defaults = BacktestConfig()
|
||||
_only_known(
|
||||
raw,
|
||||
{
|
||||
"initial_cash",
|
||||
"symbols",
|
||||
"universe",
|
||||
"fees",
|
||||
"execution",
|
||||
"strategy",
|
||||
"synthetic",
|
||||
},
|
||||
"config",
|
||||
)
|
||||
nested = {
|
||||
name: raw.get(name, {})
|
||||
for name in (
|
||||
"universe",
|
||||
"fees",
|
||||
"execution",
|
||||
"strategy",
|
||||
"synthetic",
|
||||
)
|
||||
}
|
||||
for name, value in nested.items():
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{name} must be an object")
|
||||
universe_raw = dict(nested["universe"])
|
||||
fee_raw = dict(nested["fees"])
|
||||
execution_raw = dict(nested["execution"])
|
||||
strategy_raw = dict(nested["strategy"])
|
||||
synthetic_raw = dict(nested["synthetic"])
|
||||
_only_known(
|
||||
universe_raw,
|
||||
set(UniverseConfig.__dataclass_fields__),
|
||||
"universe",
|
||||
)
|
||||
_only_known(fee_raw, set(FeeConfig.__dataclass_fields__), "fees")
|
||||
_only_known(
|
||||
execution_raw, set(ExecutionConfig.__dataclass_fields__), "execution"
|
||||
)
|
||||
_only_known(
|
||||
strategy_raw, set(StrategyConfig.__dataclass_fields__), "strategy"
|
||||
)
|
||||
_only_known(
|
||||
synthetic_raw, set(SyntheticConfig.__dataclass_fields__), "synthetic"
|
||||
)
|
||||
symbols_raw = raw.get("symbols", defaults.symbols)
|
||||
if not isinstance(symbols_raw, (list, tuple)) or any(
|
||||
type(symbol) is not str for symbol in symbols_raw
|
||||
):
|
||||
raise ValueError("symbols must be an array of strings")
|
||||
config = BacktestConfig(
|
||||
initial_cash=raw.get("initial_cash", defaults.initial_cash),
|
||||
symbols=tuple(normalize_symbol(symbol) for symbol in symbols_raw),
|
||||
universe=UniverseConfig(**universe_raw),
|
||||
fees=FeeConfig(**fee_raw),
|
||||
execution=ExecutionConfig(**execution_raw),
|
||||
strategy=StrategyConfig(**strategy_raw),
|
||||
synthetic=SyntheticConfig(**synthetic_raw),
|
||||
)
|
||||
config.validate()
|
||||
return config
|
||||
|
||||
|
||||
def load_config(path: str | Path) -> BacktestConfig:
|
||||
with Path(path).open("r", encoding="utf-8") as handle:
|
||||
raw = json.load(handle)
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError("configuration root must be an object")
|
||||
return config_from_dict(raw)
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Dependency-free validation for the repository's JSON contract subset."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
|
||||
class ContractValidationError(ValueError):
|
||||
"""A value does not satisfy its versioned JSON contract."""
|
||||
|
||||
|
||||
def _matches_type(value: Any, name: str) -> bool:
|
||||
if name == "null":
|
||||
return value is None
|
||||
if name == "object":
|
||||
return isinstance(value, dict)
|
||||
if name == "array":
|
||||
return isinstance(value, list)
|
||||
if name == "string":
|
||||
return isinstance(value, str)
|
||||
if name == "boolean":
|
||||
return isinstance(value, bool)
|
||||
if name == "integer":
|
||||
return type(value) is int
|
||||
if name == "number":
|
||||
return (
|
||||
isinstance(value, (int, float))
|
||||
and not isinstance(value, bool)
|
||||
and math.isfinite(float(value))
|
||||
)
|
||||
raise ContractValidationError(f"unsupported schema type {name!r}")
|
||||
|
||||
|
||||
def validate_contract(
|
||||
value: Any,
|
||||
schema: Mapping[str, Any],
|
||||
*,
|
||||
root: Mapping[str, Any] | None = None,
|
||||
path: str = "$",
|
||||
) -> None:
|
||||
"""Validate the JSON-Schema 2020-12 subset used by quant60."""
|
||||
root_schema = schema if root is None else root
|
||||
reference = schema.get("$ref")
|
||||
if reference is not None:
|
||||
if not str(reference).startswith("#/"):
|
||||
raise ContractValidationError(
|
||||
f"{path}: only local schema references are supported"
|
||||
)
|
||||
target: Any = root_schema
|
||||
for component in str(reference)[2:].split("/"):
|
||||
key = component.replace("~1", "/").replace("~0", "~")
|
||||
target = target[key]
|
||||
validate_contract(value, target, root=root_schema, path=path)
|
||||
return
|
||||
|
||||
expected_type = schema.get("type")
|
||||
if expected_type is not None:
|
||||
names = (
|
||||
list(expected_type)
|
||||
if isinstance(expected_type, list)
|
||||
else [expected_type]
|
||||
)
|
||||
if not any(_matches_type(value, str(name)) for name in names):
|
||||
raise ContractValidationError(
|
||||
f"{path}: expected {names}, got {type(value).__name__}"
|
||||
)
|
||||
if "const" in schema and value != schema["const"]:
|
||||
raise ContractValidationError(
|
||||
f"{path}: expected constant {schema['const']!r}"
|
||||
)
|
||||
if "enum" in schema and value not in schema["enum"]:
|
||||
raise ContractValidationError(f"{path}: value is outside enum")
|
||||
if (
|
||||
value is not None
|
||||
and "minLength" in schema
|
||||
and len(value) < int(schema["minLength"])
|
||||
):
|
||||
raise ContractValidationError(f"{path}: string is too short")
|
||||
if (
|
||||
value is not None
|
||||
and "pattern" in schema
|
||||
and re.search(str(schema["pattern"]), value) is None
|
||||
):
|
||||
raise ContractValidationError(f"{path}: string does not match pattern")
|
||||
if value is not None and "minimum" in schema and value < schema["minimum"]:
|
||||
raise ContractValidationError(f"{path}: value is below minimum")
|
||||
if value is not None and "maximum" in schema and value > schema["maximum"]:
|
||||
raise ContractValidationError(f"{path}: value is above maximum")
|
||||
if (
|
||||
value is not None
|
||||
and "exclusiveMinimum" in schema
|
||||
and value <= schema["exclusiveMinimum"]
|
||||
):
|
||||
raise ContractValidationError(
|
||||
f"{path}: value is not above exclusive minimum"
|
||||
)
|
||||
|
||||
value_format = schema.get("format")
|
||||
if value_format == "date":
|
||||
try:
|
||||
date.fromisoformat(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ContractValidationError(
|
||||
f"{path}: value is not an ISO date"
|
||||
) from exc
|
||||
elif value_format == "date-time" and value is not None:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except (AttributeError, ValueError) as exc:
|
||||
raise ContractValidationError(
|
||||
f"{path}: value is not an ISO date-time"
|
||||
) from exc
|
||||
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
||||
raise ContractValidationError(
|
||||
f"{path}: date-time has no timezone offset"
|
||||
)
|
||||
|
||||
if isinstance(value, dict):
|
||||
missing = [
|
||||
key for key in schema.get("required", []) if key not in value
|
||||
]
|
||||
if missing:
|
||||
raise ContractValidationError(
|
||||
f"{path}: missing required fields {missing}"
|
||||
)
|
||||
properties = schema.get("properties", {})
|
||||
if schema.get("additionalProperties") is False:
|
||||
unknown = sorted(set(value) - set(properties))
|
||||
if unknown:
|
||||
raise ContractValidationError(
|
||||
f"{path}: unsupported fields {unknown}"
|
||||
)
|
||||
for key, child_schema in properties.items():
|
||||
if key in value:
|
||||
validate_contract(
|
||||
value[key],
|
||||
child_schema,
|
||||
root=root_schema,
|
||||
path=f"{path}.{key}",
|
||||
)
|
||||
for trigger, dependencies in schema.get(
|
||||
"dependentRequired",
|
||||
{},
|
||||
).items():
|
||||
if trigger in value:
|
||||
absent = [key for key in dependencies if key not in value]
|
||||
if absent:
|
||||
raise ContractValidationError(
|
||||
f"{path}: {trigger} requires {absent}"
|
||||
)
|
||||
|
||||
if isinstance(value, list) and "items" in schema:
|
||||
for index, item in enumerate(value):
|
||||
validate_contract(
|
||||
item,
|
||||
schema["items"],
|
||||
root=root_schema,
|
||||
path=f"{path}[{index}]",
|
||||
)
|
||||
|
||||
if "oneOf" in schema:
|
||||
matches = 0
|
||||
failures = []
|
||||
for candidate in schema["oneOf"]:
|
||||
try:
|
||||
validate_contract(
|
||||
value,
|
||||
candidate,
|
||||
root=root_schema,
|
||||
path=path,
|
||||
)
|
||||
matches += 1
|
||||
except ContractValidationError as exc:
|
||||
failures.append(str(exc))
|
||||
if matches != 1:
|
||||
raise ContractValidationError(
|
||||
f"{path}: expected one schema branch, got {matches}: {failures}"
|
||||
)
|
||||
|
||||
|
||||
def schema_directory() -> Path:
|
||||
directory = Path(__file__).resolve().parents[2] / "schemas"
|
||||
if not directory.is_dir():
|
||||
raise FileNotFoundError(f"quant60 schema directory not found: {directory}")
|
||||
return directory
|
||||
|
||||
|
||||
def load_schema(filename: str) -> dict[str, Any]:
|
||||
name = Path(filename).name
|
||||
if name != filename or not name.endswith(".schema.json"):
|
||||
raise ValueError("schema filename must be a local *.schema.json name")
|
||||
raw = json.loads(
|
||||
(schema_directory() / name).read_text(encoding="utf-8"),
|
||||
parse_constant=lambda constant: (_ for _ in ()).throw(
|
||||
ValueError(f"non-finite JSON constant {constant}")
|
||||
),
|
||||
)
|
||||
if not isinstance(raw, dict):
|
||||
raise ContractValidationError("schema root must be an object")
|
||||
return raw
|
||||
|
||||
|
||||
def validate_records(
|
||||
records: Iterable[Mapping[str, Any]],
|
||||
schema_filename: str,
|
||||
) -> int:
|
||||
schema = load_schema(schema_filename)
|
||||
count = 0
|
||||
for count, record in enumerate(records, 1):
|
||||
validate_contract(record, schema, path=f"$[{count - 1}]")
|
||||
return count
|
||||
|
||||
|
||||
def validate_broker_snapshot(snapshot: Mapping[str, Any]) -> None:
|
||||
validate_contract(
|
||||
dict(snapshot),
|
||||
load_schema("broker_snapshot.schema.json"),
|
||||
)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Versioned A-share fee rules and transparent spread/impact forecasts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
|
||||
from .config import FeeConfig
|
||||
from .domain import Side
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DatedFeeRule:
|
||||
effective_from: date
|
||||
effective_to: date | None
|
||||
fees: FeeConfig
|
||||
version: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.effective_to is not None and self.effective_to < self.effective_from:
|
||||
raise ValueError("fee effective_to precedes effective_from")
|
||||
if not str(self.version).strip():
|
||||
raise ValueError("fee rule version is required")
|
||||
self.fees.validate()
|
||||
|
||||
def contains(self, trading_date: date) -> bool:
|
||||
return self.effective_from <= trading_date and (
|
||||
self.effective_to is None or trading_date <= self.effective_to
|
||||
)
|
||||
|
||||
|
||||
class VersionedFeeSchedule:
|
||||
def __init__(self, rules: list[DatedFeeRule]) -> None:
|
||||
if not rules:
|
||||
raise ValueError("at least one fee rule is required")
|
||||
ordered = sorted(rules, key=lambda rule: rule.effective_from)
|
||||
for previous, current in zip(ordered, ordered[1:]):
|
||||
if previous.effective_to is None:
|
||||
raise ValueError("open-ended fee rule must be last")
|
||||
if current.effective_from <= previous.effective_to:
|
||||
raise ValueError("fee rule effective intervals overlap")
|
||||
self.rules = tuple(ordered)
|
||||
|
||||
def at(self, trading_date: date) -> DatedFeeRule:
|
||||
matches = [rule for rule in self.rules if rule.contains(trading_date)]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(
|
||||
f"expected one fee rule for {trading_date.isoformat()}, "
|
||||
f"found {len(matches)}"
|
||||
)
|
||||
return matches[0]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CostModelParameters:
|
||||
half_spread_bps: float = 2.0
|
||||
impact_coefficient: float = 0.10
|
||||
max_participation: float = 0.10
|
||||
|
||||
def validate(self) -> None:
|
||||
for name in ("half_spread_bps", "impact_coefficient"):
|
||||
value = float(getattr(self, name))
|
||||
if not math.isfinite(value) or value < 0:
|
||||
raise ValueError(f"{name} must be finite and non-negative")
|
||||
participation = float(self.max_participation)
|
||||
if not math.isfinite(participation) or not (0 < participation <= 1):
|
||||
raise ValueError("max_participation must lie in (0, 1]")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CostForecast:
|
||||
symbol: str
|
||||
side: Side
|
||||
quantity: int
|
||||
notional: float
|
||||
explicit_fees: float
|
||||
spread_cost: float
|
||||
impact_cost: float
|
||||
low: float
|
||||
base: float
|
||||
high: float
|
||||
base_bps: float
|
||||
participation: float
|
||||
capacity_breached: bool
|
||||
fee_version: str
|
||||
|
||||
|
||||
def forecast_trade_cost(
|
||||
*,
|
||||
symbol: str,
|
||||
side: Side,
|
||||
quantity: int,
|
||||
price: float,
|
||||
adv_quantity: float,
|
||||
daily_volatility: float,
|
||||
fee_rule: DatedFeeRule,
|
||||
parameters: CostModelParameters = CostModelParameters(),
|
||||
) -> CostForecast:
|
||||
parameters.validate()
|
||||
if not isinstance(side, Side):
|
||||
raise ValueError("side must be a Side")
|
||||
if type(quantity) is not int or quantity <= 0:
|
||||
raise ValueError("quantity must be a positive integer")
|
||||
numeric = {
|
||||
"price": float(price),
|
||||
"adv_quantity": float(adv_quantity),
|
||||
"daily_volatility": float(daily_volatility),
|
||||
}
|
||||
for name, value in numeric.items():
|
||||
if not math.isfinite(value) or value <= 0:
|
||||
raise ValueError(f"{name} must be finite and positive")
|
||||
notional = numeric["price"] * quantity
|
||||
fees = fee_rule.fees
|
||||
commission = max(
|
||||
float(fees.minimum_commission),
|
||||
notional * float(fees.commission_rate),
|
||||
)
|
||||
stamp = (
|
||||
notional * float(fees.stamp_duty_rate)
|
||||
if side == Side.SELL
|
||||
else 0.0
|
||||
)
|
||||
transfer = notional * float(fees.transfer_fee_rate)
|
||||
explicit = commission + stamp + transfer
|
||||
spread = notional * float(parameters.half_spread_bps) / 10_000.0
|
||||
participation = quantity / numeric["adv_quantity"]
|
||||
impact_rate = (
|
||||
float(parameters.impact_coefficient)
|
||||
* numeric["daily_volatility"]
|
||||
* math.sqrt(participation)
|
||||
)
|
||||
impact = notional * impact_rate
|
||||
low = explicit + spread * 0.5 + impact * 0.5
|
||||
base = explicit + spread + impact
|
||||
high = explicit + spread * 1.5 + impact * 2.0
|
||||
return CostForecast(
|
||||
symbol=normalize_symbol(symbol),
|
||||
side=side,
|
||||
quantity=quantity,
|
||||
notional=notional,
|
||||
explicit_fees=explicit,
|
||||
spread_cost=spread,
|
||||
impact_cost=impact,
|
||||
low=low,
|
||||
base=base,
|
||||
high=high,
|
||||
base_bps=base / notional * 10_000.0,
|
||||
participation=participation,
|
||||
capacity_breached=participation > parameters.max_participation,
|
||||
fee_version=fee_rule.version,
|
||||
)
|
||||
@@ -0,0 +1,467 @@
|
||||
"""Canonical local market-data snapshots with quality and hash manifests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .ledger import canonical_json, jsonable
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
class DataQualityError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CanonicalBarRecord:
|
||||
trading_date: date
|
||||
symbol: str
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: int
|
||||
money: float
|
||||
paused: bool
|
||||
source: str
|
||||
is_st: bool = False
|
||||
adjustment: str = "none"
|
||||
adjustment_factor: float | None = None
|
||||
previous_close: float | None = None
|
||||
limit_up: float | None = None
|
||||
limit_down: float | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "symbol", normalize_symbol(self.symbol))
|
||||
if not isinstance(self.trading_date, date):
|
||||
raise DataQualityError("trading_date must be a date")
|
||||
for name in ("open", "high", "low", "close"):
|
||||
value = float(getattr(self, name))
|
||||
if not math.isfinite(value) or value <= 0:
|
||||
raise DataQualityError(f"{name} must be finite and positive")
|
||||
if not (self.low <= self.open <= self.high):
|
||||
raise DataQualityError("open must lie within [low, high]")
|
||||
if not (self.low <= self.close <= self.high):
|
||||
raise DataQualityError("close must lie within [low, high]")
|
||||
if type(self.volume) is not int or self.volume < 0:
|
||||
raise DataQualityError("volume must be a non-negative integer")
|
||||
if not math.isfinite(float(self.money)) or self.money < 0:
|
||||
raise DataQualityError("money must be finite and non-negative")
|
||||
if type(self.paused) is not bool:
|
||||
raise DataQualityError("paused must be a bool")
|
||||
if type(self.is_st) is not bool:
|
||||
raise DataQualityError("is_st must be a bool")
|
||||
if not str(self.source).strip():
|
||||
raise DataQualityError("bar source is required")
|
||||
if self.adjustment not in {"none", "front", "back"}:
|
||||
raise DataQualityError("unsupported adjustment")
|
||||
if self.paused and self.volume != 0:
|
||||
raise DataQualityError("paused bar must have zero volume")
|
||||
for name in (
|
||||
"adjustment_factor",
|
||||
"previous_close",
|
||||
"limit_up",
|
||||
"limit_down",
|
||||
):
|
||||
value = getattr(self, name)
|
||||
if value is not None and (
|
||||
not math.isfinite(float(value)) or float(value) <= 0
|
||||
):
|
||||
raise DataQualityError(
|
||||
f"{name} must be finite and positive when present"
|
||||
)
|
||||
if (
|
||||
self.limit_up is not None
|
||||
and self.limit_down is not None
|
||||
and float(self.limit_down) > float(self.limit_up)
|
||||
):
|
||||
raise DataQualityError("limit_down must not exceed limit_up")
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return jsonable(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class IndexMembershipRecord:
|
||||
effective_date: date
|
||||
index_symbol: str
|
||||
member_symbol: str
|
||||
source: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.effective_date, date):
|
||||
raise DataQualityError("effective_date must be a date")
|
||||
object.__setattr__(
|
||||
self,
|
||||
"index_symbol",
|
||||
normalize_symbol(self.index_symbol),
|
||||
)
|
||||
object.__setattr__(
|
||||
self,
|
||||
"member_symbol",
|
||||
normalize_symbol(self.member_symbol),
|
||||
)
|
||||
if not str(self.source).strip():
|
||||
raise DataQualityError("membership source is required")
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return jsonable(self)
|
||||
|
||||
|
||||
def _hash_rows(rows: Iterable[Any]) -> str:
|
||||
return hashlib.sha256(canonical_json(list(rows)).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_snapshot_payload(
|
||||
*,
|
||||
bars: Iterable[CanonicalBarRecord],
|
||||
memberships: Iterable[IndexMembershipRecord],
|
||||
provider: str,
|
||||
provider_version: str,
|
||||
retrieved_at: datetime,
|
||||
query: dict[str, Any],
|
||||
next_trading_session: date,
|
||||
) -> dict[str, Any]:
|
||||
if retrieved_at.tzinfo is None or retrieved_at.utcoffset() is None:
|
||||
raise DataQualityError("retrieved_at must include a timezone")
|
||||
provider_name = str(provider).strip()
|
||||
version = str(provider_version).strip()
|
||||
if not provider_name or not version:
|
||||
raise DataQualityError("provider and provider_version are required")
|
||||
if not isinstance(next_trading_session, date):
|
||||
raise DataQualityError("next_trading_session must be a date")
|
||||
ordered_bars = sorted(
|
||||
tuple(bars),
|
||||
key=lambda item: (item.trading_date, item.symbol),
|
||||
)
|
||||
ordered_memberships = sorted(
|
||||
tuple(memberships),
|
||||
key=lambda item: (
|
||||
item.effective_date,
|
||||
item.index_symbol,
|
||||
item.member_symbol,
|
||||
),
|
||||
)
|
||||
if not ordered_bars or not ordered_memberships:
|
||||
raise DataQualityError("snapshot bars and memberships must be non-empty")
|
||||
bar_keys: set[tuple[date, str]] = set()
|
||||
for bar in ordered_bars:
|
||||
key = (bar.trading_date, bar.symbol)
|
||||
if key in bar_keys:
|
||||
raise DataQualityError(
|
||||
f"duplicate bar {bar.trading_date} {bar.symbol}"
|
||||
)
|
||||
bar_keys.add(key)
|
||||
membership_keys: set[tuple[date, str, str]] = set()
|
||||
for membership in ordered_memberships:
|
||||
key = (
|
||||
membership.effective_date,
|
||||
membership.index_symbol,
|
||||
membership.member_symbol,
|
||||
)
|
||||
if key in membership_keys:
|
||||
raise DataQualityError(
|
||||
"duplicate membership "
|
||||
f"{membership.effective_date} {membership.member_symbol}"
|
||||
)
|
||||
membership_keys.add(key)
|
||||
if (membership.effective_date, membership.member_symbol) not in bar_keys:
|
||||
raise DataQualityError(
|
||||
"index membership has no same-session canonical bar: "
|
||||
f"{membership.effective_date} {membership.member_symbol}"
|
||||
)
|
||||
query_start = query.get("start_date")
|
||||
query_end = query.get("end_date")
|
||||
if query_start is not None and query_end is not None:
|
||||
start_date = date.fromisoformat(str(query_start))
|
||||
end_date = date.fromisoformat(str(query_end))
|
||||
if start_date > end_date:
|
||||
raise DataQualityError("query start_date must not follow end_date")
|
||||
for bar in ordered_bars:
|
||||
if not start_date <= bar.trading_date <= end_date:
|
||||
raise DataQualityError(
|
||||
"canonical bar falls outside the declared query range"
|
||||
)
|
||||
for membership in ordered_memberships:
|
||||
if not start_date <= membership.effective_date <= end_date:
|
||||
raise DataQualityError(
|
||||
"index membership falls outside the declared query range"
|
||||
)
|
||||
if next_trading_session <= end_date:
|
||||
raise DataQualityError(
|
||||
"next_trading_session must follow query end_date"
|
||||
)
|
||||
trading_sessions_raw = query.get("trading_sessions")
|
||||
if not isinstance(trading_sessions_raw, list):
|
||||
raise DataQualityError(
|
||||
"query.trading_sessions must freeze the provider calendar"
|
||||
)
|
||||
try:
|
||||
trading_sessions = [
|
||||
date.fromisoformat(str(item)) for item in trading_sessions_raw
|
||||
]
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise DataQualityError(
|
||||
"query.trading_sessions contains an invalid date"
|
||||
) from exc
|
||||
if (
|
||||
len(trading_sessions) < 2
|
||||
or trading_sessions != sorted(set(trading_sessions))
|
||||
):
|
||||
raise DataQualityError(
|
||||
"query.trading_sessions must be unique and strictly increasing"
|
||||
)
|
||||
if trading_sessions[-1] != next_trading_session:
|
||||
raise DataQualityError(
|
||||
"query.trading_sessions must end at next_trading_session"
|
||||
)
|
||||
observed_sessions = {
|
||||
item.trading_date for item in ordered_bars
|
||||
} | {
|
||||
item.effective_date for item in ordered_memberships
|
||||
}
|
||||
if set(trading_sessions[:-1]) != observed_sessions:
|
||||
raise DataQualityError(
|
||||
"query.trading_sessions does not exactly match snapshot sessions"
|
||||
)
|
||||
bar_rows = [bar.as_dict() for bar in ordered_bars]
|
||||
membership_rows = [item.as_dict() for item in ordered_memberships]
|
||||
quality = {
|
||||
"ok": True,
|
||||
"bar_count": len(bar_rows),
|
||||
"membership_count": len(membership_rows),
|
||||
"symbol_count": len({bar.symbol for bar in ordered_bars}),
|
||||
"session_count": len({bar.trading_date for bar in ordered_bars}),
|
||||
"paused_bar_count": sum(1 for bar in ordered_bars if bar.paused),
|
||||
"membership_bar_coverage_ok": True,
|
||||
"start_date": ordered_bars[0].trading_date.isoformat(),
|
||||
"end_date": ordered_bars[-1].trading_date.isoformat(),
|
||||
}
|
||||
manifest_body = {
|
||||
"schema_version": "1.0",
|
||||
"provider": provider_name,
|
||||
"provider_version": version,
|
||||
"retrieved_at": retrieved_at.isoformat(),
|
||||
"next_trading_session": next_trading_session.isoformat(),
|
||||
"query": jsonable(query),
|
||||
"bars_sha256": _hash_rows(bar_rows),
|
||||
"memberships_sha256": _hash_rows(membership_rows),
|
||||
"quality": quality,
|
||||
}
|
||||
manifest_body["data_version"] = _hash_rows([manifest_body])
|
||||
return {
|
||||
"bars": bar_rows,
|
||||
"memberships": membership_rows,
|
||||
"quality": quality,
|
||||
"manifest": manifest_body,
|
||||
}
|
||||
|
||||
|
||||
def _atomic_write(path: Path, payload: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temporary = tempfile.mkstemp(
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
dir=str(path.parent),
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _json_bytes(payload: Any) -> bytes:
|
||||
return (
|
||||
json.dumps(
|
||||
jsonable(payload),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _jsonl_bytes(rows: list[dict[str, Any]]) -> bytes:
|
||||
return (
|
||||
"".join(canonical_json(row) + "\n" for row in rows)
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def write_data_snapshot(
|
||||
payload: dict[str, Any],
|
||||
output_dir: str | Path,
|
||||
) -> dict[str, str]:
|
||||
root = Path(output_dir)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
marker = root / ".quant60-incomplete"
|
||||
_atomic_write(
|
||||
marker,
|
||||
(payload["manifest"]["data_version"] + "\n").encode("ascii"),
|
||||
)
|
||||
bars_path = root / "bars.jsonl"
|
||||
memberships_path = root / "memberships.jsonl"
|
||||
quality_path = root / "quality.json"
|
||||
_atomic_write(bars_path, _jsonl_bytes(payload["bars"]))
|
||||
_atomic_write(memberships_path, _jsonl_bytes(payload["memberships"]))
|
||||
_atomic_write(quality_path, _json_bytes(payload["quality"]))
|
||||
manifest = dict(payload["manifest"])
|
||||
manifest["artifacts"] = {
|
||||
path.name: hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in (bars_path, memberships_path, quality_path)
|
||||
}
|
||||
manifest_path = root / "manifest.json"
|
||||
_atomic_write(manifest_path, _json_bytes(manifest))
|
||||
marker.unlink()
|
||||
return {
|
||||
"bars": str(bars_path.resolve()),
|
||||
"memberships": str(memberships_path.resolve()),
|
||||
"quality": str(quality_path.resolve()),
|
||||
"manifest": str(manifest_path.resolve()),
|
||||
}
|
||||
|
||||
|
||||
def verify_data_snapshot(manifest_file: str | Path) -> dict[str, Any]:
|
||||
path = Path(manifest_file)
|
||||
root = path.parent
|
||||
if (root / ".quant60-incomplete").exists():
|
||||
raise DataQualityError("data snapshot publication is incomplete")
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
expected = set(manifest.get("artifacts", {})) | {path.name}
|
||||
actual = {entry.name for entry in root.iterdir()}
|
||||
if actual != expected:
|
||||
raise DataQualityError("data snapshot directory is not an exact set")
|
||||
for name, expected_hash in manifest["artifacts"].items():
|
||||
digest = hashlib.sha256((root / name).read_bytes()).hexdigest()
|
||||
if digest != expected_hash:
|
||||
raise DataQualityError(f"data snapshot artifact hash mismatch: {name}")
|
||||
bars = [
|
||||
json.loads(line)
|
||||
for line in (root / "bars.jsonl").read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
memberships = [
|
||||
json.loads(line)
|
||||
for line in (root / "memberships.jsonl")
|
||||
.read_text(encoding="utf-8")
|
||||
.splitlines()
|
||||
if line
|
||||
]
|
||||
if _hash_rows(bars) != manifest["bars_sha256"]:
|
||||
raise DataQualityError("canonical bars content hash mismatch")
|
||||
if _hash_rows(memberships) != manifest["memberships_sha256"]:
|
||||
raise DataQualityError("membership content hash mismatch")
|
||||
try:
|
||||
bar_records = [
|
||||
CanonicalBarRecord(
|
||||
trading_date=date.fromisoformat(row["trading_date"]),
|
||||
symbol=row["symbol"],
|
||||
open=row["open"],
|
||||
high=row["high"],
|
||||
low=row["low"],
|
||||
close=row["close"],
|
||||
volume=row["volume"],
|
||||
money=row["money"],
|
||||
paused=row["paused"],
|
||||
source=row["source"],
|
||||
is_st=row["is_st"],
|
||||
adjustment=row.get("adjustment", "none"),
|
||||
adjustment_factor=row.get("adjustment_factor"),
|
||||
previous_close=row.get("previous_close"),
|
||||
limit_up=row.get("limit_up"),
|
||||
limit_down=row.get("limit_down"),
|
||||
)
|
||||
for row in bars
|
||||
]
|
||||
membership_records = [
|
||||
IndexMembershipRecord(
|
||||
effective_date=date.fromisoformat(row["effective_date"]),
|
||||
index_symbol=row["index_symbol"],
|
||||
member_symbol=row["member_symbol"],
|
||||
source=row["source"],
|
||||
)
|
||||
for row in memberships
|
||||
]
|
||||
rebuilt = build_snapshot_payload(
|
||||
bars=bar_records,
|
||||
memberships=membership_records,
|
||||
provider=manifest["provider"],
|
||||
provider_version=manifest["provider_version"],
|
||||
retrieved_at=datetime.fromisoformat(manifest["retrieved_at"]),
|
||||
query=manifest["query"],
|
||||
next_trading_session=date.fromisoformat(
|
||||
manifest["next_trading_session"]
|
||||
),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise DataQualityError(
|
||||
f"data snapshot semantic validation failed: {exc}"
|
||||
) from exc
|
||||
for field in (
|
||||
"schema_version",
|
||||
"provider",
|
||||
"provider_version",
|
||||
"retrieved_at",
|
||||
"next_trading_session",
|
||||
"query",
|
||||
"bars_sha256",
|
||||
"memberships_sha256",
|
||||
"quality",
|
||||
"data_version",
|
||||
):
|
||||
if rebuilt["manifest"][field] != manifest.get(field):
|
||||
raise DataQualityError(
|
||||
f"data snapshot manifest semantic mismatch: {field}"
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"data_version": manifest["data_version"],
|
||||
"bar_count": len(bars),
|
||||
"membership_count": len(memberships),
|
||||
}
|
||||
|
||||
|
||||
def load_verified_data_snapshot(
|
||||
manifest_file: str | Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Return a verified snapshot; no caller can bypass its quality gate."""
|
||||
|
||||
path = Path(manifest_file).expanduser().resolve()
|
||||
verification = verify_data_snapshot(path)
|
||||
root = path.parent
|
||||
return {
|
||||
"verification": verification,
|
||||
"manifest": json.loads(path.read_text(encoding="utf-8")),
|
||||
"bars": [
|
||||
json.loads(line)
|
||||
for line in (root / "bars.jsonl")
|
||||
.read_text(encoding="utf-8")
|
||||
.splitlines()
|
||||
if line
|
||||
],
|
||||
"memberships": [
|
||||
json.loads(line)
|
||||
for line in (root / "memberships.jsonl")
|
||||
.read_text(encoding="utf-8")
|
||||
.splitlines()
|
||||
if line
|
||||
],
|
||||
"manifest_file": str(path),
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
"""Modern domain objects and invariants for the local Python >=3.10 runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
from enum import Enum
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
MONEY_QUANT = Decimal("0.000001")
|
||||
PRICE_QUANT = Decimal("0.0001")
|
||||
|
||||
|
||||
def decimal(value: Decimal | str | int | float) -> Decimal:
|
||||
if isinstance(value, Decimal):
|
||||
return value
|
||||
return Decimal(str(value))
|
||||
|
||||
|
||||
def money(value: Decimal | str | int | float) -> Decimal:
|
||||
return decimal(value).quantize(MONEY_QUANT, rounding=ROUND_HALF_UP)
|
||||
|
||||
|
||||
class Side(str, Enum):
|
||||
BUY = "BUY"
|
||||
SELL = "SELL"
|
||||
|
||||
|
||||
class OrderStatus(str, Enum):
|
||||
NEW = "NEW"
|
||||
ACCEPTED = "ACCEPTED"
|
||||
PARTIALLY_FILLED = "PARTIALLY_FILLED"
|
||||
FILLED = "FILLED"
|
||||
CANCEL_PENDING = "CANCEL_PENDING"
|
||||
REJECTED = "REJECTED"
|
||||
CANCELLED = "CANCELLED"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
TERMINAL_ORDER_STATUSES = {
|
||||
OrderStatus.FILLED,
|
||||
OrderStatus.REJECTED,
|
||||
OrderStatus.CANCELLED,
|
||||
}
|
||||
|
||||
ORDER_TRANSITIONS: dict[OrderStatus, set[OrderStatus]] = {
|
||||
OrderStatus.NEW: {
|
||||
OrderStatus.ACCEPTED,
|
||||
OrderStatus.REJECTED,
|
||||
OrderStatus.CANCELLED,
|
||||
OrderStatus.UNKNOWN,
|
||||
},
|
||||
OrderStatus.ACCEPTED: {
|
||||
OrderStatus.PARTIALLY_FILLED,
|
||||
OrderStatus.FILLED,
|
||||
OrderStatus.CANCEL_PENDING,
|
||||
OrderStatus.CANCELLED,
|
||||
OrderStatus.REJECTED,
|
||||
OrderStatus.UNKNOWN,
|
||||
},
|
||||
OrderStatus.PARTIALLY_FILLED: {
|
||||
OrderStatus.PARTIALLY_FILLED,
|
||||
OrderStatus.FILLED,
|
||||
OrderStatus.CANCEL_PENDING,
|
||||
OrderStatus.CANCELLED,
|
||||
OrderStatus.UNKNOWN,
|
||||
},
|
||||
OrderStatus.CANCEL_PENDING: {
|
||||
OrderStatus.PARTIALLY_FILLED,
|
||||
OrderStatus.FILLED,
|
||||
OrderStatus.CANCELLED,
|
||||
OrderStatus.UNKNOWN,
|
||||
},
|
||||
OrderStatus.FILLED: set(),
|
||||
OrderStatus.REJECTED: set(),
|
||||
OrderStatus.CANCELLED: set(),
|
||||
OrderStatus.UNKNOWN: {
|
||||
OrderStatus.ACCEPTED,
|
||||
OrderStatus.PARTIALLY_FILLED,
|
||||
OrderStatus.FILLED,
|
||||
OrderStatus.CANCEL_PENDING,
|
||||
OrderStatus.REJECTED,
|
||||
OrderStatus.CANCELLED,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Bar:
|
||||
trading_date: date
|
||||
symbol: str
|
||||
open: Decimal
|
||||
high: Decimal
|
||||
low: Decimal
|
||||
close: Decimal
|
||||
volume: int
|
||||
previous_close: Decimal | None = None
|
||||
limit_up: Decimal | None = None
|
||||
limit_down: Decimal | None = None
|
||||
suspended: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "symbol", normalize_symbol(self.symbol))
|
||||
for name in ("open", "high", "low", "close"):
|
||||
value = decimal(getattr(self, name)).quantize(PRICE_QUANT)
|
||||
if value <= 0:
|
||||
raise ValueError(f"{name} must be positive")
|
||||
object.__setattr__(self, name, value)
|
||||
for name in ("previous_close", "limit_up", "limit_down"):
|
||||
value = getattr(self, name)
|
||||
if value is not None:
|
||||
object.__setattr__(
|
||||
self, name, decimal(value).quantize(PRICE_QUANT)
|
||||
)
|
||||
if self.volume < 0:
|
||||
raise ValueError("volume must be non-negative")
|
||||
if not (self.low <= self.open <= self.high):
|
||||
raise ValueError("open must lie within [low, high]")
|
||||
if not (self.low <= self.close <= self.high):
|
||||
raise ValueError("close must lie within [low, high]")
|
||||
|
||||
@property
|
||||
def locked_limit_up(self) -> bool:
|
||||
return (
|
||||
self.limit_up is not None
|
||||
and self.low >= self.limit_up
|
||||
and self.high >= self.limit_up
|
||||
)
|
||||
|
||||
@property
|
||||
def locked_limit_down(self) -> bool:
|
||||
return (
|
||||
self.limit_down is not None
|
||||
and self.high <= self.limit_down
|
||||
and self.low <= self.limit_down
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Order:
|
||||
order_id: str
|
||||
symbol: str
|
||||
side: Side
|
||||
quantity: int
|
||||
submitted_at: date
|
||||
status: OrderStatus = OrderStatus.NEW
|
||||
filled_quantity: int = 0
|
||||
average_fill_price: Decimal = Decimal("0")
|
||||
status_reason: str | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.symbol = normalize_symbol(self.symbol)
|
||||
if self.quantity <= 0:
|
||||
raise ValueError("order quantity must be positive")
|
||||
if self.filled_quantity < 0 or self.filled_quantity > self.quantity:
|
||||
raise ValueError("invalid filled quantity")
|
||||
|
||||
@property
|
||||
def remaining_quantity(self) -> int:
|
||||
return self.quantity - self.filled_quantity
|
||||
|
||||
@property
|
||||
def is_terminal(self) -> bool:
|
||||
return self.status in TERMINAL_ORDER_STATUSES
|
||||
|
||||
def transition(
|
||||
self, next_status: OrderStatus, reason: str | None = None
|
||||
) -> None:
|
||||
# Broker callbacks are at-least-once. Repeating the same observed
|
||||
# state must not make a replay fail or create a second accounting
|
||||
# effect.
|
||||
if next_status == self.status:
|
||||
if reason is not None:
|
||||
self.status_reason = reason
|
||||
return
|
||||
if next_status not in ORDER_TRANSITIONS[self.status]:
|
||||
raise ValueError(
|
||||
f"illegal order transition {self.status.value}"
|
||||
f" -> {next_status.value}"
|
||||
)
|
||||
self.status = next_status
|
||||
self.status_reason = reason
|
||||
|
||||
def apply_fill(self, quantity: int, price: Decimal) -> None:
|
||||
if self.status not in {
|
||||
OrderStatus.ACCEPTED,
|
||||
OrderStatus.PARTIALLY_FILLED,
|
||||
OrderStatus.CANCEL_PENDING,
|
||||
}:
|
||||
raise ValueError(f"cannot fill an order in {self.status.value}")
|
||||
if quantity <= 0 or quantity > self.remaining_quantity:
|
||||
raise ValueError("invalid fill quantity")
|
||||
price = decimal(price)
|
||||
total_notional = (
|
||||
self.average_fill_price * self.filled_quantity + price * quantity
|
||||
)
|
||||
self.filled_quantity += quantity
|
||||
self.average_fill_price = (
|
||||
total_notional / self.filled_quantity
|
||||
).quantize(PRICE_QUANT, rounding=ROUND_HALF_UP)
|
||||
next_status = (
|
||||
OrderStatus.FILLED
|
||||
if self.filled_quantity == self.quantity
|
||||
else OrderStatus.PARTIALLY_FILLED
|
||||
)
|
||||
self.transition(next_status)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Fill:
|
||||
fill_id: str
|
||||
order_id: str
|
||||
trading_date: date
|
||||
symbol: str
|
||||
side: Side
|
||||
quantity: int
|
||||
price: Decimal
|
||||
commission: Decimal
|
||||
stamp_duty: Decimal
|
||||
transfer_fee: Decimal
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "symbol", normalize_symbol(self.symbol))
|
||||
object.__setattr__(self, "price", decimal(self.price))
|
||||
object.__setattr__(self, "commission", money(self.commission))
|
||||
object.__setattr__(self, "stamp_duty", money(self.stamp_duty))
|
||||
object.__setattr__(self, "transfer_fee", money(self.transfer_fee))
|
||||
if self.quantity <= 0:
|
||||
raise ValueError("fill quantity must be positive")
|
||||
|
||||
@property
|
||||
def notional(self) -> Decimal:
|
||||
return money(self.price * self.quantity)
|
||||
|
||||
@property
|
||||
def total_fees(self) -> Decimal:
|
||||
return money(self.commission + self.stamp_duty + self.transfer_fee)
|
||||
|
||||
@property
|
||||
def cash_delta(self) -> Decimal:
|
||||
if self.side == Side.BUY:
|
||||
return money(-self.notional - self.total_fees)
|
||||
return money(self.notional - self.total_fees)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Position:
|
||||
symbol: str
|
||||
quantity: int = 0
|
||||
sellable_quantity: int = 0
|
||||
today_bought: int = 0
|
||||
average_cost: Decimal = Decimal("0")
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.symbol = normalize_symbol(self.symbol)
|
||||
self.average_cost = decimal(self.average_cost)
|
||||
self._validate()
|
||||
|
||||
def _validate(self) -> None:
|
||||
if min(self.quantity, self.sellable_quantity, self.today_bought) < 0:
|
||||
raise ValueError("position quantities must be non-negative")
|
||||
if self.sellable_quantity + self.today_bought != self.quantity:
|
||||
raise ValueError("sellable + today_bought must equal total quantity")
|
||||
|
||||
def release_t_plus_one(self) -> None:
|
||||
self.sellable_quantity += self.today_bought
|
||||
self.today_bought = 0
|
||||
self._validate()
|
||||
|
||||
def buy(self, quantity: int, price: Decimal, fees: Decimal) -> None:
|
||||
if quantity <= 0:
|
||||
raise ValueError("buy quantity must be positive")
|
||||
prior_cost = self.average_cost * self.quantity
|
||||
total_cost = prior_cost + decimal(price) * quantity + decimal(fees)
|
||||
self.quantity += quantity
|
||||
self.today_bought += quantity
|
||||
self.average_cost = (
|
||||
total_cost / self.quantity
|
||||
).quantize(PRICE_QUANT, rounding=ROUND_HALF_UP)
|
||||
self._validate()
|
||||
|
||||
def sell(self, quantity: int) -> None:
|
||||
if quantity <= 0 or quantity > self.sellable_quantity:
|
||||
raise ValueError("sell exceeds T+1 sellable quantity")
|
||||
self.quantity -= quantity
|
||||
self.sellable_quantity -= quantity
|
||||
if self.quantity == 0:
|
||||
self.average_cost = Decimal("0")
|
||||
self._validate()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Portfolio:
|
||||
cash: Decimal
|
||||
positions: dict[str, Position] = field(default_factory=dict)
|
||||
last_prices: dict[str, Decimal] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.cash = money(self.cash)
|
||||
|
||||
def position(self, symbol: str) -> Position:
|
||||
canonical = normalize_symbol(symbol)
|
||||
if canonical not in self.positions:
|
||||
self.positions[canonical] = Position(canonical)
|
||||
return self.positions[canonical]
|
||||
|
||||
def start_session(self) -> None:
|
||||
for position in self.positions.values():
|
||||
position.release_t_plus_one()
|
||||
|
||||
def mark(self, bars: Iterable[Bar]) -> None:
|
||||
for bar in bars:
|
||||
self.last_prices[bar.symbol] = bar.close
|
||||
|
||||
@property
|
||||
def market_value(self) -> Decimal:
|
||||
total = Decimal("0")
|
||||
for symbol, position in self.positions.items():
|
||||
price = self.last_prices.get(symbol, position.average_cost)
|
||||
total += price * position.quantity
|
||||
return money(total)
|
||||
|
||||
@property
|
||||
def equity(self) -> Decimal:
|
||||
return money(self.cash + self.market_value)
|
||||
|
||||
def apply_fill(self, fill: Fill) -> None:
|
||||
position = self.position(fill.symbol)
|
||||
if fill.side == Side.BUY:
|
||||
required = -fill.cash_delta
|
||||
if required > self.cash:
|
||||
raise ValueError("insufficient cash")
|
||||
self.cash = money(self.cash + fill.cash_delta)
|
||||
position.buy(fill.quantity, fill.price, fill.total_fees)
|
||||
else:
|
||||
position.sell(fill.quantity)
|
||||
self.cash = money(self.cash + fill.cash_delta)
|
||||
self.last_prices[fill.symbol] = fill.price
|
||||
|
||||
def quantities(self) -> dict[str, int]:
|
||||
return {
|
||||
symbol: position.quantity
|
||||
for symbol, position in self.positions.items()
|
||||
if position.quantity
|
||||
}
|
||||
|
||||
def sellable_quantities(self) -> dict[str, int]:
|
||||
return {
|
||||
symbol: position.sellable_quantity
|
||||
for symbol, position in self.positions.items()
|
||||
if position.sellable_quantity
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Deterministic parent/child planning for a guarded TWAP/POV executor."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
|
||||
from .domain import Side
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
class ResidualPolicy(str, Enum):
|
||||
DEFER = "DEFER"
|
||||
CANCEL = "CANCEL"
|
||||
|
||||
|
||||
def _aware(value: datetime, name: str) -> datetime:
|
||||
if not isinstance(value, datetime):
|
||||
raise ValueError(f"{name} must be a datetime")
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError(f"{name} must include a timezone")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExecutionBucket:
|
||||
scheduled_at: datetime
|
||||
expected_market_volume: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_aware(self.scheduled_at, "scheduled_at")
|
||||
if (
|
||||
type(self.expected_market_volume) is not int
|
||||
or self.expected_market_volume < 0
|
||||
):
|
||||
raise ValueError(
|
||||
"expected_market_volume must be a non-negative integer"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParentOrderIntent:
|
||||
decision_id: str
|
||||
symbol: str
|
||||
side: Side
|
||||
quantity: int
|
||||
limit_price: float
|
||||
window_start: datetime
|
||||
window_end: datetime
|
||||
market_data_as_of: datetime
|
||||
lot_size: int = 100
|
||||
max_participation: float = 0.10
|
||||
max_child_quantity: int | None = None
|
||||
max_child_notional: float | None = None
|
||||
residual_policy: ResidualPolicy = ResidualPolicy.DEFER
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
decision = str(self.decision_id).strip()
|
||||
if not decision:
|
||||
raise ValueError("decision_id is required")
|
||||
object.__setattr__(self, "decision_id", decision)
|
||||
object.__setattr__(self, "symbol", normalize_symbol(self.symbol))
|
||||
if not isinstance(self.side, Side):
|
||||
raise ValueError("side must be a Side")
|
||||
if type(self.quantity) is not int or self.quantity <= 0:
|
||||
raise ValueError("quantity must be a positive integer")
|
||||
if type(self.lot_size) is not int or self.lot_size <= 0:
|
||||
raise ValueError("lot_size must be positive")
|
||||
if self.quantity % self.lot_size:
|
||||
raise ValueError("parent quantity must be a complete board lot")
|
||||
if (
|
||||
not math.isfinite(float(self.limit_price))
|
||||
or self.limit_price <= 0
|
||||
):
|
||||
raise ValueError("limit_price must be finite and positive")
|
||||
_aware(self.window_start, "window_start")
|
||||
_aware(self.window_end, "window_end")
|
||||
_aware(self.market_data_as_of, "market_data_as_of")
|
||||
if self.window_end <= self.window_start:
|
||||
raise ValueError("execution window must have positive duration")
|
||||
if not math.isfinite(float(self.max_participation)) or not (
|
||||
0 < self.max_participation <= 1
|
||||
):
|
||||
raise ValueError("max_participation must lie in (0, 1]")
|
||||
if self.max_child_quantity is not None and (
|
||||
type(self.max_child_quantity) is not int
|
||||
or self.max_child_quantity < self.lot_size
|
||||
):
|
||||
raise ValueError("max_child_quantity is invalid")
|
||||
if self.max_child_notional is not None and (
|
||||
not math.isfinite(float(self.max_child_notional))
|
||||
or self.max_child_notional <= 0
|
||||
):
|
||||
raise ValueError("max_child_notional must be finite and positive")
|
||||
if not isinstance(self.residual_policy, ResidualPolicy):
|
||||
raise ValueError("residual_policy must be ResidualPolicy")
|
||||
|
||||
@property
|
||||
def idempotency_key(self) -> str:
|
||||
return (
|
||||
f"{self.decision_id}:{self.symbol}:{self.side.value}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChildOrderIntent:
|
||||
child_id: str
|
||||
parent_key: str
|
||||
sequence: int
|
||||
symbol: str
|
||||
side: Side
|
||||
quantity: int
|
||||
limit_price: float
|
||||
scheduled_at: datetime
|
||||
expected_market_volume: int
|
||||
participation_cap_quantity: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExecutionPlan:
|
||||
parent_key: str
|
||||
children: tuple[ChildOrderIntent, ...]
|
||||
planned_quantity: int
|
||||
residual_quantity: int
|
||||
residual_policy: ResidualPolicy
|
||||
diagnostics: dict[str, object]
|
||||
|
||||
|
||||
def market_data_is_fresh(
|
||||
*,
|
||||
as_of: datetime,
|
||||
now: datetime,
|
||||
max_age: timedelta = timedelta(seconds=5),
|
||||
max_future: timedelta = timedelta(seconds=1),
|
||||
) -> bool:
|
||||
_aware(as_of, "as_of")
|
||||
_aware(now, "now")
|
||||
if max_age.total_seconds() < 0 or max_future.total_seconds() < 0:
|
||||
raise ValueError("freshness tolerances must be non-negative")
|
||||
age = now - as_of
|
||||
return -max_future <= age <= max_age
|
||||
|
||||
|
||||
def plan_guarded_twap_pov(
|
||||
parent: ParentOrderIntent,
|
||||
buckets: list[ExecutionBucket],
|
||||
) -> ExecutionPlan:
|
||||
if not buckets:
|
||||
raise ValueError("execution buckets are required")
|
||||
ordered = sorted(buckets, key=lambda item: item.scheduled_at)
|
||||
if len({bucket.scheduled_at for bucket in ordered}) != len(ordered):
|
||||
raise ValueError("execution bucket timestamps must be unique")
|
||||
for bucket in ordered:
|
||||
if not (
|
||||
parent.window_start <= bucket.scheduled_at <= parent.window_end
|
||||
):
|
||||
raise ValueError("execution bucket lies outside parent window")
|
||||
if parent.market_data_as_of > parent.window_start:
|
||||
raise ValueError(
|
||||
"parent market_data_as_of cannot follow execution window start"
|
||||
)
|
||||
|
||||
remaining = parent.quantity
|
||||
children: list[ChildOrderIntent] = []
|
||||
skipped_zero_capacity = 0
|
||||
for bucket_index, bucket in enumerate(ordered, 1):
|
||||
if remaining <= 0:
|
||||
break
|
||||
buckets_left = len(ordered) - bucket_index + 1
|
||||
twap_quantity = (
|
||||
math.ceil(remaining / buckets_left / parent.lot_size)
|
||||
* parent.lot_size
|
||||
)
|
||||
pov_cap = (
|
||||
int(bucket.expected_market_volume * parent.max_participation)
|
||||
// parent.lot_size
|
||||
* parent.lot_size
|
||||
)
|
||||
caps = [remaining, twap_quantity, pov_cap]
|
||||
if parent.max_child_quantity is not None:
|
||||
caps.append(
|
||||
parent.max_child_quantity
|
||||
// parent.lot_size
|
||||
* parent.lot_size
|
||||
)
|
||||
if parent.max_child_notional is not None:
|
||||
caps.append(
|
||||
int(parent.max_child_notional / parent.limit_price)
|
||||
// parent.lot_size
|
||||
* parent.lot_size
|
||||
)
|
||||
quantity = min(caps)
|
||||
quantity = quantity // parent.lot_size * parent.lot_size
|
||||
if quantity <= 0:
|
||||
skipped_zero_capacity += 1
|
||||
continue
|
||||
sequence = len(children) + 1
|
||||
digest = hashlib.sha256(
|
||||
(
|
||||
f"{parent.idempotency_key}:"
|
||||
f"{bucket.scheduled_at.isoformat()}:{sequence}"
|
||||
).encode("utf-8")
|
||||
).hexdigest()[:16]
|
||||
children.append(
|
||||
ChildOrderIntent(
|
||||
child_id=f"QOS-{digest}",
|
||||
parent_key=parent.idempotency_key,
|
||||
sequence=sequence,
|
||||
symbol=parent.symbol,
|
||||
side=parent.side,
|
||||
quantity=quantity,
|
||||
limit_price=float(parent.limit_price),
|
||||
scheduled_at=bucket.scheduled_at,
|
||||
expected_market_volume=bucket.expected_market_volume,
|
||||
participation_cap_quantity=pov_cap,
|
||||
)
|
||||
)
|
||||
remaining -= quantity
|
||||
planned = sum(child.quantity for child in children)
|
||||
if planned + remaining != parent.quantity:
|
||||
raise AssertionError("execution quantity conservation failed")
|
||||
return ExecutionPlan(
|
||||
parent_key=parent.idempotency_key,
|
||||
children=tuple(children),
|
||||
planned_quantity=planned,
|
||||
residual_quantity=remaining,
|
||||
residual_policy=parent.residual_policy,
|
||||
diagnostics={
|
||||
"bucket_count": len(ordered),
|
||||
"child_count": len(children),
|
||||
"skipped_zero_capacity": skipped_zero_capacity,
|
||||
"fully_planned": remaining == 0,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,623 @@
|
||||
"""Deterministic research-to-target experiment for local and Colab execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .contracts import validate_records
|
||||
from .domain import Bar
|
||||
from .features import (
|
||||
TrainOnlyFeaturePreprocessor,
|
||||
build_executable_label,
|
||||
compute_price_features,
|
||||
)
|
||||
from .ledger import canonical_json, jsonable
|
||||
from .optimizer import (
|
||||
PortfolioCandidate,
|
||||
PortfolioConstraints,
|
||||
optimize_deterministic,
|
||||
)
|
||||
from .portable_core import target_quantities
|
||||
from .research import (
|
||||
DeterministicRidge,
|
||||
pearson_correlation,
|
||||
purged_walk_forward,
|
||||
spearman_correlation,
|
||||
)
|
||||
from .synthetic import generate_synthetic_bars
|
||||
from .universe import SecurityEligibility, classify_security
|
||||
|
||||
|
||||
RESEARCH_SYMBOLS = (
|
||||
"600000.XSHG",
|
||||
"600519.XSHG",
|
||||
"601318.XSHG",
|
||||
"601012.XSHG",
|
||||
"000001.XSHE",
|
||||
"000333.XSHE",
|
||||
"000651.XSHE",
|
||||
"002415.XSHE",
|
||||
"002594.XSHE",
|
||||
"300059.XSHE",
|
||||
"300750.XSHE",
|
||||
"688981.XSHG",
|
||||
)
|
||||
|
||||
INDUSTRY_BY_SYMBOL = {
|
||||
symbol: ("industry_%d" % (index % 4))
|
||||
for index, symbol in enumerate(RESEARCH_SYMBOLS)
|
||||
}
|
||||
|
||||
|
||||
def _sha256(value: Any) -> str:
|
||||
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _atomic_json(path: Path, payload: Any) -> None:
|
||||
encoded = (
|
||||
json.dumps(
|
||||
jsonable(payload),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
).encode("utf-8")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temporary = tempfile.mkstemp(
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
dir=str(path.parent),
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(encoded)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _source_identity() -> tuple[str, dict[str, str]]:
|
||||
directory = Path(__file__).resolve().parent
|
||||
sources = {
|
||||
path.name: hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in sorted(directory.glob("*.py"), key=lambda item: item.name)
|
||||
}
|
||||
return _sha256(sources), sources
|
||||
|
||||
|
||||
def _bar_data_version(bars: list[Bar]) -> str:
|
||||
payload = [
|
||||
{
|
||||
"date": bar.trading_date.isoformat(),
|
||||
"symbol": bar.symbol,
|
||||
"open": format(bar.open, "f"),
|
||||
"high": format(bar.high, "f"),
|
||||
"low": format(bar.low, "f"),
|
||||
"close": format(bar.close, "f"),
|
||||
"volume": bar.volume,
|
||||
"suspended": bar.suspended,
|
||||
}
|
||||
for bar in bars
|
||||
]
|
||||
return _sha256(payload)
|
||||
|
||||
|
||||
def _first_sessions_by_week(dates: list[date]) -> list[date]:
|
||||
result: list[date] = []
|
||||
prior_week = None
|
||||
for session in dates:
|
||||
week = session.isocalendar()[:2]
|
||||
if week != prior_week:
|
||||
result.append(session)
|
||||
prior_week = week
|
||||
return result
|
||||
|
||||
|
||||
def _matrix(
|
||||
rows: list[dict[str, Any]],
|
||||
processor: TrainOnlyFeaturePreprocessor,
|
||||
) -> list[list[float]]:
|
||||
transformed = processor.transform([row["features"] for row in rows])
|
||||
names = processor.output_names
|
||||
return [[item[name] for name in names] for item in transformed]
|
||||
|
||||
|
||||
def _daily_correlations(
|
||||
rows: list[dict[str, Any]],
|
||||
predictions: list[float],
|
||||
) -> list[dict[str, Any]]:
|
||||
grouped: dict[str, list[tuple[float, float]]] = defaultdict(list)
|
||||
for row, prediction in zip(rows, predictions):
|
||||
grouped[row["decision_date"]].append((prediction, row["label"]))
|
||||
output = []
|
||||
for decision_date, values in sorted(grouped.items()):
|
||||
predicted = [value[0] for value in values]
|
||||
observed = [value[1] for value in values]
|
||||
output.append(
|
||||
{
|
||||
"decision_date": decision_date,
|
||||
"count": len(values),
|
||||
"ic": pearson_correlation(predicted, observed),
|
||||
"rank_ic": spearman_correlation(predicted, observed),
|
||||
}
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def _research_rows(
|
||||
bars: list[Bar],
|
||||
*,
|
||||
warmup_sessions: int,
|
||||
horizon_sessions: int,
|
||||
) -> tuple[list[dict[str, Any]], list[date], dict[str, list[Bar]]]:
|
||||
by_symbol: dict[str, list[Bar]] = defaultdict(list)
|
||||
for bar in bars:
|
||||
by_symbol[bar.symbol].append(bar)
|
||||
sessions = sorted({bar.trading_date for bar in bars})
|
||||
session_position = {session: index for index, session in enumerate(sessions)}
|
||||
weekly = [
|
||||
session
|
||||
for session in _first_sessions_by_week(sessions)
|
||||
if session_position[session] >= warmup_sessions
|
||||
and session_position[session] + horizon_sessions + 1 < len(sessions)
|
||||
]
|
||||
rows: list[dict[str, Any]] = []
|
||||
for decision_date in weekly:
|
||||
date_rows: list[dict[str, Any]] = []
|
||||
labels = []
|
||||
for symbol in RESEARCH_SYMBOLS:
|
||||
snapshot = compute_price_features(
|
||||
by_symbol[symbol],
|
||||
as_of=decision_date,
|
||||
)
|
||||
label = build_executable_label(
|
||||
by_symbol[symbol],
|
||||
decision_date=decision_date,
|
||||
horizon_sessions=horizon_sessions,
|
||||
)
|
||||
if label.value is None:
|
||||
continue
|
||||
labels.append(label.value)
|
||||
date_rows.append(
|
||||
{
|
||||
"sample_id": (
|
||||
f"{decision_date.strftime('%Y%m%d')}-{symbol}"
|
||||
),
|
||||
"decision_date": decision_date.isoformat(),
|
||||
"symbol": symbol,
|
||||
"features": snapshot.values,
|
||||
"raw_label": label.value,
|
||||
"label_status": label.status,
|
||||
"entry_date": label.entry_date.isoformat(),
|
||||
"exit_date": label.exit_date.isoformat(),
|
||||
}
|
||||
)
|
||||
if not date_rows:
|
||||
continue
|
||||
cross_section_mean = statistics.fmean(labels)
|
||||
for row in date_rows:
|
||||
row["label"] = row.pop("raw_label") - cross_section_mean
|
||||
rows.append(row)
|
||||
used_dates = sorted({date.fromisoformat(row["decision_date"]) for row in rows})
|
||||
return rows, used_dates, by_symbol
|
||||
|
||||
|
||||
def run_research_smoke(
|
||||
*,
|
||||
trading_days: int = 320,
|
||||
seed: int = 60,
|
||||
warmup_sessions: int = 130,
|
||||
horizon_sessions: int = 5,
|
||||
) -> dict[str, Any]:
|
||||
"""Run a chronological Ridge experiment and generate a final target."""
|
||||
|
||||
if trading_days < 220:
|
||||
raise ValueError("research smoke requires at least 220 trading days")
|
||||
bars = generate_synthetic_bars(
|
||||
RESEARCH_SYMBOLS,
|
||||
start_date="2023-01-03",
|
||||
trading_days_count=trading_days,
|
||||
seed=seed,
|
||||
base_volume=500_000,
|
||||
include_market_events=True,
|
||||
)
|
||||
rows, decision_dates, by_symbol = _research_rows(
|
||||
bars,
|
||||
warmup_sessions=warmup_sessions,
|
||||
horizon_sessions=horizon_sessions,
|
||||
)
|
||||
if len(decision_dates) < 22:
|
||||
raise ValueError("research smoke did not produce enough weekly dates")
|
||||
rows_by_date: dict[date, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
rows_by_date[date.fromisoformat(row["decision_date"])].append(row)
|
||||
|
||||
train_size = 16
|
||||
test_size = 4
|
||||
folds = list(
|
||||
purged_walk_forward(
|
||||
len(decision_dates),
|
||||
train_size=train_size,
|
||||
test_size=test_size,
|
||||
purge=1,
|
||||
embargo=1,
|
||||
)
|
||||
)
|
||||
if not folds:
|
||||
raise ValueError("research smoke produced no walk-forward fold")
|
||||
forecasts: list[dict[str, Any]] = []
|
||||
fold_reports = []
|
||||
all_daily_metrics = []
|
||||
for fold_number, fold in enumerate(folds, 1):
|
||||
train_rows = [
|
||||
row
|
||||
for index in fold.train
|
||||
for row in rows_by_date[decision_dates[index]]
|
||||
]
|
||||
test_rows = [
|
||||
row
|
||||
for index in fold.test
|
||||
for row in rows_by_date[decision_dates[index]]
|
||||
]
|
||||
processor = TrainOnlyFeaturePreprocessor().fit(
|
||||
[row["features"] for row in train_rows]
|
||||
)
|
||||
model = DeterministicRidge(alpha=5.0).fit(
|
||||
_matrix(train_rows, processor),
|
||||
[row["label"] for row in train_rows],
|
||||
)
|
||||
predictions = model.predict(_matrix(test_rows, processor))
|
||||
daily_metrics = _daily_correlations(test_rows, predictions)
|
||||
all_daily_metrics.extend(daily_metrics)
|
||||
fold_reports.append(
|
||||
{
|
||||
"fold": fold_number,
|
||||
"train_start": decision_dates[min(fold.train)].isoformat(),
|
||||
"train_end": decision_dates[max(fold.train)].isoformat(),
|
||||
"test_start": decision_dates[min(fold.test)].isoformat(),
|
||||
"test_end": decision_dates[max(fold.test)].isoformat(),
|
||||
"purged_dates": [
|
||||
decision_dates[index].isoformat() for index in fold.purged
|
||||
],
|
||||
"embargoed_dates": [
|
||||
decision_dates[index].isoformat() for index in fold.embargoed
|
||||
],
|
||||
"train_samples": len(train_rows),
|
||||
"test_samples": len(test_rows),
|
||||
"mean_ic": statistics.fmean(
|
||||
item["ic"] for item in daily_metrics
|
||||
),
|
||||
"mean_rank_ic": statistics.fmean(
|
||||
item["rank_ic"] for item in daily_metrics
|
||||
),
|
||||
}
|
||||
)
|
||||
for row, prediction in zip(test_rows, predictions):
|
||||
forecasts.append(
|
||||
{
|
||||
"decision_date": row["decision_date"],
|
||||
"symbol": row["symbol"],
|
||||
"prediction": prediction,
|
||||
"realized_excess_proxy": row["label"],
|
||||
"fold": fold_number,
|
||||
}
|
||||
)
|
||||
|
||||
# Train strictly before the final decision and emit a point-in-time target.
|
||||
target_date = decision_dates[-1]
|
||||
final_train = [
|
||||
row
|
||||
for row in rows
|
||||
if date.fromisoformat(row["decision_date"]) < target_date
|
||||
]
|
||||
final_rows = rows_by_date[target_date]
|
||||
final_processor = TrainOnlyFeaturePreprocessor().fit(
|
||||
[row["features"] for row in final_train]
|
||||
)
|
||||
final_model = DeterministicRidge(alpha=5.0).fit(
|
||||
_matrix(final_train, final_processor),
|
||||
[row["label"] for row in final_train],
|
||||
)
|
||||
final_predictions = final_model.predict(_matrix(final_rows, final_processor))
|
||||
centered = statistics.fmean(final_predictions)
|
||||
scores = {
|
||||
row["symbol"]: prediction - centered
|
||||
for row, prediction in zip(final_rows, final_predictions)
|
||||
}
|
||||
candidates = []
|
||||
current_weights = {symbol: 0.0 for symbol in RESEARCH_SYMBOLS}
|
||||
latest_prices = {}
|
||||
for symbol in RESEARCH_SYMBOLS:
|
||||
completed = [
|
||||
bar for bar in by_symbol[symbol] if bar.trading_date <= target_date
|
||||
]
|
||||
latest = completed[-1]
|
||||
latest_prices[symbol] = float(latest.close)
|
||||
returns = [
|
||||
float(completed[index].close)
|
||||
/ float(completed[index - 1].close)
|
||||
- 1.0
|
||||
for index in range(max(1, len(completed) - 60), len(completed))
|
||||
]
|
||||
variance = (
|
||||
statistics.variance(returns) if len(returns) > 1 else 1e-4
|
||||
)
|
||||
feature_row = next(
|
||||
row for row in final_rows if row["symbol"] == symbol
|
||||
)
|
||||
turnover = feature_row["features"]["median_turnover_20"] or 0.0
|
||||
eligibility = SecurityEligibility(
|
||||
symbol=symbol,
|
||||
index_member=True,
|
||||
listing_sessions=500,
|
||||
median_turnover=turnover,
|
||||
minimum_turnover=0.0,
|
||||
held_quantity=0,
|
||||
sellable_quantity=0,
|
||||
)
|
||||
state = classify_security(eligibility).state
|
||||
candidates.append(
|
||||
PortfolioCandidate(
|
||||
symbol=symbol,
|
||||
score=scores[symbol],
|
||||
variance=max(1e-6, variance),
|
||||
current_weight=current_weights[symbol],
|
||||
cost_bps=15.0,
|
||||
max_adv_weight=max(
|
||||
0.0,
|
||||
min(0.10, turnover * 0.03 / 1_000_000.0),
|
||||
),
|
||||
industry=INDUSTRY_BY_SYMBOL[symbol],
|
||||
state=state,
|
||||
)
|
||||
)
|
||||
solution = optimize_deterministic(
|
||||
candidates,
|
||||
PortfolioConstraints(
|
||||
gross_target=0.90,
|
||||
single_name_cap=0.10,
|
||||
industry_cap=0.30,
|
||||
one_way_turnover_cap=0.45,
|
||||
risk_aversion=1.0,
|
||||
cost_aversion=1.0,
|
||||
),
|
||||
)
|
||||
quantities = target_quantities(
|
||||
solution.weights,
|
||||
latest_prices,
|
||||
1_000_000.0,
|
||||
lot_size=100,
|
||||
cash_buffer=0.05,
|
||||
)
|
||||
|
||||
data_version = _bar_data_version(bars)
|
||||
source_sha256, sources = _source_identity()
|
||||
config = {
|
||||
"trading_days": trading_days,
|
||||
"seed": seed,
|
||||
"warmup_sessions": warmup_sessions,
|
||||
"horizon_sessions": horizon_sessions,
|
||||
"symbols": list(RESEARCH_SYMBOLS),
|
||||
"feature_version": "transparent-price-v1",
|
||||
"model": "deterministic-ridge-alpha-5",
|
||||
"label": "t_plus_1_open_to_t_plus_6_open_cross_section_excess_proxy",
|
||||
"validation": {
|
||||
"train_dates": train_size,
|
||||
"test_dates": test_size,
|
||||
"purge_dates": 1,
|
||||
"embargo_dates": 1,
|
||||
},
|
||||
}
|
||||
config_hash = _sha256(config)
|
||||
run_id = "q60-research-" + _sha256(
|
||||
{
|
||||
"data": data_version,
|
||||
"source": source_sha256,
|
||||
"config": config_hash,
|
||||
}
|
||||
)[:16]
|
||||
signal_records = [
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"run_id": run_id,
|
||||
"signal_id": f"RS-{target_date.strftime('%Y%m%d')}-{symbol}",
|
||||
"as_of": f"{target_date.isoformat()}T15:00:00+08:00",
|
||||
"symbol": symbol,
|
||||
"horizon_trading_days": horizon_sessions,
|
||||
"score": scores[symbol],
|
||||
"expected_excess_return": scores[symbol],
|
||||
"confidence": None,
|
||||
"model_version": "deterministic-ridge-alpha-5",
|
||||
"data_version": data_version,
|
||||
"feature_version": "transparent-price-v1",
|
||||
"metadata": {
|
||||
"evidence_class": "synthetic-research",
|
||||
"label_proxy": config["label"],
|
||||
},
|
||||
}
|
||||
for symbol in sorted(scores)
|
||||
]
|
||||
target_records = [
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"run_id": run_id,
|
||||
"decision_id": f"RD-{target_date.strftime('%Y%m%d')}",
|
||||
"as_of": f"{target_date.isoformat()}T15:00:00+08:00",
|
||||
"symbol": symbol,
|
||||
"target_weight": solution.weights[symbol],
|
||||
"target_quantity": quantities[symbol],
|
||||
"lot_size": 100,
|
||||
"signal_id": f"RS-{target_date.strftime('%Y%m%d')}-{symbol}",
|
||||
"constraint_state": "FEASIBLE",
|
||||
"config_hash": config_hash,
|
||||
"metadata": {
|
||||
"solver": solution.solver,
|
||||
"industry": INDUSTRY_BY_SYMBOL[symbol],
|
||||
},
|
||||
}
|
||||
for symbol in sorted(solution.weights)
|
||||
]
|
||||
validate_records(signal_records, "signal.schema.json")
|
||||
validate_records(target_records, "target.schema.json")
|
||||
rank_ics = [item["rank_ic"] for item in all_daily_metrics]
|
||||
rank_ic_std = statistics.stdev(rank_ics) if len(rank_ics) > 1 else 0.0
|
||||
report = {
|
||||
"run_id": run_id,
|
||||
"evidence_class": "synthetic-research",
|
||||
"investment_value_claim": False,
|
||||
"config_hash": config_hash,
|
||||
"data_version": data_version,
|
||||
"engine_source_sha256": source_sha256,
|
||||
"sample_count": len(rows),
|
||||
"decision_date_count": len(decision_dates),
|
||||
"fold_count": len(folds),
|
||||
"folds": fold_reports,
|
||||
"daily_oos_metrics": all_daily_metrics,
|
||||
"mean_oos_ic": statistics.fmean(
|
||||
item["ic"] for item in all_daily_metrics
|
||||
),
|
||||
"mean_oos_rank_ic": statistics.fmean(rank_ics),
|
||||
"oos_rank_icir": (
|
||||
statistics.fmean(rank_ics) / rank_ic_std
|
||||
if rank_ic_std > 0
|
||||
else 0.0
|
||||
),
|
||||
"target_date": target_date.isoformat(),
|
||||
"target_solver": solution.solver,
|
||||
"target_gross_weight": solution.gross_weight,
|
||||
"target_cash_weight": solution.cash_weight,
|
||||
"target_one_way_turnover": solution.one_way_turnover,
|
||||
"target_predicted_variance": solution.predicted_variance,
|
||||
"label_warning": (
|
||||
"Synthetic daily-open cross-sectional excess proxy; not the "
|
||||
"production CSI500 T+1/T+6 VWAP label."
|
||||
),
|
||||
}
|
||||
sample_index = [
|
||||
{
|
||||
"sample_id": row["sample_id"],
|
||||
"decision_date": row["decision_date"],
|
||||
"symbol": row["symbol"],
|
||||
"entry_date": row["entry_date"],
|
||||
"exit_date": row["exit_date"],
|
||||
"label_status": row["label_status"],
|
||||
"feature_sha256": _sha256(row["features"]),
|
||||
"label": row["label"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
manifest = {
|
||||
"schema_version": "1.0",
|
||||
"run_id": run_id,
|
||||
"evidence_class": "synthetic-research",
|
||||
"config": config,
|
||||
"config_hash": config_hash,
|
||||
"data_version": data_version,
|
||||
"engine_source_sha256": source_sha256,
|
||||
"engine_sources": sources,
|
||||
"deterministic": True,
|
||||
"artifacts": {},
|
||||
}
|
||||
return {
|
||||
"manifest": manifest,
|
||||
"report": report,
|
||||
"signals": signal_records,
|
||||
"targets": target_records,
|
||||
"forecasts": forecasts,
|
||||
"samples": sample_index,
|
||||
}
|
||||
|
||||
|
||||
RESEARCH_ARTIFACTS = {
|
||||
"report": "report.json",
|
||||
"signals": "signals.json",
|
||||
"targets": "targets.json",
|
||||
"forecasts": "forecasts.json",
|
||||
"samples": "samples.json",
|
||||
}
|
||||
|
||||
|
||||
def write_research_artifacts(
|
||||
result: dict[str, Any],
|
||||
output_dir: str | Path,
|
||||
) -> dict[str, str]:
|
||||
root = Path(output_dir)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
marker = root / ".quant60-incomplete"
|
||||
marker.write_text(str(result["manifest"]["run_id"]) + "\n", encoding="ascii")
|
||||
paths: dict[str, Path] = {}
|
||||
for name, filename in RESEARCH_ARTIFACTS.items():
|
||||
path = root / filename
|
||||
_atomic_json(path, result[name])
|
||||
paths[name] = path
|
||||
manifest = dict(result["manifest"])
|
||||
manifest["artifacts"] = {
|
||||
name: {
|
||||
"file": path.name,
|
||||
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
}
|
||||
for name, path in sorted(paths.items())
|
||||
}
|
||||
manifest_path = root / "manifest.json"
|
||||
_atomic_json(manifest_path, manifest)
|
||||
marker.unlink()
|
||||
paths["manifest"] = manifest_path
|
||||
return {name: str(path.resolve()) for name, path in paths.items()}
|
||||
|
||||
|
||||
def verify_research_manifest(path: str | Path) -> dict[str, Any]:
|
||||
manifest_path = Path(path)
|
||||
root = manifest_path.parent
|
||||
if (root / ".quant60-incomplete").exists():
|
||||
raise ValueError("research artifact publication is incomplete")
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
expected_files = {
|
||||
entry["file"] for entry in manifest.get("artifacts", {}).values()
|
||||
} | {manifest_path.name}
|
||||
actual_files = {entry.name for entry in root.iterdir()}
|
||||
if actual_files != expected_files:
|
||||
raise ValueError("research artifact directory is not an exact set")
|
||||
verified = {}
|
||||
for name, entry in manifest["artifacts"].items():
|
||||
artifact = root / entry["file"]
|
||||
digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
if digest != entry["sha256"]:
|
||||
raise ValueError(f"research artifact hash mismatch: {name}")
|
||||
verified[name] = digest
|
||||
source_sha256, sources = _source_identity()
|
||||
if _sha256(manifest["engine_sources"]) != manifest["engine_source_sha256"]:
|
||||
raise ValueError("saved research source aggregate mismatch")
|
||||
if sources != manifest["engine_sources"] or source_sha256 != manifest[
|
||||
"engine_source_sha256"
|
||||
]:
|
||||
raise ValueError("current research source does not match manifest")
|
||||
signals = json.loads((root / RESEARCH_ARTIFACTS["signals"]).read_text())
|
||||
targets = json.loads((root / RESEARCH_ARTIFACTS["targets"]).read_text())
|
||||
validate_records(signals, "signal.schema.json")
|
||||
validate_records(targets, "target.schema.json")
|
||||
report = json.loads((root / RESEARCH_ARTIFACTS["report"]).read_text())
|
||||
if report["run_id"] != manifest["run_id"]:
|
||||
raise ValueError("research report/manifest run_id mismatch")
|
||||
return {
|
||||
"ok": True,
|
||||
"run_id": manifest["run_id"],
|
||||
"artifacts": verified,
|
||||
"source_matches": True,
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Transparent industry/style factor risk reference implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import statistics
|
||||
from dataclasses import dataclass
|
||||
from typing import Mapping, Sequence
|
||||
|
||||
from .portable_core import normalize_symbol
|
||||
from .research import DeterministicRidge
|
||||
|
||||
|
||||
class RiskModelError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _finite(value: float, name: str) -> float:
|
||||
numeric = float(value)
|
||||
if not math.isfinite(numeric):
|
||||
raise RiskModelError(f"{name} must be finite")
|
||||
return numeric
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FactorExposureSnapshot:
|
||||
as_of: str
|
||||
exposures: dict[str, dict[str, float]]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not str(self.as_of).strip():
|
||||
raise RiskModelError("exposure as_of is required")
|
||||
if not self.exposures:
|
||||
raise RiskModelError("factor exposures are empty")
|
||||
normalized: dict[str, dict[str, float]] = {}
|
||||
factor_set = None
|
||||
for raw_symbol, raw_factors in self.exposures.items():
|
||||
symbol = normalize_symbol(raw_symbol)
|
||||
factors = {
|
||||
str(name): _finite(value, f"{symbol}.{name}")
|
||||
for name, value in raw_factors.items()
|
||||
}
|
||||
if not factors:
|
||||
raise RiskModelError(f"factor exposures are empty for {symbol}")
|
||||
if factor_set is None:
|
||||
factor_set = set(factors)
|
||||
elif set(factors) != factor_set:
|
||||
raise RiskModelError("factor exposure columns are inconsistent")
|
||||
normalized[symbol] = dict(sorted(factors.items()))
|
||||
object.__setattr__(self, "exposures", dict(sorted(normalized.items())))
|
||||
|
||||
@property
|
||||
def factor_names(self) -> tuple[str, ...]:
|
||||
first = next(iter(self.exposures.values()))
|
||||
return tuple(sorted(first))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CrossSectionalFactorFit:
|
||||
factor_returns: dict[str, float]
|
||||
residuals: dict[str, float]
|
||||
observation_count: int
|
||||
|
||||
|
||||
def estimate_factor_returns(
|
||||
*,
|
||||
returns: Mapping[str, float],
|
||||
exposure: FactorExposureSnapshot,
|
||||
ridge_alpha: float = 1e-4,
|
||||
) -> CrossSectionalFactorFit:
|
||||
normalized_returns = {
|
||||
normalize_symbol(symbol): _finite(value, f"return.{symbol}")
|
||||
for symbol, value in returns.items()
|
||||
}
|
||||
symbols = sorted(set(normalized_returns) & set(exposure.exposures))
|
||||
factors = exposure.factor_names
|
||||
if len(symbols) <= len(factors):
|
||||
raise RiskModelError(
|
||||
"cross-sectional regression needs more securities than factors"
|
||||
)
|
||||
matrix = [
|
||||
[exposure.exposures[symbol][factor] for factor in factors]
|
||||
for symbol in symbols
|
||||
]
|
||||
target = [normalized_returns[symbol] for symbol in symbols]
|
||||
model = DeterministicRidge(
|
||||
alpha=ridge_alpha,
|
||||
fit_intercept=False,
|
||||
).fit(matrix, target)
|
||||
predictions = model.predict(matrix)
|
||||
assert model.coef_ is not None
|
||||
return CrossSectionalFactorFit(
|
||||
factor_returns={
|
||||
factor: model.coef_[index]
|
||||
for index, factor in enumerate(factors)
|
||||
},
|
||||
residuals={
|
||||
symbol: observed - predicted
|
||||
for symbol, observed, predicted in zip(
|
||||
symbols,
|
||||
target,
|
||||
predictions,
|
||||
)
|
||||
},
|
||||
observation_count=len(symbols),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FactorRiskForecast:
|
||||
factor_names: tuple[str, ...]
|
||||
factor_covariance: dict[str, dict[str, float]]
|
||||
specific_variance: dict[str, float]
|
||||
half_life: float
|
||||
shrinkage: float
|
||||
variance_floor: float
|
||||
|
||||
|
||||
def _ewma_weights(count: int, half_life: float) -> list[float]:
|
||||
if count <= 0:
|
||||
raise RiskModelError("EWMA history is empty")
|
||||
if not math.isfinite(float(half_life)) or half_life <= 0:
|
||||
raise RiskModelError("half_life must be finite and positive")
|
||||
decay = math.exp(math.log(0.5) / float(half_life))
|
||||
raw = [decay ** (count - 1 - index) for index in range(count)]
|
||||
total = sum(raw)
|
||||
return [value / total for value in raw]
|
||||
|
||||
|
||||
def build_factor_risk_forecast(
|
||||
*,
|
||||
factor_return_history: Sequence[Mapping[str, float]],
|
||||
residual_history: Mapping[str, Sequence[float]],
|
||||
half_life: float = 20.0,
|
||||
shrinkage: float = 0.20,
|
||||
variance_floor: float = 1e-8,
|
||||
) -> FactorRiskForecast:
|
||||
if not factor_return_history:
|
||||
raise RiskModelError("factor return history is empty")
|
||||
if not math.isfinite(float(shrinkage)) or not (0 <= shrinkage <= 1):
|
||||
raise RiskModelError("shrinkage must lie in [0, 1]")
|
||||
if not math.isfinite(float(variance_floor)) or variance_floor <= 0:
|
||||
raise RiskModelError("variance_floor must be finite and positive")
|
||||
factors = tuple(sorted(factor_return_history[0]))
|
||||
if not factors:
|
||||
raise RiskModelError("factor return columns are empty")
|
||||
rows = []
|
||||
for raw_row in factor_return_history:
|
||||
if set(raw_row) != set(factors):
|
||||
raise RiskModelError("factor return columns are inconsistent")
|
||||
rows.append(
|
||||
{
|
||||
factor: _finite(raw_row[factor], f"factor_return.{factor}")
|
||||
for factor in factors
|
||||
}
|
||||
)
|
||||
weights = _ewma_weights(len(rows), half_life)
|
||||
means = {
|
||||
factor: sum(
|
||||
weight * row[factor] for weight, row in zip(weights, rows)
|
||||
)
|
||||
for factor in factors
|
||||
}
|
||||
covariance: dict[str, dict[str, float]] = {}
|
||||
for left in factors:
|
||||
covariance[left] = {}
|
||||
for right in factors:
|
||||
sample = sum(
|
||||
weight
|
||||
* (row[left] - means[left])
|
||||
* (row[right] - means[right])
|
||||
for weight, row in zip(weights, rows)
|
||||
)
|
||||
if left == right:
|
||||
value = max(variance_floor, sample)
|
||||
else:
|
||||
value = (1.0 - shrinkage) * sample
|
||||
covariance[left][right] = value
|
||||
|
||||
specific: dict[str, float] = {}
|
||||
for raw_symbol, raw_values in residual_history.items():
|
||||
symbol = normalize_symbol(raw_symbol)
|
||||
values = [
|
||||
_finite(value, f"specific_residual.{symbol}")
|
||||
for value in raw_values
|
||||
]
|
||||
if len(values) < 2:
|
||||
specific[symbol] = variance_floor
|
||||
else:
|
||||
specific[symbol] = max(
|
||||
variance_floor,
|
||||
statistics.variance(values),
|
||||
)
|
||||
if not specific:
|
||||
raise RiskModelError("specific residual history is empty")
|
||||
return FactorRiskForecast(
|
||||
factor_names=factors,
|
||||
factor_covariance=covariance,
|
||||
specific_variance=dict(sorted(specific.items())),
|
||||
half_life=float(half_life),
|
||||
shrinkage=float(shrinkage),
|
||||
variance_floor=float(variance_floor),
|
||||
)
|
||||
|
||||
|
||||
def portfolio_factor_variance(
|
||||
*,
|
||||
portfolio_weights: Mapping[str, float],
|
||||
exposure: FactorExposureSnapshot,
|
||||
forecast: FactorRiskForecast,
|
||||
) -> float:
|
||||
weights = {
|
||||
normalize_symbol(symbol): _finite(value, f"weight.{symbol}")
|
||||
for symbol, value in portfolio_weights.items()
|
||||
}
|
||||
if any(value < 0 for value in weights.values()):
|
||||
raise RiskModelError("portfolio weights must be non-negative")
|
||||
missing_exposure = set(weights) - set(exposure.exposures)
|
||||
missing_specific = set(weights) - set(forecast.specific_variance)
|
||||
if missing_exposure or missing_specific:
|
||||
raise RiskModelError(
|
||||
"risk inputs missing for "
|
||||
f"exposure={sorted(missing_exposure)}, "
|
||||
f"specific={sorted(missing_specific)}"
|
||||
)
|
||||
if set(exposure.factor_names) != set(forecast.factor_names):
|
||||
raise RiskModelError("exposure/forecast factors differ")
|
||||
factor_weight = {
|
||||
factor: sum(
|
||||
weight * exposure.exposures[symbol][factor]
|
||||
for symbol, weight in weights.items()
|
||||
)
|
||||
for factor in forecast.factor_names
|
||||
}
|
||||
factor_variance = sum(
|
||||
factor_weight[left]
|
||||
* forecast.factor_covariance[left][right]
|
||||
* factor_weight[right]
|
||||
for left in forecast.factor_names
|
||||
for right in forecast.factor_names
|
||||
)
|
||||
specific_variance = sum(
|
||||
weight * weight * forecast.specific_variance[symbol]
|
||||
for symbol, weight in weights.items()
|
||||
)
|
||||
result = factor_variance + specific_variance
|
||||
if not math.isfinite(result) or result < -1e-12:
|
||||
raise RiskModelError("portfolio variance is invalid")
|
||||
return max(0.0, result)
|
||||
|
||||
|
||||
def factor_stress_loss(
|
||||
*,
|
||||
portfolio_weights: Mapping[str, float],
|
||||
exposure: FactorExposureSnapshot,
|
||||
factor_shocks: Mapping[str, float],
|
||||
) -> float:
|
||||
shocks = {
|
||||
str(factor): _finite(value, f"shock.{factor}")
|
||||
for factor, value in factor_shocks.items()
|
||||
}
|
||||
unknown = set(shocks) - set(exposure.factor_names)
|
||||
if unknown:
|
||||
raise RiskModelError(f"unknown stress factors: {sorted(unknown)}")
|
||||
loss = 0.0
|
||||
for raw_symbol, raw_weight in portfolio_weights.items():
|
||||
symbol = normalize_symbol(raw_symbol)
|
||||
if symbol not in exposure.exposures:
|
||||
raise RiskModelError(f"missing exposure for {symbol}")
|
||||
weight = _finite(raw_weight, f"weight.{symbol}")
|
||||
return_shock = sum(
|
||||
exposure.exposures[symbol][factor] * shock
|
||||
for factor, shock in shocks.items()
|
||||
)
|
||||
loss -= weight * return_shock
|
||||
return loss
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Transparent daily-bar features and train-only preprocessing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import statistics
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from typing import Iterable, Mapping, Sequence
|
||||
|
||||
from .domain import Bar
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
class FeatureIntegrityError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FeatureSnapshot:
|
||||
symbol: str
|
||||
as_of: date
|
||||
values: dict[str, float | None]
|
||||
history_start: date
|
||||
history_end: date
|
||||
feature_version: str = "transparent-price-v1"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExecutableLabel:
|
||||
symbol: str
|
||||
decision_date: date
|
||||
entry_date: date | None
|
||||
exit_date: date | None
|
||||
value: float | None
|
||||
status: str
|
||||
horizon_sessions: int
|
||||
|
||||
|
||||
def _ordered_symbol_bars(bars: Iterable[Bar]) -> list[Bar]:
|
||||
ordered = sorted(tuple(bars), key=lambda item: item.trading_date)
|
||||
if not ordered:
|
||||
raise FeatureIntegrityError("feature history is empty")
|
||||
symbol = ordered[0].symbol
|
||||
seen: set[date] = set()
|
||||
for bar in ordered:
|
||||
if bar.symbol != symbol:
|
||||
raise FeatureIntegrityError("feature history must contain one symbol")
|
||||
if bar.trading_date in seen:
|
||||
raise FeatureIntegrityError("duplicate feature history date")
|
||||
seen.add(bar.trading_date)
|
||||
return ordered
|
||||
|
||||
|
||||
def _returns(closes: Sequence[float]) -> list[float]:
|
||||
return [
|
||||
closes[index] / closes[index - 1] - 1.0
|
||||
for index in range(1, len(closes))
|
||||
if closes[index - 1] > 0
|
||||
]
|
||||
|
||||
|
||||
def _sample_std(values: Sequence[float]) -> float:
|
||||
return statistics.stdev(values) if len(values) > 1 else 0.0
|
||||
|
||||
|
||||
def compute_price_features(
|
||||
bars: Iterable[Bar],
|
||||
*,
|
||||
as_of: date | None = None,
|
||||
) -> FeatureSnapshot:
|
||||
"""Compute only from bars completed by ``as_of``.
|
||||
|
||||
Missing long-window values remain explicit ``None`` and are handled by a
|
||||
train-fitted preprocessor rather than by silently dropping the security.
|
||||
"""
|
||||
|
||||
ordered = _ordered_symbol_bars(bars)
|
||||
decision_date = ordered[-1].trading_date if as_of is None else as_of
|
||||
history = [bar for bar in ordered if bar.trading_date <= decision_date]
|
||||
if not history:
|
||||
raise FeatureIntegrityError("no completed bar exists at feature as_of")
|
||||
closes = [float(bar.close) for bar in history]
|
||||
daily_returns = _returns(closes)
|
||||
|
||||
def momentum(window: int) -> float | None:
|
||||
if len(closes) < window + 1:
|
||||
return None
|
||||
return closes[-1] / closes[-1 - window] - 1.0
|
||||
|
||||
def volatility(window: int, *, downside: bool = False) -> float | None:
|
||||
if len(daily_returns) < window:
|
||||
return None
|
||||
values = daily_returns[-window:]
|
||||
if downside:
|
||||
values = [min(0.0, value) for value in values]
|
||||
return _sample_std(values) * math.sqrt(252.0)
|
||||
|
||||
def median_turnover(window: int) -> float | None:
|
||||
if len(history) < window:
|
||||
return None
|
||||
return statistics.median(
|
||||
float(bar.close) * int(bar.volume) for bar in history[-window:]
|
||||
)
|
||||
|
||||
def amihud(window: int) -> float | None:
|
||||
if len(history) < window + 1:
|
||||
return None
|
||||
recent_bars = history[-window:]
|
||||
recent_returns = daily_returns[-window:]
|
||||
values = []
|
||||
for bar, daily_return in zip(recent_bars, recent_returns):
|
||||
turnover = float(bar.close) * int(bar.volume)
|
||||
if turnover > 0:
|
||||
values.append(abs(daily_return) / turnover)
|
||||
return statistics.fmean(values) if values else None
|
||||
|
||||
def distance_to_high(window: int) -> float | None:
|
||||
if len(closes) < window:
|
||||
return None
|
||||
high = max(closes[-window:])
|
||||
return closes[-1] / high - 1.0 if high > 0 else None
|
||||
|
||||
gap_values = []
|
||||
for bar in history[-20:]:
|
||||
if bar.previous_close is not None and float(bar.previous_close) > 0:
|
||||
gap_values.append(float(bar.open) / float(bar.previous_close) - 1.0)
|
||||
|
||||
values = {
|
||||
"momentum_5": momentum(5),
|
||||
"momentum_20": momentum(20),
|
||||
"momentum_60": momentum(60),
|
||||
"momentum_120": momentum(120),
|
||||
"reversal_5": (
|
||||
-momentum(5) if momentum(5) is not None else None
|
||||
),
|
||||
"volatility_20": volatility(20),
|
||||
"volatility_60": volatility(60),
|
||||
"downside_volatility_20": volatility(20, downside=True),
|
||||
"distance_to_high_60": distance_to_high(60),
|
||||
"median_turnover_20": median_turnover(20),
|
||||
"amihud_20": amihud(20),
|
||||
"mean_open_gap_20": (
|
||||
statistics.fmean(gap_values) if gap_values else None
|
||||
),
|
||||
}
|
||||
for name, value in values.items():
|
||||
if value is not None and not math.isfinite(float(value)):
|
||||
raise FeatureIntegrityError(f"non-finite feature {name}")
|
||||
return FeatureSnapshot(
|
||||
symbol=normalize_symbol(history[-1].symbol),
|
||||
as_of=decision_date,
|
||||
values=values,
|
||||
history_start=history[0].trading_date,
|
||||
history_end=history[-1].trading_date,
|
||||
)
|
||||
|
||||
|
||||
def build_executable_label(
|
||||
bars: Iterable[Bar],
|
||||
*,
|
||||
decision_date: date,
|
||||
horizon_sessions: int = 5,
|
||||
benchmark_bars: Iterable[Bar] | None = None,
|
||||
) -> ExecutableLabel:
|
||||
"""Build a T+1-open to T+1+horizon-open research label.
|
||||
|
||||
Daily open is an explicit proxy for the article's VWAP label. Suspended or
|
||||
zero-volume endpoints are marked censored instead of silently deleted.
|
||||
"""
|
||||
|
||||
if type(horizon_sessions) is not int or horizon_sessions <= 0:
|
||||
raise ValueError("horizon_sessions must be a positive integer")
|
||||
ordered = _ordered_symbol_bars(bars)
|
||||
future = [bar for bar in ordered if bar.trading_date > decision_date]
|
||||
if len(future) <= horizon_sessions:
|
||||
return ExecutableLabel(
|
||||
symbol=ordered[0].symbol,
|
||||
decision_date=decision_date,
|
||||
entry_date=None,
|
||||
exit_date=None,
|
||||
value=None,
|
||||
status="INSUFFICIENT_FUTURE",
|
||||
horizon_sessions=horizon_sessions,
|
||||
)
|
||||
entry = future[0]
|
||||
exit_bar = future[horizon_sessions]
|
||||
if (
|
||||
entry.suspended
|
||||
or exit_bar.suspended
|
||||
or entry.volume <= 0
|
||||
or exit_bar.volume <= 0
|
||||
):
|
||||
return ExecutableLabel(
|
||||
symbol=ordered[0].symbol,
|
||||
decision_date=decision_date,
|
||||
entry_date=entry.trading_date,
|
||||
exit_date=exit_bar.trading_date,
|
||||
value=None,
|
||||
status="NON_EXECUTABLE_CENSORED",
|
||||
horizon_sessions=horizon_sessions,
|
||||
)
|
||||
value = float(exit_bar.open) / float(entry.open) - 1.0
|
||||
if benchmark_bars is not None:
|
||||
benchmark = build_executable_label(
|
||||
benchmark_bars,
|
||||
decision_date=decision_date,
|
||||
horizon_sessions=horizon_sessions,
|
||||
)
|
||||
if benchmark.value is None:
|
||||
return ExecutableLabel(
|
||||
symbol=ordered[0].symbol,
|
||||
decision_date=decision_date,
|
||||
entry_date=entry.trading_date,
|
||||
exit_date=exit_bar.trading_date,
|
||||
value=None,
|
||||
status="BENCHMARK_CENSORED",
|
||||
horizon_sessions=horizon_sessions,
|
||||
)
|
||||
value -= benchmark.value
|
||||
if not math.isfinite(value):
|
||||
raise FeatureIntegrityError("label is non-finite")
|
||||
return ExecutableLabel(
|
||||
symbol=ordered[0].symbol,
|
||||
decision_date=decision_date,
|
||||
entry_date=entry.trading_date,
|
||||
exit_date=exit_bar.trading_date,
|
||||
value=value,
|
||||
status="OK",
|
||||
horizon_sessions=horizon_sessions,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FeatureTransform:
|
||||
lower: float
|
||||
upper: float
|
||||
median: float
|
||||
mean: float
|
||||
scale: float
|
||||
|
||||
|
||||
def _quantile(values: Sequence[float], fraction: float) -> float:
|
||||
ordered = sorted(values)
|
||||
if not ordered:
|
||||
raise ValueError("quantile requires observations")
|
||||
position = (len(ordered) - 1) * fraction
|
||||
lower = int(math.floor(position))
|
||||
upper = int(math.ceil(position))
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
weight = position - lower
|
||||
return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
|
||||
|
||||
|
||||
class TrainOnlyFeaturePreprocessor:
|
||||
"""Winsorize, impute and standardize from training observations only."""
|
||||
|
||||
def __init__(self, winsor_fraction: float = 0.01) -> None:
|
||||
if not math.isfinite(float(winsor_fraction)) or not (
|
||||
0 <= winsor_fraction < 0.5
|
||||
):
|
||||
raise ValueError("winsor_fraction must lie in [0, 0.5)")
|
||||
self.winsor_fraction = float(winsor_fraction)
|
||||
self.transforms: dict[str, FeatureTransform] | None = None
|
||||
|
||||
def fit(
|
||||
self,
|
||||
rows: Sequence[Mapping[str, float | None]],
|
||||
) -> "TrainOnlyFeaturePreprocessor":
|
||||
if not rows:
|
||||
raise ValueError("training feature rows are empty")
|
||||
names = sorted({name for row in rows for name in row})
|
||||
if not names:
|
||||
raise ValueError("training features are empty")
|
||||
transforms: dict[str, FeatureTransform] = {}
|
||||
for name in names:
|
||||
values = [
|
||||
float(row[name])
|
||||
for row in rows
|
||||
if row.get(name) is not None
|
||||
]
|
||||
if any(not math.isfinite(value) for value in values):
|
||||
raise ValueError(f"training feature {name} is non-finite")
|
||||
if not values:
|
||||
raise ValueError(f"training feature {name} is entirely missing")
|
||||
lower = _quantile(values, self.winsor_fraction)
|
||||
upper = _quantile(values, 1.0 - self.winsor_fraction)
|
||||
median = statistics.median(values)
|
||||
clipped = [min(upper, max(lower, value)) for value in values]
|
||||
mean = statistics.fmean(clipped)
|
||||
scale = _sample_std(clipped)
|
||||
transforms[name] = FeatureTransform(
|
||||
lower=lower,
|
||||
upper=upper,
|
||||
median=median,
|
||||
mean=mean,
|
||||
scale=scale if scale > 1e-12 else 1.0,
|
||||
)
|
||||
self.transforms = transforms
|
||||
return self
|
||||
|
||||
@property
|
||||
def output_names(self) -> tuple[str, ...]:
|
||||
if self.transforms is None:
|
||||
raise ValueError("preprocessor is not fitted")
|
||||
return tuple(
|
||||
item
|
||||
for name in sorted(self.transforms)
|
||||
for item in (name, f"{name}__missing")
|
||||
)
|
||||
|
||||
def transform_one(
|
||||
self,
|
||||
row: Mapping[str, float | None],
|
||||
) -> dict[str, float]:
|
||||
if self.transforms is None:
|
||||
raise ValueError("preprocessor is not fitted")
|
||||
unknown = set(row) - set(self.transforms)
|
||||
if unknown:
|
||||
raise ValueError(f"unknown feature fields: {sorted(unknown)}")
|
||||
output: dict[str, float] = {}
|
||||
for name, spec in sorted(self.transforms.items()):
|
||||
raw = row.get(name)
|
||||
missing = raw is None
|
||||
value = spec.median if missing else float(raw)
|
||||
if not math.isfinite(value):
|
||||
raise ValueError(f"feature {name} is non-finite")
|
||||
clipped = min(spec.upper, max(spec.lower, value))
|
||||
output[name] = (clipped - spec.mean) / spec.scale
|
||||
output[f"{name}__missing"] = 1.0 if missing else 0.0
|
||||
return output
|
||||
|
||||
def transform(
|
||||
self,
|
||||
rows: Sequence[Mapping[str, float | None]],
|
||||
) -> list[dict[str, float]]:
|
||||
return [self.transform_one(row) for row in rows]
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Append-only hash-chain ledger with deterministic JSONL serialization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import asdict, dataclass, is_dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
GENESIS_HASH = "0" * 64
|
||||
|
||||
|
||||
class LedgerIntegrityError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class DuplicateEventError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def jsonable(value: Any) -> Any:
|
||||
if is_dataclass(value):
|
||||
return jsonable(asdict(value))
|
||||
if isinstance(value, dict):
|
||||
return {str(key): jsonable(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [jsonable(item) for item in value]
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, Decimal):
|
||||
return format(value, "f")
|
||||
if isinstance(value, Enum):
|
||||
return value.value
|
||||
if isinstance(value, Path):
|
||||
return str(value)
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
return value
|
||||
raise TypeError(f"value is not JSON serializable: {type(value).__name__}")
|
||||
|
||||
|
||||
def canonical_json(value: Any) -> str:
|
||||
return json.dumps(
|
||||
jsonable(value),
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LedgerRecord:
|
||||
sequence: int
|
||||
timestamp: str
|
||||
event_id: str
|
||||
event_type: str
|
||||
payload: dict[str, Any]
|
||||
previous_hash: str
|
||||
hash: str
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return jsonable(asdict(self))
|
||||
|
||||
|
||||
class HashChainLedger:
|
||||
def __init__(self, records: Iterable[LedgerRecord] = ()) -> None:
|
||||
self.records: list[LedgerRecord] = []
|
||||
self._by_event_id: dict[str, LedgerRecord] = {}
|
||||
for record in records:
|
||||
self._accept_loaded(record)
|
||||
self.verify()
|
||||
|
||||
@property
|
||||
def head_hash(self) -> str:
|
||||
return self.records[-1].hash if self.records else GENESIS_HASH
|
||||
|
||||
def _body(
|
||||
self,
|
||||
sequence: int,
|
||||
timestamp: str,
|
||||
event_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any],
|
||||
previous_hash: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"sequence": sequence,
|
||||
"timestamp": timestamp,
|
||||
"event_id": event_id,
|
||||
"event_type": event_type,
|
||||
"payload": jsonable(payload),
|
||||
"previous_hash": previous_hash,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _digest(body: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(canonical_json(body).encode("utf-8")).hexdigest()
|
||||
|
||||
def append(
|
||||
self,
|
||||
event_type: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
timestamp: str,
|
||||
event_id: str,
|
||||
) -> LedgerRecord:
|
||||
if not event_type or not event_id or not timestamp:
|
||||
raise ValueError("event_type, event_id and timestamp are required")
|
||||
normalized_payload = jsonable(payload)
|
||||
prior = self._by_event_id.get(event_id)
|
||||
if prior is not None:
|
||||
if (
|
||||
prior.event_type == event_type
|
||||
and prior.payload == normalized_payload
|
||||
and prior.timestamp == timestamp
|
||||
):
|
||||
return prior
|
||||
raise DuplicateEventError(
|
||||
f"event_id {event_id!r} already exists with different content"
|
||||
)
|
||||
sequence = len(self.records) + 1
|
||||
previous_hash = self.head_hash
|
||||
body = self._body(
|
||||
sequence,
|
||||
timestamp,
|
||||
event_id,
|
||||
event_type,
|
||||
normalized_payload,
|
||||
previous_hash,
|
||||
)
|
||||
record = LedgerRecord(
|
||||
**body,
|
||||
hash=self._digest(body),
|
||||
)
|
||||
self.records.append(record)
|
||||
self._by_event_id[event_id] = record
|
||||
return record
|
||||
|
||||
def _accept_loaded(self, record: LedgerRecord) -> None:
|
||||
if record.event_id in self._by_event_id:
|
||||
raise LedgerIntegrityError(
|
||||
f"duplicate event_id in ledger: {record.event_id}"
|
||||
)
|
||||
self.records.append(record)
|
||||
self._by_event_id[record.event_id] = record
|
||||
|
||||
def verify(self) -> bool:
|
||||
previous_hash = GENESIS_HASH
|
||||
seen: set[str] = set()
|
||||
for expected_sequence, record in enumerate(self.records, 1):
|
||||
if record.sequence != expected_sequence:
|
||||
raise LedgerIntegrityError(
|
||||
f"sequence gap at {expected_sequence}"
|
||||
)
|
||||
if record.previous_hash != previous_hash:
|
||||
raise LedgerIntegrityError(
|
||||
f"previous hash mismatch at {expected_sequence}"
|
||||
)
|
||||
if record.event_id in seen:
|
||||
raise LedgerIntegrityError(
|
||||
f"duplicate event_id at {expected_sequence}"
|
||||
)
|
||||
seen.add(record.event_id)
|
||||
body = self._body(
|
||||
record.sequence,
|
||||
record.timestamp,
|
||||
record.event_id,
|
||||
record.event_type,
|
||||
record.payload,
|
||||
record.previous_hash,
|
||||
)
|
||||
expected_hash = self._digest(body)
|
||||
if record.hash != expected_hash:
|
||||
raise LedgerIntegrityError(
|
||||
f"hash mismatch at sequence {expected_sequence}"
|
||||
)
|
||||
previous_hash = record.hash
|
||||
return True
|
||||
|
||||
def write_jsonl(self, path: str | Path) -> Path:
|
||||
destination = Path(path)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{destination.name}.",
|
||||
suffix=".tmp",
|
||||
dir=str(destination.parent),
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
for record in self.records:
|
||||
handle.write(canonical_json(record.as_dict()))
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_name, destination)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temporary_name)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
return destination
|
||||
|
||||
@classmethod
|
||||
def read_jsonl(cls, path: str | Path) -> HashChainLedger:
|
||||
records: list[LedgerRecord] = []
|
||||
with Path(path).open("r", encoding="utf-8") as handle:
|
||||
for line_number, line in enumerate(handle, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(line)
|
||||
records.append(LedgerRecord(**raw))
|
||||
except (TypeError, json.JSONDecodeError) as error:
|
||||
raise LedgerIntegrityError(
|
||||
f"invalid ledger line {line_number}: {error}"
|
||||
) from error
|
||||
return cls(records)
|
||||
@@ -0,0 +1,425 @@
|
||||
"""Long-only portfolio construction with deterministic and CVXPY paths."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .portable_core import normalize_symbol
|
||||
from .universe import UniverseState
|
||||
|
||||
|
||||
class OptimizationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortfolioCandidate:
|
||||
symbol: str
|
||||
score: float
|
||||
variance: float
|
||||
current_weight: float
|
||||
cost_bps: float
|
||||
max_adv_weight: float
|
||||
industry: str
|
||||
state: UniverseState = UniverseState.OPENABLE
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "symbol", normalize_symbol(self.symbol))
|
||||
if not isinstance(self.state, UniverseState):
|
||||
raise OptimizationError("candidate state must be UniverseState")
|
||||
if not str(self.industry).strip():
|
||||
raise OptimizationError("industry is required")
|
||||
for name in (
|
||||
"score",
|
||||
"variance",
|
||||
"current_weight",
|
||||
"cost_bps",
|
||||
"max_adv_weight",
|
||||
):
|
||||
value = float(getattr(self, name))
|
||||
if not math.isfinite(value):
|
||||
raise OptimizationError(f"{name} must be finite")
|
||||
if self.variance <= 0:
|
||||
raise OptimizationError("variance must be positive")
|
||||
if self.current_weight < 0 or self.max_adv_weight < 0:
|
||||
raise OptimizationError("weights/capacity must be non-negative")
|
||||
if self.cost_bps < 0:
|
||||
raise OptimizationError("cost_bps must be non-negative")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortfolioConstraints:
|
||||
gross_target: float = 0.95
|
||||
single_name_cap: float = 0.03
|
||||
industry_cap: float = 0.25
|
||||
one_way_turnover_cap: float = 0.20
|
||||
risk_aversion: float = 1.0
|
||||
cost_aversion: float = 1.0
|
||||
turnover_aversion: float = 0.01
|
||||
variance_floor: float = 1e-8
|
||||
|
||||
def validate(self) -> None:
|
||||
fractions = (
|
||||
"gross_target",
|
||||
"single_name_cap",
|
||||
"industry_cap",
|
||||
"one_way_turnover_cap",
|
||||
)
|
||||
for name in fractions:
|
||||
value = float(getattr(self, name))
|
||||
if not math.isfinite(value) or not (0 <= value <= 1):
|
||||
raise OptimizationError(f"{name} must lie in [0, 1]")
|
||||
if self.single_name_cap <= 0 or self.industry_cap <= 0:
|
||||
raise OptimizationError("single-name and industry caps must be positive")
|
||||
for name in (
|
||||
"risk_aversion",
|
||||
"cost_aversion",
|
||||
"turnover_aversion",
|
||||
):
|
||||
value = float(getattr(self, name))
|
||||
if not math.isfinite(value) or value < 0:
|
||||
raise OptimizationError(f"{name} must be non-negative")
|
||||
if (
|
||||
not math.isfinite(float(self.variance_floor))
|
||||
or self.variance_floor <= 0
|
||||
):
|
||||
raise OptimizationError("variance_floor must be positive")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortfolioSolution:
|
||||
weights: dict[str, float]
|
||||
cash_weight: float
|
||||
gross_weight: float
|
||||
one_way_turnover: float
|
||||
predicted_variance: float
|
||||
expected_score: float
|
||||
estimated_cost_bps_weighted: float
|
||||
solver: str
|
||||
status: str
|
||||
diagnostics: dict[str, Any]
|
||||
|
||||
|
||||
def _validate_candidates(
|
||||
candidates: Iterable[PortfolioCandidate],
|
||||
) -> tuple[PortfolioCandidate, ...]:
|
||||
ordered = tuple(sorted(candidates, key=lambda item: item.symbol))
|
||||
if not ordered:
|
||||
raise OptimizationError("candidate set is empty")
|
||||
seen: set[str] = set()
|
||||
for item in ordered:
|
||||
if item.symbol in seen:
|
||||
raise OptimizationError(f"duplicate candidate {item.symbol}")
|
||||
seen.add(item.symbol)
|
||||
if item.state == UniverseState.EXCLUDED and item.current_weight > 1e-12:
|
||||
raise OptimizationError(
|
||||
f"held position {item.symbol} cannot disappear as EXCLUDED"
|
||||
)
|
||||
return ordered
|
||||
|
||||
|
||||
def _solution(
|
||||
candidates: tuple[PortfolioCandidate, ...],
|
||||
weights: dict[str, float],
|
||||
*,
|
||||
solver: str,
|
||||
status: str,
|
||||
diagnostics: dict[str, Any],
|
||||
) -> PortfolioSolution:
|
||||
gross = sum(weights.values())
|
||||
current = {item.symbol: item.current_weight for item in candidates}
|
||||
turnover = 0.5 * sum(
|
||||
abs(weights.get(symbol, 0.0) - current.get(symbol, 0.0))
|
||||
for symbol in set(weights) | set(current)
|
||||
)
|
||||
predicted_variance = sum(
|
||||
item.variance * weights[item.symbol] ** 2 for item in candidates
|
||||
)
|
||||
expected_score = sum(
|
||||
item.score * weights[item.symbol] for item in candidates
|
||||
)
|
||||
weighted_cost = sum(
|
||||
item.cost_bps
|
||||
* abs(weights[item.symbol] - item.current_weight)
|
||||
for item in candidates
|
||||
)
|
||||
return PortfolioSolution(
|
||||
weights=dict(sorted(weights.items())),
|
||||
cash_weight=max(0.0, 1.0 - gross),
|
||||
gross_weight=gross,
|
||||
one_way_turnover=turnover,
|
||||
predicted_variance=predicted_variance,
|
||||
expected_score=expected_score,
|
||||
estimated_cost_bps_weighted=weighted_cost,
|
||||
solver=solver,
|
||||
status=status,
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
|
||||
|
||||
def _post_check(
|
||||
candidates: tuple[PortfolioCandidate, ...],
|
||||
constraints: PortfolioConstraints,
|
||||
weights: dict[str, float],
|
||||
) -> None:
|
||||
tolerance = 1e-8
|
||||
by_symbol = {item.symbol: item for item in candidates}
|
||||
for symbol, weight in weights.items():
|
||||
item = by_symbol[symbol]
|
||||
if not math.isfinite(weight) or weight < -tolerance:
|
||||
raise OptimizationError(f"invalid target weight for {symbol}")
|
||||
cap = min(constraints.single_name_cap, item.max_adv_weight)
|
||||
if item.state == UniverseState.OPENABLE and weight > cap + tolerance:
|
||||
raise OptimizationError(f"target exceeds cap for {symbol}")
|
||||
if item.state == UniverseState.FROZEN and (
|
||||
abs(weight - item.current_weight) > tolerance
|
||||
):
|
||||
raise OptimizationError(f"frozen weight changed for {symbol}")
|
||||
if item.state == UniverseState.HOLD_ONLY and (
|
||||
weight > item.current_weight + tolerance
|
||||
):
|
||||
raise OptimizationError(f"hold-only weight increased for {symbol}")
|
||||
if item.state in {UniverseState.SELL_ONLY, UniverseState.EXCLUDED} and (
|
||||
weight > item.current_weight + tolerance
|
||||
):
|
||||
raise OptimizationError(f"exit-only weight increased for {symbol}")
|
||||
industries: dict[str, float] = {}
|
||||
for item in candidates:
|
||||
industries[item.industry] = (
|
||||
industries.get(item.industry, 0.0) + weights[item.symbol]
|
||||
)
|
||||
if any(
|
||||
weight > constraints.industry_cap + tolerance
|
||||
for weight in industries.values()
|
||||
):
|
||||
raise OptimizationError("industry cap violated")
|
||||
gross = sum(weights.values())
|
||||
current_gross = sum(item.current_weight for item in candidates)
|
||||
# A pre-existing breach may only decline; it cannot always be removed in a
|
||||
# single turnover-constrained rebalance.
|
||||
allowed_gross = max(constraints.gross_target, current_gross)
|
||||
if gross > allowed_gross + tolerance:
|
||||
raise OptimizationError("gross cap violated")
|
||||
turnover = 0.5 * sum(
|
||||
abs(weights[item.symbol] - item.current_weight) for item in candidates
|
||||
)
|
||||
if turnover > constraints.one_way_turnover_cap + tolerance:
|
||||
raise OptimizationError("turnover cap violated")
|
||||
|
||||
|
||||
def optimize_deterministic(
|
||||
candidates: Iterable[PortfolioCandidate],
|
||||
constraints: PortfolioConstraints = PortfolioConstraints(),
|
||||
) -> PortfolioSolution:
|
||||
"""Deterministic risk/cost-adjusted allocation with conservative fallbacks."""
|
||||
|
||||
constraints.validate()
|
||||
ordered = _validate_candidates(candidates)
|
||||
weights = {item.symbol: 0.0 for item in ordered}
|
||||
|
||||
# Frozen and hold-only positions survive the decision. Sell-only positions
|
||||
# are eligible to reduce toward zero; excluded unheld names stay zero.
|
||||
for item in ordered:
|
||||
if item.state in {UniverseState.FROZEN, UniverseState.HOLD_ONLY}:
|
||||
weights[item.symbol] = item.current_weight
|
||||
|
||||
fixed_gross = sum(weights.values())
|
||||
available = max(0.0, constraints.gross_target - fixed_gross)
|
||||
active: dict[str, float] = {}
|
||||
caps: dict[str, float] = {}
|
||||
by_symbol = {item.symbol: item for item in ordered}
|
||||
for item in ordered:
|
||||
if item.state != UniverseState.OPENABLE:
|
||||
continue
|
||||
net_score = item.score - (
|
||||
constraints.cost_aversion * item.cost_bps / 10_000.0
|
||||
)
|
||||
utility = max(0.0, net_score) / max(
|
||||
constraints.variance_floor,
|
||||
item.variance * max(constraints.risk_aversion, 1e-12),
|
||||
)
|
||||
if utility <= 0:
|
||||
continue
|
||||
active[item.symbol] = utility
|
||||
caps[item.symbol] = min(
|
||||
constraints.single_name_cap,
|
||||
item.max_adv_weight,
|
||||
)
|
||||
|
||||
remaining = available
|
||||
while active and remaining > 1e-12:
|
||||
utility_sum = sum(active.values())
|
||||
if utility_sum <= 0:
|
||||
break
|
||||
capped: list[str] = []
|
||||
for symbol in sorted(active):
|
||||
proposal = remaining * active[symbol] / utility_sum
|
||||
room = max(0.0, caps[symbol] - weights[symbol])
|
||||
if proposal >= room - 1e-12:
|
||||
weights[symbol] += room
|
||||
remaining -= room
|
||||
capped.append(symbol)
|
||||
if not capped:
|
||||
for symbol in sorted(active):
|
||||
weights[symbol] += remaining * active[symbol] / utility_sum
|
||||
remaining = 0.0
|
||||
break
|
||||
for symbol in capped:
|
||||
active.pop(symbol)
|
||||
remaining = max(0.0, remaining)
|
||||
|
||||
# Industry limits are hard post-allocation constraints. Scaling leaves the
|
||||
# unused amount as cash instead of inventing a second optimizer pass.
|
||||
industry_totals: dict[str, float] = {}
|
||||
for item in ordered:
|
||||
industry_totals[item.industry] = (
|
||||
industry_totals.get(item.industry, 0.0) + weights[item.symbol]
|
||||
)
|
||||
for industry, total in sorted(industry_totals.items()):
|
||||
if total <= constraints.industry_cap + 1e-12:
|
||||
continue
|
||||
fixed = sum(
|
||||
weights[item.symbol]
|
||||
for item in ordered
|
||||
if item.industry == industry
|
||||
and item.state in {UniverseState.FROZEN, UniverseState.HOLD_ONLY}
|
||||
)
|
||||
if fixed > constraints.industry_cap + 1e-12:
|
||||
raise OptimizationError(
|
||||
f"fixed {industry} exposure exceeds the industry cap"
|
||||
)
|
||||
scalable = total - fixed
|
||||
factor = (
|
||||
(constraints.industry_cap - fixed) / scalable
|
||||
if scalable > 0
|
||||
else 0.0
|
||||
)
|
||||
for item in ordered:
|
||||
if (
|
||||
item.industry == industry
|
||||
and item.state not in {UniverseState.FROZEN, UniverseState.HOLD_ONLY}
|
||||
):
|
||||
weights[item.symbol] *= max(0.0, factor)
|
||||
|
||||
raw_turnover = 0.5 * sum(
|
||||
abs(weights[item.symbol] - item.current_weight) for item in ordered
|
||||
)
|
||||
turnover_scaled = False
|
||||
if raw_turnover > constraints.one_way_turnover_cap + 1e-12:
|
||||
scale = constraints.one_way_turnover_cap / raw_turnover
|
||||
for item in ordered:
|
||||
delta = weights[item.symbol] - item.current_weight
|
||||
weights[item.symbol] = item.current_weight + delta * scale
|
||||
turnover_scaled = True
|
||||
|
||||
_post_check(ordered, constraints, weights)
|
||||
return _solution(
|
||||
ordered,
|
||||
weights,
|
||||
solver="deterministic_reference",
|
||||
status="OPTIMAL_REFERENCE",
|
||||
diagnostics={
|
||||
"unallocated_cash_from_caps": remaining,
|
||||
"raw_one_way_turnover": raw_turnover,
|
||||
"turnover_scaled": turnover_scaled,
|
||||
"candidate_count": len(ordered),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def optimize_cvxpy(
|
||||
candidates: Iterable[PortfolioCandidate],
|
||||
constraints: PortfolioConstraints = PortfolioConstraints(),
|
||||
*,
|
||||
solver: str | None = None,
|
||||
) -> PortfolioSolution:
|
||||
"""Solve the declared convex surrogate when CVXPY is available."""
|
||||
|
||||
constraints.validate()
|
||||
ordered = _validate_candidates(candidates)
|
||||
try:
|
||||
import cvxpy as cp
|
||||
except ImportError as exc:
|
||||
raise OptimizationError(
|
||||
"CVXPY is unavailable; install the full research environment "
|
||||
"or use optimize_deterministic"
|
||||
) from exc
|
||||
|
||||
count = len(ordered)
|
||||
weight = cp.Variable(count)
|
||||
current = [item.current_weight for item in ordered]
|
||||
alpha = [item.score for item in ordered]
|
||||
variance = [item.variance for item in ordered]
|
||||
cost_rate = [item.cost_bps / 10_000.0 for item in ordered]
|
||||
delta = weight - current
|
||||
objective = cp.Maximize(
|
||||
alpha @ weight
|
||||
- constraints.risk_aversion
|
||||
* cp.sum(cp.multiply(variance, cp.square(weight)))
|
||||
- constraints.cost_aversion
|
||||
* cp.sum(cp.multiply(cost_rate, cp.abs(delta)))
|
||||
- constraints.turnover_aversion * cp.norm1(delta)
|
||||
)
|
||||
limits = [
|
||||
weight >= 0,
|
||||
cp.sum(weight) <= constraints.gross_target,
|
||||
0.5 * cp.norm1(delta) <= constraints.one_way_turnover_cap,
|
||||
]
|
||||
for index, item in enumerate(ordered):
|
||||
if item.state == UniverseState.OPENABLE:
|
||||
limits.append(
|
||||
weight[index]
|
||||
<= min(constraints.single_name_cap, item.max_adv_weight)
|
||||
)
|
||||
elif item.state == UniverseState.FROZEN:
|
||||
limits.append(weight[index] == item.current_weight)
|
||||
elif item.state == UniverseState.HOLD_ONLY:
|
||||
limits.append(weight[index] <= item.current_weight)
|
||||
elif item.state in {UniverseState.SELL_ONLY, UniverseState.EXCLUDED}:
|
||||
limits.append(weight[index] <= item.current_weight)
|
||||
for industry in sorted({item.industry for item in ordered}):
|
||||
indexes = [
|
||||
index
|
||||
for index, item in enumerate(ordered)
|
||||
if item.industry == industry
|
||||
]
|
||||
limits.append(cp.sum(weight[indexes]) <= constraints.industry_cap)
|
||||
problem = cp.Problem(objective, limits)
|
||||
selected_solver = solver
|
||||
if selected_solver is None:
|
||||
installed = set(cp.installed_solvers())
|
||||
selected_solver = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in ("CLARABEL", "ECOS", "SCS")
|
||||
if candidate in installed
|
||||
),
|
||||
None,
|
||||
)
|
||||
solve_kwargs: dict[str, Any] = {"warm_start": False, "verbose": False}
|
||||
if selected_solver is not None:
|
||||
solve_kwargs["solver"] = selected_solver
|
||||
try:
|
||||
problem.solve(**solve_kwargs)
|
||||
except Exception as exc:
|
||||
raise OptimizationError(f"CVXPY solve failed: {exc}") from exc
|
||||
if problem.status not in {"optimal", "optimal_inaccurate"}:
|
||||
raise OptimizationError(f"CVXPY returned {problem.status}")
|
||||
values = [float(item) for item in weight.value]
|
||||
weights = {
|
||||
item.symbol: max(0.0, values[index])
|
||||
for index, item in enumerate(ordered)
|
||||
}
|
||||
_post_check(ordered, constraints, weights)
|
||||
return _solution(
|
||||
ordered,
|
||||
weights,
|
||||
solver=f"cvxpy:{selected_solver or 'default'}",
|
||||
status=str(problem.status).upper(),
|
||||
diagnostics={
|
||||
"objective": float(problem.value),
|
||||
"candidate_count": count,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Point-in-time records, joins, quality checks, and deterministic versions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .ledger import canonical_json, jsonable
|
||||
|
||||
|
||||
class PITIntegrityError(ValueError):
|
||||
"""Point-in-time data is ambiguous, malformed, or future contaminated."""
|
||||
|
||||
|
||||
def _aware_datetime(value: datetime | str, name: str) -> datetime:
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise PITIntegrityError(f"{name} must be an ISO date-time") from exc
|
||||
elif isinstance(value, datetime):
|
||||
parsed = value
|
||||
else:
|
||||
raise PITIntegrityError(f"{name} must be a datetime or ISO string")
|
||||
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
||||
raise PITIntegrityError(f"{name} must include a timezone")
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PITRecord:
|
||||
"""One field value with separate economic and information timestamps."""
|
||||
|
||||
entity_id: str
|
||||
field: str
|
||||
effective_time: datetime
|
||||
available_time: datetime
|
||||
value: Any
|
||||
source: str
|
||||
source_version: str
|
||||
revision: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
entity = str(self.entity_id).strip()
|
||||
field_name = str(self.field).strip()
|
||||
source = str(self.source).strip()
|
||||
source_version = str(self.source_version).strip()
|
||||
if not entity or not field_name or not source or not source_version:
|
||||
raise PITIntegrityError(
|
||||
"entity_id, field, source and source_version are required"
|
||||
)
|
||||
if type(self.revision) is not int or self.revision < 0:
|
||||
raise PITIntegrityError("revision must be a non-negative integer")
|
||||
if isinstance(self.value, float) and not math.isfinite(self.value):
|
||||
raise PITIntegrityError("PIT numeric values must be finite")
|
||||
try:
|
||||
jsonable(self.value)
|
||||
except TypeError as exc:
|
||||
raise PITIntegrityError("PIT value must be JSON serializable") from exc
|
||||
object.__setattr__(self, "entity_id", entity)
|
||||
object.__setattr__(self, "field", field_name)
|
||||
object.__setattr__(self, "source", source)
|
||||
object.__setattr__(self, "source_version", source_version)
|
||||
object.__setattr__(
|
||||
self,
|
||||
"effective_time",
|
||||
_aware_datetime(self.effective_time, "effective_time"),
|
||||
)
|
||||
object.__setattr__(
|
||||
self,
|
||||
"available_time",
|
||||
_aware_datetime(self.available_time, "available_time"),
|
||||
)
|
||||
|
||||
@property
|
||||
def identity(self) -> tuple[str, str, datetime, datetime, int]:
|
||||
return (
|
||||
self.entity_id,
|
||||
self.field,
|
||||
self.effective_time,
|
||||
self.available_time,
|
||||
self.revision,
|
||||
)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"entity_id": self.entity_id,
|
||||
"field": self.field,
|
||||
"effective_time": self.effective_time.isoformat(),
|
||||
"available_time": self.available_time.isoformat(),
|
||||
"value": jsonable(self.value),
|
||||
"source": self.source,
|
||||
"source_version": self.source_version,
|
||||
"revision": self.revision,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PITValue:
|
||||
entity_id: str
|
||||
field: str
|
||||
value: Any
|
||||
effective_time: datetime
|
||||
available_time: datetime
|
||||
source: str
|
||||
source_version: str
|
||||
revision: int
|
||||
|
||||
@classmethod
|
||||
def from_record(cls, record: PITRecord) -> "PITValue":
|
||||
return cls(
|
||||
entity_id=record.entity_id,
|
||||
field=record.field,
|
||||
value=record.value,
|
||||
effective_time=record.effective_time,
|
||||
available_time=record.available_time,
|
||||
source=record.source,
|
||||
source_version=record.source_version,
|
||||
revision=record.revision,
|
||||
)
|
||||
|
||||
|
||||
class PointInTimeTable:
|
||||
"""Immutable in-memory reference table used by adapters and tests."""
|
||||
|
||||
def __init__(self, records: Iterable[PITRecord]) -> None:
|
||||
ordered = sorted(
|
||||
tuple(records),
|
||||
key=lambda record: (
|
||||
record.entity_id,
|
||||
record.field,
|
||||
record.effective_time,
|
||||
record.available_time,
|
||||
record.revision,
|
||||
record.source,
|
||||
record.source_version,
|
||||
),
|
||||
)
|
||||
seen: set[tuple[str, str, datetime, datetime, int]] = set()
|
||||
by_key: dict[tuple[str, str], list[PITRecord]] = {}
|
||||
for record in ordered:
|
||||
if not isinstance(record, PITRecord):
|
||||
raise PITIntegrityError("all entries must be PITRecord instances")
|
||||
if record.identity in seen:
|
||||
raise PITIntegrityError(
|
||||
"duplicate PIT identity for "
|
||||
f"{record.entity_id}.{record.field}"
|
||||
)
|
||||
seen.add(record.identity)
|
||||
by_key.setdefault((record.entity_id, record.field), []).append(record)
|
||||
self._records = ordered
|
||||
self._by_key = by_key
|
||||
self._data_version = hashlib.sha256(
|
||||
canonical_json([record.as_dict() for record in ordered]).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
@property
|
||||
def records(self) -> tuple[PITRecord, ...]:
|
||||
return self._records
|
||||
|
||||
@property
|
||||
def data_version(self) -> str:
|
||||
return self._data_version
|
||||
|
||||
def query(
|
||||
self,
|
||||
entity_id: str,
|
||||
field: str,
|
||||
*,
|
||||
as_of: datetime | str,
|
||||
effective_at: datetime | str | None = None,
|
||||
) -> PITValue | None:
|
||||
information_clock = _aware_datetime(as_of, "as_of")
|
||||
economic_clock = (
|
||||
information_clock
|
||||
if effective_at is None
|
||||
else _aware_datetime(effective_at, "effective_at")
|
||||
)
|
||||
candidates = [
|
||||
record
|
||||
for record in self._by_key.get(
|
||||
(str(entity_id).strip(), str(field).strip()),
|
||||
(),
|
||||
)
|
||||
if record.available_time <= information_clock
|
||||
and record.effective_time <= economic_clock
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
selected = max(
|
||||
candidates,
|
||||
key=lambda record: (
|
||||
record.effective_time,
|
||||
record.available_time,
|
||||
record.revision,
|
||||
record.source_version,
|
||||
),
|
||||
)
|
||||
return PITValue.from_record(selected)
|
||||
|
||||
def snapshot(
|
||||
self,
|
||||
*,
|
||||
as_of: datetime | str,
|
||||
effective_at: datetime | str | None = None,
|
||||
entities: Iterable[str] | None = None,
|
||||
fields: Iterable[str] | None = None,
|
||||
) -> dict[str, dict[str, PITValue]]:
|
||||
requested_entities = (
|
||||
{str(entity).strip() for entity in entities}
|
||||
if entities is not None
|
||||
else {entity for entity, unused in self._by_key}
|
||||
)
|
||||
requested_fields = (
|
||||
{str(field).strip() for field in fields}
|
||||
if fields is not None
|
||||
else {field for unused, field in self._by_key}
|
||||
)
|
||||
result: dict[str, dict[str, PITValue]] = {}
|
||||
for entity in sorted(requested_entities):
|
||||
for field in sorted(requested_fields):
|
||||
value = self.query(
|
||||
entity,
|
||||
field,
|
||||
as_of=as_of,
|
||||
effective_at=effective_at,
|
||||
)
|
||||
if value is not None:
|
||||
result.setdefault(entity, {})[field] = value
|
||||
return result
|
||||
|
||||
def quality_report(self) -> dict[str, Any]:
|
||||
sources = sorted(
|
||||
{
|
||||
f"{record.source}@{record.source_version}"
|
||||
for record in self._records
|
||||
}
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"record_count": len(self._records),
|
||||
"entity_count": len({record.entity_id for record in self._records}),
|
||||
"field_count": len({record.field for record in self._records}),
|
||||
"sources": sources,
|
||||
"data_version": self.data_version,
|
||||
}
|
||||
|
||||
|
||||
def assert_information_available(
|
||||
values: Iterable[PITRecord | PITValue],
|
||||
*,
|
||||
as_of: datetime | str,
|
||||
) -> None:
|
||||
"""Fail if any supplied value was unavailable at the decision timestamp."""
|
||||
|
||||
decision_time = _aware_datetime(as_of, "as_of")
|
||||
future = [
|
||||
f"{value.entity_id}.{value.field}@{value.available_time.isoformat()}"
|
||||
for value in values
|
||||
if value.available_time > decision_time
|
||||
]
|
||||
if future:
|
||||
raise PITIntegrityError(
|
||||
"future-available values reached the decision: " + ", ".join(future)
|
||||
)
|
||||
@@ -0,0 +1,374 @@
|
||||
"""Small strategy core that also runs on Python 3.6 platform runtimes.
|
||||
|
||||
Keep this module deliberately boring: no dataclasses, no third-party imports,
|
||||
no modern annotation syntax. JoinQuant/QMT/Qlib adapters can vendor this
|
||||
single file when importing the complete ``quant60`` package is not possible.
|
||||
"""
|
||||
|
||||
from __future__ import division
|
||||
|
||||
import math
|
||||
import re
|
||||
|
||||
|
||||
_CANONICAL_EXCHANGES = {
|
||||
"SH": "XSHG",
|
||||
"SSE": "XSHG",
|
||||
"XSHG": "XSHG",
|
||||
"SZ": "XSHE",
|
||||
"SZE": "XSHE",
|
||||
"SZSE": "XSHE",
|
||||
"XSHE": "XSHE",
|
||||
"BJ": "XBSE",
|
||||
"BSE": "XBSE",
|
||||
"XBSE": "XBSE",
|
||||
}
|
||||
|
||||
_SHORT_EXCHANGES = {
|
||||
"XSHG": "SH",
|
||||
"XSHE": "SZ",
|
||||
"XBSE": "BJ",
|
||||
}
|
||||
|
||||
|
||||
def _sum_numeric(values):
|
||||
"""Sum numerics without trusting a hosted runtime's global ``sum`` name."""
|
||||
|
||||
total = 0.0
|
||||
for value in values:
|
||||
total += value
|
||||
return total
|
||||
|
||||
|
||||
def _infer_exchange(code):
|
||||
if code.startswith(("4", "8")):
|
||||
return "XBSE"
|
||||
if code.startswith(("5", "6", "9")):
|
||||
return "XSHG"
|
||||
return "XSHE"
|
||||
|
||||
|
||||
def _split_symbol(symbol):
|
||||
if not isinstance(symbol, str):
|
||||
raise TypeError("symbol must be a string")
|
||||
value = symbol.strip().upper().replace(" ", "")
|
||||
if not value:
|
||||
raise ValueError("symbol must not be empty")
|
||||
|
||||
match = re.match(r"^(\d{6})\.(XSHG|XSHE|XBSE|SH|SZ|BJ)$", value)
|
||||
if match:
|
||||
return match.group(1), _CANONICAL_EXCHANGES[match.group(2)]
|
||||
|
||||
match = re.match(r"^(XSHG|XSHE|XBSE|SH|SZ|BJ)[\.:]?(\d{6})$", value)
|
||||
if match:
|
||||
return match.group(2), _CANONICAL_EXCHANGES[match.group(1)]
|
||||
|
||||
match = re.match(r"^(\d{6})[\.:](SSE|SZE|SZSE|BSE)$", value)
|
||||
if match:
|
||||
return match.group(1), _CANONICAL_EXCHANGES[match.group(2)]
|
||||
|
||||
if re.match(r"^\d{6}$", value):
|
||||
return value, _infer_exchange(value)
|
||||
raise ValueError("unsupported A-share symbol: %s" % symbol)
|
||||
|
||||
|
||||
def normalize_symbol(symbol, target="canonical"):
|
||||
"""Convert common A-share codes to canonical/JQ, QMT, or Qlib format."""
|
||||
|
||||
code, exchange = _split_symbol(symbol)
|
||||
target_name = str(target).strip().lower()
|
||||
if target_name in ("canonical", "joinquant", "jq"):
|
||||
return "%s.%s" % (code, exchange)
|
||||
if target_name in ("qmt", "xtquant"):
|
||||
return "%s.%s" % (code, _SHORT_EXCHANGES[exchange])
|
||||
if target_name == "qlib":
|
||||
return "%s%s" % (_SHORT_EXCHANGES[exchange], code)
|
||||
raise ValueError("unsupported target platform: %s" % target)
|
||||
|
||||
|
||||
def convert_symbol(symbol, target="canonical"):
|
||||
"""Alias kept as the stable adapter-facing name."""
|
||||
|
||||
return normalize_symbol(symbol, target)
|
||||
|
||||
|
||||
def momentum_score(prices, lookback=20, skip=0):
|
||||
"""Return trailing simple momentum, or ``None`` if history is insufficient.
|
||||
|
||||
``skip=0`` uses the latest completed decision bar. Set ``skip=1`` only
|
||||
when intentionally implementing a 20-1 convention. A lookback of N needs
|
||||
N+skip+1 observations.
|
||||
"""
|
||||
|
||||
lookback = int(lookback)
|
||||
skip = int(skip)
|
||||
if lookback <= 0 or skip < 0:
|
||||
raise ValueError("lookback must be positive and skip non-negative")
|
||||
values = list(prices)
|
||||
if len(values) < lookback + skip + 1:
|
||||
return None
|
||||
end_index = len(values) - 1 - skip
|
||||
start_index = end_index - lookback
|
||||
start = float(values[start_index])
|
||||
end = float(values[end_index])
|
||||
if (
|
||||
start <= 0.0
|
||||
or end <= 0.0
|
||||
or not math.isfinite(start)
|
||||
or not math.isfinite(end)
|
||||
):
|
||||
return None
|
||||
return end / start - 1.0
|
||||
|
||||
|
||||
def rank_momentum(price_history, lookback=20, skip=0, top_n=None):
|
||||
"""Rank ``{symbol: closes}`` by momentum with deterministic tie-breaking."""
|
||||
|
||||
ranked = []
|
||||
for symbol in sorted(price_history):
|
||||
score = momentum_score(price_history[symbol], lookback, skip)
|
||||
if score is not None and math.isfinite(score):
|
||||
ranked.append((normalize_symbol(symbol), score))
|
||||
ranked.sort(key=lambda item: (-item[1], item[0]))
|
||||
if top_n is not None:
|
||||
ranked = ranked[: max(0, int(top_n))]
|
||||
return ranked
|
||||
|
||||
|
||||
def capped_target_weights(
|
||||
scores, max_weight=0.20, gross_target=1.0, min_score=0.0
|
||||
):
|
||||
"""Long-only score-proportional weights with iterative cap redistribution."""
|
||||
|
||||
max_weight = float(max_weight)
|
||||
gross_target = float(gross_target)
|
||||
min_score = float(min_score)
|
||||
if not math.isfinite(max_weight) or not 0.0 < max_weight <= 1.0:
|
||||
raise ValueError("max_weight must be finite and lie in (0, 1]")
|
||||
if not math.isfinite(gross_target) or not 0.0 <= gross_target <= 1.0:
|
||||
raise ValueError("gross_target must be finite and lie in [0, 1]")
|
||||
if not math.isfinite(min_score):
|
||||
raise ValueError("min_score must be finite")
|
||||
|
||||
if hasattr(scores, "items"):
|
||||
items = list(scores.items())
|
||||
else:
|
||||
items = list(scores)
|
||||
normalized = {}
|
||||
for symbol, raw_score in items:
|
||||
name = normalize_symbol(symbol)
|
||||
score = float(raw_score)
|
||||
if not math.isfinite(score):
|
||||
raise ValueError("score must be finite for %s" % name)
|
||||
normalized[name] = score
|
||||
|
||||
weights = dict((symbol, 0.0) for symbol in normalized)
|
||||
active = dict(
|
||||
(symbol, score)
|
||||
for symbol, score in normalized.items()
|
||||
if score > min_score and score > 0.0
|
||||
)
|
||||
remaining = min(gross_target, len(active) * max_weight)
|
||||
|
||||
while active and remaining > 1e-15:
|
||||
score_sum = _sum_numeric(active.values())
|
||||
if score_sum <= 0.0:
|
||||
break
|
||||
capped = []
|
||||
for symbol in sorted(active):
|
||||
proposed = remaining * active[symbol] / score_sum
|
||||
if proposed >= max_weight - 1e-15:
|
||||
weights[symbol] = max_weight
|
||||
capped.append(symbol)
|
||||
if not capped:
|
||||
for symbol in sorted(active):
|
||||
weights[symbol] = remaining * active[symbol] / score_sum
|
||||
remaining = 0.0
|
||||
break
|
||||
for symbol in capped:
|
||||
active.pop(symbol)
|
||||
remaining -= max_weight
|
||||
remaining = max(0.0, remaining)
|
||||
return weights
|
||||
|
||||
|
||||
def round_board_lot(quantity, lot_size=100):
|
||||
"""Round an absolute long position down to a board-lot quantity."""
|
||||
|
||||
lot_size = int(lot_size)
|
||||
if lot_size <= 0:
|
||||
raise ValueError("lot_size must be positive")
|
||||
if not math.isfinite(float(quantity)):
|
||||
raise ValueError("quantity must be finite")
|
||||
quantity = max(0, int(quantity))
|
||||
return quantity // lot_size * lot_size
|
||||
|
||||
|
||||
def _is_star_market(symbol):
|
||||
code, exchange = _split_symbol(symbol)
|
||||
return exchange == "XSHG" and code.startswith(("688", "689"))
|
||||
|
||||
|
||||
def target_quantities(
|
||||
weights, prices, equity, lot_size=100, cash_buffer=0.02
|
||||
):
|
||||
"""Convert target weights to affordable board-lot long quantities."""
|
||||
|
||||
equity = float(equity)
|
||||
cash_buffer = float(cash_buffer)
|
||||
if (
|
||||
not math.isfinite(equity)
|
||||
or equity < 0.0
|
||||
or not math.isfinite(cash_buffer)
|
||||
or not 0.0 <= cash_buffer < 1.0
|
||||
):
|
||||
raise ValueError("invalid equity or cash_buffer")
|
||||
positive_weights = []
|
||||
for raw_symbol, raw_weight in weights.items():
|
||||
weight = float(raw_weight)
|
||||
if not math.isfinite(weight):
|
||||
raise ValueError(
|
||||
"weight must be finite for %s" % normalize_symbol(raw_symbol)
|
||||
)
|
||||
positive_weights.append(max(0.0, weight))
|
||||
requested_gross = _sum_numeric(positive_weights)
|
||||
gross_ceiling = 1.0 - cash_buffer
|
||||
scale = (
|
||||
min(1.0, gross_ceiling / requested_gross)
|
||||
if requested_gross > 0.0
|
||||
else 1.0
|
||||
)
|
||||
result = {}
|
||||
for raw_symbol in sorted(weights):
|
||||
symbol = normalize_symbol(raw_symbol)
|
||||
weight = float(weights[raw_symbol])
|
||||
if not math.isfinite(weight):
|
||||
raise ValueError("weight must be finite for %s" % symbol)
|
||||
weight = max(0.0, weight)
|
||||
price = prices.get(raw_symbol)
|
||||
if price is None:
|
||||
price = prices.get(symbol)
|
||||
if price is None:
|
||||
raise KeyError("missing price for %s" % symbol)
|
||||
price = float(price)
|
||||
if price <= 0.0 or not math.isfinite(price):
|
||||
raise ValueError("invalid price for %s" % symbol)
|
||||
# Weights are absolute equity weights. cash_buffer is a ceiling, not
|
||||
# a second multiplicative haircut: gross=.95 and buffer=.02 stays .95.
|
||||
budget = equity * weight * scale
|
||||
quantity = round_board_lot(budget / price, lot_size)
|
||||
# STAR Market auction orders must contain at least 200 shares. Keep
|
||||
# the portable planner conservative and on the configured 100-share
|
||||
# grid even though quantities above 200 may increment by one share.
|
||||
if _is_star_market(symbol) and 0 < quantity < 200:
|
||||
quantity = 0
|
||||
result[symbol] = quantity
|
||||
return result
|
||||
|
||||
|
||||
def order_deltas(
|
||||
targets, current, sellable=None, lot_size=100
|
||||
):
|
||||
"""Return signed board-lot orders; negative sells respect sellable quantity."""
|
||||
|
||||
canonical_targets = {}
|
||||
canonical_current = {}
|
||||
canonical_sellable = {}
|
||||
for symbol, quantity in targets.items():
|
||||
canonical_targets[normalize_symbol(symbol)] = max(0, int(quantity))
|
||||
for symbol, quantity in current.items():
|
||||
canonical_current[normalize_symbol(symbol)] = max(0, int(quantity))
|
||||
if sellable is not None:
|
||||
for symbol, quantity in sellable.items():
|
||||
canonical_sellable[normalize_symbol(symbol)] = max(0, int(quantity))
|
||||
|
||||
result = {}
|
||||
symbols = sorted(set(canonical_targets) | set(canonical_current))
|
||||
for symbol in symbols:
|
||||
target = round_board_lot(canonical_targets.get(symbol, 0), lot_size)
|
||||
if _is_star_market(symbol) and 0 < target < 200:
|
||||
target = 0
|
||||
held = canonical_current.get(symbol, 0)
|
||||
delta = target - held
|
||||
if delta > 0:
|
||||
delta = round_board_lot(delta, lot_size)
|
||||
if _is_star_market(symbol) and 0 < delta < 200:
|
||||
delta = 0
|
||||
elif delta < 0:
|
||||
available = held
|
||||
if sellable is not None:
|
||||
available = min(held, canonical_sellable.get(symbol, 0))
|
||||
requested = min(-delta, available)
|
||||
# A complete liquidation may submit the entire odd-lot balance.
|
||||
# This also covers the STAR exception for a remaining balance
|
||||
# below 200 shares. Partial orders stay on the configured grid.
|
||||
full_liquidation = (
|
||||
target == 0
|
||||
and requested == held
|
||||
and available >= held
|
||||
)
|
||||
if full_liquidation:
|
||||
delta = -held
|
||||
else:
|
||||
delta = -round_board_lot(requested, lot_size)
|
||||
if _is_star_market(symbol) and 0 < -delta < 200:
|
||||
delta = 0
|
||||
if delta:
|
||||
result[symbol] = delta
|
||||
return result
|
||||
|
||||
|
||||
def build_rebalance_plan(
|
||||
price_history,
|
||||
current,
|
||||
sellable,
|
||||
equity,
|
||||
lookback=20,
|
||||
skip=0,
|
||||
top_n=10,
|
||||
max_weight=0.20,
|
||||
gross_target=1.0,
|
||||
cash_buffer=0.02,
|
||||
lot_size=100,
|
||||
):
|
||||
"""Build the complete portable signal -> weight -> quantity -> order plan."""
|
||||
|
||||
ranked = rank_momentum(price_history, lookback, skip, top_n)
|
||||
scores = dict(ranked)
|
||||
weights = capped_target_weights(scores, max_weight, gross_target, 0.0)
|
||||
for raw_symbol in price_history:
|
||||
symbol = normalize_symbol(raw_symbol)
|
||||
if symbol not in scores:
|
||||
scores[symbol] = momentum_score(
|
||||
price_history[raw_symbol], lookback, skip
|
||||
)
|
||||
if symbol not in weights:
|
||||
weights[symbol] = 0.0
|
||||
prices = {}
|
||||
for raw_symbol, history in price_history.items():
|
||||
if not history:
|
||||
continue
|
||||
prices[normalize_symbol(raw_symbol)] = history[-1]
|
||||
targets = target_quantities(
|
||||
weights, prices, equity, lot_size, cash_buffer
|
||||
)
|
||||
orders = order_deltas(targets, current, sellable, lot_size)
|
||||
return {
|
||||
"scores": scores,
|
||||
"weights": weights,
|
||||
"targets": targets,
|
||||
"orders": orders,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_rebalance_plan",
|
||||
"capped_target_weights",
|
||||
"convert_symbol",
|
||||
"momentum_score",
|
||||
"normalize_symbol",
|
||||
"order_deltas",
|
||||
"rank_momentum",
|
||||
"round_board_lot",
|
||||
"target_quantities",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,846 @@
|
||||
"""Broker-snapshot reconciliation with explicit fail-closed semantics.
|
||||
|
||||
The original cash/position-only keyword API remains supported. Passing
|
||||
``local_snapshot``/``broker_snapshot`` (or ``require_complete=True``) enables
|
||||
the complete contract: cash, positions, sellable quantities, open orders and
|
||||
fresh, timezone-aware snapshot timestamps must all be present.
|
||||
|
||||
This module validates normalized snapshots. It is not evidence that a real
|
||||
broker's proprietary fields or callback lifecycle have been certified.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
from .domain import decimal
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
_OPEN_STATUSES = {
|
||||
"NEW",
|
||||
"ACCEPTED",
|
||||
"PARTIALLY_FILLED",
|
||||
"CANCEL_PENDING",
|
||||
"UNKNOWN",
|
||||
}
|
||||
_TERMINAL_STATUSES = {"FILLED", "REJECTED", "CANCELLED"}
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReconciliationDifference:
|
||||
kind: str
|
||||
key: str
|
||||
expected: str
|
||||
observed: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReconciliationReport:
|
||||
balanced: bool
|
||||
differences: tuple[ReconciliationDifference, ...]
|
||||
complete: bool = False
|
||||
|
||||
@property
|
||||
def safe_to_open(self) -> bool:
|
||||
"""New opening orders are permitted only for an exact safe report."""
|
||||
|
||||
return self.complete and self.balanced
|
||||
|
||||
|
||||
def _render(value: Any) -> str:
|
||||
if isinstance(value, Decimal):
|
||||
return format(value, "f")
|
||||
if isinstance(value, (dict, list, tuple)):
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
return str(value)
|
||||
|
||||
|
||||
def _difference(
|
||||
differences: list[ReconciliationDifference],
|
||||
kind: str,
|
||||
key: str,
|
||||
expected: Any,
|
||||
observed: Any,
|
||||
) -> None:
|
||||
differences.append(
|
||||
ReconciliationDifference(
|
||||
kind=kind,
|
||||
key=key,
|
||||
expected=_render(expected),
|
||||
observed=_render(observed),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _mapping_value(
|
||||
snapshot: Mapping[str, Any] | None,
|
||||
*names: str,
|
||||
) -> Any:
|
||||
if snapshot is None:
|
||||
return _MISSING
|
||||
for name in names:
|
||||
if name in snapshot:
|
||||
return snapshot[name]
|
||||
return _MISSING
|
||||
|
||||
|
||||
def _quantity(value: Any, label: str) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(f"{label} must be an integer")
|
||||
result = int(value)
|
||||
if result != value and not (
|
||||
isinstance(value, str) and str(result) == value.strip()
|
||||
):
|
||||
raise ValueError(f"{label} must be an integer")
|
||||
if result < 0:
|
||||
raise ValueError(f"{label} must be non-negative")
|
||||
return result
|
||||
|
||||
|
||||
def _position_maps(
|
||||
raw: Any,
|
||||
*,
|
||||
label: str,
|
||||
) -> tuple[dict[str, int], dict[str, int], bool]:
|
||||
"""Return total positions, embedded sellable and sellable-presence."""
|
||||
|
||||
if raw is _MISSING or raw is None:
|
||||
raise ValueError(f"{label} positions are missing")
|
||||
positions: dict[str, int] = {}
|
||||
sellable: dict[str, int] = {}
|
||||
sellable_present = True
|
||||
|
||||
if isinstance(raw, Mapping):
|
||||
if all(not isinstance(value, Mapping) for value in raw.values()):
|
||||
for symbol, value in raw.items():
|
||||
canonical = normalize_symbol(str(symbol))
|
||||
quantity = _quantity(value, f"{label} position {canonical}")
|
||||
if quantity:
|
||||
positions[canonical] = quantity
|
||||
# With no holdings, an empty sellable map is unambiguous. A
|
||||
# non-empty scalar position map still needs an explicit sellable
|
||||
# snapshot in complete mode.
|
||||
return positions, sellable, not raw
|
||||
records: Iterable[Any] = (
|
||||
dict(value, symbol=value.get("symbol", key))
|
||||
if isinstance(value, Mapping)
|
||||
else {"symbol": key, "quantity": value}
|
||||
for key, value in raw.items()
|
||||
)
|
||||
elif isinstance(raw, Iterable) and not isinstance(raw, (str, bytes)):
|
||||
records = raw
|
||||
else:
|
||||
raise ValueError(f"{label} positions must be a mapping or iterable")
|
||||
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(records):
|
||||
if not isinstance(item, Mapping):
|
||||
raise ValueError(f"{label} position #{index} must be an object")
|
||||
symbol = item.get("symbol", item.get("stock_code"))
|
||||
if not symbol:
|
||||
raise ValueError(f"{label} position #{index} has no symbol")
|
||||
canonical = normalize_symbol(str(symbol))
|
||||
if canonical in seen:
|
||||
raise ValueError(f"duplicate {label} position {canonical}")
|
||||
seen.add(canonical)
|
||||
total_raw = item.get(
|
||||
"quantity",
|
||||
item.get("volume", item.get("total_amount", _MISSING)),
|
||||
)
|
||||
if total_raw is _MISSING:
|
||||
raise ValueError(f"{label} position {canonical} has no quantity")
|
||||
total = _quantity(total_raw, f"{label} position {canonical}")
|
||||
if total:
|
||||
positions[canonical] = total
|
||||
available_raw = item.get(
|
||||
"sellable_quantity",
|
||||
item.get(
|
||||
"sellable",
|
||||
item.get("can_use_volume", item.get("closeable_amount", _MISSING)),
|
||||
),
|
||||
)
|
||||
if available_raw is _MISSING:
|
||||
sellable_present = False
|
||||
continue
|
||||
available = _quantity(
|
||||
available_raw,
|
||||
f"{label} sellable {canonical}",
|
||||
)
|
||||
if available > total:
|
||||
raise ValueError(
|
||||
f"{label} sellable {canonical} exceeds total position"
|
||||
)
|
||||
if available:
|
||||
sellable[canonical] = available
|
||||
return positions, sellable, sellable_present
|
||||
|
||||
|
||||
def _quantity_map(raw: Any, *, label: str) -> dict[str, int]:
|
||||
if raw is _MISSING or raw is None:
|
||||
raise ValueError(f"{label} is missing")
|
||||
if not isinstance(raw, Mapping):
|
||||
raise ValueError(f"{label} must be a mapping")
|
||||
output: dict[str, int] = {}
|
||||
for symbol, value in raw.items():
|
||||
canonical = normalize_symbol(str(symbol))
|
||||
quantity = _quantity(value, f"{label} {canonical}")
|
||||
if quantity:
|
||||
output[canonical] = quantity
|
||||
return output
|
||||
|
||||
|
||||
def _normalise_open_orders(
|
||||
raw: Any,
|
||||
*,
|
||||
label: str,
|
||||
differences: list[ReconciliationDifference],
|
||||
require_intent_fields: bool = False,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
if raw is _MISSING or raw is None:
|
||||
raise ValueError(f"{label} open_orders are missing")
|
||||
def record_from(value: Any, default_id: Any = None) -> Any:
|
||||
if isinstance(value, Mapping):
|
||||
result = dict(value)
|
||||
if default_id is not None:
|
||||
result.setdefault("order_id", default_id)
|
||||
return result
|
||||
if hasattr(value, "order_id"):
|
||||
side = getattr(value, "side", None)
|
||||
status = getattr(value, "status", None)
|
||||
return {
|
||||
"order_id": getattr(value, "order_id", default_id),
|
||||
"symbol": getattr(value, "symbol", None),
|
||||
"side": getattr(side, "value", side),
|
||||
"quantity": getattr(value, "quantity", None),
|
||||
"filled_quantity": getattr(value, "filled_quantity", 0),
|
||||
"status": getattr(status, "value", status),
|
||||
"broker_order_id": getattr(value, "broker_order_id", None),
|
||||
"limit_price": getattr(value, "limit_price", None),
|
||||
"order_type": getattr(value, "order_type", None),
|
||||
"time_in_force": getattr(value, "time_in_force", None),
|
||||
}
|
||||
return {"order_id": default_id, "status": value}
|
||||
|
||||
if isinstance(raw, Mapping):
|
||||
records: Iterable[Any] = (
|
||||
record_from(value, key) for key, value in raw.items()
|
||||
)
|
||||
elif isinstance(raw, Iterable) and not isinstance(raw, (str, bytes)):
|
||||
records = (record_from(value) for value in raw)
|
||||
else:
|
||||
raise ValueError(f"{label} open_orders must be a mapping or iterable")
|
||||
|
||||
output: dict[str, dict[str, Any]] = {}
|
||||
for index, item in enumerate(records):
|
||||
if not isinstance(item, Mapping):
|
||||
_difference(
|
||||
differences,
|
||||
"INVALID_OPEN_ORDER",
|
||||
f"{label}[{index}]",
|
||||
"object",
|
||||
type(item).__name__,
|
||||
)
|
||||
continue
|
||||
identity = item.get(
|
||||
"client_order_id",
|
||||
item.get("order_id", item.get("broker_order_id")),
|
||||
)
|
||||
if identity in {None, ""}:
|
||||
_difference(
|
||||
differences,
|
||||
"INVALID_OPEN_ORDER",
|
||||
f"{label}[{index}]",
|
||||
"non-empty order identity",
|
||||
"missing",
|
||||
)
|
||||
continue
|
||||
order_id = str(identity)
|
||||
try:
|
||||
symbol = normalize_symbol(str(item["symbol"]))
|
||||
side = str(item["side"]).upper()
|
||||
if side not in {"BUY", "SELL"}:
|
||||
raise ValueError("side must be BUY or SELL")
|
||||
quantity = _quantity(
|
||||
item.get("order_quantity", item.get("quantity")),
|
||||
f"{label} order {order_id} quantity",
|
||||
)
|
||||
filled = _quantity(
|
||||
item.get(
|
||||
"cumulative_filled_quantity",
|
||||
item.get("filled_quantity", 0),
|
||||
),
|
||||
f"{label} order {order_id} filled quantity",
|
||||
)
|
||||
if quantity <= 0:
|
||||
raise ValueError("quantity must be positive")
|
||||
if filled > quantity:
|
||||
raise ValueError("filled quantity exceeds order quantity")
|
||||
status = str(item.get("status", item.get("state", ""))).upper()
|
||||
if status not in _OPEN_STATUSES | _TERMINAL_STATUSES:
|
||||
raise ValueError(f"unsupported status {status!r}")
|
||||
broker_order_id = item.get("broker_order_id")
|
||||
limit_price_raw = item.get("limit_price")
|
||||
order_type_raw = item.get("order_type")
|
||||
time_in_force_raw = item.get("time_in_force")
|
||||
if require_intent_fields:
|
||||
if broker_order_id in {None, ""}:
|
||||
raise ValueError("broker_order_id is required")
|
||||
if limit_price_raw is None:
|
||||
raise ValueError("limit_price is required")
|
||||
if order_type_raw is None:
|
||||
raise ValueError("order_type is required")
|
||||
if time_in_force_raw is None:
|
||||
raise ValueError("time_in_force is required")
|
||||
limit_price = None
|
||||
if limit_price_raw is not None:
|
||||
limit_price = decimal(limit_price_raw)
|
||||
if not limit_price.is_finite() or limit_price <= 0:
|
||||
raise ValueError(
|
||||
"limit_price must be finite and positive"
|
||||
)
|
||||
order_type = (
|
||||
None
|
||||
if order_type_raw is None
|
||||
else str(order_type_raw).upper()
|
||||
)
|
||||
if order_type is not None and order_type != "LIMIT":
|
||||
raise ValueError("order_type must be LIMIT")
|
||||
time_in_force = (
|
||||
None
|
||||
if time_in_force_raw is None
|
||||
else str(time_in_force_raw).upper()
|
||||
)
|
||||
if time_in_force is not None and time_in_force != "DAY":
|
||||
raise ValueError("time_in_force must be DAY")
|
||||
except (
|
||||
InvalidOperation,
|
||||
KeyError,
|
||||
OverflowError,
|
||||
TypeError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
_difference(
|
||||
differences,
|
||||
"INVALID_OPEN_ORDER",
|
||||
order_id,
|
||||
"valid normalized open order",
|
||||
f"{type(exc).__name__}: {exc}",
|
||||
)
|
||||
continue
|
||||
|
||||
normalized = {
|
||||
"symbol": symbol,
|
||||
"side": side,
|
||||
"quantity": quantity,
|
||||
"filled_quantity": filled,
|
||||
"status": status,
|
||||
"broker_order_id": (
|
||||
None if broker_order_id is None else str(broker_order_id)
|
||||
),
|
||||
"limit_price": (
|
||||
None if limit_price is None else format(limit_price, "f")
|
||||
),
|
||||
"order_type": order_type,
|
||||
"time_in_force": time_in_force,
|
||||
}
|
||||
prior = output.get(order_id)
|
||||
if prior is not None:
|
||||
_difference(
|
||||
differences,
|
||||
"DUPLICATE_OPEN_ORDER",
|
||||
order_id,
|
||||
"unique order identity",
|
||||
(
|
||||
"identical duplicate"
|
||||
if prior == normalized
|
||||
else normalized
|
||||
),
|
||||
)
|
||||
continue
|
||||
output[order_id] = normalized
|
||||
if status == "UNKNOWN":
|
||||
_difference(
|
||||
differences,
|
||||
"UNKNOWN_ORDER",
|
||||
order_id,
|
||||
"known broker state",
|
||||
"UNKNOWN",
|
||||
)
|
||||
elif status in _TERMINAL_STATUSES:
|
||||
_difference(
|
||||
differences,
|
||||
"TERMINAL_OPEN_ORDER",
|
||||
order_id,
|
||||
"active state",
|
||||
status,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def _parse_timestamp(value: Any, label: str) -> datetime:
|
||||
if isinstance(value, datetime):
|
||||
result = value
|
||||
elif isinstance(value, str):
|
||||
text = value.strip()
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
try:
|
||||
result = datetime.fromisoformat(text)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{label} must be ISO-8601") from exc
|
||||
else:
|
||||
raise ValueError(f"{label} must be a datetime or ISO-8601 string")
|
||||
if result.tzinfo is None or result.utcoffset() is None:
|
||||
raise ValueError(f"{label} must include a timezone offset")
|
||||
return result.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _finite_non_negative_seconds(value: Any, label: str) -> float:
|
||||
"""Return a usable tolerance or reject unsafe numeric edge cases."""
|
||||
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(f"{label} must be a finite non-negative number")
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(
|
||||
f"{label} must be a finite non-negative number"
|
||||
) from exc
|
||||
if not math.isfinite(result) or result < 0:
|
||||
raise ValueError(f"{label} must be a finite non-negative number")
|
||||
return result
|
||||
|
||||
|
||||
def _check_freshness(
|
||||
*,
|
||||
differences: list[ReconciliationDifference],
|
||||
key: str,
|
||||
value: Any,
|
||||
now: datetime,
|
||||
max_age_seconds: float,
|
||||
max_clock_skew_seconds: float,
|
||||
) -> datetime | None:
|
||||
try:
|
||||
timestamp = _parse_timestamp(value, key)
|
||||
except ValueError as exc:
|
||||
_difference(
|
||||
differences,
|
||||
"INVALID_TIMESTAMP",
|
||||
key,
|
||||
"timezone-aware ISO-8601",
|
||||
str(exc),
|
||||
)
|
||||
return None
|
||||
age = (now - timestamp).total_seconds()
|
||||
if age < -max_clock_skew_seconds:
|
||||
_difference(
|
||||
differences,
|
||||
"FUTURE_TIMESTAMP",
|
||||
key,
|
||||
f"<= {max_clock_skew_seconds:g}s in future",
|
||||
f"{-age:g}s in future",
|
||||
)
|
||||
elif age > max_age_seconds:
|
||||
_difference(
|
||||
differences,
|
||||
(
|
||||
"STALE_SOURCE"
|
||||
if key == "broker.source_time"
|
||||
else "STALE_SNAPSHOT"
|
||||
),
|
||||
key,
|
||||
f"age <= {max_age_seconds:g}s",
|
||||
f"age={age:g}s",
|
||||
)
|
||||
return timestamp
|
||||
|
||||
|
||||
def reconcile_snapshot(
|
||||
*,
|
||||
local_cash: Decimal | float | str | None = None,
|
||||
local_positions: Mapping[str, int] | Iterable[Mapping[str, Any]] | None = None,
|
||||
broker_cash: Decimal | float | str | None = None,
|
||||
broker_positions: Mapping[str, int] | Iterable[Mapping[str, Any]] | None = None,
|
||||
cash_tolerance: Decimal | float | str = "0.01",
|
||||
local_sellable: Mapping[str, int] | None = None,
|
||||
broker_sellable: Mapping[str, int] | None = None,
|
||||
local_open_orders: Mapping[str, Any] | Iterable[Mapping[str, Any]] | None = None,
|
||||
broker_open_orders: Mapping[str, Any] | Iterable[Mapping[str, Any]] | None = None,
|
||||
local_as_of: datetime | str | None = None,
|
||||
broker_as_of: datetime | str | None = None,
|
||||
broker_source_time: datetime | str | None = None,
|
||||
as_of: datetime | str | None = None,
|
||||
source_time: datetime | str | None = None,
|
||||
now: datetime | str | None = None,
|
||||
max_snapshot_age_seconds: float = 60.0,
|
||||
max_source_age_seconds: float | None = None,
|
||||
max_age_seconds: float | None = None,
|
||||
max_clock_skew_seconds: float = 5.0,
|
||||
local_snapshot: Mapping[str, Any] | None = None,
|
||||
broker_snapshot: Mapping[str, Any] | None = None,
|
||||
require_complete: bool | None = None,
|
||||
) -> ReconciliationReport:
|
||||
"""Compare local intent/accounting with a broker snapshot.
|
||||
|
||||
Legacy callers may continue to provide only the original four
|
||||
cash/position arguments. Complete mode is selected automatically when a
|
||||
snapshot object is provided, or explicitly with ``require_complete=True``.
|
||||
Complete mode fails closed on any missing sellable/open-order/timestamp
|
||||
field and on an ``UNKNOWN`` order even when both sides report it.
|
||||
|
||||
``as_of`` and ``source_time`` are aliases for the broker timestamps. They
|
||||
exist for direct use with the JSON broker-snapshot contract.
|
||||
"""
|
||||
|
||||
differences: list[ReconciliationDifference] = []
|
||||
complete = (
|
||||
bool(require_complete)
|
||||
if require_complete is not None
|
||||
else local_snapshot is not None or broker_snapshot is not None
|
||||
)
|
||||
snapshot_age_limit = _finite_non_negative_seconds(
|
||||
max_snapshot_age_seconds,
|
||||
"max_snapshot_age_seconds",
|
||||
)
|
||||
source_age_limit = (
|
||||
None
|
||||
if max_source_age_seconds is None
|
||||
else _finite_non_negative_seconds(
|
||||
max_source_age_seconds,
|
||||
"max_source_age_seconds",
|
||||
)
|
||||
)
|
||||
clock_skew_limit = _finite_non_negative_seconds(
|
||||
max_clock_skew_seconds,
|
||||
"max_clock_skew_seconds",
|
||||
)
|
||||
if max_age_seconds is not None:
|
||||
alias_age_limit = _finite_non_negative_seconds(
|
||||
max_age_seconds,
|
||||
"max_age_seconds",
|
||||
)
|
||||
snapshot_age_limit = alias_age_limit
|
||||
if source_age_limit is None:
|
||||
source_age_limit = alias_age_limit
|
||||
if source_age_limit is None:
|
||||
source_age_limit = snapshot_age_limit
|
||||
|
||||
def pick(explicit: Any, snapshot: Mapping[str, Any] | None, *names: str) -> Any:
|
||||
if explicit is not None:
|
||||
return explicit
|
||||
return _mapping_value(snapshot, *names)
|
||||
|
||||
local_cash_raw = pick(local_cash, local_snapshot, "cash")
|
||||
broker_cash_raw = pick(broker_cash, broker_snapshot, "cash")
|
||||
local_positions_raw = pick(local_positions, local_snapshot, "positions")
|
||||
broker_positions_raw = pick(broker_positions, broker_snapshot, "positions")
|
||||
|
||||
required = {
|
||||
"local.cash": local_cash_raw,
|
||||
"broker.cash": broker_cash_raw,
|
||||
"local.positions": local_positions_raw,
|
||||
"broker.positions": broker_positions_raw,
|
||||
}
|
||||
missing_required = [
|
||||
key for key, value in required.items() if value is _MISSING or value is None
|
||||
]
|
||||
if missing_required and not complete:
|
||||
raise ValueError("missing reconciliation inputs: " + ", ".join(missing_required))
|
||||
for key in missing_required:
|
||||
_difference(differences, "MISSING_FIELD", key, "present", "missing")
|
||||
|
||||
tolerance = decimal(cash_tolerance)
|
||||
if not tolerance.is_finite() or tolerance < 0:
|
||||
raise ValueError("cash_tolerance must be a finite non-negative number")
|
||||
if not missing_required:
|
||||
local_cash_value = decimal(local_cash_raw)
|
||||
broker_cash_value = decimal(broker_cash_raw)
|
||||
if not local_cash_value.is_finite() or not broker_cash_value.is_finite():
|
||||
raise ValueError("cash values must be finite")
|
||||
if abs(local_cash_value - broker_cash_value) > tolerance:
|
||||
_difference(
|
||||
differences,
|
||||
"CASH",
|
||||
"CNY",
|
||||
local_cash_value,
|
||||
broker_cash_value,
|
||||
)
|
||||
|
||||
local_position_map: dict[str, int] = {}
|
||||
broker_position_map: dict[str, int] = {}
|
||||
embedded_local_sellable: dict[str, int] = {}
|
||||
embedded_broker_sellable: dict[str, int] = {}
|
||||
local_has_sellable = False
|
||||
broker_has_sellable = False
|
||||
if local_positions_raw is not _MISSING and local_positions_raw is not None:
|
||||
(
|
||||
local_position_map,
|
||||
embedded_local_sellable,
|
||||
local_has_sellable,
|
||||
) = _position_maps(local_positions_raw, label="local")
|
||||
if broker_positions_raw is not _MISSING and broker_positions_raw is not None:
|
||||
(
|
||||
broker_position_map,
|
||||
embedded_broker_sellable,
|
||||
broker_has_sellable,
|
||||
) = _position_maps(broker_positions_raw, label="broker")
|
||||
for symbol in sorted(set(local_position_map) | set(broker_position_map)):
|
||||
expected = local_position_map.get(symbol, 0)
|
||||
observed = broker_position_map.get(symbol, 0)
|
||||
if expected != observed:
|
||||
_difference(differences, "POSITION", symbol, expected, observed)
|
||||
|
||||
local_sellable_raw = pick(
|
||||
local_sellable,
|
||||
local_snapshot,
|
||||
"sellable",
|
||||
"sellable_positions",
|
||||
)
|
||||
broker_sellable_raw = pick(
|
||||
broker_sellable,
|
||||
broker_snapshot,
|
||||
"sellable",
|
||||
"sellable_positions",
|
||||
)
|
||||
if local_sellable_raw is _MISSING and local_has_sellable:
|
||||
local_sellable_map = embedded_local_sellable
|
||||
elif local_sellable_raw is not _MISSING and local_sellable_raw is not None:
|
||||
local_sellable_map = _quantity_map(
|
||||
local_sellable_raw,
|
||||
label="local sellable",
|
||||
)
|
||||
else:
|
||||
local_sellable_map = None
|
||||
if broker_sellable_raw is _MISSING and broker_has_sellable:
|
||||
broker_sellable_map = embedded_broker_sellable
|
||||
elif broker_sellable_raw is not _MISSING and broker_sellable_raw is not None:
|
||||
broker_sellable_map = _quantity_map(
|
||||
broker_sellable_raw,
|
||||
label="broker sellable",
|
||||
)
|
||||
else:
|
||||
broker_sellable_map = None
|
||||
|
||||
if complete and local_sellable_map is None:
|
||||
_difference(
|
||||
differences,
|
||||
"MISSING_FIELD",
|
||||
"local.sellable",
|
||||
"present",
|
||||
"missing",
|
||||
)
|
||||
if complete and broker_sellable_map is None:
|
||||
_difference(
|
||||
differences,
|
||||
"MISSING_FIELD",
|
||||
"broker.sellable",
|
||||
"present",
|
||||
"missing",
|
||||
)
|
||||
if local_sellable_map is not None:
|
||||
for symbol, quantity in local_sellable_map.items():
|
||||
if quantity > local_position_map.get(symbol, 0):
|
||||
_difference(
|
||||
differences,
|
||||
"INVALID_SELLABLE",
|
||||
f"local:{symbol}",
|
||||
f"<= {local_position_map.get(symbol, 0)}",
|
||||
quantity,
|
||||
)
|
||||
if broker_sellable_map is not None:
|
||||
for symbol, quantity in broker_sellable_map.items():
|
||||
if quantity > broker_position_map.get(symbol, 0):
|
||||
_difference(
|
||||
differences,
|
||||
"INVALID_SELLABLE",
|
||||
f"broker:{symbol}",
|
||||
f"<= {broker_position_map.get(symbol, 0)}",
|
||||
quantity,
|
||||
)
|
||||
if local_sellable_map is not None and broker_sellable_map is not None:
|
||||
for symbol in sorted(set(local_sellable_map) | set(broker_sellable_map)):
|
||||
expected = local_sellable_map.get(symbol, 0)
|
||||
observed = broker_sellable_map.get(symbol, 0)
|
||||
if expected != observed:
|
||||
_difference(differences, "SELLABLE", symbol, expected, observed)
|
||||
|
||||
local_orders_raw = pick(
|
||||
local_open_orders,
|
||||
local_snapshot,
|
||||
"open_orders",
|
||||
)
|
||||
broker_orders_raw = pick(
|
||||
broker_open_orders,
|
||||
broker_snapshot,
|
||||
"open_orders",
|
||||
)
|
||||
local_orders: dict[str, dict[str, Any]] | None = None
|
||||
broker_orders: dict[str, dict[str, Any]] | None = None
|
||||
if local_orders_raw is not _MISSING and local_orders_raw is not None:
|
||||
local_orders = _normalise_open_orders(
|
||||
local_orders_raw,
|
||||
label="local",
|
||||
differences=differences,
|
||||
require_intent_fields=complete,
|
||||
)
|
||||
if broker_orders_raw is not _MISSING and broker_orders_raw is not None:
|
||||
broker_orders = _normalise_open_orders(
|
||||
broker_orders_raw,
|
||||
label="broker",
|
||||
differences=differences,
|
||||
require_intent_fields=complete,
|
||||
)
|
||||
if complete and local_orders is None:
|
||||
_difference(
|
||||
differences,
|
||||
"MISSING_FIELD",
|
||||
"local.open_orders",
|
||||
"present",
|
||||
"missing",
|
||||
)
|
||||
if complete and broker_orders is None:
|
||||
_difference(
|
||||
differences,
|
||||
"MISSING_FIELD",
|
||||
"broker.open_orders",
|
||||
"present",
|
||||
"missing",
|
||||
)
|
||||
if local_orders is not None and broker_orders is not None:
|
||||
for order_id in sorted(set(local_orders) | set(broker_orders)):
|
||||
expected = local_orders.get(order_id)
|
||||
observed = broker_orders.get(order_id)
|
||||
if expected != observed:
|
||||
_difference(
|
||||
differences,
|
||||
"OPEN_ORDER",
|
||||
order_id,
|
||||
expected if expected is not None else "missing",
|
||||
observed if observed is not None else "missing",
|
||||
)
|
||||
|
||||
local_as_of_raw = pick(local_as_of, local_snapshot, "as_of")
|
||||
broker_as_of_explicit = broker_as_of if broker_as_of is not None else as_of
|
||||
broker_as_of_raw = pick(
|
||||
broker_as_of_explicit,
|
||||
broker_snapshot,
|
||||
"as_of",
|
||||
)
|
||||
source_explicit = (
|
||||
broker_source_time if broker_source_time is not None else source_time
|
||||
)
|
||||
source_time_raw = pick(
|
||||
source_explicit,
|
||||
broker_snapshot,
|
||||
"source_time",
|
||||
)
|
||||
if (
|
||||
broker_source_time is not None
|
||||
and source_time is not None
|
||||
and _parse_timestamp(broker_source_time, "broker_source_time")
|
||||
!= _parse_timestamp(source_time, "source_time")
|
||||
):
|
||||
raise ValueError("broker_source_time and source_time disagree")
|
||||
if (
|
||||
broker_as_of is not None
|
||||
and as_of is not None
|
||||
and _parse_timestamp(broker_as_of, "broker_as_of")
|
||||
!= _parse_timestamp(as_of, "as_of")
|
||||
):
|
||||
raise ValueError("broker_as_of and as_of disagree")
|
||||
|
||||
freshness_requested = complete or any(
|
||||
value is not _MISSING and value is not None
|
||||
for value in (local_as_of_raw, broker_as_of_raw, source_time_raw)
|
||||
)
|
||||
if freshness_requested:
|
||||
now_value = (
|
||||
datetime.now(timezone.utc)
|
||||
if now is None
|
||||
else _parse_timestamp(now, "now")
|
||||
)
|
||||
timestamp_values = {
|
||||
"local.as_of": local_as_of_raw,
|
||||
"broker.as_of": broker_as_of_raw,
|
||||
"broker.source_time": source_time_raw,
|
||||
}
|
||||
parsed: dict[str, datetime] = {}
|
||||
for key, value in timestamp_values.items():
|
||||
if value is _MISSING or value is None:
|
||||
if complete:
|
||||
_difference(
|
||||
differences,
|
||||
"MISSING_FIELD",
|
||||
key,
|
||||
"present",
|
||||
"missing",
|
||||
)
|
||||
continue
|
||||
limit = (
|
||||
source_age_limit
|
||||
if key == "broker.source_time"
|
||||
else snapshot_age_limit
|
||||
)
|
||||
timestamp = _check_freshness(
|
||||
differences=differences,
|
||||
key=key,
|
||||
value=value,
|
||||
now=now_value,
|
||||
max_age_seconds=limit,
|
||||
max_clock_skew_seconds=clock_skew_limit,
|
||||
)
|
||||
if timestamp is not None:
|
||||
parsed[key] = timestamp
|
||||
|
||||
local_time = parsed.get("local.as_of")
|
||||
broker_time = parsed.get("broker.as_of")
|
||||
broker_source = parsed.get("broker.source_time")
|
||||
if (
|
||||
local_time is not None
|
||||
and broker_time is not None
|
||||
and abs((local_time - broker_time).total_seconds())
|
||||
> clock_skew_limit
|
||||
):
|
||||
_difference(
|
||||
differences,
|
||||
"AS_OF_SKEW",
|
||||
"local_vs_broker",
|
||||
f"<= {clock_skew_limit:g}s",
|
||||
f"{abs((local_time - broker_time).total_seconds()):g}s",
|
||||
)
|
||||
if (
|
||||
broker_time is not None
|
||||
and broker_source is not None
|
||||
and broker_source > broker_time
|
||||
):
|
||||
lead = (broker_source - broker_time).total_seconds()
|
||||
if lead > clock_skew_limit:
|
||||
_difference(
|
||||
differences,
|
||||
"SOURCE_AFTER_AS_OF",
|
||||
"broker.source_time",
|
||||
f"<= broker.as_of + {clock_skew_limit:g}s",
|
||||
f"lead={lead:g}s",
|
||||
)
|
||||
|
||||
differences = sorted(
|
||||
set(differences),
|
||||
key=lambda item: (item.kind, item.key, item.expected, item.observed)
|
||||
)
|
||||
return ReconciliationReport(
|
||||
not differences,
|
||||
tuple(differences),
|
||||
complete=complete,
|
||||
)
|
||||
@@ -0,0 +1,837 @@
|
||||
"""Strict projector for normalized ``ORDER_STATUS`` and ``FILL`` events.
|
||||
|
||||
The projector rebuilds order cumulative fills, cash delta and position delta.
|
||||
It accepts at-least-once *identical* callbacks idempotently, but rejects
|
||||
conflicting duplicates, unknown-order fills, overfills, sequence/time
|
||||
regressions and illegal order-state regressions.
|
||||
|
||||
The canonical JSON wire shape is the ledger envelope documented by
|
||||
``schemas/order_event.schema.json``: ``sequence``, ``timestamp``, ``event_id``,
|
||||
``event_type`` and ``payload``, plus an optional *paired* ``previous_hash`` /
|
||||
``hash``. ``SimBroker`` emits that shape through ``LedgerRecord.as_dict()``.
|
||||
Legacy quantity aliases remain accepted at the Python boundary, but are not
|
||||
part of the canonical schema.
|
||||
|
||||
It operates on local normalized ledger events. Passing its tests is not
|
||||
evidence that a real broker callback mapping has been certified.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
from .domain import (
|
||||
ORDER_TRANSITIONS,
|
||||
PRICE_QUANT,
|
||||
OrderStatus,
|
||||
Side,
|
||||
decimal,
|
||||
money,
|
||||
)
|
||||
from .ledger import HashChainLedger, LedgerRecord, canonical_json
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
class LedgerReplayError(ValueError):
|
||||
"""Base class for events that cannot be projected safely."""
|
||||
|
||||
|
||||
class OutOfOrderError(LedgerReplayError):
|
||||
"""A sequence, event-time or state transition moved backwards."""
|
||||
|
||||
|
||||
class ConflictingDuplicateError(LedgerReplayError):
|
||||
"""One event/fill identity was reused for different content."""
|
||||
|
||||
|
||||
class UnknownOrderError(LedgerReplayError):
|
||||
"""A fill referred to an order that has not been registered."""
|
||||
|
||||
|
||||
class OverfillError(LedgerReplayError):
|
||||
"""Cumulative fills exceeded the submitted order quantity."""
|
||||
|
||||
|
||||
# Descriptive aliases kept for callers that prefer the longer names.
|
||||
ReplayError = LedgerReplayError
|
||||
ReplayOrderingError = OutOfOrderError
|
||||
ReplayConflictError = ConflictingDuplicateError
|
||||
|
||||
|
||||
_OPEN_STATUSES = {
|
||||
OrderStatus.NEW,
|
||||
OrderStatus.ACCEPTED,
|
||||
OrderStatus.PARTIALLY_FILLED,
|
||||
OrderStatus.CANCEL_PENDING,
|
||||
OrderStatus.UNKNOWN,
|
||||
}
|
||||
_ENVELOPE_KEYS = {
|
||||
"sequence",
|
||||
"timestamp",
|
||||
"event_id",
|
||||
"event_type",
|
||||
"payload",
|
||||
"previous_hash",
|
||||
"hash",
|
||||
}
|
||||
_ORDER_STATUS_PAYLOAD_KEYS = {
|
||||
"order_id",
|
||||
"broker_order_id",
|
||||
"symbol",
|
||||
"side",
|
||||
"quantity",
|
||||
"order_quantity",
|
||||
"filled_quantity",
|
||||
"cumulative_filled_quantity",
|
||||
"average_fill_price",
|
||||
"status",
|
||||
"reason",
|
||||
"submitted_at",
|
||||
"metadata",
|
||||
}
|
||||
_FILL_PAYLOAD_KEYS = {
|
||||
"fill_id",
|
||||
"order_id",
|
||||
"trading_date",
|
||||
"symbol",
|
||||
"side",
|
||||
"quantity",
|
||||
"last_fill_quantity",
|
||||
"price",
|
||||
"commission",
|
||||
"stamp_duty",
|
||||
"transfer_fee",
|
||||
"cash_delta",
|
||||
}
|
||||
_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
def _is_present(value: Any) -> bool:
|
||||
return value is not None and value != ""
|
||||
|
||||
|
||||
def _timestamp(value: Any, label: str) -> datetime:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
elif isinstance(value, str):
|
||||
text = value.strip()
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text)
|
||||
except ValueError as exc:
|
||||
raise LedgerReplayError(f"{label} must be ISO-8601") from exc
|
||||
else:
|
||||
raise LedgerReplayError(
|
||||
f"{label} must be a datetime or ISO-8601 string"
|
||||
)
|
||||
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
||||
raise LedgerReplayError(f"{label} must include a timezone offset")
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _required(payload: Mapping[str, Any], key: str, event_id: str) -> Any:
|
||||
if key not in payload or not _is_present(payload[key]):
|
||||
raise LedgerReplayError(f"{event_id}: missing {key}")
|
||||
return payload[key]
|
||||
|
||||
|
||||
def _positive_int(value: Any, label: str) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise LedgerReplayError(f"{label} must be an integer")
|
||||
try:
|
||||
result = int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise LedgerReplayError(f"{label} must be an integer") from exc
|
||||
if isinstance(value, str):
|
||||
exact = value.strip() == str(result)
|
||||
else:
|
||||
exact = value == result
|
||||
if not exact or result <= 0:
|
||||
raise LedgerReplayError(f"{label} must be a positive integer")
|
||||
return result
|
||||
|
||||
|
||||
def _nonnegative_int(value: Any, label: str) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise LedgerReplayError(f"{label} must be an integer")
|
||||
try:
|
||||
result = int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise LedgerReplayError(f"{label} must be an integer") from exc
|
||||
if isinstance(value, str):
|
||||
exact = value.strip() == str(result)
|
||||
else:
|
||||
exact = value == result
|
||||
if not exact or result < 0:
|
||||
raise LedgerReplayError(f"{label} must be a non-negative integer")
|
||||
return result
|
||||
|
||||
|
||||
def _aliased_int(
|
||||
payload: Mapping[str, Any],
|
||||
keys: tuple[str, ...],
|
||||
event_id: str,
|
||||
*,
|
||||
positive: bool,
|
||||
) -> int:
|
||||
"""Read canonical/legacy integer names and reject conflicting copies."""
|
||||
|
||||
present = [
|
||||
(key, payload[key])
|
||||
for key in keys
|
||||
if key in payload and _is_present(payload[key])
|
||||
]
|
||||
if not present:
|
||||
raise LedgerReplayError(
|
||||
f"{event_id}: missing one of {', '.join(keys)}"
|
||||
)
|
||||
parser = _positive_int if positive else _nonnegative_int
|
||||
parsed = [
|
||||
(key, parser(value, f"{event_id}.{key}"))
|
||||
for key, value in present
|
||||
]
|
||||
if any(value != parsed[0][1] for _, value in parsed[1:]):
|
||||
names = ", ".join(key for key, _ in parsed)
|
||||
raise ConflictingDuplicateError(
|
||||
f"{event_id}: conflicting alias values for {names}"
|
||||
)
|
||||
return parsed[0][1]
|
||||
|
||||
|
||||
def _reject_unknown_keys(
|
||||
payload: Mapping[str, Any],
|
||||
allowed: set[str],
|
||||
event_id: str,
|
||||
) -> None:
|
||||
unknown = sorted(str(key) for key in set(payload) - allowed)
|
||||
if unknown:
|
||||
raise LedgerReplayError(
|
||||
f"{event_id}: unsupported payload fields {unknown}"
|
||||
)
|
||||
|
||||
|
||||
def _finite_decimal(value: Any, label: str) -> Decimal:
|
||||
try:
|
||||
result = decimal(value)
|
||||
except Exception as exc:
|
||||
raise LedgerReplayError(f"{label} must be numeric") from exc
|
||||
if not result.is_finite():
|
||||
raise LedgerReplayError(f"{label} must be finite")
|
||||
return result
|
||||
|
||||
|
||||
def _record_value(record: Any, name: str, default: Any = None) -> Any:
|
||||
if isinstance(record, Mapping):
|
||||
return record.get(name, default)
|
||||
return getattr(record, name, default)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayedOrder:
|
||||
order_id: str
|
||||
symbol: str
|
||||
side: str
|
||||
quantity: int
|
||||
status: str
|
||||
filled_quantity: int
|
||||
average_fill_price: Decimal
|
||||
cash_delta: Decimal
|
||||
position_delta: int
|
||||
fill_ids: tuple[str, ...]
|
||||
last_sequence: int
|
||||
last_timestamp: str
|
||||
|
||||
@property
|
||||
def remaining_quantity(self) -> int:
|
||||
return self.quantity - self.filled_quantity
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
return OrderStatus(self.status) in _OPEN_STATUSES
|
||||
|
||||
def accounting_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"order_id": self.order_id,
|
||||
"symbol": self.symbol,
|
||||
"side": self.side,
|
||||
"quantity": self.quantity,
|
||||
"status": self.status,
|
||||
"filled_quantity": self.filled_quantity,
|
||||
"average_fill_price": format(self.average_fill_price, "f"),
|
||||
"cash_delta": format(self.cash_delta, "f"),
|
||||
"position_delta": self.position_delta,
|
||||
"fill_ids": list(self.fill_ids),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LedgerProjection:
|
||||
orders: dict[str, ReplayedOrder]
|
||||
cash_delta: Decimal
|
||||
position_deltas: dict[str, int]
|
||||
fill_ids: tuple[str, ...]
|
||||
source_last_sequence: int
|
||||
relevant_event_count: int
|
||||
|
||||
@property
|
||||
def has_unknown_orders(self) -> bool:
|
||||
return any(
|
||||
order.status == OrderStatus.UNKNOWN.value
|
||||
for order in self.orders.values()
|
||||
)
|
||||
|
||||
@property
|
||||
def safe_to_reconcile(self) -> bool:
|
||||
return not self.has_unknown_orders
|
||||
|
||||
@property
|
||||
def open_orders(self) -> dict[str, ReplayedOrder]:
|
||||
return {
|
||||
order_id: order
|
||||
for order_id, order in self.orders.items()
|
||||
if order.is_open
|
||||
}
|
||||
|
||||
def accounting_dict(self) -> dict[str, Any]:
|
||||
"""Stable accounting state, excluding delivery-order metadata."""
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"cash_delta": format(self.cash_delta, "f"),
|
||||
"position_deltas": dict(sorted(self.position_deltas.items())),
|
||||
"orders": [
|
||||
self.orders[order_id].accounting_dict()
|
||||
for order_id in sorted(self.orders)
|
||||
],
|
||||
"fill_ids": list(self.fill_ids),
|
||||
"has_unknown_orders": self.has_unknown_orders,
|
||||
}
|
||||
|
||||
@property
|
||||
def digest(self) -> str:
|
||||
return hashlib.sha256(
|
||||
canonical_json(self.accounting_dict()).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _MutableOrder:
|
||||
order_id: str
|
||||
symbol: str
|
||||
side: Side
|
||||
quantity: int
|
||||
status: OrderStatus
|
||||
filled_quantity: int = 0
|
||||
average_fill_price: Decimal = Decimal("0")
|
||||
cash_delta: Decimal = Decimal("0")
|
||||
position_delta: int = 0
|
||||
fill_ids: list[str] = field(default_factory=list)
|
||||
last_sequence: int = 0
|
||||
last_timestamp: datetime | None = None
|
||||
last_status_signature: str | None = None
|
||||
|
||||
|
||||
class LedgerProjector:
|
||||
"""Stateful strict projector; one instance processes one ordered stream."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._orders: dict[str, _MutableOrder] = {}
|
||||
self._event_signatures: dict[str, tuple[int, str]] = {}
|
||||
self._fill_signatures: dict[str, str] = {}
|
||||
self._fill_ids: list[str] = []
|
||||
self._cash_delta = Decimal("0")
|
||||
self._position_deltas: dict[str, int] = {}
|
||||
self._last_sequence = 0
|
||||
self._relevant_event_count = 0
|
||||
self._failure: LedgerReplayError | None = None
|
||||
|
||||
def _check_order_time(
|
||||
self,
|
||||
order: _MutableOrder,
|
||||
event_time: datetime,
|
||||
event_id: str,
|
||||
) -> None:
|
||||
if order.last_timestamp is not None and event_time < order.last_timestamp:
|
||||
raise OutOfOrderError(
|
||||
f"{event_id}: timestamp precedes prior event for {order.order_id}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_status_semantics(
|
||||
order: _MutableOrder,
|
||||
event_id: str,
|
||||
) -> None:
|
||||
if (
|
||||
order.status == OrderStatus.FILLED
|
||||
and order.filled_quantity != order.quantity
|
||||
):
|
||||
raise LedgerReplayError(
|
||||
f"{event_id}: FILLED requires cumulative quantity == order quantity"
|
||||
)
|
||||
if order.status == OrderStatus.PARTIALLY_FILLED and not (
|
||||
0 < order.filled_quantity < order.quantity
|
||||
):
|
||||
raise LedgerReplayError(
|
||||
f"{event_id}: PARTIALLY_FILLED has invalid cumulative quantity"
|
||||
)
|
||||
|
||||
def _status_event(
|
||||
self,
|
||||
*,
|
||||
payload: Mapping[str, Any],
|
||||
event_id: str,
|
||||
sequence: int,
|
||||
event_time: datetime,
|
||||
) -> None:
|
||||
_reject_unknown_keys(
|
||||
payload,
|
||||
_ORDER_STATUS_PAYLOAD_KEYS,
|
||||
event_id,
|
||||
)
|
||||
order_id = str(_required(payload, "order_id", event_id))
|
||||
symbol = normalize_symbol(str(_required(payload, "symbol", event_id)))
|
||||
try:
|
||||
side = Side(str(_required(payload, "side", event_id)).upper())
|
||||
except ValueError as exc:
|
||||
raise LedgerReplayError(f"{event_id}: unsupported order side") from exc
|
||||
quantity = _aliased_int(
|
||||
payload,
|
||||
("quantity", "order_quantity"),
|
||||
event_id,
|
||||
positive=True,
|
||||
)
|
||||
filled = _aliased_int(
|
||||
payload,
|
||||
("filled_quantity", "cumulative_filled_quantity"),
|
||||
event_id,
|
||||
positive=False,
|
||||
)
|
||||
if filled > quantity:
|
||||
raise OverfillError(
|
||||
f"{event_id}: reported cumulative fill exceeds order quantity"
|
||||
)
|
||||
try:
|
||||
status = OrderStatus(
|
||||
str(_required(payload, "status", event_id)).upper()
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise LedgerReplayError(f"{event_id}: unsupported order status") from exc
|
||||
|
||||
order = self._orders.get(order_id)
|
||||
if order is None:
|
||||
if filled:
|
||||
raise UnknownOrderError(
|
||||
f"{event_id}: initial order status refers to unprojected fills"
|
||||
)
|
||||
order = _MutableOrder(
|
||||
order_id=order_id,
|
||||
symbol=symbol,
|
||||
side=side,
|
||||
quantity=quantity,
|
||||
status=status,
|
||||
last_sequence=sequence,
|
||||
last_timestamp=event_time,
|
||||
)
|
||||
self._orders[order_id] = order
|
||||
else:
|
||||
if (
|
||||
order.symbol != symbol
|
||||
or order.side != side
|
||||
or order.quantity != quantity
|
||||
):
|
||||
raise ConflictingDuplicateError(
|
||||
f"{event_id}: immutable order fields changed for {order_id}"
|
||||
)
|
||||
if filled != order.filled_quantity:
|
||||
relation = "ahead of" if filled > order.filled_quantity else "behind"
|
||||
raise OutOfOrderError(
|
||||
f"{event_id}: status cumulative fill is {relation} projected fills"
|
||||
)
|
||||
|
||||
reported_average = payload.get("average_fill_price")
|
||||
if reported_average is not None:
|
||||
observed_average = _finite_decimal(
|
||||
reported_average,
|
||||
f"{event_id}.average_fill_price",
|
||||
).quantize(PRICE_QUANT, rounding=ROUND_HALF_UP)
|
||||
if observed_average != order.average_fill_price:
|
||||
raise ConflictingDuplicateError(
|
||||
f"{event_id}: average fill price disagrees with FILL events"
|
||||
)
|
||||
signature = canonical_json(
|
||||
{
|
||||
"order_id": order_id,
|
||||
"symbol": symbol,
|
||||
"side": side.value,
|
||||
"quantity": quantity,
|
||||
"status": status.value,
|
||||
"filled_quantity": filled,
|
||||
"average_fill_price": format(order.average_fill_price, "f"),
|
||||
}
|
||||
)
|
||||
if order.last_status_signature == signature:
|
||||
return
|
||||
if order.last_sequence:
|
||||
self._check_order_time(order, event_time, event_id)
|
||||
if status != order.status:
|
||||
if status not in ORDER_TRANSITIONS[order.status]:
|
||||
raise OutOfOrderError(
|
||||
f"{event_id}: illegal state regression "
|
||||
f"{order.status.value}->{status.value}"
|
||||
)
|
||||
order.status = status
|
||||
order.last_sequence = sequence
|
||||
order.last_timestamp = event_time
|
||||
self._validate_status_semantics(order, event_id)
|
||||
if order.last_status_signature != signature:
|
||||
self._relevant_event_count += 1
|
||||
order.last_status_signature = signature
|
||||
|
||||
def _fill_event(
|
||||
self,
|
||||
*,
|
||||
payload: Mapping[str, Any],
|
||||
event_id: str,
|
||||
sequence: int,
|
||||
event_time: datetime,
|
||||
) -> None:
|
||||
_reject_unknown_keys(
|
||||
payload,
|
||||
_FILL_PAYLOAD_KEYS,
|
||||
event_id,
|
||||
)
|
||||
fill_id = str(_required(payload, "fill_id", event_id))
|
||||
order_id = str(_required(payload, "order_id", event_id))
|
||||
order = self._orders.get(order_id)
|
||||
if order is None:
|
||||
raise UnknownOrderError(
|
||||
f"{event_id}: fill {fill_id} references unknown order {order_id}"
|
||||
)
|
||||
symbol = normalize_symbol(str(_required(payload, "symbol", event_id)))
|
||||
try:
|
||||
side = Side(str(_required(payload, "side", event_id)).upper())
|
||||
except ValueError as exc:
|
||||
raise LedgerReplayError(f"{event_id}: unsupported fill side") from exc
|
||||
if symbol != order.symbol or side != order.side:
|
||||
raise ConflictingDuplicateError(
|
||||
f"{event_id}: fill identity disagrees with order {order_id}"
|
||||
)
|
||||
quantity = _aliased_int(
|
||||
payload,
|
||||
("quantity", "last_fill_quantity"),
|
||||
event_id,
|
||||
positive=True,
|
||||
)
|
||||
price = _finite_decimal(
|
||||
_required(payload, "price", event_id),
|
||||
f"{event_id}.price",
|
||||
)
|
||||
if price <= 0:
|
||||
raise LedgerReplayError(f"{event_id}: fill price must be positive")
|
||||
fees: dict[str, Decimal] = {}
|
||||
for name in ("commission", "stamp_duty", "transfer_fee"):
|
||||
value = _finite_decimal(payload.get(name, 0), f"{event_id}.{name}")
|
||||
if value < 0:
|
||||
raise LedgerReplayError(f"{event_id}: {name} must be non-negative")
|
||||
fees[name] = money(value)
|
||||
total_fees = money(sum(fees.values(), Decimal("0")))
|
||||
notional = money(price * quantity)
|
||||
cash_delta = (
|
||||
money(-notional - total_fees)
|
||||
if side == Side.BUY
|
||||
else money(notional - total_fees)
|
||||
)
|
||||
if "cash_delta" in payload:
|
||||
reported_cash = money(
|
||||
_finite_decimal(payload["cash_delta"], f"{event_id}.cash_delta")
|
||||
)
|
||||
if reported_cash != cash_delta:
|
||||
raise ConflictingDuplicateError(
|
||||
f"{event_id}: cash_delta disagrees with price, quantity and fees"
|
||||
)
|
||||
fill_signature = canonical_json(
|
||||
{
|
||||
"fill_id": fill_id,
|
||||
"order_id": order_id,
|
||||
"symbol": symbol,
|
||||
"side": side.value,
|
||||
"quantity": quantity,
|
||||
"price": format(price, "f"),
|
||||
"commission": format(fees["commission"], "f"),
|
||||
"stamp_duty": format(fees["stamp_duty"], "f"),
|
||||
"transfer_fee": format(fees["transfer_fee"], "f"),
|
||||
"cash_delta": format(cash_delta, "f"),
|
||||
}
|
||||
)
|
||||
prior_fill = self._fill_signatures.get(fill_id)
|
||||
if prior_fill is not None:
|
||||
if prior_fill != fill_signature:
|
||||
raise ConflictingDuplicateError(
|
||||
f"{event_id}: fill_id {fill_id} has conflicting content"
|
||||
)
|
||||
return
|
||||
self._check_order_time(order, event_time, event_id)
|
||||
if order.status not in {
|
||||
OrderStatus.ACCEPTED,
|
||||
OrderStatus.PARTIALLY_FILLED,
|
||||
OrderStatus.CANCEL_PENDING,
|
||||
OrderStatus.UNKNOWN,
|
||||
}:
|
||||
raise OutOfOrderError(
|
||||
f"{event_id}: fill cannot follow order state {order.status.value}"
|
||||
)
|
||||
if order.filled_quantity + quantity > order.quantity:
|
||||
raise OverfillError(
|
||||
f"{event_id}: fill would exceed order quantity {order.quantity}"
|
||||
)
|
||||
|
||||
previous_quantity = order.filled_quantity
|
||||
weighted = order.average_fill_price * previous_quantity + price * quantity
|
||||
order.filled_quantity += quantity
|
||||
order.average_fill_price = (
|
||||
weighted / order.filled_quantity
|
||||
).quantize(PRICE_QUANT, rounding=ROUND_HALF_UP)
|
||||
order.cash_delta = money(order.cash_delta + cash_delta)
|
||||
signed_quantity = quantity if side == Side.BUY else -quantity
|
||||
order.position_delta += signed_quantity
|
||||
order.fill_ids.append(fill_id)
|
||||
order.last_sequence = sequence
|
||||
order.last_timestamp = event_time
|
||||
inferred = (
|
||||
OrderStatus.FILLED
|
||||
if order.filled_quantity == order.quantity
|
||||
else OrderStatus.PARTIALLY_FILLED
|
||||
)
|
||||
if inferred != order.status:
|
||||
if inferred not in ORDER_TRANSITIONS[order.status]:
|
||||
raise OutOfOrderError(
|
||||
f"{event_id}: fill implies illegal state "
|
||||
f"{order.status.value}->{inferred.value}"
|
||||
)
|
||||
order.status = inferred
|
||||
|
||||
self._fill_signatures[fill_id] = fill_signature
|
||||
self._fill_ids.append(fill_id)
|
||||
self._cash_delta = money(self._cash_delta + cash_delta)
|
||||
self._position_deltas[symbol] = (
|
||||
self._position_deltas.get(symbol, 0) + signed_quantity
|
||||
)
|
||||
if not self._position_deltas[symbol]:
|
||||
del self._position_deltas[symbol]
|
||||
self._relevant_event_count += 1
|
||||
|
||||
def process(self, record: LedgerRecord | Mapping[str, Any]) -> None:
|
||||
"""Process one event, permanently poisoning the instance on failure."""
|
||||
|
||||
if self._failure is not None:
|
||||
raise LedgerReplayError(
|
||||
"projector is fail-closed after a prior replay error"
|
||||
) from self._failure
|
||||
try:
|
||||
self._process(record)
|
||||
except LedgerReplayError as exc:
|
||||
self._failure = exc
|
||||
raise
|
||||
except Exception as exc:
|
||||
wrapped = LedgerReplayError(
|
||||
f"replay rejected malformed event: {type(exc).__name__}: {exc}"
|
||||
)
|
||||
self._failure = wrapped
|
||||
raise wrapped from exc
|
||||
|
||||
def _process(self, record: LedgerRecord | Mapping[str, Any]) -> None:
|
||||
if isinstance(record, Mapping):
|
||||
unknown_envelope = sorted(
|
||||
str(key) for key in set(record) - _ENVELOPE_KEYS
|
||||
)
|
||||
if unknown_envelope:
|
||||
raise LedgerReplayError(
|
||||
f"record has unsupported envelope fields {unknown_envelope}"
|
||||
)
|
||||
event_id_value = _record_value(record, "event_id")
|
||||
event_type_value = _record_value(record, "event_type")
|
||||
timestamp_value = _record_value(record, "timestamp")
|
||||
sequence_value = _record_value(record, "sequence")
|
||||
payload = _record_value(record, "payload")
|
||||
previous_hash = _record_value(record, "previous_hash", _MISSING)
|
||||
record_hash = _record_value(record, "hash", _MISSING)
|
||||
if (previous_hash is _MISSING) != (record_hash is _MISSING):
|
||||
raise LedgerReplayError(
|
||||
"record previous_hash and hash must be supplied together"
|
||||
)
|
||||
for name, value in (
|
||||
("previous_hash", previous_hash),
|
||||
("hash", record_hash),
|
||||
):
|
||||
if value is not _MISSING and (
|
||||
not isinstance(value, str)
|
||||
or _SHA256_PATTERN.fullmatch(value) is None
|
||||
):
|
||||
raise LedgerReplayError(
|
||||
f"record {name} must be a lowercase SHA-256 hex string"
|
||||
)
|
||||
if not isinstance(event_id_value, str) or not event_id_value:
|
||||
raise LedgerReplayError("record event_id must be a non-empty string")
|
||||
if not isinstance(event_type_value, str) or not event_type_value:
|
||||
raise LedgerReplayError(f"{event_id_value}: record has no event_type")
|
||||
if (
|
||||
event_type_value.upper() in {"ORDER_STATUS", "FILL"}
|
||||
and event_type_value not in {"ORDER_STATUS", "FILL"}
|
||||
):
|
||||
raise LedgerReplayError(
|
||||
f"{event_id_value}: event_type must use canonical uppercase"
|
||||
)
|
||||
if (
|
||||
isinstance(sequence_value, bool)
|
||||
or not isinstance(sequence_value, int)
|
||||
or sequence_value <= 0
|
||||
):
|
||||
raise LedgerReplayError(
|
||||
f"{event_id_value}.sequence must be a positive integer"
|
||||
)
|
||||
sequence = sequence_value
|
||||
if not isinstance(timestamp_value, str) or not timestamp_value:
|
||||
raise LedgerReplayError(
|
||||
f"{event_id_value}: timestamp must be a non-empty string"
|
||||
)
|
||||
if not isinstance(payload, Mapping):
|
||||
raise LedgerReplayError(f"{event_id_value}: payload must be an object")
|
||||
event_id = str(event_id_value)
|
||||
event_type = event_type_value
|
||||
event_time = _timestamp(timestamp_value, f"{event_id}.timestamp")
|
||||
event_signature = canonical_json(
|
||||
{
|
||||
"event_type": event_type,
|
||||
"timestamp": timestamp_value,
|
||||
"payload": dict(payload),
|
||||
}
|
||||
)
|
||||
prior_event = self._event_signatures.get(event_id)
|
||||
expected_sequence = self._last_sequence + 1
|
||||
if sequence < self._last_sequence:
|
||||
raise OutOfOrderError(
|
||||
f"{event_id}: sequence {sequence} follows {self._last_sequence}"
|
||||
)
|
||||
if sequence > expected_sequence:
|
||||
raise OutOfOrderError(
|
||||
f"{event_id}: sequence gap; expected {expected_sequence}, "
|
||||
f"observed {sequence}"
|
||||
)
|
||||
if sequence == self._last_sequence:
|
||||
if prior_event is None or prior_event[0] != sequence:
|
||||
raise OutOfOrderError(
|
||||
f"{event_id}: sequence {sequence} is already occupied"
|
||||
)
|
||||
if prior_event[1] != event_signature:
|
||||
raise ConflictingDuplicateError(
|
||||
f"event_id {event_id} has conflicting content"
|
||||
)
|
||||
return
|
||||
if prior_event is not None:
|
||||
raise ConflictingDuplicateError(
|
||||
f"event_id {event_id} was first observed at sequence "
|
||||
f"{prior_event[0]}, not {sequence}"
|
||||
)
|
||||
self._event_signatures[event_id] = (sequence, event_signature)
|
||||
self._last_sequence = sequence
|
||||
if event_type not in {"ORDER_STATUS", "FILL"}:
|
||||
return
|
||||
if event_type == "ORDER_STATUS":
|
||||
self._status_event(
|
||||
payload=payload,
|
||||
event_id=event_id,
|
||||
sequence=sequence,
|
||||
event_time=event_time,
|
||||
)
|
||||
else:
|
||||
self._fill_event(
|
||||
payload=payload,
|
||||
event_id=event_id,
|
||||
sequence=sequence,
|
||||
event_time=event_time,
|
||||
)
|
||||
|
||||
def projection(self) -> LedgerProjection:
|
||||
if self._failure is not None:
|
||||
raise LedgerReplayError(
|
||||
"cannot materialize a projection after a replay error"
|
||||
) from self._failure
|
||||
orders = {
|
||||
order_id: ReplayedOrder(
|
||||
order_id=order.order_id,
|
||||
symbol=order.symbol,
|
||||
side=order.side.value,
|
||||
quantity=order.quantity,
|
||||
status=order.status.value,
|
||||
filled_quantity=order.filled_quantity,
|
||||
average_fill_price=order.average_fill_price,
|
||||
cash_delta=money(order.cash_delta),
|
||||
position_delta=order.position_delta,
|
||||
fill_ids=tuple(order.fill_ids),
|
||||
last_sequence=order.last_sequence,
|
||||
last_timestamp=(
|
||||
order.last_timestamp.isoformat()
|
||||
if order.last_timestamp is not None
|
||||
else ""
|
||||
),
|
||||
)
|
||||
for order_id, order in sorted(self._orders.items())
|
||||
}
|
||||
return LedgerProjection(
|
||||
orders=orders,
|
||||
cash_delta=money(self._cash_delta),
|
||||
position_deltas=dict(sorted(self._position_deltas.items())),
|
||||
fill_ids=tuple(self._fill_ids),
|
||||
source_last_sequence=self._last_sequence,
|
||||
relevant_event_count=self._relevant_event_count,
|
||||
)
|
||||
|
||||
|
||||
def project_ledger(
|
||||
records: HashChainLedger
|
||||
| Iterable[LedgerRecord | Mapping[str, Any]],
|
||||
) -> LedgerProjection:
|
||||
"""Project a verified ledger or an explicitly ordered event iterable."""
|
||||
|
||||
if isinstance(records, HashChainLedger):
|
||||
records.verify()
|
||||
source: Iterable[LedgerRecord | Mapping[str, Any]] = records.records
|
||||
else:
|
||||
source = records
|
||||
projector = LedgerProjector()
|
||||
for record in source:
|
||||
projector.process(record)
|
||||
return projector.projection()
|
||||
|
||||
|
||||
def replay_ledger(
|
||||
records: HashChainLedger
|
||||
| Iterable[LedgerRecord | Mapping[str, Any]],
|
||||
) -> LedgerProjection:
|
||||
"""Compatibility-friendly verb alias for :func:`project_ledger`."""
|
||||
|
||||
return project_ledger(records)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConflictingDuplicateError",
|
||||
"LedgerProjection",
|
||||
"LedgerProjector",
|
||||
"LedgerReplayError",
|
||||
"OutOfOrderError",
|
||||
"OverfillError",
|
||||
"ReplayConflictError",
|
||||
"ReplayError",
|
||||
"ReplayOrderingError",
|
||||
"ReplayedOrder",
|
||||
"UnknownOrderError",
|
||||
"project_ledger",
|
||||
"replay_ledger",
|
||||
]
|
||||
@@ -0,0 +1,274 @@
|
||||
"""Small research utilities with explicit time-ordering semantics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterator, Sequence
|
||||
|
||||
|
||||
class ResearchDependencyUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WalkForwardFold:
|
||||
train: tuple[int, ...]
|
||||
test: tuple[int, ...]
|
||||
purged: tuple[int, ...]
|
||||
embargoed: tuple[int, ...]
|
||||
|
||||
|
||||
def purged_walk_forward(
|
||||
sample_count: int,
|
||||
*,
|
||||
train_size: int,
|
||||
test_size: int,
|
||||
purge: int,
|
||||
embargo: int = 0,
|
||||
step: int | None = None,
|
||||
) -> Iterator[WalkForwardFold]:
|
||||
"""Yield expanding chronological folds without train/test label overlap."""
|
||||
|
||||
values = (sample_count, train_size, test_size, purge, embargo)
|
||||
if min(values) < 0 or train_size == 0 or test_size == 0:
|
||||
raise ValueError("invalid walk-forward sizes")
|
||||
stride = test_size if step is None else int(step)
|
||||
if stride <= 0:
|
||||
raise ValueError("step must be positive")
|
||||
test_start = train_size + purge
|
||||
while test_start + test_size <= sample_count:
|
||||
train_end = test_start - purge
|
||||
train = tuple(range(0, train_end))
|
||||
purged = tuple(range(train_end, test_start))
|
||||
test = tuple(range(test_start, test_start + test_size))
|
||||
embargo_end = min(sample_count, test_start + test_size + embargo)
|
||||
embargoed = tuple(range(test_start + test_size, embargo_end))
|
||||
yield WalkForwardFold(train, test, purged, embargoed)
|
||||
test_start += stride + embargo
|
||||
|
||||
|
||||
class NumpyRidge:
|
||||
"""Minimal deterministic Ridge challenger; NumPy is imported lazily."""
|
||||
|
||||
def __init__(self, alpha: float = 1.0, fit_intercept: bool = True):
|
||||
if alpha < 0:
|
||||
raise ValueError("alpha must be non-negative")
|
||||
self.alpha = float(alpha)
|
||||
self.fit_intercept = bool(fit_intercept)
|
||||
self.coef_ = None
|
||||
self.intercept_ = 0.0
|
||||
|
||||
@staticmethod
|
||||
def _numpy():
|
||||
try:
|
||||
import numpy as np
|
||||
except ImportError as exc:
|
||||
raise ResearchDependencyUnavailable(
|
||||
"NumpyRidge requires NumPy; install the research environment"
|
||||
) from exc
|
||||
return np
|
||||
|
||||
def fit(self, x: Sequence[Sequence[float]], y: Sequence[float]) -> "NumpyRidge":
|
||||
np = self._numpy()
|
||||
matrix = np.asarray(x, dtype=float)
|
||||
target = np.asarray(y, dtype=float)
|
||||
if matrix.ndim != 2 or target.ndim != 1:
|
||||
raise ValueError("x must be 2-D and y must be 1-D")
|
||||
if matrix.shape[0] != target.shape[0] or matrix.shape[0] == 0:
|
||||
raise ValueError("x/y row count mismatch or empty data")
|
||||
if not np.isfinite(matrix).all() or not np.isfinite(target).all():
|
||||
raise ValueError("x/y must be finite")
|
||||
if self.fit_intercept:
|
||||
x_mean = matrix.mean(axis=0)
|
||||
y_mean = float(target.mean())
|
||||
centered_x = matrix - x_mean
|
||||
centered_y = target - y_mean
|
||||
else:
|
||||
x_mean = np.zeros(matrix.shape[1], dtype=float)
|
||||
y_mean = 0.0
|
||||
centered_x, centered_y = matrix, target
|
||||
gram = centered_x.T @ centered_x
|
||||
penalty = np.eye(gram.shape[0], dtype=float) * self.alpha
|
||||
self.coef_ = np.linalg.solve(gram + penalty, centered_x.T @ centered_y)
|
||||
self.intercept_ = y_mean - float(x_mean @ self.coef_)
|
||||
return self
|
||||
|
||||
def predict(self, x: Sequence[Sequence[float]]):
|
||||
if self.coef_ is None:
|
||||
raise ValueError("model is not fitted")
|
||||
np = self._numpy()
|
||||
matrix = np.asarray(x, dtype=float)
|
||||
if matrix.ndim != 2 or matrix.shape[1] != len(self.coef_):
|
||||
raise ValueError("prediction feature shape mismatch")
|
||||
return matrix @ self.coef_ + self.intercept_
|
||||
|
||||
|
||||
def _solve_linear_system(
|
||||
matrix: list[list[float]],
|
||||
target: list[float],
|
||||
) -> list[float]:
|
||||
"""Solve a dense system with deterministic partial-pivot elimination."""
|
||||
|
||||
size = len(matrix)
|
||||
if size == 0 or len(target) != size or any(
|
||||
len(row) != size for row in matrix
|
||||
):
|
||||
raise ValueError("linear system must be non-empty and square")
|
||||
augmented = [
|
||||
[float(value) for value in row] + [float(target[index])]
|
||||
for index, row in enumerate(matrix)
|
||||
]
|
||||
for column in range(size):
|
||||
pivot = max(
|
||||
range(column, size),
|
||||
key=lambda row: (abs(augmented[row][column]), -row),
|
||||
)
|
||||
if abs(augmented[pivot][column]) <= 1e-14:
|
||||
raise ValueError("ridge normal equation is singular")
|
||||
if pivot != column:
|
||||
augmented[column], augmented[pivot] = (
|
||||
augmented[pivot],
|
||||
augmented[column],
|
||||
)
|
||||
divisor = augmented[column][column]
|
||||
augmented[column] = [value / divisor for value in augmented[column]]
|
||||
for row in range(size):
|
||||
if row == column:
|
||||
continue
|
||||
factor = augmented[row][column]
|
||||
if factor == 0:
|
||||
continue
|
||||
augmented[row] = [
|
||||
value - factor * pivot_value
|
||||
for value, pivot_value in zip(
|
||||
augmented[row],
|
||||
augmented[column],
|
||||
)
|
||||
]
|
||||
return [augmented[row][-1] for row in range(size)]
|
||||
|
||||
|
||||
class DeterministicRidge:
|
||||
"""Dependency-free Ridge challenger for CI and Colab smoke experiments."""
|
||||
|
||||
def __init__(self, alpha: float = 1.0, fit_intercept: bool = True):
|
||||
if not math.isfinite(float(alpha)) or alpha < 0:
|
||||
raise ValueError("alpha must be finite and non-negative")
|
||||
if type(fit_intercept) is not bool:
|
||||
raise ValueError("fit_intercept must be a bool")
|
||||
self.alpha = float(alpha)
|
||||
self.fit_intercept = fit_intercept
|
||||
self.coef_: tuple[float, ...] | None = None
|
||||
self.intercept_: float = 0.0
|
||||
|
||||
@staticmethod
|
||||
def _validate(
|
||||
x: Sequence[Sequence[float]],
|
||||
y: Sequence[float] | None = None,
|
||||
) -> tuple[list[list[float]], list[float] | None]:
|
||||
matrix = [[float(value) for value in row] for row in x]
|
||||
if not matrix or not matrix[0]:
|
||||
raise ValueError("x must be a non-empty 2-D matrix")
|
||||
width = len(matrix[0])
|
||||
if any(len(row) != width for row in matrix):
|
||||
raise ValueError("x rows have inconsistent widths")
|
||||
if any(not math.isfinite(value) for row in matrix for value in row):
|
||||
raise ValueError("x must be finite")
|
||||
if y is None:
|
||||
return matrix, None
|
||||
target = [float(value) for value in y]
|
||||
if len(target) != len(matrix):
|
||||
raise ValueError("x/y row count mismatch")
|
||||
if any(not math.isfinite(value) for value in target):
|
||||
raise ValueError("y must be finite")
|
||||
return matrix, target
|
||||
|
||||
def fit(
|
||||
self,
|
||||
x: Sequence[Sequence[float]],
|
||||
y: Sequence[float],
|
||||
) -> "DeterministicRidge":
|
||||
matrix, target = self._validate(x, y)
|
||||
assert target is not None
|
||||
design = (
|
||||
[[1.0] + row for row in matrix]
|
||||
if self.fit_intercept
|
||||
else matrix
|
||||
)
|
||||
width = len(design[0])
|
||||
gram = [[0.0 for unused in range(width)] for unused in range(width)]
|
||||
rhs = [0.0 for unused in range(width)]
|
||||
for row, observed in zip(design, target):
|
||||
for left in range(width):
|
||||
rhs[left] += row[left] * observed
|
||||
for right in range(width):
|
||||
gram[left][right] += row[left] * row[right]
|
||||
for index in range(width):
|
||||
if not (self.fit_intercept and index == 0):
|
||||
gram[index][index] += self.alpha
|
||||
solution = _solve_linear_system(gram, rhs)
|
||||
if self.fit_intercept:
|
||||
self.intercept_ = solution[0]
|
||||
self.coef_ = tuple(solution[1:])
|
||||
else:
|
||||
self.intercept_ = 0.0
|
||||
self.coef_ = tuple(solution)
|
||||
return self
|
||||
|
||||
def predict(
|
||||
self,
|
||||
x: Sequence[Sequence[float]],
|
||||
) -> list[float]:
|
||||
if self.coef_ is None:
|
||||
raise ValueError("model is not fitted")
|
||||
matrix, unused = self._validate(x)
|
||||
del unused
|
||||
if any(len(row) != len(self.coef_) for row in matrix):
|
||||
raise ValueError("prediction feature shape mismatch")
|
||||
return [
|
||||
self.intercept_
|
||||
+ sum(coefficient * value for coefficient, value in zip(self.coef_, row))
|
||||
for row in matrix
|
||||
]
|
||||
|
||||
|
||||
def pearson_correlation(left: Sequence[float], right: Sequence[float]) -> float:
|
||||
if len(left) != len(right) or len(left) < 2:
|
||||
raise ValueError("correlation needs equal vectors with at least two rows")
|
||||
x = [float(value) for value in left]
|
||||
y = [float(value) for value in right]
|
||||
if any(not math.isfinite(value) for value in x + y):
|
||||
raise ValueError("correlation inputs must be finite")
|
||||
x_mean = sum(x) / len(x)
|
||||
y_mean = sum(y) / len(y)
|
||||
numerator = sum(
|
||||
(x_value - x_mean) * (y_value - y_mean)
|
||||
for x_value, y_value in zip(x, y)
|
||||
)
|
||||
x_norm = math.sqrt(sum((value - x_mean) ** 2 for value in x))
|
||||
y_norm = math.sqrt(sum((value - y_mean) ** 2 for value in y))
|
||||
if x_norm <= 1e-15 or y_norm <= 1e-15:
|
||||
return 0.0
|
||||
return numerator / (x_norm * y_norm)
|
||||
|
||||
|
||||
def _average_ranks(values: Sequence[float]) -> list[float]:
|
||||
indexed = sorted(enumerate(values), key=lambda item: (item[1], item[0]))
|
||||
ranks = [0.0] * len(values)
|
||||
start = 0
|
||||
while start < len(indexed):
|
||||
end = start + 1
|
||||
while end < len(indexed) and indexed[end][1] == indexed[start][1]:
|
||||
end += 1
|
||||
average = (start + 1 + end) / 2.0
|
||||
for position in range(start, end):
|
||||
ranks[indexed[position][0]] = average
|
||||
start = end
|
||||
return ranks
|
||||
|
||||
|
||||
def spearman_correlation(left: Sequence[float], right: Sequence[float]) -> float:
|
||||
if len(left) != len(right) or len(left) < 2:
|
||||
raise ValueError("correlation needs equal vectors with at least two rows")
|
||||
return pearson_correlation(_average_ranks(left), _average_ranks(right))
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Deterministic pre-trade checks for the portable reference strategy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Mapping
|
||||
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RiskViolation:
|
||||
code: str
|
||||
message: str
|
||||
symbol: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RiskDecision:
|
||||
approved: bool
|
||||
violations: tuple[RiskViolation, ...]
|
||||
gross_weight: float
|
||||
one_way_turnover: float
|
||||
|
||||
def require_approved(self) -> None:
|
||||
if not self.approved:
|
||||
details = "; ".join(item.message for item in self.violations)
|
||||
raise ValueError(f"pre-trade risk rejected target: {details}")
|
||||
|
||||
|
||||
def weights_from_positions(
|
||||
quantities: Mapping[str, int],
|
||||
prices: Mapping[str, float],
|
||||
equity: float,
|
||||
) -> dict[str, float]:
|
||||
if not math.isfinite(float(equity)) or equity <= 0:
|
||||
raise ValueError("equity must be finite and positive")
|
||||
output: dict[str, float] = {}
|
||||
for raw_symbol, quantity in quantities.items():
|
||||
symbol = normalize_symbol(raw_symbol)
|
||||
price = prices.get(raw_symbol, prices.get(symbol))
|
||||
if (
|
||||
price is None
|
||||
or not math.isfinite(float(price))
|
||||
or float(price) <= 0
|
||||
):
|
||||
raise ValueError(f"missing finite positive price for {symbol}")
|
||||
output[symbol] = (
|
||||
max(0, int(quantity)) * float(price) / float(equity)
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def check_target_weights(
|
||||
targets: Mapping[str, float],
|
||||
current: Mapping[str, float] | None = None,
|
||||
*,
|
||||
max_single_weight: float,
|
||||
max_gross_weight: float,
|
||||
max_one_way_turnover: float = 1.0,
|
||||
tolerance: float = 1e-10,
|
||||
) -> RiskDecision:
|
||||
"""Check concentration, gross exposure and one-way turnover.
|
||||
|
||||
The function is intentionally independent of any optimizer. It is run
|
||||
after target generation so an optimizer cannot silently return a
|
||||
numerically invalid solution.
|
||||
"""
|
||||
|
||||
if not math.isfinite(float(max_single_weight)) or not (
|
||||
0 < max_single_weight <= 1
|
||||
):
|
||||
raise ValueError("max_single_weight must lie in (0, 1]")
|
||||
if not math.isfinite(float(max_gross_weight)) or not (
|
||||
0 <= max_gross_weight <= 1
|
||||
):
|
||||
raise ValueError("max_gross_weight must lie in [0, 1]")
|
||||
if not math.isfinite(float(max_one_way_turnover)) or not (
|
||||
0 <= max_one_way_turnover <= 1
|
||||
):
|
||||
raise ValueError("max_one_way_turnover must lie in [0, 1]")
|
||||
if not math.isfinite(float(tolerance)) or tolerance < 0:
|
||||
raise ValueError("tolerance must be finite and non-negative")
|
||||
|
||||
normalized = {
|
||||
normalize_symbol(symbol): float(weight)
|
||||
for symbol, weight in targets.items()
|
||||
}
|
||||
prior = {
|
||||
normalize_symbol(symbol): float(weight)
|
||||
for symbol, weight in (current or {}).items()
|
||||
}
|
||||
violations: list[RiskViolation] = []
|
||||
for symbol, weight in sorted(normalized.items()):
|
||||
if not math.isfinite(weight):
|
||||
violations.append(
|
||||
RiskViolation(
|
||||
"NON_FINITE_WEIGHT",
|
||||
f"{symbol} target weight is not finite",
|
||||
symbol,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if weight < -tolerance:
|
||||
violations.append(
|
||||
RiskViolation("NEGATIVE_WEIGHT", f"{symbol} weight is negative", symbol)
|
||||
)
|
||||
if weight > max_single_weight + tolerance:
|
||||
violations.append(
|
||||
RiskViolation(
|
||||
"SINGLE_NAME_CAP",
|
||||
f"{symbol} weight {weight:.12f} exceeds "
|
||||
f"{max_single_weight:.12f}",
|
||||
symbol,
|
||||
)
|
||||
)
|
||||
for symbol, weight in sorted(prior.items()):
|
||||
if not math.isfinite(weight):
|
||||
violations.append(
|
||||
RiskViolation(
|
||||
"NON_FINITE_CURRENT_WEIGHT",
|
||||
f"{symbol} current weight is not finite",
|
||||
symbol,
|
||||
)
|
||||
)
|
||||
has_non_finite = any(
|
||||
not math.isfinite(weight)
|
||||
for weight in list(normalized.values()) + list(prior.values())
|
||||
)
|
||||
gross = (
|
||||
float("inf")
|
||||
if has_non_finite
|
||||
else sum(max(0.0, weight) for weight in normalized.values())
|
||||
)
|
||||
if gross > max_gross_weight + tolerance:
|
||||
violations.append(
|
||||
RiskViolation(
|
||||
"GROSS_CAP",
|
||||
f"gross weight {gross:.12f} exceeds {max_gross_weight:.12f}",
|
||||
)
|
||||
)
|
||||
symbols = set(normalized) | set(prior)
|
||||
turnover = (
|
||||
float("inf")
|
||||
if has_non_finite
|
||||
else 0.5
|
||||
* sum(
|
||||
abs(normalized.get(symbol, 0.0) - prior.get(symbol, 0.0))
|
||||
for symbol in symbols
|
||||
)
|
||||
)
|
||||
if turnover > max_one_way_turnover + tolerance:
|
||||
violations.append(
|
||||
RiskViolation(
|
||||
"TURNOVER_CAP",
|
||||
f"one-way turnover {turnover:.12f} exceeds "
|
||||
f"{max_one_way_turnover:.12f}",
|
||||
)
|
||||
)
|
||||
return RiskDecision(
|
||||
approved=not violations,
|
||||
violations=tuple(violations),
|
||||
gross_weight=gross,
|
||||
one_way_turnover=turnover,
|
||||
)
|
||||
|
||||
|
||||
def diagonal_variance(
|
||||
returns_by_symbol: Mapping[str, list[float]],
|
||||
*,
|
||||
variance_floor: float = 1e-8,
|
||||
) -> dict[str, float]:
|
||||
"""Transparent risk fallback used when a full factor model is unavailable."""
|
||||
|
||||
if not math.isfinite(float(variance_floor)) or variance_floor <= 0:
|
||||
raise ValueError("variance_floor must be finite and positive")
|
||||
output: dict[str, float] = {}
|
||||
for raw_symbol, values in returns_by_symbol.items():
|
||||
symbol = normalize_symbol(raw_symbol)
|
||||
observations = [float(value) for value in values]
|
||||
if any(not math.isfinite(value) for value in observations):
|
||||
raise ValueError(f"returns must be finite for {symbol}")
|
||||
if len(observations) < 2:
|
||||
output[symbol] = variance_floor
|
||||
continue
|
||||
mean = sum(observations) / len(observations)
|
||||
variance = sum((value - mean) ** 2 for value in observations) / (
|
||||
len(observations) - 1
|
||||
)
|
||||
output[symbol] = max(variance_floor, variance)
|
||||
return output
|
||||
@@ -0,0 +1,341 @@
|
||||
"""Historical local backtest driven by a verified provider data snapshot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .backtest import (
|
||||
BacktestResult,
|
||||
_contract_manifest,
|
||||
_engine_source_manifest,
|
||||
_metrics,
|
||||
)
|
||||
from .broker import SimBroker
|
||||
from .config import BacktestConfig
|
||||
from .data_snapshot import load_verified_data_snapshot
|
||||
from .domain import Bar, Portfolio, Side
|
||||
from .ledger import HashChainLedger, canonical_json
|
||||
from .snapshot_pipeline import (
|
||||
_build_snapshot_decision_from_verified_snapshot,
|
||||
first_executable_window,
|
||||
)
|
||||
from .synthetic import group_bars_by_date
|
||||
|
||||
|
||||
def _sha256(payload: Any) -> str:
|
||||
return hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _decimal_or_none(value: Any) -> Decimal | None:
|
||||
return Decimal(str(value)) if value is not None else None
|
||||
|
||||
|
||||
def _domain_bars(rows: list[dict[str, Any]]) -> list[Bar]:
|
||||
bars = []
|
||||
for row in rows:
|
||||
bars.append(
|
||||
Bar(
|
||||
trading_date=date.fromisoformat(row["trading_date"]),
|
||||
symbol=row["symbol"],
|
||||
open=Decimal(str(row["open"])),
|
||||
high=Decimal(str(row["high"])),
|
||||
low=Decimal(str(row["low"])),
|
||||
close=Decimal(str(row["close"])),
|
||||
volume=int(row["volume"]),
|
||||
previous_close=_decimal_or_none(row.get("previous_close")),
|
||||
limit_up=_decimal_or_none(row.get("limit_up")),
|
||||
limit_down=_decimal_or_none(row.get("limit_down")),
|
||||
suspended=bool(row["paused"]),
|
||||
)
|
||||
)
|
||||
return bars
|
||||
|
||||
|
||||
def run_snapshot_backtest(
|
||||
*,
|
||||
snapshot_manifest: str | Path,
|
||||
config: BacktestConfig,
|
||||
) -> BacktestResult:
|
||||
"""Run portable-momentum with daily PIT membership and provider rules."""
|
||||
|
||||
config.validate()
|
||||
snapshot = load_verified_data_snapshot(snapshot_manifest)
|
||||
data_manifest = snapshot["manifest"]
|
||||
if data_manifest.get("query", {}).get("adjustment") != "none":
|
||||
raise ValueError(
|
||||
"snapshot backtest requires raw OHLC plus adjustment factors"
|
||||
)
|
||||
if config.universe.mode != "pit_index":
|
||||
raise ValueError(
|
||||
"provider snapshot backtest requires universe.mode=pit_index"
|
||||
)
|
||||
all_bars = _domain_bars(snapshot["bars"])
|
||||
sessions = group_bars_by_date(all_bars)
|
||||
if len(sessions) <= config.strategy.lookback + config.strategy.skip + 1:
|
||||
raise ValueError("provider snapshot is too short for the strategy")
|
||||
|
||||
config_body = config.as_dict()
|
||||
config_body["symbols"] = ["<PIT_INDEX_MEMBERSHIP>"]
|
||||
config_body["universe_query"] = data_manifest.get("query", {})
|
||||
config_hash = _sha256(config_body)
|
||||
engine_source_sha256, engine_sources = _engine_source_manifest()
|
||||
contracts_sha256, contract_schemas = _contract_manifest()
|
||||
portable_path = Path(__file__).with_name("portable_core.py")
|
||||
portable_core_sha256 = hashlib.sha256(
|
||||
portable_path.read_bytes()
|
||||
).hexdigest()
|
||||
strategy_sha256 = _sha256(
|
||||
{
|
||||
"version": "portable-momentum-v1",
|
||||
"parameters": config_body["strategy"],
|
||||
"portable_core_sha256": portable_core_sha256,
|
||||
}
|
||||
)
|
||||
rules_sha256 = _sha256(
|
||||
{
|
||||
"version": "provider-cn-a-v1",
|
||||
"provider": data_manifest["provider"],
|
||||
"provider_version": data_manifest["provider_version"],
|
||||
"fees": config_body["fees"],
|
||||
"execution": config_body["execution"],
|
||||
"membership": "exact_effective_date",
|
||||
"price_adjustment": "PIT_FRONT_FROM_RAW_AND_FACTOR",
|
||||
"price_limits": "provider_daily_fields",
|
||||
"clock": "weekly_first_close_to_next_open",
|
||||
}
|
||||
)
|
||||
run_id = "q60p-" + _sha256(
|
||||
{
|
||||
"config_hash": config_hash,
|
||||
"data_version": data_manifest["data_version"],
|
||||
"engine_source_sha256": engine_source_sha256,
|
||||
"strategy_sha256": strategy_sha256,
|
||||
"rules_sha256": rules_sha256,
|
||||
"contracts_sha256": contracts_sha256,
|
||||
}
|
||||
)[:16]
|
||||
|
||||
portfolio = Portfolio(Decimal(str(config.initial_cash)))
|
||||
ledger = HashChainLedger()
|
||||
broker = SimBroker(
|
||||
portfolio,
|
||||
config.execution,
|
||||
config.fees,
|
||||
ledger,
|
||||
)
|
||||
signals: list[dict[str, Any]] = []
|
||||
targets: list[dict[str, Any]] = []
|
||||
equity_curve: list[dict[str, Any]] = []
|
||||
decision_count = 0
|
||||
session_items = list(sessions.items())
|
||||
|
||||
for session_index, (trading_date, session_bars) in enumerate(session_items):
|
||||
portfolio.start_session()
|
||||
broker.execute_session(session_bars)
|
||||
if config.execution.cancel_open_orders_at_end:
|
||||
broker.cancel_open_orders(
|
||||
trading_date,
|
||||
"DAILY_WINDOW_END",
|
||||
phase="post_open",
|
||||
)
|
||||
portfolio.mark(session_bars.values())
|
||||
snapshot_row = {
|
||||
"date": trading_date.isoformat(),
|
||||
"cash": float(portfolio.cash),
|
||||
"market_value": float(portfolio.market_value),
|
||||
"equity": float(portfolio.equity),
|
||||
"positions": portfolio.quantities(),
|
||||
}
|
||||
equity_curve.append(snapshot_row)
|
||||
ledger.append(
|
||||
"PORTFOLIO_SNAPSHOT",
|
||||
snapshot_row,
|
||||
timestamp=f"{trading_date.isoformat()}T15:00:00+08:00",
|
||||
event_id=f"portfolio:{trading_date.isoformat()}",
|
||||
)
|
||||
|
||||
first_session_of_week = (
|
||||
session_index == 0
|
||||
or session_items[session_index - 1][0].isocalendar()[:2]
|
||||
!= trading_date.isocalendar()[:2]
|
||||
)
|
||||
warmed_up = (
|
||||
session_index
|
||||
>= config.strategy.lookback + config.strategy.skip
|
||||
)
|
||||
if (
|
||||
not first_session_of_week
|
||||
or not warmed_up
|
||||
or session_index >= len(session_items) - 1
|
||||
):
|
||||
continue
|
||||
positions = []
|
||||
for symbol, quantity in sorted(portfolio.quantities().items()):
|
||||
position = portfolio.position(symbol)
|
||||
positions.append(
|
||||
{
|
||||
"symbol": symbol,
|
||||
"quantity": quantity,
|
||||
"sellable_quantity": position.sellable_quantity,
|
||||
"average_cost": (
|
||||
float(position.average_cost)
|
||||
if quantity > 0
|
||||
else None
|
||||
),
|
||||
"market_value": None,
|
||||
}
|
||||
)
|
||||
as_of_timestamp = (
|
||||
f"{trading_date.isoformat()}T15:00:00+08:00"
|
||||
)
|
||||
next_trading_session = session_items[session_index + 1][0]
|
||||
executable_window = first_executable_window(
|
||||
next_trading_session
|
||||
)
|
||||
observation_timestamp = executable_window["start"]
|
||||
broker_snapshot = {
|
||||
"schema_version": "1.0",
|
||||
"snapshot_id": (
|
||||
f"local-backtest-{run_id}-{trading_date.strftime('%Y%m%d')}"
|
||||
),
|
||||
"signal_as_of": as_of_timestamp,
|
||||
"next_trading_session": next_trading_session.isoformat(),
|
||||
"first_executable_window": executable_window,
|
||||
"observation_as_of": observation_timestamp,
|
||||
"as_of": observation_timestamp,
|
||||
"source_time": observation_timestamp,
|
||||
"broker": "quant-os-local-sim",
|
||||
"account_hash": _sha256({"run_id": run_id})[:16],
|
||||
"cash": float(portfolio.cash),
|
||||
"total_asset": float(portfolio.equity),
|
||||
"market_value": float(portfolio.market_value),
|
||||
"positions": positions,
|
||||
"open_orders": [],
|
||||
"metadata": {
|
||||
"evidence_class": "provider_snapshot_local_simulation"
|
||||
},
|
||||
}
|
||||
decision_result = _build_snapshot_decision_from_verified_snapshot(
|
||||
snapshot=snapshot,
|
||||
config=config,
|
||||
as_of=trading_date,
|
||||
broker_snapshot=broker_snapshot,
|
||||
)
|
||||
decision = decision_result["decision"]
|
||||
decision_count += 1
|
||||
for record in decision_result["signals"]:
|
||||
normalized = dict(record)
|
||||
normalized["run_id"] = run_id
|
||||
signals.append(normalized)
|
||||
for record in decision_result["targets"]:
|
||||
normalized = dict(record)
|
||||
normalized["run_id"] = run_id
|
||||
normalized["config_hash"] = config_hash
|
||||
targets.append(normalized)
|
||||
ledger.append(
|
||||
"DECISION",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"decision_id": decision["decision_id"],
|
||||
"as_of": decision["as_of"],
|
||||
"provider_decision_run_id": decision["run_id"],
|
||||
"weights": decision["weights"],
|
||||
"targets": decision["targets"],
|
||||
"orders": decision["orders"],
|
||||
"risk_approved": decision["risk"]["approved"],
|
||||
"data_version": decision["data_version"],
|
||||
},
|
||||
timestamp=decision["as_of"],
|
||||
event_id=f"decision:{decision['decision_id']}",
|
||||
)
|
||||
for symbol, delta in sorted(
|
||||
decision["orders"].items(),
|
||||
key=lambda item: (item[1] > 0, item[0]),
|
||||
):
|
||||
if int(delta) == 0:
|
||||
continue
|
||||
broker.submit(
|
||||
symbol,
|
||||
Side.BUY if int(delta) > 0 else Side.SELL,
|
||||
abs(int(delta)),
|
||||
trading_date,
|
||||
metadata={
|
||||
"run_id": run_id,
|
||||
"decision_id": decision["decision_id"],
|
||||
"data_version": data_manifest["data_version"],
|
||||
},
|
||||
)
|
||||
|
||||
if decision_count == 0:
|
||||
raise ValueError("provider snapshot produced no weekly decision")
|
||||
last_date = session_items[-1][0]
|
||||
broker.cancel_open_orders(last_date, "BACKTEST_END")
|
||||
ledger.verify()
|
||||
report = _metrics(
|
||||
equity_curve,
|
||||
broker,
|
||||
float(config.initial_cash),
|
||||
)
|
||||
report.update(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"config_hash": config_hash,
|
||||
"data_version": data_manifest["data_version"],
|
||||
"engine_source_sha256": engine_source_sha256,
|
||||
"rules_sha256": rules_sha256,
|
||||
"strategy_sha256": strategy_sha256,
|
||||
"contracts_sha256": contracts_sha256,
|
||||
"final_cash": float(portfolio.cash),
|
||||
"final_positions": portfolio.quantities(),
|
||||
"ledger_records": len(ledger.records),
|
||||
"ledger_head_hash": ledger.head_hash,
|
||||
"decision_count": decision_count,
|
||||
"provider": data_manifest["provider"],
|
||||
"provider_version": data_manifest["provider_version"],
|
||||
"evidence_class": "provider-snapshot-local-backtest",
|
||||
"real_platform_backtest": False,
|
||||
"investment_value_claim": False,
|
||||
}
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": "1.0",
|
||||
"run_id": run_id,
|
||||
"engine": "local",
|
||||
"mode": "provider_snapshot",
|
||||
"evidence_class": "provider-snapshot-local-backtest",
|
||||
"config_hash": config_hash,
|
||||
"data_version": data_manifest["data_version"],
|
||||
"input_data_manifest_sha256": hashlib.sha256(
|
||||
Path(snapshot["manifest_file"]).read_bytes()
|
||||
).hexdigest(),
|
||||
"engine_source_sha256": engine_source_sha256,
|
||||
"engine_sources": engine_sources,
|
||||
"portable_core_sha256": portable_core_sha256,
|
||||
"rules_sha256": rules_sha256,
|
||||
"strategy_sha256": strategy_sha256,
|
||||
"contracts_sha256": contracts_sha256,
|
||||
"contract_schemas": contract_schemas,
|
||||
"strategy_version": "portable-momentum-v1",
|
||||
"rules_version": "provider-cn-a-v1",
|
||||
"timezone": "Asia/Shanghai",
|
||||
"signal_clock": "T_CLOSE_COMPLETE",
|
||||
"first_executable_clock": "T_PLUS_1_OPEN",
|
||||
"price_adjustment": "PIT_FRONT_FROM_RAW_AND_FACTOR",
|
||||
"fill_model": "prior_day_provider_volume_capped_open_slippage_v2",
|
||||
"deterministic": True,
|
||||
}
|
||||
return BacktestResult(
|
||||
run_id=run_id,
|
||||
config_hash=config_hash,
|
||||
data_version=data_manifest["data_version"],
|
||||
signals=signals,
|
||||
targets=targets,
|
||||
equity_curve=equity_curve,
|
||||
report=report,
|
||||
manifest=manifest,
|
||||
ledger=ledger,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,194 @@
|
||||
"""Deterministic synthetic A-share daily data for offline smoke tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
from typing import Iterable
|
||||
|
||||
from .domain import Bar
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
CENT = Decimal("0.01")
|
||||
|
||||
|
||||
class _StableRng:
|
||||
"""Tiny fixed LCG whose output does not depend on Python's random module."""
|
||||
|
||||
def __init__(self, seed: int) -> None:
|
||||
self.state = int(seed) & 0x7FFFFFFF
|
||||
|
||||
def uniform(self) -> float:
|
||||
self.state = (1103515245 * self.state + 12345) & 0x7FFFFFFF
|
||||
return self.state / 2147483648.0
|
||||
|
||||
def centered(self) -> float:
|
||||
return sum(self.uniform() for _ in range(6)) - 3.0
|
||||
|
||||
|
||||
def trading_dates(start: date, count: int) -> list[date]:
|
||||
if count <= 0:
|
||||
return []
|
||||
result: list[date] = []
|
||||
current = start
|
||||
while len(result) < count:
|
||||
if current.weekday() < 5:
|
||||
result.append(current)
|
||||
current += timedelta(days=1)
|
||||
return result
|
||||
|
||||
|
||||
def _limit_ratio(symbol: str) -> Decimal:
|
||||
code = symbol[:6]
|
||||
if symbol.endswith(".XBSE"):
|
||||
return Decimal("0.30")
|
||||
if code.startswith(("300", "301", "688", "689")):
|
||||
return Decimal("0.20")
|
||||
return Decimal("0.10")
|
||||
|
||||
|
||||
def _tick(value: Decimal | float) -> Decimal:
|
||||
return Decimal(str(value)).quantize(CENT, rounding=ROUND_HALF_UP)
|
||||
|
||||
|
||||
def generate_synthetic_bars(
|
||||
symbols: Iterable[str],
|
||||
start_date: date | str = date(2024, 1, 2),
|
||||
trading_days_count: int = 90,
|
||||
seed: int = 60,
|
||||
base_volume: int = 100_000,
|
||||
include_market_events: bool = True,
|
||||
) -> list[Bar]:
|
||||
"""Generate stable OHLCV with suspensions and locked limit events.
|
||||
|
||||
Each symbol gets one suspension, one locked limit-up and one locked
|
||||
limit-down session when the requested horizon is long enough. The event
|
||||
placement is deterministic and staggered across symbols.
|
||||
"""
|
||||
|
||||
canonical_symbols = sorted({normalize_symbol(symbol) for symbol in symbols})
|
||||
if not canonical_symbols:
|
||||
raise ValueError("at least one symbol is required")
|
||||
if isinstance(start_date, str):
|
||||
start_date = date.fromisoformat(start_date)
|
||||
if trading_days_count <= 1 or base_volume < 100:
|
||||
raise ValueError("invalid synthetic horizon/base_volume")
|
||||
|
||||
dates = trading_dates(start_date, trading_days_count)
|
||||
rng = _StableRng(seed)
|
||||
result: list[Bar] = []
|
||||
previous: dict[str, Decimal] = {
|
||||
symbol: _tick(8 + index * 7.25)
|
||||
for index, symbol in enumerate(canonical_symbols)
|
||||
}
|
||||
|
||||
for day_index, trading_date in enumerate(dates):
|
||||
for symbol_index, symbol in enumerate(canonical_symbols):
|
||||
previous_close = previous[symbol]
|
||||
ratio = _limit_ratio(symbol)
|
||||
limit_up = _tick(previous_close * (Decimal("1") + ratio))
|
||||
limit_down = _tick(previous_close * (Decimal("1") - ratio))
|
||||
suspended_day = 11 + symbol_index * 3
|
||||
limit_up_day = 24 + symbol_index * 2
|
||||
limit_down_day = 39 + symbol_index * 2
|
||||
|
||||
if include_market_events and day_index == suspended_day:
|
||||
bar = Bar(
|
||||
trading_date=trading_date,
|
||||
symbol=symbol,
|
||||
open=previous_close,
|
||||
high=previous_close,
|
||||
low=previous_close,
|
||||
close=previous_close,
|
||||
volume=0,
|
||||
previous_close=previous_close,
|
||||
limit_up=limit_up,
|
||||
limit_down=limit_down,
|
||||
suspended=True,
|
||||
)
|
||||
elif include_market_events and day_index == limit_up_day:
|
||||
bar = Bar(
|
||||
trading_date=trading_date,
|
||||
symbol=symbol,
|
||||
open=limit_up,
|
||||
high=limit_up,
|
||||
low=limit_up,
|
||||
close=limit_up,
|
||||
volume=max(100, base_volume // 20),
|
||||
previous_close=previous_close,
|
||||
limit_up=limit_up,
|
||||
limit_down=limit_down,
|
||||
)
|
||||
elif include_market_events and day_index == limit_down_day:
|
||||
bar = Bar(
|
||||
trading_date=trading_date,
|
||||
symbol=symbol,
|
||||
open=limit_down,
|
||||
high=limit_down,
|
||||
low=limit_down,
|
||||
close=limit_down,
|
||||
volume=max(100, base_volume // 20),
|
||||
previous_close=previous_close,
|
||||
limit_up=limit_up,
|
||||
limit_down=limit_down,
|
||||
)
|
||||
else:
|
||||
drift = (symbol_index - (len(canonical_symbols) - 1) / 2) * 0.0007
|
||||
cycle = math.sin((day_index + symbol_index * 5) / 8.0) * 0.003
|
||||
noise = rng.centered() * 0.004
|
||||
close_return = max(
|
||||
-float(ratio) * 0.85,
|
||||
min(float(ratio) * 0.85, drift + cycle + noise),
|
||||
)
|
||||
gap = rng.centered() * 0.0015
|
||||
open_price = _tick(previous_close * Decimal(str(1 + gap)))
|
||||
close_price = _tick(
|
||||
previous_close * Decimal(str(1 + close_return))
|
||||
)
|
||||
open_price = min(limit_up, max(limit_down, open_price))
|
||||
close_price = min(limit_up, max(limit_down, close_price))
|
||||
intraday = Decimal(
|
||||
str(abs(rng.centered()) * 0.0025 + 0.001)
|
||||
)
|
||||
high = _tick(
|
||||
max(open_price, close_price)
|
||||
* (Decimal("1") + intraday)
|
||||
)
|
||||
low = _tick(
|
||||
min(open_price, close_price)
|
||||
* (Decimal("1") - intraday)
|
||||
)
|
||||
high = min(limit_up, max(high, open_price, close_price))
|
||||
low = max(limit_down, min(low, open_price, close_price))
|
||||
volume_factor = 0.7 + rng.uniform() * 0.8
|
||||
volume = int(base_volume * volume_factor) // 100 * 100
|
||||
bar = Bar(
|
||||
trading_date=trading_date,
|
||||
symbol=symbol,
|
||||
open=open_price,
|
||||
high=high,
|
||||
low=low,
|
||||
close=close_price,
|
||||
volume=volume,
|
||||
previous_close=previous_close,
|
||||
limit_up=limit_up,
|
||||
limit_down=limit_down,
|
||||
)
|
||||
result.append(bar)
|
||||
previous[symbol] = bar.close
|
||||
result.sort(key=lambda bar: (bar.trading_date, bar.symbol))
|
||||
return result
|
||||
|
||||
|
||||
def group_bars_by_date(bars: Iterable[Bar]) -> dict[date, dict[str, Bar]]:
|
||||
grouped: dict[date, dict[str, Bar]] = {}
|
||||
for bar in bars:
|
||||
session = grouped.setdefault(bar.trading_date, {})
|
||||
if bar.symbol in session:
|
||||
raise ValueError(
|
||||
f"duplicate bar for {bar.symbol} on {bar.trading_date}"
|
||||
)
|
||||
session[bar.symbol] = bar
|
||||
return dict(sorted(grouped.items()))
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Point-in-time A-share universe state machine with explicit reason codes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
from .portable_core import normalize_symbol
|
||||
|
||||
|
||||
class UniverseState(str, Enum):
|
||||
OPENABLE = "openable"
|
||||
HOLD_ONLY = "hold_only"
|
||||
SELL_ONLY = "sell_only"
|
||||
FROZEN = "frozen"
|
||||
EXCLUDED = "excluded"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SecurityEligibility:
|
||||
symbol: str
|
||||
index_member: bool
|
||||
listing_sessions: int
|
||||
median_turnover: float
|
||||
minimum_turnover: float
|
||||
is_st: bool = False
|
||||
delisting: bool = False
|
||||
suspended: bool = False
|
||||
valid_quote: bool = True
|
||||
locked_limit_up: bool = False
|
||||
locked_limit_down: bool = False
|
||||
held_quantity: int = 0
|
||||
sellable_quantity: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "symbol", normalize_symbol(self.symbol))
|
||||
for name in (
|
||||
"index_member",
|
||||
"is_st",
|
||||
"delisting",
|
||||
"suspended",
|
||||
"valid_quote",
|
||||
"locked_limit_up",
|
||||
"locked_limit_down",
|
||||
):
|
||||
if type(getattr(self, name)) is not bool:
|
||||
raise ValueError(f"{name} must be a bool")
|
||||
if type(self.listing_sessions) is not int or self.listing_sessions < 0:
|
||||
raise ValueError("listing_sessions must be a non-negative integer")
|
||||
if type(self.held_quantity) is not int or self.held_quantity < 0:
|
||||
raise ValueError("held_quantity must be a non-negative integer")
|
||||
if (
|
||||
type(self.sellable_quantity) is not int
|
||||
or self.sellable_quantity < 0
|
||||
or self.sellable_quantity > self.held_quantity
|
||||
):
|
||||
raise ValueError("sellable_quantity must lie within held quantity")
|
||||
for name in ("median_turnover", "minimum_turnover"):
|
||||
value = float(getattr(self, name))
|
||||
if not math.isfinite(value) or value < 0:
|
||||
raise ValueError(f"{name} must be finite and non-negative")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UniverseDecision:
|
||||
symbol: str
|
||||
state: UniverseState
|
||||
reasons: tuple[str, ...]
|
||||
can_open: bool
|
||||
can_hold: bool
|
||||
can_sell: bool
|
||||
|
||||
|
||||
def classify_security(
|
||||
eligibility: SecurityEligibility,
|
||||
*,
|
||||
seasoning_sessions: int = 120,
|
||||
) -> UniverseDecision:
|
||||
if type(seasoning_sessions) is not int or seasoning_sessions < 0:
|
||||
raise ValueError("seasoning_sessions must be a non-negative integer")
|
||||
item = eligibility
|
||||
held = item.held_quantity > 0
|
||||
reasons: list[str] = []
|
||||
|
||||
if not item.index_member:
|
||||
reasons.append("NOT_PIT_INDEX_MEMBER")
|
||||
if item.listing_sessions < seasoning_sessions:
|
||||
reasons.append("IPO_SEASONING")
|
||||
if item.is_st:
|
||||
reasons.append("RISK_WARNING")
|
||||
if item.delisting:
|
||||
reasons.append("DELISTING")
|
||||
if item.median_turnover < item.minimum_turnover:
|
||||
reasons.append("LIQUIDITY_BELOW_FLOOR")
|
||||
if item.suspended:
|
||||
reasons.append("SUSPENDED")
|
||||
if not item.valid_quote:
|
||||
reasons.append("NO_VALID_QUOTE")
|
||||
if item.locked_limit_up:
|
||||
reasons.append("LOCKED_LIMIT_UP")
|
||||
if item.locked_limit_down:
|
||||
reasons.append("LOCKED_LIMIT_DOWN")
|
||||
if held and item.sellable_quantity <= 0:
|
||||
reasons.append("NO_SELLABLE_QUANTITY")
|
||||
|
||||
completely_frozen = (
|
||||
item.suspended
|
||||
or not item.valid_quote
|
||||
or (held and item.locked_limit_down)
|
||||
or (held and item.sellable_quantity <= 0)
|
||||
)
|
||||
structural_exit = (
|
||||
not item.index_member
|
||||
or item.is_st
|
||||
or item.delisting
|
||||
)
|
||||
disallow_new = (
|
||||
structural_exit
|
||||
or item.listing_sessions < seasoning_sessions
|
||||
or item.median_turnover < item.minimum_turnover
|
||||
or item.suspended
|
||||
or not item.valid_quote
|
||||
or item.locked_limit_up
|
||||
or item.locked_limit_down
|
||||
)
|
||||
|
||||
if held and completely_frozen:
|
||||
state = UniverseState.FROZEN
|
||||
can_open = False
|
||||
can_hold = True
|
||||
can_sell = False
|
||||
elif held and disallow_new:
|
||||
state = (
|
||||
UniverseState.SELL_ONLY
|
||||
if structural_exit
|
||||
else UniverseState.HOLD_ONLY
|
||||
)
|
||||
can_open = False
|
||||
can_hold = True
|
||||
can_sell = item.sellable_quantity > 0
|
||||
elif held:
|
||||
state = UniverseState.OPENABLE
|
||||
can_open = True
|
||||
can_hold = True
|
||||
can_sell = item.sellable_quantity > 0
|
||||
elif disallow_new:
|
||||
state = UniverseState.EXCLUDED
|
||||
can_open = False
|
||||
can_hold = False
|
||||
can_sell = False
|
||||
else:
|
||||
state = UniverseState.OPENABLE
|
||||
can_open = True
|
||||
can_hold = True
|
||||
can_sell = False
|
||||
|
||||
return UniverseDecision(
|
||||
symbol=item.symbol,
|
||||
state=state,
|
||||
reasons=tuple(reasons or ["ELIGIBLE"]),
|
||||
can_open=can_open,
|
||||
can_hold=can_hold,
|
||||
can_sell=can_sell,
|
||||
)
|
||||
|
||||
|
||||
def build_universe(
|
||||
entries: list[SecurityEligibility],
|
||||
*,
|
||||
seasoning_sessions: int = 120,
|
||||
) -> dict[str, UniverseDecision]:
|
||||
result: dict[str, UniverseDecision] = {}
|
||||
for entry in sorted(entries, key=lambda item: item.symbol):
|
||||
if entry.symbol in result:
|
||||
raise ValueError(f"duplicate universe symbol {entry.symbol}")
|
||||
result[entry.symbol] = classify_security(
|
||||
entry,
|
||||
seasoning_sessions=seasoning_sessions,
|
||||
)
|
||||
return result
|
||||
Reference in New Issue
Block a user