From 48c5f64bbd330739142e78a0b298b28647e431b5 Mon Sep 17 00:00:00 2001 From: zebzhang Date: Sun, 26 Jul 2026 12:54:04 +0800 Subject: [PATCH] feat: add Quant OS A-share baseline --- .env.example | 17 + .gitignore | 44 + CLAUDE.md | 54 + IMPLEMENTATION_PLAN.md | 163 + Makefile | 42 + README.md | 494 +++ adapters/__init__.py | 1 + adapters/jqdata_local.py | 411 ++ adapters/xttrader_live.py | 1304 +++++++ configs/baseline.json | 44 + dist/bundle_manifest.json | 26 + dist/joinquant_strategy.py | 822 ++++ dist/qmt_builtin_strategy.py | 843 ++++ docs/ARCHITECTURE.md | 274 ++ docs/AUDIT_2026-07-25.md | 250 ++ docs/CROSS_ENGINE_CONSISTENCY.md | 252 ++ docs/JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md | 73 + docs/PLATFORM_MATRIX.md | 116 + docs/SECURITY.md | 201 + gate_scorecard.json | 177 + notebooks/quant_os_colab.ipynb | 235 ++ platforms/__init__.py | 1 + platforms/fake_joinquant_harness.py | 189 + platforms/fake_qmt_harness.py | 158 + platforms/joinquant_strategy.py | 465 +++ platforms/qlib_runner.py | 866 +++++ platforms/qmt_builtin_strategy.py | 485 +++ platforms/qmt_research_runner.py | 344 ++ pyproject.toml | 22 + requirements/base.txt | 3 + requirements/dev.txt | 4 + requirements/jqdata.txt | 4 + requirements/qmt.txt | 8 + requirements/research-py312.txt | 14 + runbooks/INCIDENTS.md | 209 + runbooks/PLATFORM_DEPLOYMENT.md | 551 +++ schemas/broker_snapshot.schema.json | 236 ++ schemas/order_event.schema.json | 259 ++ schemas/signal.schema.json | 88 + schemas/target.schema.json | 80 + scripts/validate_all.sh | 32 + src/quant60/__init__.py | 23 + src/quant60/__main__.py | 5 + src/quant60/backtest.py | 672 ++++ src/quant60/broker.py | 368 ++ src/quant60/cli.py | 299 ++ src/quant60/config.py | 263 ++ src/quant60/contracts.py | 223 ++ src/quant60/costs.py | 153 + src/quant60/data_snapshot.py | 467 +++ src/quant60/domain.py | 357 ++ src/quant60/execution.py | 237 ++ src/quant60/experiment.py | 623 +++ src/quant60/factor_risk.py | 276 ++ src/quant60/features.py | 338 ++ src/quant60/ledger.py | 225 ++ src/quant60/optimizer.py | 425 +++ src/quant60/pit.py | 268 ++ src/quant60/portable_core.py | 374 ++ src/quant60/qmt_shadow.py | 3595 ++++++++++++++++++ src/quant60/reconcile.py | 846 +++++ src/quant60/replay.py | 837 ++++ src/quant60/research.py | 274 ++ src/quant60/risk.py | 192 + src/quant60/snapshot_backtest.py | 341 ++ src/quant60/snapshot_pipeline.py | 1050 +++++ src/quant60/synthetic.py | 194 + src/quant60/universe.py | 181 + tests/platform_bundle_probe_test.py | 132 + tests/platform_joinquant_qmt_parity_test.py | 460 +++ tests/platform_qlib_test.py | 363 ++ tests/platform_qmt_runner_test.py | 155 + tests/platform_xttrader_test.py | 859 +++++ tests/test_backtest_cli.py | 212 ++ tests/test_cost_optimizer.py | 155 + tests/test_data_jqdata.py | 418 ++ tests/test_domain_broker.py | 247 ++ tests/test_execution_planner.py | 104 + tests/test_experiment.py | 46 + tests/test_factor_risk.py | 91 + tests/test_features.py | 83 + tests/test_ledger_reconcile.py | 987 +++++ tests/test_mock_parity_report.py | 35 + tests/test_pit_universe.py | 192 + tests/test_portable_core.py | 171 + tests/test_project_scaffold.py | 49 + tests/test_qmt_shadow.py | 2179 +++++++++++ tests/test_qmt_shadow_tool.py | 102 + tests/test_research_risk.py | 99 + tests/test_snapshot_pipeline.py | 544 +++ tools/__init__.py | 1 + tools/build_qlib_tiny_fixture.py | 164 + tools/bundle_platforms.py | 276 ++ tools/capability_probe.py | 209 + tools/export_mock_parity.py | 208 + tools/jqdata_snapshot.py | 37 + tools/qmt_shadow_plan.py | 255 ++ tools/run_colab_notebook.py | 74 + 98 files changed, 31874 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 IMPLEMENTATION_PLAN.md create mode 100644 Makefile create mode 100644 README.md create mode 100644 adapters/__init__.py create mode 100644 adapters/jqdata_local.py create mode 100644 adapters/xttrader_live.py create mode 100644 configs/baseline.json create mode 100644 dist/bundle_manifest.json create mode 100644 dist/joinquant_strategy.py create mode 100644 dist/qmt_builtin_strategy.py create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/AUDIT_2026-07-25.md create mode 100644 docs/CROSS_ENGINE_CONSISTENCY.md create mode 100644 docs/JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md create mode 100644 docs/PLATFORM_MATRIX.md create mode 100644 docs/SECURITY.md create mode 100644 gate_scorecard.json create mode 100644 notebooks/quant_os_colab.ipynb create mode 100644 platforms/__init__.py create mode 100644 platforms/fake_joinquant_harness.py create mode 100644 platforms/fake_qmt_harness.py create mode 100644 platforms/joinquant_strategy.py create mode 100644 platforms/qlib_runner.py create mode 100644 platforms/qmt_builtin_strategy.py create mode 100644 platforms/qmt_research_runner.py create mode 100644 pyproject.toml create mode 100644 requirements/base.txt create mode 100644 requirements/dev.txt create mode 100644 requirements/jqdata.txt create mode 100644 requirements/qmt.txt create mode 100644 requirements/research-py312.txt create mode 100644 runbooks/INCIDENTS.md create mode 100644 runbooks/PLATFORM_DEPLOYMENT.md create mode 100644 schemas/broker_snapshot.schema.json create mode 100644 schemas/order_event.schema.json create mode 100644 schemas/signal.schema.json create mode 100644 schemas/target.schema.json create mode 100755 scripts/validate_all.sh create mode 100644 src/quant60/__init__.py create mode 100644 src/quant60/__main__.py create mode 100644 src/quant60/backtest.py create mode 100644 src/quant60/broker.py create mode 100644 src/quant60/cli.py create mode 100644 src/quant60/config.py create mode 100644 src/quant60/contracts.py create mode 100644 src/quant60/costs.py create mode 100644 src/quant60/data_snapshot.py create mode 100644 src/quant60/domain.py create mode 100644 src/quant60/execution.py create mode 100644 src/quant60/experiment.py create mode 100644 src/quant60/factor_risk.py create mode 100644 src/quant60/features.py create mode 100644 src/quant60/ledger.py create mode 100644 src/quant60/optimizer.py create mode 100644 src/quant60/pit.py create mode 100644 src/quant60/portable_core.py create mode 100644 src/quant60/qmt_shadow.py create mode 100644 src/quant60/reconcile.py create mode 100644 src/quant60/replay.py create mode 100644 src/quant60/research.py create mode 100644 src/quant60/risk.py create mode 100644 src/quant60/snapshot_backtest.py create mode 100644 src/quant60/snapshot_pipeline.py create mode 100644 src/quant60/synthetic.py create mode 100644 src/quant60/universe.py create mode 100644 tests/platform_bundle_probe_test.py create mode 100644 tests/platform_joinquant_qmt_parity_test.py create mode 100644 tests/platform_qlib_test.py create mode 100644 tests/platform_qmt_runner_test.py create mode 100644 tests/platform_xttrader_test.py create mode 100644 tests/test_backtest_cli.py create mode 100644 tests/test_cost_optimizer.py create mode 100644 tests/test_data_jqdata.py create mode 100644 tests/test_domain_broker.py create mode 100644 tests/test_execution_planner.py create mode 100644 tests/test_experiment.py create mode 100644 tests/test_factor_risk.py create mode 100644 tests/test_features.py create mode 100644 tests/test_ledger_reconcile.py create mode 100644 tests/test_mock_parity_report.py create mode 100644 tests/test_pit_universe.py create mode 100644 tests/test_portable_core.py create mode 100644 tests/test_project_scaffold.py create mode 100644 tests/test_qmt_shadow.py create mode 100644 tests/test_qmt_shadow_tool.py create mode 100644 tests/test_research_risk.py create mode 100644 tests/test_snapshot_pipeline.py create mode 100644 tools/__init__.py create mode 100644 tools/build_qlib_tiny_fixture.py create mode 100644 tools/bundle_platforms.py create mode 100644 tools/capability_probe.py create mode 100644 tools/export_mock_parity.py create mode 100644 tools/jqdata_snapshot.py create mode 100644 tools/qmt_shadow_plan.py create mode 100644 tools/run_colab_notebook.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..05b4f69 --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# Copy to .env only on the execution machine. Never commit the populated file. + +# Optional local JQData adapter. Prefer an interactive prompt; if environment +# injection is used, keep the populated file outside Git and Obsidian. +JQDATA_USERNAME=replace-at-runtime +JQDATA_PASSWORD=replace-at-runtime + +# QMT/MiniQMT supplies authentication through its licensed logged-in client. +QMT_USERDATA_PATH=/absolute/path/to/your/qmt/userdata_mini +QMT_SESSION_ID=600001 +QMT_ACCOUNT_ID=replace-with-local-account-id +# Path only. The tool creates this local 0600 HMAC key; never copy its +# contents into Git, Obsidian, logs, notebooks, or chat. +QMT_ACCOUNT_HASH_KEY_FILE=/absolute/local/path/account-hash.key + +# Safety defaults. The built-in QMT strategy is always backtest-only. +QMT_ALLOW_LIVE_ORDERS=false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d2d2d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,44 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.coverage +htmlcov/ +.mypy_cache/ +.ruff_cache/ + +# Local environments +.venv/ +.venv-*/ +venv/ + +# Generated evidence and local data +artifacts/ +reports/ +runs/ +data/raw/ +data/canonical/ +data/snapshots/ +data/qlib/ +mlruns/ +*.parquet +*.duckdb +*.db + +# Credentials and proprietary QMT/JQ material +.env +.env.* +!.env.example +secrets/ +credentials/ +userdata_mini/ +xtquant/ +*.key +*.pem +*.p12 + +# Editors and operating systems +.DS_Store +.idea/ +.vscode/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..3666851 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,54 @@ +# CLAUDE.md + +This directory is the executable source of truth for Quant OS. `quant60` is +its first A-share Baseline kernel/package; the project itself also owns data, +research, portfolio, execution, evidence, and platform adapters. Obsidian +contains research and status documentation only. + +## Development commands + +Run all commands from this directory: + +```bash +make test +make bundle +make smoke +make verify +make doctor +make local +``` + +The zero-install path is `PYTHONPATH=src:. python3 ...`. Do not require a +virtual environment for the deterministic local smoke. + +## Platform boundaries + +- `dist/joinquant_strategy.py` is the generated single-file JoinQuant upload. +- `dist/qmt_builtin_strategy.py` is the generated QMT built-in backtest file. +- `platforms/qmt_research_runner.py` drives qmttools backtests. +- `adapters/xttrader_live.py` is the guarded external QMT/XtTrader boundary. +- `platforms/qlib_runner.py` is an isolated Qlib research bridge. +- `notebooks/quant_os_colab.ipynb` must remain executable locally and in Colab. + +Never make QMT built-in code live-capable. Real broker mutation stays disabled +unless the external XtTrader adapter receives a fresh, account-bound guard. + +## Security + +- Never commit passwords, tokens, cookies, broker userdata, account exports, + private keys, or reusable credentials. +- JoinQuant authentication uses an interactive browser session. +- QMT configuration is injected through local environment variables; only + placeholder names belong in `.env.example`. +- Fake, synthetic, and unit-test evidence must never be described as a real + platform or broker pass. + +## Definition of done + +A change is complete only when: + +1. unit/integration tests pass; +2. hosted bundles rebuild and pass Python 3.6 syntax checks; +3. the deterministic smoke and manifest verifier pass; +4. docs state the exact evidence class and remaining external gate; +5. no secret or machine-specific absolute path is added to tracked artifacts. diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..bce1317 --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,163 @@ +# Quant OS implementation plan + +Quant OS 是项目名；`quant60` 是第一版 A 股 Baseline 内核、CLI namespace +和 artifact 协议前缀。终态不是“仓库里有若干 adapter”，而是本地研究证据链 +可无人值守重放、聚宽登录后可直接回测、QMT 授权环境就绪后无需改源码即可完成 +回测/影子验证，并且所有 G1—G10 都由版本绑定的真实证据支持。 + +当前结论仍为 `NOT_BASELINE_60`。最近一次标准库全量测试记录为 +`Ran 211 tests`、`OK (skipped=3)`；三个 skip 是可选依赖/运行时路径。 +该结果证明代码合同，不证明真实平台、券商或合规 Gate。 + +## Stage 1: 独立仓库与确定性执行 + +**目标**：Quant OS 代码独立于 Obsidian 文档，并有一条可重复的本地验证入口。 + +**已实现**： + +- `quant-os/` 是独立项目目录，Obsidian 只保留项目说明和验证文档； +- `make local` / `scripts/validate_all.sh` 串联测试、bundle、execution smoke、 + research smoke、manifest verifier、mock parity、capability probe 和 Colab + notebook 执行； +- 聚宽与 QMT 内置策略由同一个 Python 3.6-compatible portable core 生成； +- Gitea CI 定义和 Colab notebook 已提供。 + +**状态**：本地功能实现完成；CI 首次远端运行记录仍需随提交保存。Stage 1 +完成不等于任一 Gate 自动通过。 + +## Stage 2: 数据快照、研究与目标生成 + +**目标**：把 PIT 数据、Universe、透明特征、时间有序验证、风险/成本和约束 +组合连接成可审计的 research-to-target 链。 + +**已实现**： + +- PIT `effective_time` / `available_time`、修订版本和未来值 fail-closed； +- openable / hold-only / sell-only / frozen / excluded 状态与原因码； +- 透明价量特征、可执行时钟标签、train-only 预处理； +- purged walk-forward、deterministic Ridge challenger、IC/RankIC 报告； +- 日期化费用、low/base/high 冲击、透明因子风险、约束优化与失败诊断； +- JQData adapter 通过 `get_bars` 获取 raw OHLCV/money、factor、`pre_close`、 + 每日涨跌停/停牌字段，通过 `get_extras` 获取 PIT `is_st`，并逐交易日 + 获取 PIT 指数成员；区间 `get_trade_days` 必须与 `get_all_trade_days` + 一致，并冻结唯一下一交易日；当日未到 24:00 的日线和任何覆盖缺口均 + fail closed； +- snapshot verifier 会重建领域对象、重算质量汇总和 `data_version`； +- 已验证 snapshot 可直接进入 `snapshot-backtest` 和 + `snapshot-decision`，后者支持 canonical broker snapshot 的 + fail-closed 账户状态输入； +- synthetic research artifacts 和 snapshot decision artifacts 都有精确 + manifest verifier。 + +**仍缺**： + +- 授权 JQData 账户上的真实快照、长样本质量报告和许可记录； +- 完整 Security Master、基本面公告时点、公司行动与退市链； +- 真实样本风险/冲击/容量校准； +- Ridge 的冻结 model bundle、真实 OOS 报告与跨引擎推理验证。 + +**状态**：功能实现和合成/fake-provider 合同验证完成，生产证据未完成。 +Ridge 仍是 research challenger，不是已部署模型。 + +## Stage 3: 平台回测与跨引擎证据 + +**目标**：在相同冻结信息集和策略版本上保存本地、聚宽、QMT 与 Qlib 的 +逐层结果，而不是只比较净值。 + +**已验证**： + +- 本地 deterministic execution/research 与 manifest 重放； +- JoinQuant/QMT fake wrapper 合同和 mock parity exporter； +- 2026-07-26 使用生成 bundle 完成 2024 全年真实 JoinQuant hosted + backtest，可见完成日志无 ERROR；该运行还促成 hosted namespace、 + 科创板保护价与 200 股最小申报的真实修复； +- CPython 3.12 + `pyqlib==0.9.7` native momentum fixture smoke， + 实际生成 24 条 signal，并保存 signal/report hash、表边界和重算指标； +- Qlib Alpha158 + LightGBM + Recorder fixture smoke 已真实成功， + runner 会为本地 MLflow file store 设置 + `MLFLOW_ALLOW_FILE_STORE=true`，并保存 model、SignalRecord、 + SigAnaRecord 和 PortAnaRecord，同时读取 portfolio report 保存 hash、 + 表边界、重算指标与 artifact path。 + +Qlib 两次 smoke 使用的是确定性两股票 synthetic fixture，性能没有投资意义， +也不证明 A 股逐日涨跌停/T+1/目标订单语义。mock parity 的 +`evidence_class=mock_contract_only`、`real_platform_pass=false`， +因此不能计入 G9。 + +**仍缺**： + +- 聚宽完整输入/plan/orders/fills 导出和相同输入本地 peer； +- 授权 QMT built-in 与 qmttools 回测导出； +- 至少 20 个交易日的真实 QMT/broker shadow； +- 同一 canonical 输入的 L1—L5 差异报告。 + +**状态**：部分完成；JoinQuant runtime smoke 已执行，但 strict comparison、 +真实 QMT 和 broker 运行仍未完成，G9 保持 `not_passed`。 + +## Stage 4: Broker-safe shadow operation + +**目标**：完成查询、callback replay、重启恢复、持续对账、 +guarded TWAP/POV 和影子证据。 + +**已有基础**： + +- XtTrader adapter 默认禁止 live mutation； +- operator CLI 严格只读，能生成 HMAC 账户绑定的 broker + observation/snapshot、target-diff shadow plan 与 exact manifest； +- T 日完整 Signal 只能与冻结 provider calendar 的唯一下一交易日 + Asia/Shanghai `[09:00,09:30)` broker observation 显式绑定，保留两个 + 时钟；同日、周末近似、错窗、错误 session 和迟到补查均失败关闭； +- 严格类型、提交前幂等 reservation、单调状态回调、fresh + account-bound guard 和审计记录； +- hash-chain ledger、重复/乱序重放和完整 broker snapshot 对账合同。 + +**仍缺**： + +- 券商实际状态码映射与脱敏回调； +- 实际 QMT query API 的失败/空列表合同探针； +- durable scheduler/recovery daemon； +- 20 个交易日 shadow、日终零未知账差和故障恢复演练； +- 券商权限、频率、软件和程序化交易报告确认。 + +**状态**：仅有未认证的安全边界；不可视为 live-ready。 + +## Stage 5: Baseline-60 evidence + +**目标**：证据评分至少 60，且 G1—G10 十道硬门全部 `passed`。 + +**状态**：未开始计分。当前 [`gate_scorecard.json`](gate_scorecard.json) +中 G1—G10 全部 `not_passed`。源文件、单测、synthetic、fixture、fake +harness 和成功的 Qlib smoke 都不能替代真实数据、平台、券商和合规证据。 + +## 当前验证入口 + +```bash +# 核心、本地两条 synthetic 链、bundle、mock parity、Colab +make local + +# JQData 授权后：快照 -> 语义校验 -> PIT 回测/决策 +python3.12 -m venv .venv-jqdata +source .venv-jqdata/bin/activate +python -m pip install -r requirements/jqdata.txt +PYTHONPATH=src:. python tools/jqdata_snapshot.py \ + --index 000905.XSHG --start 2024-01-02 --end 2024-12-31 \ + --output data/snapshots/csi500-2024 +PYTHONPATH=src:. python -c \ + "from quant60.data_snapshot import verify_data_snapshot; verify_data_snapshot('data/snapshots/csi500-2024/manifest.json')" +PYTHONPATH=src:. python -m quant60 snapshot-backtest \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json --output artifacts/csi500-2024-backtest +PYTHONPATH=src:. python -m quant60 snapshot-decision \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json --as-of 2024-12-31 \ + --equity 1000000 --output artifacts/csi500-2024-decision + +# Qlib 0.9.7 独立环境 +python3.12 -m venv .venv-qlib312 +source .venv-qlib312/bin/activate +python -m pip install -r requirements/research-py312.txt +python tools/build_qlib_tiny_fixture.py artifacts/qlib-fixture --days 400 +``` + +平台真实运行和证据保存步骤见 +[`runbooks/PLATFORM_DEPLOYMENT.md`](runbooks/PLATFORM_DEPLOYMENT.md)。 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ba11481 --- /dev/null +++ b/Makefile @@ -0,0 +1,42 @@ +PYTHON ?= python3 +PROJECT_PYTHONPATH := src:. +SMOKE_DIR := artifacts/local-smoke +RESEARCH_DIR := artifacts/research-smoke +PARITY_REPORT := artifacts/mock-parity/report.json + +.PHONY: test bundle smoke verify research research-verify parity parity-verify doctor notebook-check notebook-smoke local + +test: + PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m unittest discover -s tests -p '*.py' -q + +bundle: + $(PYTHON) tools/bundle_platforms.py + +smoke: + PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m quant60 smoke --config configs/baseline.json --output $(SMOKE_DIR) + +verify: + PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m quant60 verify-manifest $(SMOKE_DIR)/manifest.json + +research: + PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m quant60 research-smoke --output $(RESEARCH_DIR) + +research-verify: + PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m quant60 verify-research-manifest $(RESEARCH_DIR)/manifest.json + +parity: + PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) tools/export_mock_parity.py export --output $(PARITY_REPORT) + +parity-verify: + PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) tools/export_mock_parity.py verify $(PARITY_REPORT) + +doctor: + PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) tools/capability_probe.py + +notebook-check: + $(PYTHON) tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb + +notebook-smoke: + $(PYTHON) tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb --execute + +local: test bundle smoke verify research research-verify parity parity-verify doctor notebook-check notebook-smoke diff --git a/README.md b/README.md new file mode 100644 index 0000000..5221529 --- /dev/null +++ b/README.md @@ -0,0 +1,494 @@ +# Quant OS：个人 A 股 60 分 Baseline 工程 + +Quant OS 是项目名；`quant60` 是第一版 A 股 Baseline 的 Python +包、CLI namespace 与 artifact 协议前缀。 + +本项目把 2026-07-22 的技术选型文章落成一个可执行、可审计的模块化单体： +本地与 Colab 可零账号运行工程 smoke；JQData 登录后可生成不可变快照并直接 +运行 PIT 本地回测/目标决策；聚宽、QMT 与 Qlib 有各自明确的运行入口。 + +> 当前状态仍是 `NOT_BASELINE_60`。代码存在、单测、synthetic、fake harness、 +> Qlib fixture 和单次真实聚宽 hosted smoke，都不能代替同输入跨引擎对比、 +> 真实 QMT、券商影子盘、连续对账与合规证据。 +> 原文章复评为设计覆盖约 70/100、可实施规格 49/100、原实现证据 6/100、 +> G1—G10 为 0/10。详见 +> [`docs/AUDIT_2026-07-25.md`](docs/AUDIT_2026-07-25.md) 与 +> [`gate_scorecard.json`](gate_scorecard.json)。 + +## 当前交付边界 + +| 路径 | 当前能做什么 | 已有证据 | 仍缺什么 | +| --- | --- | --- | --- | +| 本地 synthetic execution | 周度信号、T+1 开盘撮合、费用、涨跌停、部分成交、ledger、重放 | 标准库全量测试、确定性 smoke、manifest verifier | 不代表真实市场或投资价值 | +| 本地 synthetic research | PIT/Universe、透明特征、purged walk-forward、Ridge、成本/风险、约束 target | research artifact 与 verifier | 仍是 synthetic；Ridge 尚未冻结为跨平台生产模型 | +| JQData → 本地 | 原始价、复权因子、前收、涨跌停、停牌、PIT ST 状态与历史指数成员快照；PIT 本地回测与单日目标 | fake-provider 端到端回归测试、完整收盘日 fail-closed 合同 | 需要用户授权登录后保存真实数据运行；基本面/公司行动尚未全接 | +| Colab | 自动 clone、测试、bundle、两套 synthetic 流程；可选 JQData/Qlib | notebook 本地逐 cell 执行 | JQData/Qlib 可选 cell 需对应账号/运行时 | +| 聚宽 hosted | 可上传的 Python 单文件，直接回测 portable baseline | 2024 全年、100 万资金的真实 hosted run 已完成且可见日志无 ERROR；另有 bundle/fake/mock 合约 | 仍需完整导出、相同输入本地 peer 与逐层差异报告 | +| QMT built-in/qmttools | 可上传/驱动的硬 backtest-only portable baseline；固定中证 500 benchmark、10% 参与率、PIT 成分/ST | Python 3.6 语法、fake/qmttools 合约 | 需要已授权 QMT 客户端、历史 ST 数据权限/正例探针与实跑导出 | +| Qlib 0.9.7 | native momentum backtest；Alpha158/LightGBM CLI/Recorder 工作流 | Python 3.12 两条 fixture 已真实跑通，并保存表级 hash、行列/时间边界和重算指标 | synthetic 两股票没有投资价值；无真实 OOS、L3/L4 parity | +| XtTrader shadow | 只读账户查询、HMAC 账户绑定、broker observation/snapshot、零下单 target-diff 计划与完整 verifier | fake broker/QMT 合约；所有 broker mutation 均不可达 | 需要授权 QMT 环境的账户合同探针、20 日影子、恢复/对账和券商确认 | + +跨四个引擎当前共同的、可冻结生产基线是 +`portable-momentum-v1`。本地 Ridge 与 Qlib Alpha158/LightGBM 是研究候选， +在出现版本化 model bundle、真实 OOS 证据和跨引擎推理验证前，不会冒充已经部署。 + +## 一条命令验证本地工程 + +要求 Python 3.10 或更高；核心没有第三方运行依赖： + +```bash +make local +``` + +它依次执行： + +1. 全量单元/合约测试； +2. 聚宽与 QMT 单文件 bundle； +3. synthetic execution smoke 与完整 manifest verifier； +4. synthetic research-to-target 与 research manifest verifier； +5. JoinQuant/QMT mock parity 报告及校验； +6. capability probe； +7. Colab notebook 语法检查和逐 cell 本地执行。 + +等价的脚本入口： + +```bash +bash scripts/validate_all.sh +``` + +主要本地产物位于被 Git 忽略的 `artifacts/`： + +```text +artifacts/ +├── local-smoke/ +│ ├── manifest.json +│ ├── signal.json +│ ├── target.json +│ ├── events.jsonl +│ ├── equity.json +│ └── report.json +├── research-smoke/ +│ ├── manifest.json +│ ├── signals.json +│ ├── targets.json +│ ├── forecasts.json +│ ├── samples.json +│ └── report.json +└── mock-parity/report.json +``` + +`verify-ledger` 只验证事件 hash chain；`verify-manifest` 还验证原子发布标记、 +精确 artifact 集合与 SHA-256、ledger head、report/manifest 身份、 +当前 source/schema 聚合 hash，以及 Signal、Target、OrderEvent JSON Schema。 + +## JQData：登录后直接跑真实数据本地回测 + +账号和密码只从无回显交互或本地环境变量读取，绝不写入 snapshot、notebook、 +OB 或 Git。先创建独立环境： + +```bash +python3.12 -m venv .venv-jqdata +source .venv-jqdata/bin/activate +python -m pip install -r requirements/jqdata.txt +``` + +生成一个中证 500 历史快照： + +```bash +PYTHONPATH=src:. python tools/jqdata_snapshot.py \ + --index 000905.XSHG \ + --start 2024-01-02 \ + --end 2024-12-31 \ + --output data/snapshots/csi500-2024 +``` + +adapter 使用官方 `get_bars` 的未复权 OHLCV/money、复权因子、 +`pre_close`、`high_limit`、`low_limit`、`paused`，并通过 +`get_extras("is_st", ...)` 保存每日 PIT ST 状态和每日历史指数成员。为避免 +把尚未完成的当日线误标为完整收盘，`--end` 必须早于 Asia/Shanghai 的抓取 +日期；JQData 日线 24:00 完成后才允许进入快照。字段、日期或成员覆盖不完整 +都会 fail closed。快照 verifier 会重新构造领域对象、重算质量汇总与 +`data_version`，不是只比文件 hash。 + +直接运行动态 PIT 成分的本地事件回测： + +```bash +PYTHONPATH=src:. python -m quant60 snapshot-backtest \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json \ + --output artifacts/csi500-2024-backtest + +PYTHONPATH=src:. python -m quant60 verify-manifest \ + artifacts/csi500-2024-backtest/manifest.json +``` + +生成某个决策日的可移交 Signal/Target/Order Delta： + +```bash +PYTHONPATH=src:. python -m quant60 snapshot-decision \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json \ + --as-of 2024-12-31 \ + --equity 1000000 \ + --output artifacts/csi500-2024-decision + +PYTHONPATH=src:. python -m quant60 verify-snapshot-decision \ + artifacts/csi500-2024-decision/manifest.json +``` + +生产账户应把 `--equity` 换成 `--broker-snapshot`。该 canonical broker +snapshot 必须绑定同一个 T 日 Signal/decision，但其观察时钟只能位于冻结 +provider calendar 的唯一下一交易日 Asia/Shanghai `[09:00,09:30)`；它还 +必须没有未对账 open orders，并只保存加盐/带密钥的账户标识。不完整或错窗 +状态会 fail closed。 + +这个本地回测采用： + +- 每日精确 PIT 指数成员； +- 每日 PIT ST 状态；ST 证券不进入新目标，已持有 ST 只允许卖出； +- 原始价作为执行/名义价格； +- 仅使用截至每个决策日的 raw price + factor 构造定点前复权动量； +- 供应商每日停牌、涨跌停与前收字段； +- 第一交易日完整收盘形成信号，紧接着下一可用交易日开盘尝试成交； +- 上一交易日成交量参与率、显式费用、T+1、百股交易单位和未成交撤销。 + +它仍是本地 fill model，不是聚宽/QMT 的真实成交证据。 + +## Colab + +打开 [`notebooks/quant_os_colab.ipynb`](notebooks/quant_os_colab.ipynb) +并执行全部默认 cell。默认流程不联网安装依赖，也不需要账号。 + +可选 cell： + +- `RUN_JQDATA=True`：无回显登录，生成 snapshot、本地 PIT backtest 与决策； +- `RUN_QLIB=True`：只在 Python 3.12 安装固定 Qlib 栈并跑 native fixture。 + +不要把密码、token 或带凭据的 clone URL 写进 cell。默认流程最终生成两个可下载 +zip；启用 JQData 后再增加 backtest 与 decision zip。 + +本地可验证 notebook 本身： + +```bash +python3 tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb +python3 tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb --execute +``` + +## 聚宽：直接上传回测 + +先构建： + +```bash +python3 tools/bundle_platforms.py +``` + +上传完整的 [`dist/joinquant_strategy.py`](dist/joinquant_strategy.py)。 +wrapper 已启用： + +```python +set_option("avoid_future_data", True) +set_option("use_real_price", True) +run_daily(rebalance, time="open") +``` + +canonical 周时钟不是固定星期几：某 ISO 周第一实际交易日收盘形成计划， +下一实际交易日开盘执行；单交易日长假周会自然跨到下一周。wrapper 用 +`context.previous_date` 查询 PIT 指数成员和 `is_st`，用批量 +`history(..., fq="pre")` 取得已完成日线；卖单先行，以 `order_target` +提交 `当前数量 + 可执行 delta`，并在提交点重新检查 current data。手续费、 +滑点和最大成交量占比 10% 都在策略内显式设置。科创板买卖使用当日涨/跌停价 +作为限价保护；共享核心拒绝不足 200 股的普通科创板申报，只保留一次性卖清 +不足 200 股余额的交易所例外。 + +2026-07-26 已用生成 bundle 完成一次真实聚宽全年 hosted backtest。运行记录、 +bundle hash、指标、真实平台发现的问题和证据边界见 +[`docs/JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md`](docs/JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md)。 +该次运行的可见完成日志没有 `ERROR`，但尚未导出相同输入、plan、orders/fills， +所以不计为 G9 通过。 + +真实验收必须保存： + +- Git commit 与 `dist/bundle_manifest.json`； +- 聚宽运行环境、日期、资金、benchmark、费率/滑点； +- `g.quant60_last_plan`、订单/成交、日志和完整回测导出； +- 与本地相同输入 close array 或其 hash。 + +本地 fake harness 与 `artifacts/mock-parity/report.json` 只证明 wrapper 合约。 +单次 hosted smoke 是外部运行证据，但没有同输入跨引擎 manifest 时仍不算 +真实聚宽 Gate 通过。 + +## QMT built-in 与 qmttools + +同一 bundler 生成： + +```text +dist/qmt_builtin_strategy.py +dist/bundle_manifest.json +``` + +在 QMT Python 策略编辑器导入，选择 `1d` 与 **backtest**，主图/基准设为 +`000905.SH`，初始资金与日期按证据计划填写，并把 GUI 的最大成交量占比设为 +**10%**。bundle 兼容 Python 3.6，使用模块全局 `g` 保存策略状态，以适配 +QMT 文档所述的 `ContextInfo` 用户属性回滚。它硬编码 backtest-only；参数和 +环境变量均不能把它提升成实盘 `passorder`。 + +策略按原始毫秒 timetag 查询历史中证 500 成分，并通过 +`ContextInfo.get_his_st_data` 读取决策日 ST/*ST/PT 区间；接口缺失、格式异常 +或覆盖不足会阻止计划。首次真实验收还必须用一只已知历史 ST 股票做正例探针， +证明客户端权限和本地历史 ST 数据确实可用，不能把空结果直接解释成“当天无 +ST”。 + +Windows 上、已登录且安装券商分发 `xtquant` 的 qmttools 入口： + +```powershell +$env:PYTHONPATH = "src;." +python -m platforms.qmt_research_runner ` + dist/qmt_builtin_strategy.py ` + --stock-code 000905.SH ` + --start 20240102 ` + --end 20241231 ` + --period 1d ` + --asset 1000000 ` + --account-id test +``` + +runner 固定 `trade_mode=backtest`、`quote_mode=history`，真实证据不得跳过 +只读数据 preflight；它从 `configs/baseline.json` 固定 benchmark +`000905.SH`，并程序化传入 10% `max_vol_rate`。QMT 没有单独过户费参数， +本项目明确把它折入佣金；最低佣金与 fill 行为仍属于必须单列的引擎差异。 + +QMT 并不是“只给账号密码”即可远程完成：还必须有券商授权的 QMT/MiniQMT +客户端、匹配的 `xtquant`、userdata path、行情/历史数据权限和已经登录的账户。 +这些就绪后，代码不需要改源文件，只注入 `.env.example` 中列出的本地参数。 + +## XtTrader：严格只读影子 + +外部 adapter 是 [`adapters/xttrader_live.py`](adapters/xttrader_live.py)。 +`tools/qmt_shadow_plan.py` 只暴露查询接口，并始终以 +`allow_live_orders=False` 创建 adapter；该工具没有报单或撤单命令。它会 +二次检查 adapter 状态、账户归属、资产恒等式、持仓、委托、成交以及查询时长， +保存脱敏的 `broker_observation.json`、可校验的 `broker_snapshot.json` 和 +全部 delta 为零或只读建议的 shadow plan。任一事实不可信就 fail closed。 +账户绑定 decision 的绝对目标股数只在账户规模未变化时有效，因此 planner +还会比较 `decision.equity` 与当前 `total_asset`：允许差额固定为 +`max(1 元, 0.01% × decision equity)`，边界包含；超过即产生 +`DECISION_EQUITY_DRIFT`，状态为 `BLOCKED` 且 proposed delta 全部归零。 +容差、计算规则、实际差额和结果全部写入 plan，并由 verifier 重新计算，不能 +通过改 JSON 放宽。 + +read-only observation 还冻结 adapter query 后状态、live flag、callback +error 数、live-authorization history 数、query 起止时钟、原始记录数和事实 +校验结果。query duration 的硬上限是 10 秒，资产/持仓市值及 +`cash + market_value = total_asset` 的绝对容差硬上限是 1 元；CLI 只允许 +收紧，builder 和 verifier 都拒绝放宽。verify 必须同时提供原始 decision +manifest。verifier 不相信 plan 中的 blockers、status、snapshot-valid、 +target/weight/lot、feasible/proposed delta 或 cash/portfolio 派生值，而是从 +原始 decision + `broker_observation.json` 重建唯一语义投影并逐项 exact +compare。`SHADOW_READY` 必须有可重建且内容完全一致的 broker snapshot； +删除 blocker、注入假 blocker、同步改大目标量或把 snapshot 改成 `null` 后 +重新计算普通 hash 都不能通过。 + +同一个本地 account HMAC key 还以独立 domain 签署 authenticated evidence +envelope；签名同时覆盖完整 plan、broker observation、broker snapshot 的 +canonical 内容 hash 和三份确定性发布文件的原始 byte SHA-256，以及原始 +decision manifest、planner/engine/`tools/qmt_shadow_plan.py` source 和固定 +policy。四份发布 JSON（含 manifest）都必须逐 byte 等于唯一 writer 编码； +只做 minify、换序或空白改写也会失败。 +verify 必须使用生成时的**既有** key；缺失/错误 key 失败，verify 不会生成 +替代 key。该 MAC 只证明持 key 的 Quant OS 发布后未被无 key 改写，不是券商 +对 API 原始事实的签名或 attestation。 + +在已登录的 Windows QMT/MiniQMT 环境先注入本地参数；不要在聊天、Git 或 OB +中发送它们： + +```powershell +$env:PYTHONPATH = "src;." +$env:QMT_USERDATA_PATH = "C:\local\qmt\userdata_mini" +$env:QMT_ACCOUNT_ID = "" +$env:QMT_SESSION_ID = "" +$env:QMT_ACCOUNT_HASH_KEY_FILE = "C:\local\quant-os\account-hash.key" +$env:DECISION_DATE = "" +``` + +首次对一个尚未绑定账户的 snapshot decision 运行： + +```powershell +python tools/qmt_shadow_plan.py plan ` + --decision artifacts\csi500-2024-decision\manifest.json ` + --output artifacts\qmt-shadow-bootstrap +``` + +若 broker 查询全部可信，首次运行仍会因 `DECISION_NOT_ACCOUNT_BOUND` 预期返回 +退出码 2，但会输出可信的 `broker_snapshot.json`。若该文件内容为 JSON +`null`，说明观察事实不可信，禁止继续。用**同一个决策日**重新生成账户绑定的 +decision： + +```powershell +python -m quant60 snapshot-decision ` + --snapshot data\snapshots\latest\manifest.json ` + --config configs\baseline.json ` + --as-of $env:DECISION_DATE ` + --broker-snapshot artifacts\qmt-shadow-bootstrap\broker_snapshot.json ` + --output artifacts\qmt-bound-decision + +python tools/qmt_shadow_plan.py plan ` + --decision artifacts\qmt-bound-decision\manifest.json ` + --output artifacts\qmt-shadow + +python tools/qmt_shadow_plan.py verify ` + --manifest artifacts\qmt-shadow\manifest.json ` + --decision artifacts\qmt-bound-decision\manifest.json +``` + +Signal 仍绑定 T 日 15:00 完整收盘。JQData snapshot 冻结经过两套交易日历 +API 一致性校验的 session 序列，并保存唯一 `next_trading_session`；broker +查询只能在该 session 的 Asia/Shanghai `[09:00,09:30)` 盘前窗口完成。 +周末、同日收盘后、09:30 边界、错误 session 和迟到多日都会 fail closed； +国庆等长假按冻结交易日历接受，绝不使用 elapsed-day 阈值或 weekday 近似。 + +持仓变化不会沿用 decision 形成时的本地数量：target diff 每次都以当前券商 +持仓和可卖量重新计算，逐 symbol 的 target quantity/weight/lot size 必须与 +原始 decision 完全一致，未完成或查询期间变化的委托会直接阻断。当前 cash、 +market value、total asset 和持仓市值恒等式由 verifier 独立重算。 +现金/持仓变化仍可能影响购买力； +shadow 没有实时执行价，明确不做 buying-power 认定，所以 +`SHADOW_READY` 只表示可人工审阅，始终不是可实盘提交。 + +plan 首次运行可自动生成账户 HMAC key，Unix 要求 0600；verify 只读取既有 +key。它必须稳定备份、限制访问且绝不 +进入 Git、OB、notebook、日志或聊天。更换 key 会改变账户绑定标识并使旧决策 +失配，也会使旧 evidence 无法通过 HMAC。当前流程仍是 +`uncertified / not live-ready`；真实账户还必须先验证 +官方 query API 对“失败”和“空列表”的返回合同，并记录实际 +terminal/xtquant build、确认 `XtTrade.traded_amount` 成交金额语义与允许 +误差。当前 amount 只是被认证的 informational fact，不参与 target quantity +或 proposed delta。之后还要累计至少 20 个交易日、恢复演练和零未知账差。 + +## Qlib 0.9.7 + +研究环境隔离在 CPython 3.12，关键数值/工作流依赖已固定： + +```bash +python3.12 -m venv .venv-qlib312 +source .venv-qlib312/bin/activate +python -m pip install -r requirements/research-py312.txt +python -c "import qlib; assert qlib.__version__ == '0.9.7'" +``` + +native momentum smoke： + +```bash +python tools/build_qlib_tiny_fixture.py /tmp/quant-os-qlib --days 80 +PYTHONPATH=src:. python -m platforms.qlib_runner \ + --provider-uri /tmp/quant-os-qlib \ + --market csi300 \ + --benchmark SH000300 \ + --start 2024-02-01 \ + --end 2024-04-19 \ + --lookback 20 \ + --topk 1 \ + --n-drop 1 \ + --rebalance weekly \ + --output-json artifacts/qlib-native/result.json +``` + +本机实际运行 `pyqlib==0.9.7` 得到 24 条 signal，使用 +`SimulatorExecutor + TopkDropoutStrategy`。结果 JSON 保存 signal 与 +portfolio report 的稳定 hash、行列/时间边界，以及从 report 重算的累计收益、 +最大回撤、基准收益、成本和换手；`investment_value_claim=false`。fixture +只有 smoke 意义。 +Qlib 的统一 `limit_threshold` 不能表示逐日板块/ST 规则，因此只参加 +L1/L2 与诊断 L6，不声称 L3 target/L4 order parity。 + +Alpha158/LightGBM CLI： + +```bash +PYTHONPATH=src:. python -m platforms.qlib_runner \ + --workflow alpha158 \ + --provider-uri /path/to/authorized/qlib-provider \ + --market csi500 \ + --benchmark SH000905 \ + --train-start 2018-01-01 \ + --train-end 2020-12-31 \ + --valid-start 2021-01-01 \ + --valid-end 2021-12-31 \ + --test-start 2022-01-01 \ + --test-end 2022-12-31 \ + --experiment-name quant-os-alpha158 \ + --output-json artifacts/qlib-alpha158/result.json +``` + +它通过 Qlib Recorder 保存 model、SignalRecord、SigAnaRecord 和 +PortAnaRecord；runner 会读取 `report_normal_1day.pkl`，保存表级 hash、 +行列/时间边界和有限值指标。真实研究必须换成授权且版本不可变的 provider。 + +## 跨引擎与证据 + +生成本地 wrapper 合同 parity： + +```bash +PYTHONPATH=src:. python tools/export_mock_parity.py \ + export \ + --output artifacts/mock-parity/report.json +PYTHONPATH=src:. python tools/export_mock_parity.py \ + verify \ + artifacts/mock-parity/report.json +``` + +报告逐层比较 score、weight、target quantity 与 order delta，并固定 source +hash；它明确写入： + +```text +evidence_class = mock_contract_only +real_platform_pass = false +gate_credit = [] +``` + +真实 G9 仍要求聚宽/QMT 导出与至少 20 个交易日 QMT 影子记录。完整层级、 +容差和 reason code 见 +[`docs/CROSS_ENGINE_CONSISTENCY.md`](docs/CROSS_ENGINE_CONSISTENCY.md)。 + +## 已实现的系统模块 + +- `pit.py`：`effective_time` / `available_time`、修订版本、未来值拒绝； +- `universe.py`：openable / hold-only / sell-only / frozen / excluded； +- `features.py`：透明价量特征、可执行时钟标签、train-only 预处理； +- `research.py` / `experiment.py`：purge/embargo、Ridge、OOS IC/RankIC； +- `factor_risk.py`：行业/风格暴露、EWMA 收缩因子协方差、特异风险、stress； +- `costs.py`：日期化费率、最低佣金、low/base/high impact 与 capacity； +- `optimizer.py`：deterministic fallback 与 lazy CVXPY 约束优化； +- `execution.py`：guarded TWAP/POV parent-child planner； +- `domain.py` / `broker.py`：A 股事件撮合与订单状态； +- `ledger.py` / `replay.py` / `reconcile.py`：hash-chain、幂等重放、fail-closed 对账； +- `data_snapshot.py` / `snapshot_pipeline.py` / `snapshot_backtest.py`： + provider snapshot 到 PIT target 与本地历史回测； +- `platforms/` / `adapters/`：聚宽、QMT、Qlib 与 XtTrader 边界。 + +这些模块已有自动测试，聚宽 hosted 路径也已有一次真实运行；真实数据校准、 +QMT/券商运行和运营证据仍分别计分。 + +## 仍未达到 60 的核心原因 + +- 基本面公告时点、完整公司行动、Security Master 与退市链尚未形成生产数据湖； +- Ridge/LightGBM 候选尚未冻结成四引擎共同加载的 model bundle； +- 风险协方差、冲击参数和容量尚未用真实长样本校准； +- 聚宽已有真实 hosted run 但尚无完整同输入导出；QMT 尚无真实导出，QMT + 历史 ST 权限正例、账户 query 合同、 + 20 日只读影子、回调/恢复/对账尚未形成真实证据； +- 自动 scheduler、日终对账服务、监控告警与 kill-switch 演练未形成连续证据； +- 实际券商程序化交易报告、权限、频率与软件要求尚未书面确认。 + +因此当前是可以继续填真实证据的 Quant OS 生产候选框架，不是可直接投入资金的 +成品，也不构成投资建议。 + +## 安全与进一步文档 + +凭据、cookie、QMT userdata、broker statement、licensed `xtquant` 与无转授权 +行情数据一律不得进入 Git。 + +- 架构：[`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) +- 平台矩阵：[`docs/PLATFORM_MATRIX.md`](docs/PLATFORM_MATRIX.md) +- 部署：[`runbooks/PLATFORM_DEPLOYMENT.md`](runbooks/PLATFORM_DEPLOYMENT.md) +- 安全：[`docs/SECURITY.md`](docs/SECURITY.md) +- 故障：[`runbooks/INCIDENTS.md`](runbooks/INCIDENTS.md) +- 复评：[`docs/AUDIT_2026-07-25.md`](docs/AUDIT_2026-07-25.md) diff --git a/adapters/__init__.py b/adapters/__init__.py new file mode 100644 index 0000000..b6a897e --- /dev/null +++ b/adapters/__init__.py @@ -0,0 +1 @@ +"""Broker and platform boundary adapters.""" diff --git a/adapters/jqdata_local.py b/adapters/jqdata_local.py new file mode 100644 index 0000000..81e8ac3 --- /dev/null +++ b/adapters/jqdata_local.py @@ -0,0 +1,411 @@ +"""JQData login and canonical daily snapshot adapter. + +The password is accepted only from a caller or an interactive prompt. It is +never written to the snapshot, logs, Obsidian, or Git. +""" + +from __future__ import annotations + +import getpass +import importlib +import math +import os +from bisect import bisect_right +from datetime import date, datetime, time, timezone +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +from quant60.data_snapshot import ( + CanonicalBarRecord, + IndexMembershipRecord, + build_snapshot_payload, + write_data_snapshot, +) +from quant60.portable_core import normalize_symbol + + +class JQDataUnavailableError(RuntimeError): + pass + + +def load_jqdata() -> Any: + try: + return importlib.import_module("jqdatasdk") + except ImportError as exc: + raise JQDataUnavailableError( + "jqdatasdk is unavailable; install requirements/jqdata.txt" + ) from exc + + +def authenticate( + sdk: Any, + *, + username: str | None = None, + password: str | None = None, + interactive: bool = True, +) -> None: + account = username or os.environ.get("JQDATA_USERNAME") + secret = password or os.environ.get("JQDATA_PASSWORD") + if interactive and not account: + account = input("JQData account: ").strip() + if interactive and not secret: + secret = getpass.getpass("JQData password: ") + if not account or not secret: + raise JQDataUnavailableError( + "JQData credentials are required through environment or prompt" + ) + auth = getattr(sdk, "auth", None) + if not callable(auth): + raise JQDataUnavailableError("jqdatasdk.auth is unavailable") + auth(account, secret) + is_auth = getattr(sdk, "is_auth", None) + if callable(is_auth) and is_auth() is not True: + raise JQDataUnavailableError("JQData authentication was not accepted") + + +def _row_value(row: Any, name: str, default: Any = None) -> Any: + if isinstance(row, dict): + return row.get(name, default) + try: + return row[name] + except (KeyError, TypeError): + return getattr(row, name, default) + + +def _required_row_value(row: Any, name: str) -> Any: + marker = object() + value = _row_value(row, name, marker) + if value is marker or value is None: + raise JQDataUnavailableError( + f"JQData get_bars omitted required field {name}" + ) + return value + + +def _as_date(value: Any) -> date: + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + parser = getattr(value, "date", None) + if callable(parser): + parsed = parser() + if isinstance(parsed, date): + return parsed + return date.fromisoformat(str(value)[:10]) + + +def _optional_positive_float(value: Any) -> float | None: + if value is None: + return None + numeric = float(value) + if not math.isfinite(numeric) or numeric <= 0: + return None + return numeric + + +def _required_positive_float( + row: Any, + name: str, + *, + symbol: str, + timestamp: Any, +) -> float: + numeric = _optional_positive_float(_required_row_value(row, name)) + if numeric is None: + raise JQDataUnavailableError( + f"invalid {name} field for {symbol} at {timestamp}" + ) + return numeric + + +def _required_bool( + value: Any, + name: str, + *, + symbol: str, + timestamp: Any, +) -> bool: + if isinstance(value, str): + raise JQDataUnavailableError( + f"invalid {name} field for {symbol} at {timestamp}" + ) + try: + numeric = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise JQDataUnavailableError( + f"invalid {name} field for {symbol} at {timestamp}" + ) from exc + if not math.isfinite(numeric) or numeric not in (0.0, 1.0): + raise JQDataUnavailableError( + f"invalid {name} field for {symbol} at {timestamp}" + ) + return bool(numeric) + + +def fetch_index_daily_snapshot( + sdk: Any, + *, + index_symbol: str, + start_date: date, + end_date: date, + output_dir: str | Path, + retrieved_at: datetime | None = None, +) -> dict[str, str]: + """Download raw, unadjusted daily bars plus PIT index membership.""" + + if start_date > end_date: + raise ValueError("start_date must not follow end_date") + retrieval_time = retrieved_at or datetime.now(timezone.utc) + if ( + retrieval_time.tzinfo is None + or retrieval_time.utcoffset() is None + ): + raise ValueError("retrieved_at must include a timezone") + # JQData documents a 24:00 reconciliation for daily bars. Requiring the + # requested end date to precede the Shanghai retrieval date prevents an + # intraday/preliminary daily bar from being labelled T_CLOSE_COMPLETE. + shanghai_retrieval_date = retrieval_time.astimezone( + ZoneInfo("Asia/Shanghai") + ).date() + if end_date >= shanghai_retrieval_date: + raise JQDataUnavailableError( + "end_date must precede the Asia/Shanghai retrieval date so the " + "daily bar has passed JQData's 24:00 finalization boundary" + ) + canonical_index = normalize_symbol(index_symbol) + trade_days = [ + _as_date(item) + for item in sdk.get_trade_days( + start_date=start_date, + end_date=end_date, + ) + ] + if not trade_days: + raise JQDataUnavailableError("JQData returned no trading sessions") + if trade_days != sorted(set(trade_days)): + raise JQDataUnavailableError( + "JQData returned a non-unique or unordered trading calendar" + ) + get_all_trade_days = getattr(sdk, "get_all_trade_days", None) + if not callable(get_all_trade_days): + raise JQDataUnavailableError( + "jqdatasdk.get_all_trade_days is required to resolve the " + "next trading session" + ) + full_calendar = [_as_date(item) for item in get_all_trade_days()] + if full_calendar != sorted(set(full_calendar)): + raise JQDataUnavailableError( + "JQData returned a non-unique or unordered full trading calendar" + ) + expected_trade_days = [ + item for item in full_calendar if start_date <= item <= end_date + ] + if trade_days != expected_trade_days: + raise JQDataUnavailableError( + "JQData trading-calendar APIs returned inconsistent sessions" + ) + next_index = bisect_right(full_calendar, end_date) + if next_index >= len(full_calendar): + raise JQDataUnavailableError( + "JQData did not return the unique next trading session " + "after end_date" + ) + next_trading_session = full_calendar[next_index] + memberships: list[IndexMembershipRecord] = [] + symbols: set[str] = set() + for trading_date in trade_days: + members = sdk.get_index_stocks( + canonical_index, + date=trading_date, + ) + if not members: + raise JQDataUnavailableError( + f"no index members on {trading_date.isoformat()}" + ) + for member in sorted(members): + canonical = normalize_symbol(member) + symbols.add(canonical) + memberships.append( + IndexMembershipRecord( + effective_date=trading_date, + index_symbol=canonical_index, + member_symbol=canonical, + source="jqdata", + ) + ) + + get_extras = getattr(sdk, "get_extras", None) + if not callable(get_extras): + raise JQDataUnavailableError( + "jqdatasdk.get_extras is required for PIT is_st status" + ) + status_frame = get_extras( + "is_st", + sorted(symbols), + start_date=start_date, + end_date=end_date, + df=True, + ) + if status_frame is None or not hasattr(status_frame, "iterrows"): + raise JQDataUnavailableError("invalid get_extras is_st result") + pit_is_st: dict[tuple[date, str], bool] = {} + for timestamp, row in status_frame.iterrows(): + trading_date = _as_date(timestamp) + for symbol in sorted(symbols): + value = _required_row_value(row, symbol) + pit_is_st[(trading_date, symbol)] = _required_bool( + value, + "is_st", + symbol=symbol, + timestamp=timestamp, + ) + + bars: list[CanonicalBarRecord] = [] + fields = [ + "date", + "open", + "high", + "low", + "close", + "volume", + "money", + "factor", + "paused", + "high_limit", + "low_limit", + "pre_close", + ] + get_bars = getattr(sdk, "get_bars", None) + if not callable(get_bars): + raise JQDataUnavailableError( + "jqdatasdk.get_bars is required for raw price, factor, " + "price-limit and previous-close fields" + ) + for symbol in sorted(symbols): + frame = get_bars( + symbol, + start_dt=start_date, + # A date-only end_dt is interpreted as midnight by data APIs and + # can exclude the requested final trading session. Make the + # inclusive daily boundary explicit. + end_dt=datetime.combine(end_date, time(23, 59, 59)), + unit="1d", + fields=fields, + skip_paused=False, + include_now=True, + fq_ref_date=None, + df=True, + ) + if frame is None or not hasattr(frame, "iterrows"): + raise JQDataUnavailableError( + f"invalid get_bars result for {symbol}" + ) + for timestamp, row in frame.iterrows(): + trading_date = _as_date(_required_row_value(row, "date")) + status_key = (trading_date, symbol) + if status_key not in pit_is_st: + raise JQDataUnavailableError( + "JQData get_extras omitted PIT is_st status for " + f"{symbol} at {trading_date.isoformat()}" + ) + raw_paused = _required_row_value(row, "paused") + try: + paused_numeric = float(raw_paused) + except (TypeError, ValueError) as exc: + raise JQDataUnavailableError( + f"invalid paused field for {symbol} at {timestamp}" + ) from exc + if ( + not math.isfinite(paused_numeric) + or paused_numeric not in (0.0, 1.0) + ): + raise JQDataUnavailableError( + f"invalid paused field for {symbol} at {timestamp}" + ) + paused = bool(paused_numeric) + volume = int(_required_row_value(row, "volume")) + money = float(_required_row_value(row, "money")) + if not math.isfinite(money) or money < 0: + raise JQDataUnavailableError( + f"invalid money field for {symbol} at {timestamp}" + ) + factor = _required_positive_float( + row, + "factor", + symbol=symbol, + timestamp=timestamp, + ) + bars.append( + CanonicalBarRecord( + trading_date=trading_date, + symbol=symbol, + open=float(_required_row_value(row, "open")), + high=float(_required_row_value(row, "high")), + low=float(_required_row_value(row, "low")), + close=float(_required_row_value(row, "close")), + volume=volume, + money=money, + paused=paused, + source="jqdata", + is_st=pit_is_st[status_key], + adjustment="none", + adjustment_factor=factor, + previous_close=_required_positive_float( + row, + "pre_close", + symbol=symbol, + timestamp=timestamp, + ), + limit_up=_required_positive_float( + row, + "high_limit", + symbol=symbol, + timestamp=timestamp, + ), + limit_down=_required_positive_float( + row, + "low_limit", + symbol=symbol, + timestamp=timestamp, + ), + ) + ) + version = str(getattr(sdk, "__version__", "unknown")) + payload = build_snapshot_payload( + bars=bars, + memberships=memberships, + provider="jqdata", + provider_version=version, + retrieved_at=retrieval_time, + next_trading_session=next_trading_session, + query={ + "index_symbol": canonical_index, + "start_date": start_date.isoformat(), + "end_date": end_date.isoformat(), + "trading_sessions": [ + *(item.isoformat() for item in trade_days), + next_trading_session.isoformat(), + ], + "calendar_api": "get_trade_days+get_all_trade_days", + "next_session_policy": ( + "first full-calendar session strictly after end_date" + ), + "api": "get_bars", + "frequency": "1d", + "adjustment": "none", + "factor_semantics": ( + "raw OHLC plus provider back-adjustment factor; " + "front adjustment is derived per decision as_of" + ), + "skip_paused": False, + "include_now": True, + "pit_status_api": "get_extras:is_st", + "finalization_policy": ( + "end_date_before_retrieval_date_Asia_Shanghai" + ), + "fields": fields, + }, + ) + return write_data_snapshot(payload, output_dir) diff --git a/adapters/xttrader_live.py b/adapters/xttrader_live.py new file mode 100644 index 0000000..63652e8 --- /dev/null +++ b/adapters/xttrader_live.py @@ -0,0 +1,1304 @@ +"""Guarded XtTrader live adapter. + +The adapter is intentionally inert by default. Real submissions require both +``allow_live_orders=True`` at construction and ``confirm_live=True`` on every +submit/cancel call. Queries, callback normalization, idempotency and fresh +instance reconnects remain available independently. +""" + +from __future__ import annotations + +import datetime as dt +import hashlib +import importlib +import json +import math +import numbers +import operator +import os +import re +import tempfile +import threading +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional + +from quant60.portable_core import convert_symbol + + +class XtTraderAdapterError(RuntimeError): + """Base adapter failure.""" + + +class NotConnectedError(XtTraderAdapterError): + """Operation requires a ready and subscribed QMT connection.""" + + +class LiveOrderBlocked(XtTraderAdapterError): + """Safety gate prevented a real broker mutation.""" + + +class IdempotencyConflict(XtTraderAdapterError): + """A client order id was reused for a different payload.""" + + +class ReconnectFailed(XtTraderAdapterError): + """Fresh-client reconnect attempts were exhausted.""" + + +STATUS_BY_CODE = { + 48: "NEW", # ORDER_UNREPORTED + 49: "NEW", # ORDER_WAIT_REPORTING + 50: "ACCEPTED", # ORDER_REPORTED + 51: "CANCEL_PENDING", # ORDER_REPORTED_CANCEL + 52: "CANCEL_PENDING", # ORDER_PARTSUCC_CANCEL + 53: "CANCELLED", # ORDER_PART_CANCEL + 54: "CANCELLED", + 55: "PARTIALLY_FILLED", + 56: "FILLED", + 57: "REJECTED", + 255: "UNKNOWN", +} + +TERMINAL_STATES = { + "CANCELLED", + "FILLED", + "REJECTED", +} + +_BROKER_STATE_TRANSITIONS = { + "SUBMITTING": { + "NEW", + "ACCEPTED", + "PARTIALLY_FILLED", + "FILLED", + "CANCEL_PENDING", + "CANCELLED", + "REJECTED", + "UNKNOWN", + }, + "NEW": { + "NEW", + "ACCEPTED", + "PARTIALLY_FILLED", + "FILLED", + "CANCEL_PENDING", + "CANCELLED", + "REJECTED", + "UNKNOWN", + }, + "ACCEPTED": { + "ACCEPTED", + "PARTIALLY_FILLED", + "FILLED", + "CANCEL_PENDING", + "CANCELLED", + "REJECTED", + "UNKNOWN", + }, + "PARTIALLY_FILLED": { + "PARTIALLY_FILLED", + "FILLED", + "CANCEL_PENDING", + "CANCELLED", + "UNKNOWN", + }, + "CANCEL_PENDING": { + "CANCEL_PENDING", + "PARTIALLY_FILLED", + "FILLED", + "CANCELLED", + "UNKNOWN", + }, + "UNKNOWN": { + "NEW", + "ACCEPTED", + "PARTIALLY_FILLED", + "FILLED", + "CANCEL_PENDING", + "CANCELLED", + "REJECTED", + "UNKNOWN", + }, + "FILLED": {"FILLED"}, + "CANCELLED": {"CANCELLED"}, + "REJECTED": {"REJECTED"}, +} + +# QMT remarks are commonly limited to 24 ASCII characters. Reserve four for +# ``q60:`` so broker round-trips preserve the full idempotency key. +_CLIENT_ID_RE = re.compile(r"^[A-Za-z0-9_.-]{1,20}$") +_REMARK_PREFIX = "q60:" +_LIVE_POLICY_FLAGS = ( + "reconciliation_safe", + "account_allowed", + "within_notional_limit", + "within_order_rate_limit", + "operator_approved", + "kill_switch_ready", + "compliance_approved", +) +_MAX_POLICY_AGE_SECONDS = 60.0 +_MAX_POLICY_FUTURE_SECONDS = 5.0 + + +def map_order_status(code: Any) -> str: + """Map the documented XtQuant order status constants to domain states.""" + try: + return STATUS_BY_CODE[int(code)] + except (KeyError, TypeError, ValueError): + return "UNKNOWN" + + +def _field(obj: Any, names: Iterable[str], default: Any = None) -> Any: + for name in names: + if isinstance(obj, Mapping) and name in obj: + value = obj[name] + elif hasattr(obj, name): + value = getattr(obj, name) + else: + continue + if value is not None: + return value + return default + + +def _payload_hash(payload: Mapping[str, Any]) -> str: + encoded = json.dumps( + dict(payload), + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + return hashlib.sha256(encoded).hexdigest() + + +def _client_id_from_remark(remark: Any) -> Optional[str]: + text = str(remark or "") + if not text.startswith(_REMARK_PREFIX): + return None + value = text[len(_REMARK_PREFIX) :] + return value or None + + +def _strict_bool(value: Any, name: str) -> bool: + if type(value) is not bool: + raise TypeError(f"{name} must be a bool, not {type(value).__name__}") + return value + + +class IdempotencyJournal: + """Thread-safe JSON journal; an omitted path gives an in-memory journal.""" + + def __init__(self, path: Optional[os.PathLike[str] | str] = None): + self.path = Path(path).expanduser().resolve() if path else None + self._lock = threading.RLock() + self._records: Dict[str, Dict[str, Any]] = {} + if self.path and self.path.exists(): + with self.path.open("r", encoding="utf-8") as handle: + data = json.load(handle) + if not isinstance(data, dict): + raise ValueError("idempotency journal must contain a JSON object") + self._records = data + + def get(self, client_order_id: str) -> Optional[Dict[str, Any]]: + with self._lock: + record = self._records.get(client_order_id) + return dict(record) if record is not None else None + + def put(self, client_order_id: str, record: Mapping[str, Any]) -> None: + with self._lock: + self._records[client_order_id] = dict(record) + self._persist() + + def update(self, client_order_id: str, **updates: Any) -> Dict[str, Any]: + with self._lock: + record = dict(self._records.get(client_order_id, {})) + record.update(updates) + self._records[client_order_id] = record + self._persist() + return dict(record) + + def find_by_seq(self, seq: Any) -> Optional[tuple[str, Dict[str, Any]]]: + with self._lock: + for client_id, record in self._records.items(): + if record.get("request_seq") == seq: + return client_id, dict(record) + return None + + def find_by_broker_order_id( + self, broker_order_id: Any + ) -> Optional[tuple[str, Dict[str, Any]]]: + with self._lock: + for client_id, record in self._records.items(): + if record.get("broker_order_id") == broker_order_id: + return client_id, dict(record) + return None + + def _persist(self) -> None: + if self.path is None: + return + self.path.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps( + self._records, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + descriptor, temp_name = tempfile.mkstemp( + prefix=self.path.name + ".", + suffix=".tmp", + dir=str(self.path.parent), + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temp_name, self.path) + finally: + if os.path.exists(temp_name): + os.unlink(temp_name) + + +class _CallbackProxy: + def __init__(self, adapter: "XtTraderLiveAdapter"): + self.adapter = adapter + + def on_connected(self) -> None: + self.adapter._callback_connected() + + def on_disconnected(self) -> None: + self.adapter._callback_disconnected() + + def on_account_status(self, status: Any) -> None: + self.adapter._account_status = status + + def on_stock_asset(self, asset: Any) -> None: + self.adapter._asset = asset + + def on_stock_order(self, order: Any) -> None: + self.adapter._ingest_order(order) + + def on_stock_trade(self, trade: Any) -> None: + self.adapter._ingest_trade(trade) + + def on_stock_position(self, position: Any) -> None: + code = str(_field(position, ["stock_code"], "")) + if code: + snapshot = self.adapter._normalize_position(position) + self.adapter._positions[snapshot["symbol"]] = snapshot + + def on_order_error(self, error: Any) -> None: + self.adapter._ingest_order_error(error) + + def on_cancel_error(self, error: Any) -> None: + self.adapter._errors.append( + { + "kind": "cancel", + "order_id": _field(error, ["order_id"], None), + "error_id": _field(error, ["error_id"], None), + "message": str(_field(error, ["error_msg"], "")), + } + ) + + def on_order_stock_async_response(self, response: Any) -> None: + self.adapter._ingest_submit_response(response) + + def on_cancel_order_stock_async_response(self, response: Any) -> None: + self.adapter._cancel_responses.append( + { + "order_id": _field(response, ["order_id"], None), + "result": _field(response, ["cancel_result"], None), + "seq": _field(response, ["seq"], None), + } + ) + + def on_smt_appointment_async_response(self, response: Any) -> None: + del response + + +class XtTraderLiveAdapter: + """Normalized, reconnectable and idempotent XtTrader boundary.""" + + def __init__( + self, + *, + trader_factory: Callable[[], Any], + account: Any, + constants: Any, + allow_live_orders: bool = False, + live_guard: Optional[ + Callable[[Mapping[str, Any]], Mapping[str, Any]] + ] = None, + journal_path: Optional[os.PathLike[str] | str] = None, + strategy_name: str = "quant60", + ): + self.allow_live_orders = _strict_bool( + allow_live_orders, "allow_live_orders" + ) + if live_guard is not None and not callable(live_guard): + raise TypeError("live_guard must be callable") + if self.allow_live_orders and live_guard is None: + raise TypeError( + "allow_live_orders=True requires a callable live_guard" + ) + self.live_guard = live_guard + self._trader_factory = trader_factory + self.account = account + self.constants = constants + self.strategy_name = str(strategy_name) + self.journal = IdempotencyJournal(journal_path) + self._lock = threading.RLock() + self._callback_lock = threading.RLock() + self._trader: Any = None + self._state = "NEW" + self._orders: Dict[Any, Dict[str, Any]] = {} + self._trades: Dict[Any, Dict[str, Any]] = {} + self._positions: Dict[str, Dict[str, Any]] = {} + self._asset: Any = None + self._account_status: Any = None + self._errors: List[Dict[str, Any]] = [] + self._cancel_responses: List[Dict[str, Any]] = [] + self._orphan_submit_responses: Dict[Any, Any] = {} + self._orphan_order_errors: List[Dict[str, Any]] = [] + self._live_authorization_audit: List[Dict[str, Any]] = [] + self._disconnect_seen = False + + @classmethod + def from_xtquant( + cls, + *, + userdata_path: os.PathLike[str] | str, + session_id: int, + account_id: str, + allow_live_orders: bool = False, + live_guard: Optional[ + Callable[[Mapping[str, Any]], Mapping[str, Any]] + ] = None, + journal_path: Optional[os.PathLike[str] | str] = None, + strategy_name: str = "quant60", + ) -> "XtTraderLiveAdapter": + """Create a lazy real-runtime adapter without connecting or ordering.""" + _strict_bool(allow_live_orders, "allow_live_orders") + if live_guard is not None and not callable(live_guard): + raise TypeError("live_guard must be callable") + if allow_live_orders and live_guard is None: + raise TypeError( + "allow_live_orders=True requires a callable live_guard" + ) + try: + xttrader = importlib.import_module("xtquant.xttrader") + xttype = importlib.import_module("xtquant.xttype") + constants = importlib.import_module("xtquant.xtconstant") + except Exception as exc: + raise XtTraderAdapterError(f"xtquant import failed: {exc}") from exc + path = str(Path(userdata_path).expanduser()) + account = xttype.StockAccount(account_id) + + def factory() -> Any: + return xttrader.XtQuantTrader(path, int(session_id)) + + return cls( + trader_factory=factory, + account=account, + constants=constants, + allow_live_orders=allow_live_orders, + live_guard=live_guard, + journal_path=journal_path, + strategy_name=strategy_name, + ) + + @property + def state(self) -> str: + return self._state + + @property + def errors(self) -> List[Dict[str, Any]]: + with self._callback_lock: + return [dict(item) for item in self._errors] + + @property + def live_authorization_audit(self) -> List[Dict[str, Any]]: + with self._callback_lock: + return [ + { + "intent": dict(item["intent"]), + "decision": dict(item["decision"]), + "outcome": item["outcome"], + } + for item in self._live_authorization_audit + ] + + def connect(self) -> None: + """Create, start, connect, subscribe and verify a fresh trader.""" + with self._lock: + if self._state == "READY": + return + self._connect_fresh() + + def _connect_fresh(self) -> None: + self._dispose_trader() + self._state = "CONNECTING" + self._disconnect_seen = False + trader = self._trader_factory() + if trader is None: + self._state = "FAILED" + raise NotConnectedError("trader_factory returned None") + self._trader = trader + trader.register_callback(_CallbackProxy(self)) + trader.start() + connect_code = trader.connect() + if connect_code != 0: + self._state = "FAILED" + self._dispose_trader() + raise NotConnectedError(f"XtTrader connect failed with code {connect_code}") + subscribe_code = trader.subscribe(self.account) + if subscribe_code != 0: + self._state = "FAILED" + self._dispose_trader() + raise NotConnectedError( + f"XtTrader account subscribe failed with code {subscribe_code}" + ) + + # Read-only broker truth is the readiness barrier, not merely TCP state. + asset = trader.query_stock_asset(self.account) + orders = trader.query_stock_orders(self.account) + if asset is None or orders is None or self._disconnect_seen: + self._state = "FAILED" + self._dispose_trader() + raise NotConnectedError( + "XtTrader connected but account snapshot query failed" + ) + try: + self._require_query_account(asset, "asset") + self._asset = asset + for order in orders: + self._ingest_order(order) + except Exception: + self._state = "FAILED" + self._dispose_trader() + raise + self._state = "READY" + + def reconnect(self, attempts: int = 1) -> None: + """Reconnect with fresh XtTrader instances or fail before any order.""" + if int(attempts) < 1: + raise ValueError("attempts must be at least one") + last_error: Optional[Exception] = None + for _ in range(int(attempts)): + try: + with self._lock: + self._connect_fresh() + return + except Exception as exc: + last_error = exc + self._state = "FAILED" + raise ReconnectFailed( + f"XtTrader reconnect failed after {attempts} fresh attempt(s): {last_error}" + ) from last_error + + def close(self) -> None: + with self._lock: + self._dispose_trader() + self._state = "CLOSED" + + def _dispose_trader(self) -> None: + trader, self._trader = self._trader, None + if trader is not None: + try: + trader.stop() + except Exception: + pass + + def _callback_connected(self) -> None: + if self._state not in {"READY", "CONNECTING"}: + self._state = "CONNECTED_UNVERIFIED" + + def _callback_disconnected(self) -> None: + # Never auto-submit/reconnect from a callback. The caller must invoke + # reconnect and pass the account snapshot readiness barrier again. + self._disconnect_seen = True + self._state = "DISCONNECTED" + + def _require_ready(self) -> Any: + if self._state != "READY" or self._trader is None: + raise NotConnectedError( + f"XtTrader is not ready (state={self._state}); no broker call made" + ) + return self._trader + + def _authorize_live_intent( + self, intent: Mapping[str, Any] + ) -> Dict[str, Any]: + guard = self.live_guard + if not callable(guard): + raise LiveOrderBlocked( + "live mutation blocked: callable live_guard is unavailable" + ) + normalized_intent = dict(intent) + try: + # The guard receives a copy so it cannot rewrite the account/action + # that the adapter subsequently binds into its audit evidence. + decision = guard(dict(normalized_intent)) + except Exception as exc: + raise LiveOrderBlocked( + f"live_guard raised {type(exc).__name__}: {exc}" + ) from exc + if not isinstance(decision, Mapping): + raise LiveOrderBlocked( + "live_guard must return a structured mapping" + ) + account_id = normalized_intent.get("account_id") + if not isinstance(account_id, str) or not account_id.strip(): + raise LiveOrderBlocked( + "live intent account_id must be a non-empty string" + ) + normalized_intent["account_id"] = account_id.strip() + + authorization_id = decision.get("authorization_id") + snapshot_id = decision.get("snapshot_id") + if not isinstance(authorization_id, str) or not authorization_id.strip(): + raise LiveOrderBlocked( + "live_guard authorization_id must be a non-empty string" + ) + if not isinstance(snapshot_id, str) or not snapshot_id.strip(): + raise LiveOrderBlocked( + "live_guard snapshot_id must be a non-empty string" + ) + + raw_as_of = decision.get("as_of") + if isinstance(raw_as_of, dt.datetime): + as_of = raw_as_of + elif isinstance(raw_as_of, str): + value = raw_as_of.strip() + if value.endswith(("Z", "z")): + value = value[:-1] + "+00:00" + try: + as_of = dt.datetime.fromisoformat(value) + except ValueError as exc: + raise LiveOrderBlocked( + "live_guard as_of must be an ISO-8601 datetime" + ) from exc + else: + raise LiveOrderBlocked( + "live_guard as_of must be a timezone-aware datetime" + ) + if as_of.tzinfo is None or as_of.utcoffset() is None: + raise LiveOrderBlocked( + "live_guard as_of must be timezone-aware" + ) + now = dt.datetime.now(dt.timezone.utc) + age_seconds = ( + now - as_of.astimezone(dt.timezone.utc) + ).total_seconds() + if age_seconds > _MAX_POLICY_AGE_SECONDS: + raise LiveOrderBlocked( + "live_guard decision is stale (older than 60 seconds)" + ) + if age_seconds < -_MAX_POLICY_FUTURE_SECONDS: + raise LiveOrderBlocked( + "live_guard as_of is more than 5 seconds in the future" + ) + + for field in _LIVE_POLICY_FLAGS: + value = decision.get(field) + if type(value) is not bool or value is not True: + raise LiveOrderBlocked( + f"live_guard {field} must be exactly bool True" + ) + normalized_decision = { + "authorization_id": authorization_id.strip(), + "snapshot_id": snapshot_id.strip(), + "as_of": as_of.astimezone(dt.timezone.utc).isoformat(), + "account_id": normalized_intent["account_id"], + **{field: True for field in _LIVE_POLICY_FLAGS}, + } + with self._callback_lock: + self._live_authorization_audit.append( + { + "intent": normalized_intent, + "decision": normalized_decision, + "outcome": "AUTHORIZED", + } + ) + return normalized_decision + + def _record_live_authorization_outcome( + self, authorization_id: str, outcome: str + ) -> None: + with self._callback_lock: + for item in reversed(self._live_authorization_audit): + if ( + item["decision"].get("authorization_id") + == authorization_id + ): + item["outcome"] = outcome + return + raise RuntimeError( + f"authorization audit record not found: {authorization_id}" + ) + + def _normalized_account_id(self) -> str: + value = _field(self.account, ["account_id"], None) + if not isinstance(value, str) or not value.strip(): + raise LiveOrderBlocked( + "live mutation requires a non-empty account_id" + ) + return value.strip() + + def _require_query_account(self, item: Any, kind: str) -> str: + expected = str( + _field(self.account, ["account_id"], "") or "" + ).strip() + observed = str(_field(item, ["account_id"], "") or "").strip() + if not expected or observed != expected: + self._state = "DEGRADED" + raise NotConnectedError( + f"{kind} account_id does not match the configured account" + ) + return observed + + def query_asset(self) -> Dict[str, Any]: + trader = self._require_ready() + raw = trader.query_stock_asset(self.account) + if raw is None: + self._state = "DEGRADED" + raise NotConnectedError("asset query returned None") + self._require_query_account(raw, "asset") + self._asset = raw + return { + "account_id": str(_field(raw, ["account_id"], "")), + "cash": float(_field(raw, ["cash"], 0.0) or 0.0), + "market_value": float(_field(raw, ["market_value"], 0.0) or 0.0), + "total_asset": float(_field(raw, ["total_asset"], 0.0) or 0.0), + } + + def query_positions(self) -> List[Dict[str, Any]]: + trader = self._require_ready() + raw = trader.query_stock_positions(self.account) + if raw is None: + self._state = "DEGRADED" + raise NotConnectedError("positions query returned None") + positions = [self._normalize_position(item) for item in raw] + self._positions = {item["symbol"]: item for item in positions} + return positions + + def query_orders(self, cancelable_only: bool = False) -> List[Dict[str, Any]]: + trader = self._require_ready() + raw = trader.query_stock_orders(self.account, bool(cancelable_only)) + if raw is None: + self._state = "DEGRADED" + raise NotConnectedError("orders query returned None") + output = [] + for item in raw: + output.append(self._ingest_order(item)) + return output + + def query_trades(self) -> List[Dict[str, Any]]: + trader = self._require_ready() + raw = trader.query_stock_trades(self.account) + if raw is None: + self._state = "DEGRADED" + raise NotConnectedError("trades query returned None") + return [self._ingest_trade(item) for item in raw] + + def _normalize_position(self, item: Any) -> Dict[str, Any]: + self._require_query_account(item, "position") + raw_symbol = str(_field(item, ["stock_code"], "")) + return { + "account_id": str(_field(item, ["account_id"], "") or ""), + "symbol": convert_symbol(raw_symbol, "canonical") if raw_symbol else "", + "volume": int(_field(item, ["volume"], 0) or 0), + "sellable": int(_field(item, ["can_use_volume"], 0) or 0), + "market_value": float(_field(item, ["market_value"], 0.0) or 0.0), + "avg_price": float(_field(item, ["open_price", "avg_price"], 0.0) or 0.0), + } + + def _normalize_order(self, item: Any) -> Dict[str, Any]: + self._require_query_account(item, "order") + order_type = int(_field(item, ["order_type"], 0) or 0) + buy_type = int(getattr(self.constants, "STOCK_BUY", 23)) + sell_type = int(getattr(self.constants, "STOCK_SELL", 24)) + side = "BUY" if order_type == buy_type else "SELL" if order_type == sell_type else "UNKNOWN" + remark = str(_field(item, ["order_remark"], "") or "") + raw_symbol = str(_field(item, ["stock_code"], "")) + return { + "account_id": str(_field(item, ["account_id"], "") or ""), + "broker_order_id": _field(item, ["order_id"], None), + "broker_system_id": str(_field(item, ["order_sysid"], "") or ""), + "client_order_id": _client_id_from_remark(remark), + "symbol": convert_symbol(raw_symbol, "canonical") if raw_symbol else "", + "side": side, + "quantity": int(_field(item, ["order_volume"], 0) or 0), + "filled_quantity": int(_field(item, ["traded_volume"], 0) or 0), + "limit_price": float(_field(item, ["price"], 0.0) or 0.0), + "average_fill_price": float( + _field(item, ["traded_price"], 0.0) or 0.0 + ), + "state": map_order_status(_field(item, ["order_status"], 255)), + "status_message": str(_field(item, ["status_msg"], "") or ""), + "order_remark": remark, + } + + def _normalize_trade(self, item: Any) -> Dict[str, Any]: + self._require_query_account(item, "trade") + trade_id = _field(item, ["traded_id", "trade_id"], None) + raw_symbol = str(_field(item, ["stock_code"], "")) + order_type = int(_field(item, ["order_type"], 0) or 0) + buy_type = int(getattr(self.constants, "STOCK_BUY", 23)) + sell_type = int(getattr(self.constants, "STOCK_SELL", 24)) + side = ( + "BUY" + if order_type == buy_type + else "SELL" + if order_type == sell_type + else "UNKNOWN" + ) + return { + "account_id": str(_field(item, ["account_id"], "") or ""), + "trade_id": trade_id, + "broker_order_id": _field(item, ["order_id"], None), + "symbol": convert_symbol(raw_symbol, "canonical") if raw_symbol else "", + "side": side, + "quantity": int(_field(item, ["traded_volume"], 0) or 0), + "price": float(_field(item, ["traded_price"], 0.0) or 0.0), + "amount": float(_field(item, ["traded_amount"], 0.0) or 0.0), + } + + def _ingest_order(self, raw: Any) -> Dict[str, Any]: + order = self._normalize_order(raw) + order_id = order["broker_order_id"] + quantity = order["quantity"] + filled_quantity = order["filled_quantity"] + if ( + quantity <= 0 + or filled_quantity < 0 + or filled_quantity > quantity + or ( + order["state"] == "FILLED" + and filled_quantity != quantity + ) + or ( + order["state"] == "PARTIALLY_FILLED" + and not 0 < filled_quantity < quantity + ) + ): + self._state = "DEGRADED" + raise IdempotencyConflict( + f"invalid broker order quantities/state for {order_id}" + ) + + prior_order = ( + self._orders.get(order_id) if order_id is not None else None + ) + client_id = order.get("client_order_id") + if prior_order is not None: + prior_client_id = prior_order.get("client_order_id") + if client_id is None: + client_id = prior_client_id + order["client_order_id"] = prior_client_id + immutable_fields = ( + "symbol", + "side", + "quantity", + "limit_price", + ) + changed = [ + field + for field in immutable_fields + if prior_order.get(field) != order.get(field) + ] + prior_system_id = prior_order.get("broker_system_id") + current_system_id = order.get("broker_system_id") + if ( + prior_client_id + and client_id + and prior_client_id != client_id + ): + changed.append("client_order_id") + if ( + prior_system_id + and current_system_id + and prior_system_id != current_system_id + ): + changed.append("broker_system_id") + if changed: + self._state = "DEGRADED" + raise IdempotencyConflict( + f"broker order {order_id} changed immutable " + f"identity/payload fields: {sorted(set(changed))}" + ) + if filled_quantity < prior_order["filled_quantity"]: + self._state = "DEGRADED" + raise IdempotencyConflict( + f"broker order {order_id} cumulative fill regressed " + f"from {prior_order['filled_quantity']} " + f"to {filled_quantity}" + ) + allowed = _BROKER_STATE_TRANSITIONS.get( + prior_order["state"], set() + ) + if order["state"] not in allowed: + self._state = "DEGRADED" + raise IdempotencyConflict( + f"broker order {order_id} state regressed illegally " + f"from {prior_order['state']} to {order['state']}" + ) + if order == prior_order: + return dict(prior_order) + + if client_id: + existing = self.journal.get(client_id) + payload = { + "symbol": order["symbol"], + "side": order["side"], + "quantity": order["quantity"], + "limit_price": order["limit_price"], + } + payload_digest = _payload_hash(payload) + if existing is None: + existing = { + "client_order_id": client_id, + "payload": payload, + "payload_hash": payload_digest, + } + self.journal.put(client_id, existing) + elif existing.get("payload_hash") != payload_digest: + self._state = "DEGRADED" + raise IdempotencyConflict( + f"broker payload conflicts with journal for {client_id}" + ) + prior_snapshot = existing.get("broker_snapshot") + prior_filled = int(existing.get("filled_quantity", 0) or 0) + if filled_quantity < prior_filled: + self._state = "DEGRADED" + raise IdempotencyConflict( + f"broker order {order_id} journal cumulative fill " + f"regressed from {prior_filled} to {filled_quantity}" + ) + prior_state = existing.get("state") + if prior_snapshot is not None and prior_state is not None: + allowed = _BROKER_STATE_TRANSITIONS.get( + str(prior_state), set() + ) + if order["state"] not in allowed: + self._state = "DEGRADED" + raise IdempotencyConflict( + f"broker order {order_id} journal state regressed " + f"illegally from {prior_state} to {order['state']}" + ) + elif ( + prior_state in TERMINAL_STATES + and order["state"] != prior_state + ): + self._state = "DEGRADED" + raise IdempotencyConflict( + f"terminal journal state {prior_state} conflicts " + f"with broker state {order['state']}" + ) + prior_broker_id = existing.get("broker_order_id") + if ( + prior_broker_id not in {None, order_id} + and order_id is not None + ): + self._state = "DEGRADED" + raise IdempotencyConflict( + f"multiple broker orders share client id {client_id}" + ) + self.journal.update( + client_id, + broker_order_id=order_id, + state=order["state"], + filled_quantity=filled_quantity, + broker_snapshot=order, + ) + if order_id is not None: + self._orders[order_id] = order + return dict(order) + + def _ingest_trade(self, raw: Any) -> Dict[str, Any]: + trade = self._normalize_trade(raw) + key = trade["trade_id"] + if key is None: + key = ( + trade["broker_order_id"], + trade["symbol"], + trade["quantity"], + trade["price"], + ) + self._trades[key] = trade + return dict(trade) + + def _apply_submit_response_locked(self, response: Any) -> None: + seq = _field(response, ["seq"], None) + found = self.journal.find_by_seq(seq) + if found is None: + # The callback may race ahead of order_stock_async returning seq. + self._orphan_submit_responses[seq] = response + return + client_id, unused_record = found + del unused_record + broker_id = _field(response, ["order_id"], None) + message = str(_field(response, ["error_msg"], "") or "") + state = "ACCEPTED" if broker_id not in {None, -1, 0} and not message else "REJECTED" + current = self.journal.get(client_id) or {} + updates = { + "broker_order_id": broker_id, + "error_message": message, + } + current_state = current.get("state") + allowed = _BROKER_STATE_TRANSITIONS.get( + str(current_state), set() + ) + if current_state is not None and state not in allowed: + self._state = "DEGRADED" + updates["callback_state_conflict"] = ( + f"submit response {state} after {current_state}" + ) + else: + updates["state"] = state + self.journal.update(client_id, **updates) + + def _ingest_submit_response(self, response: Any) -> None: + with self._callback_lock: + self._apply_submit_response_locked(response) + + def _apply_order_error_locked(self, item: Mapping[str, Any]) -> bool: + client_id = item.get("client_order_id") + found = None + if client_id: + record = self.journal.get(str(client_id)) + if record is not None: + found = (str(client_id), record) + order_id = item.get("order_id") + if found is None and order_id not in {None, -1, 0}: + found = self.journal.find_by_broker_order_id(order_id) + seq = item.get("seq") + if found is None and seq is not None: + found = self.journal.find_by_seq(seq) + if found is None: + return False + matched_client_id, record = found + if record is None: + return False + current_state = record.get("state") + allowed = _BROKER_STATE_TRANSITIONS.get( + str(current_state), set() + ) + if current_state is not None and "REJECTED" not in allowed: + self._state = "DEGRADED" + self.journal.update( + matched_client_id, + callback_state_conflict=( + f"order error REJECTED after {current_state}" + ), + error_id=item.get("error_id"), + error_message=item.get("message", ""), + ) + return True + updates = { + "state": "REJECTED", + "error_id": item.get("error_id"), + "error_message": item.get("message", ""), + } + if order_id not in {None, -1, 0}: + updates["broker_order_id"] = order_id + self.journal.update(matched_client_id, **updates) + return True + + def _ingest_order_error(self, error: Any) -> None: + remark = _field(error, ["order_remark"], "") + item = { + "kind": "order", + "order_id": _field(error, ["order_id"], None), + "seq": _field(error, ["seq"], None), + "client_order_id": _client_id_from_remark(remark), + "error_id": _field(error, ["error_id"], None), + "message": str(_field(error, ["error_msg"], "")), + } + with self._callback_lock: + self._errors.append(item) + if not self._apply_order_error_locked(item): + # XtTrader may invoke the callback synchronously before + # order_stock_async returns its request sequence. + self._orphan_order_errors.append(item) + + def _drain_order_errors_locked(self) -> None: + remaining = [] + for item in self._orphan_order_errors: + if not self._apply_order_error_locked(item): + remaining.append(item) + self._orphan_order_errors = remaining + + def _existing_broker_order( + self, client_order_id: str, payload: Mapping[str, Any] + ) -> Optional[Dict[str, Any]]: + for order in self.query_orders(): + if order.get("client_order_id") != client_order_id: + continue + comparable = { + "symbol": order["symbol"], + "side": order["side"], + "quantity": order["quantity"], + "limit_price": order["limit_price"], + } + if _payload_hash(comparable) != _payload_hash(payload): + raise IdempotencyConflict( + f"broker already has {client_order_id} with another payload" + ) + return order + return None + + def submit_order( + self, + *, + client_order_id: str, + symbol: str, + side: str, + quantity: int, + limit_price: float, + confirm_live: bool = False, + ) -> Dict[str, Any]: + """Idempotently plan or asynchronously submit one limit order.""" + confirmed_live = _strict_bool(confirm_live, "confirm_live") + if not _CLIENT_ID_RE.fullmatch(client_order_id): + raise ValueError( + "client_order_id must be 1-20 ASCII letters/digits/_.-" + ) + normalized_symbol = str(symbol).upper() + if not re.fullmatch(r"\d{6}\.(SH|SZ)", normalized_symbol): + raise ValueError("symbol must use QMT 600000.SH/000001.SZ form") + normalized_side = str(side).upper() + if normalized_side not in {"BUY", "SELL"}: + raise ValueError("side must be BUY or SELL") + if isinstance(quantity, bool): + raise ValueError( + "A-share quantity must be a positive integer 100-share lot" + ) + try: + volume = operator.index(quantity) + except TypeError as exc: + raise ValueError( + "A-share quantity must be a positive integer 100-share lot" + ) from exc + if volume <= 0 or volume % 100: + raise ValueError( + "A-share quantity must be a positive integer 100-share lot" + ) + if isinstance(limit_price, bool) or not isinstance( + limit_price, numbers.Real + ): + raise ValueError( + "limit_price must be a non-bool real number" + ) + price = float(limit_price) + if not math.isfinite(price) or price <= 0: + raise ValueError("limit_price must be finite and positive") + payload = { + "symbol": convert_symbol(normalized_symbol, "canonical"), + "side": normalized_side, + "quantity": volume, + "limit_price": price, + } + digest = _payload_hash(payload) + # The entire read/check/recover/reserve/submit/persist transaction is + # serialized. The journal's own lock only protects individual calls; + # it cannot by itself prevent two threads from both seeing "missing". + with self._lock: + existing = self.journal.get(client_order_id) + if existing: + if existing.get("payload_hash") != digest: + raise IdempotencyConflict( + f"client_order_id {client_order_id} has another payload" + ) + return existing + + if not self.allow_live_orders: + record = { + "client_order_id": client_order_id, + "payload": payload, + "payload_hash": digest, + "state": "DRY_RUN", + "broker_order_id": None, + } + self.journal.put(client_order_id, record) + return record + if not confirmed_live: + raise LiveOrderBlocked( + "real order requires confirm_live=True for this submission" + ) + + trader = self._require_ready() + broker_existing = self._existing_broker_order( + client_order_id, payload + ) + if broker_existing is not None: + record = { + "client_order_id": client_order_id, + "payload": payload, + "payload_hash": digest, + "state": broker_existing["state"], + "broker_order_id": broker_existing["broker_order_id"], + "recovered_from_broker": True, + } + self.journal.put(client_order_id, record) + return record + + order_type = ( + getattr(self.constants, "STOCK_BUY", 23) + if normalized_side == "BUY" + else getattr(self.constants, "STOCK_SELL", 24) + ) + price_type = getattr(self.constants, "FIX_PRICE", 11) + live_policy = self._authorize_live_intent( + { + "action": "SUBMIT_ORDER", + "account_id": self._normalized_account_id(), + "client_order_id": client_order_id, + "symbol": payload["symbol"], + "side": normalized_side, + "quantity": volume, + "limit_price": price, + "notional": price * volume, + "strategy_name": self.strategy_name, + } + ) + reservation = { + "client_order_id": client_order_id, + "payload": payload, + "payload_hash": digest, + "state": "SUBMITTING", + "request_seq": None, + "broker_order_id": None, + "live_policy": live_policy, + } + # Persist before crossing the broker mutation boundary. A crash or + # callback can then never turn a duplicate retry into a new order. + self.journal.put(client_order_id, reservation) + try: + seq = trader.order_stock_async( + self.account, + normalized_symbol, + order_type, + volume, + price_type, + price, + self.strategy_name, + _REMARK_PREFIX + client_order_id, + ) + except BaseException as exc: + current = self.journal.get(client_order_id) or reservation + if current.get("state") == "SUBMITTING": + self.journal.update( + client_order_id, + state="UNKNOWN", + error_message=( + f"order_stock_async raised " + f"{type(exc).__name__}: {exc}" + ), + ) + raise + try: + if isinstance(seq, bool): + raise TypeError("bool is not a request sequence") + request_seq = operator.index(seq) + except Exception as exc: + self.journal.update( + client_order_id, + state="UNKNOWN", + error_message=( + "XtTrader returned a malformed async request " + f"sequence: {seq!r}" + ), + ) + raise XtTraderAdapterError( + "XtTrader returned a malformed async request " + f"sequence: {seq!r}" + ) from exc + if request_seq <= 0: + self.journal.update( + client_order_id, + state="REJECTED", + error_message=( + "XtTrader rejected async submit before " + f"acknowledgement: {seq}" + ), + ) + raise XtTraderAdapterError( + f"XtTrader rejected async submit before acknowledgement: {seq}" + ) + + with self._callback_lock: + self.journal.update( + client_order_id, request_seq=request_seq + ) + raced_response = self._orphan_submit_responses.pop( + request_seq, None + ) + if raced_response is not None: + self._apply_submit_response_locked(raced_response) + self._drain_order_errors_locked() + current = self.journal.get(client_order_id) or reservation + if current.get("state") == "SUBMITTING": + current = self.journal.update( + client_order_id, state="NEW" + ) + return current + + def cancel_order( + self, broker_order_id: int, *, confirm_live: bool = False + ) -> int: + """Cancel synchronously behind the same explicit live safety gates.""" + confirmed_live = _strict_bool(confirm_live, "confirm_live") + if isinstance(broker_order_id, bool): + raise ValueError( + "broker_order_id must be a positive integer" + ) + try: + normalized_order_id = operator.index(broker_order_id) + except TypeError as exc: + raise ValueError( + "broker_order_id must be a positive integer" + ) from exc + if normalized_order_id <= 0: + raise ValueError( + "broker_order_id must be a positive integer" + ) + if not self.allow_live_orders or not confirmed_live: + raise LiveOrderBlocked( + "real cancel requires allow_live_orders and confirm_live=True" + ) + trader = self._require_ready() + live_policy = self._authorize_live_intent( + { + "action": "CANCEL_ORDER", + "account_id": self._normalized_account_id(), + "broker_order_id": normalized_order_id, + "strategy_name": self.strategy_name, + } + ) + try: + result = trader.cancel_order_stock( + self.account, normalized_order_id + ) + except BaseException: + self._record_live_authorization_outcome( + live_policy["authorization_id"], + "BROKER_CALL_EXCEPTION", + ) + raise + if result != 0: + self._record_live_authorization_outcome( + live_policy["authorization_id"], + "BROKER_REJECTED", + ) + raise XtTraderAdapterError( + f"XtTrader cancel failed for {normalized_order_id}: {result}" + ) + self._record_live_authorization_outcome( + live_policy["authorization_id"], + "BROKER_ACCEPTED", + ) + # A zero code accepts the cancel request; a later order snapshot or + # callback must still confirm CANCELLED before funds/quantity are reused. + return 0 diff --git a/configs/baseline.json b/configs/baseline.json new file mode 100644 index 0000000..73f31d6 --- /dev/null +++ b/configs/baseline.json @@ -0,0 +1,44 @@ +{ + "initial_cash": 1000000.0, + "symbols": [ + "600000.XSHG", + "000001.XSHE", + "300750.XSHE", + "000333.XSHE" + ], + "universe": { + "mode": "pit_index", + "index_symbol": "000905.XSHG", + "qmt_sector_name": "中证500", + "dedicated_account_required": true, + "exclude_st": true + }, + "fees": { + "commission_rate": 0.0002, + "minimum_commission": 5.0, + "stamp_duty_rate": 0.0005, + "transfer_fee_rate": 0.00001 + }, + "execution": { + "lot_size": 100, + "participation_rate": 0.1, + "slippage_bps": 2.0, + "cancel_open_orders_at_end": true + }, + "strategy": { + "lookback": 20, + "skip": 0, + "top_n": 20, + "max_weight": 0.05, + "gross_target": 0.95, + "cash_buffer": 0.02, + "rebalance_every": 5, + "rebalance_schedule": "weekly_first_close" + }, + "synthetic": { + "start_date": "2024-01-02", + "trading_days": 90, + "seed": 60, + "base_volume": 100000 + } +} diff --git a/dist/bundle_manifest.json b/dist/bundle_manifest.json new file mode 100644 index 0000000..73b8bfc --- /dev/null +++ b/dist/bundle_manifest.json @@ -0,0 +1,26 @@ +{ + "artifacts": { + "joinquant": { + "core": "src/quant60/portable_core.py", + "core_sha256": "af29e4e5e10705cf6aff89874394c1e80d3780f093c19d90ed53ef9d0188a934", + "effective_config_sha256": "3ef9b9efda9ec6cc5f57a5daca57cab3b68242e0368f18fdaaf9e249bdaa7873", + "output": "dist/joinquant_strategy.py", + "output_sha256": "22ddb0546beb3bd4e4bbedecb271452111324c3903e45be1f4ac3d39f38cce4f", + "wrapper": "platforms/joinquant_strategy.py", + "wrapper_sha256": "b5a8e3b0a774a0f45f52f6bcccbe786e234913a1a4cf5b68b85e2bd989ee9294" + }, + "qmt_builtin": { + "core": "src/quant60/portable_core.py", + "core_sha256": "af29e4e5e10705cf6aff89874394c1e80d3780f093c19d90ed53ef9d0188a934", + "effective_config_sha256": "11f189ea5c9d4eaddeb6394dc342abc871941cb3395d7ff79aae1275ce2ad98f", + "output": "dist/qmt_builtin_strategy.py", + "output_sha256": "ad503e5ebde60a5deb0ec86334c04d4be2d77ab91d9607d4fe88e2119f22d5dc", + "wrapper": "platforms/qmt_builtin_strategy.py", + "wrapper_sha256": "efb931bbd04d0abf0a576f2422a13403fd4d659dccf59f8170e8b13b1bafa640" + } + }, + "baseline_config": "configs/baseline.json", + "baseline_config_sha256": "939a7df386b06bc82869940572c35544e9361913b94d9e03027411e51b1006e1", + "portable_core_sha256": "af29e4e5e10705cf6aff89874394c1e80d3780f093c19d90ed53ef9d0188a934", + "schema_version": 1 +} diff --git a/dist/joinquant_strategy.py b/dist/joinquant_strategy.py new file mode 100644 index 0000000..47fe481 --- /dev/null +++ b/dist/joinquant_strategy.py @@ -0,0 +1,822 @@ +# coding: utf-8 +"""JoinQuant hosted wrapper. + +Upload the generated bundle, not this source file, to JoinQuant. The bundle +tool replaces the marked import block with the exact portable core source. +""" + +# __PORTABLE_CORE_BUNDLE_START__ +# Inlined by tools/bundle_platforms.py; do not edit this region. +import math +import re + + +_CANONICAL_EXCHANGES = { + "SH": "XSHG", + "SSE": "XSHG", + "XSHG": "XSHG", + "SZ": "XSHE", + "SZE": "XSHE", + "SZSE": "XSHE", + "XSHE": "XSHE", + "BJ": "XBSE", + "BSE": "XBSE", + "XBSE": "XBSE", +} + +_SHORT_EXCHANGES = { + "XSHG": "SH", + "XSHE": "SZ", + "XBSE": "BJ", +} + + +def _sum_numeric(values): + """Sum numerics without trusting a hosted runtime's global ``sum`` name.""" + + total = 0.0 + for value in values: + total += value + return total + + +def _infer_exchange(code): + if code.startswith(("4", "8")): + return "XBSE" + if code.startswith(("5", "6", "9")): + return "XSHG" + return "XSHE" + + +def _split_symbol(symbol): + if not isinstance(symbol, str): + raise TypeError("symbol must be a string") + value = symbol.strip().upper().replace(" ", "") + if not value: + raise ValueError("symbol must not be empty") + + match = re.match(r"^(\d{6})\.(XSHG|XSHE|XBSE|SH|SZ|BJ)$", value) + if match: + return match.group(1), _CANONICAL_EXCHANGES[match.group(2)] + + match = re.match(r"^(XSHG|XSHE|XBSE|SH|SZ|BJ)[\.:]?(\d{6})$", value) + if match: + return match.group(2), _CANONICAL_EXCHANGES[match.group(1)] + + match = re.match(r"^(\d{6})[\.:](SSE|SZE|SZSE|BSE)$", value) + if match: + return match.group(1), _CANONICAL_EXCHANGES[match.group(2)] + + if re.match(r"^\d{6}$", value): + return value, _infer_exchange(value) + raise ValueError("unsupported A-share symbol: %s" % symbol) + + +def normalize_symbol(symbol, target="canonical"): + """Convert common A-share codes to canonical/JQ, QMT, or Qlib format.""" + + code, exchange = _split_symbol(symbol) + target_name = str(target).strip().lower() + if target_name in ("canonical", "joinquant", "jq"): + return "%s.%s" % (code, exchange) + if target_name in ("qmt", "xtquant"): + return "%s.%s" % (code, _SHORT_EXCHANGES[exchange]) + if target_name == "qlib": + return "%s%s" % (_SHORT_EXCHANGES[exchange], code) + raise ValueError("unsupported target platform: %s" % target) + + +def convert_symbol(symbol, target="canonical"): + """Alias kept as the stable adapter-facing name.""" + + return normalize_symbol(symbol, target) + + +def momentum_score(prices, lookback=20, skip=0): + """Return trailing simple momentum, or ``None`` if history is insufficient. + + ``skip=0`` uses the latest completed decision bar. Set ``skip=1`` only + when intentionally implementing a 20-1 convention. A lookback of N needs + N+skip+1 observations. + """ + + lookback = int(lookback) + skip = int(skip) + if lookback <= 0 or skip < 0: + raise ValueError("lookback must be positive and skip non-negative") + values = list(prices) + if len(values) < lookback + skip + 1: + return None + end_index = len(values) - 1 - skip + start_index = end_index - lookback + start = float(values[start_index]) + end = float(values[end_index]) + if ( + start <= 0.0 + or end <= 0.0 + or not math.isfinite(start) + or not math.isfinite(end) + ): + return None + return end / start - 1.0 + + +def rank_momentum(price_history, lookback=20, skip=0, top_n=None): + """Rank ``{symbol: closes}`` by momentum with deterministic tie-breaking.""" + + ranked = [] + for symbol in sorted(price_history): + score = momentum_score(price_history[symbol], lookback, skip) + if score is not None and math.isfinite(score): + ranked.append((normalize_symbol(symbol), score)) + ranked.sort(key=lambda item: (-item[1], item[0])) + if top_n is not None: + ranked = ranked[: max(0, int(top_n))] + return ranked + + +def capped_target_weights( + scores, max_weight=0.20, gross_target=1.0, min_score=0.0 +): + """Long-only score-proportional weights with iterative cap redistribution.""" + + max_weight = float(max_weight) + gross_target = float(gross_target) + min_score = float(min_score) + if not math.isfinite(max_weight) or not 0.0 < max_weight <= 1.0: + raise ValueError("max_weight must be finite and lie in (0, 1]") + if not math.isfinite(gross_target) or not 0.0 <= gross_target <= 1.0: + raise ValueError("gross_target must be finite and lie in [0, 1]") + if not math.isfinite(min_score): + raise ValueError("min_score must be finite") + + if hasattr(scores, "items"): + items = list(scores.items()) + else: + items = list(scores) + normalized = {} + for symbol, raw_score in items: + name = normalize_symbol(symbol) + score = float(raw_score) + if not math.isfinite(score): + raise ValueError("score must be finite for %s" % name) + normalized[name] = score + + weights = dict((symbol, 0.0) for symbol in normalized) + active = dict( + (symbol, score) + for symbol, score in normalized.items() + if score > min_score and score > 0.0 + ) + remaining = min(gross_target, len(active) * max_weight) + + while active and remaining > 1e-15: + score_sum = _sum_numeric(active.values()) + if score_sum <= 0.0: + break + capped = [] + for symbol in sorted(active): + proposed = remaining * active[symbol] / score_sum + if proposed >= max_weight - 1e-15: + weights[symbol] = max_weight + capped.append(symbol) + if not capped: + for symbol in sorted(active): + weights[symbol] = remaining * active[symbol] / score_sum + remaining = 0.0 + break + for symbol in capped: + active.pop(symbol) + remaining -= max_weight + remaining = max(0.0, remaining) + return weights + + +def round_board_lot(quantity, lot_size=100): + """Round an absolute long position down to a board-lot quantity.""" + + lot_size = int(lot_size) + if lot_size <= 0: + raise ValueError("lot_size must be positive") + if not math.isfinite(float(quantity)): + raise ValueError("quantity must be finite") + quantity = max(0, int(quantity)) + return quantity // lot_size * lot_size + + +def _is_star_market(symbol): + code, exchange = _split_symbol(symbol) + return exchange == "XSHG" and code.startswith(("688", "689")) + + +def target_quantities( + weights, prices, equity, lot_size=100, cash_buffer=0.02 +): + """Convert target weights to affordable board-lot long quantities.""" + + equity = float(equity) + cash_buffer = float(cash_buffer) + if ( + not math.isfinite(equity) + or equity < 0.0 + or not math.isfinite(cash_buffer) + or not 0.0 <= cash_buffer < 1.0 + ): + raise ValueError("invalid equity or cash_buffer") + positive_weights = [] + for raw_symbol, raw_weight in weights.items(): + weight = float(raw_weight) + if not math.isfinite(weight): + raise ValueError( + "weight must be finite for %s" % normalize_symbol(raw_symbol) + ) + positive_weights.append(max(0.0, weight)) + requested_gross = _sum_numeric(positive_weights) + gross_ceiling = 1.0 - cash_buffer + scale = ( + min(1.0, gross_ceiling / requested_gross) + if requested_gross > 0.0 + else 1.0 + ) + result = {} + for raw_symbol in sorted(weights): + symbol = normalize_symbol(raw_symbol) + weight = float(weights[raw_symbol]) + if not math.isfinite(weight): + raise ValueError("weight must be finite for %s" % symbol) + weight = max(0.0, weight) + price = prices.get(raw_symbol) + if price is None: + price = prices.get(symbol) + if price is None: + raise KeyError("missing price for %s" % symbol) + price = float(price) + if price <= 0.0 or not math.isfinite(price): + raise ValueError("invalid price for %s" % symbol) + # Weights are absolute equity weights. cash_buffer is a ceiling, not + # a second multiplicative haircut: gross=.95 and buffer=.02 stays .95. + budget = equity * weight * scale + quantity = round_board_lot(budget / price, lot_size) + # STAR Market auction orders must contain at least 200 shares. Keep + # the portable planner conservative and on the configured 100-share + # grid even though quantities above 200 may increment by one share. + if _is_star_market(symbol) and 0 < quantity < 200: + quantity = 0 + result[symbol] = quantity + return result + + +def order_deltas( + targets, current, sellable=None, lot_size=100 +): + """Return signed board-lot orders; negative sells respect sellable quantity.""" + + canonical_targets = {} + canonical_current = {} + canonical_sellable = {} + for symbol, quantity in targets.items(): + canonical_targets[normalize_symbol(symbol)] = max(0, int(quantity)) + for symbol, quantity in current.items(): + canonical_current[normalize_symbol(symbol)] = max(0, int(quantity)) + if sellable is not None: + for symbol, quantity in sellable.items(): + canonical_sellable[normalize_symbol(symbol)] = max(0, int(quantity)) + + result = {} + symbols = sorted(set(canonical_targets) | set(canonical_current)) + for symbol in symbols: + target = round_board_lot(canonical_targets.get(symbol, 0), lot_size) + if _is_star_market(symbol) and 0 < target < 200: + target = 0 + held = canonical_current.get(symbol, 0) + delta = target - held + if delta > 0: + delta = round_board_lot(delta, lot_size) + if _is_star_market(symbol) and 0 < delta < 200: + delta = 0 + elif delta < 0: + available = held + if sellable is not None: + available = min(held, canonical_sellable.get(symbol, 0)) + requested = min(-delta, available) + # A complete liquidation may submit the entire odd-lot balance. + # This also covers the STAR exception for a remaining balance + # below 200 shares. Partial orders stay on the configured grid. + full_liquidation = ( + target == 0 + and requested == held + and available >= held + ) + if full_liquidation: + delta = -held + else: + delta = -round_board_lot(requested, lot_size) + if _is_star_market(symbol) and 0 < -delta < 200: + delta = 0 + if delta: + result[symbol] = delta + return result + + +def build_rebalance_plan( + price_history, + current, + sellable, + equity, + lookback=20, + skip=0, + top_n=10, + max_weight=0.20, + gross_target=1.0, + cash_buffer=0.02, + lot_size=100, +): + """Build the complete portable signal -> weight -> quantity -> order plan.""" + + ranked = rank_momentum(price_history, lookback, skip, top_n) + scores = dict(ranked) + weights = capped_target_weights(scores, max_weight, gross_target, 0.0) + for raw_symbol in price_history: + symbol = normalize_symbol(raw_symbol) + if symbol not in scores: + scores[symbol] = momentum_score( + price_history[raw_symbol], lookback, skip + ) + if symbol not in weights: + weights[symbol] = 0.0 + prices = {} + for raw_symbol, history in price_history.items(): + if not history: + continue + prices[normalize_symbol(raw_symbol)] = history[-1] + targets = target_quantities( + weights, prices, equity, lot_size, cash_buffer + ) + orders = order_deltas(targets, current, sellable, lot_size) + return { + "scores": scores, + "weights": weights, + "targets": targets, + "orders": orders, + } + + +__all__ = [ + "build_rebalance_plan", + "capped_target_weights", + "convert_symbol", + "momentum_score", + "normalize_symbol", + "order_deltas", + "rank_momentum", + "round_board_lot", + "target_quantities", +] +# __PORTABLE_CORE_BUNDLE_END__ + +import math + + +class PriceHistoryUnavailable(RuntimeError): + pass + + +class UnmanagedPositionError(RuntimeError): + pass + + +class ClockStateError(RuntimeError): + pass + + +# __BASELINE_CONFIG_START__ +# Generated from configs/baseline.json; do not edit this region. +CONFIG = {'universe_mode': 'pit_index', + 'index_symbol': '000905.XSHG', + 'qmt_sector_name': '\u4e2d\u8bc1500', + 'dedicated_account_required': True, + 'exclude_st': True, + 'universe': ['600000.XSHG', '000001.XSHE', '300750.XSHE', '000333.XSHE'], + 'lookback': 20, + 'skip': 0, + 'top_n': 20, + 'max_weight': 0.05, + 'gross_target': 0.95, + 'cash_buffer': 0.02, + 'lot_size': 100, + 'participation_rate': 0.1, + 'slippage_bps': 2.0, + 'commission_rate': 0.0002, + 'minimum_commission': 5.0, + 'stamp_duty_rate': 0.0005, + 'transfer_fee_rate': 1e-05, + 'rebalance_schedule': 'weekly_first_close'} +# __BASELINE_CONFIG_END__ + + +def initialize(context): + """Configure a daily-open clock that executes a first-week-close signal.""" + set_option("avoid_future_data", True) + set_option("use_real_price", True) + g.quant60_config = dict(CONFIG) + if g.quant60_config.get("rebalance_schedule") != "weekly_first_close": + raise ValueError("unsupported rebalance_schedule") + if g.quant60_config.get("universe_mode") not in ("pit_index", "fixed"): + raise ValueError("unsupported universe_mode") + if ( + g.quant60_config.get("universe_mode") == "pit_index" + and not g.quant60_config.get("dedicated_account_required") + ): + raise ValueError("pit_index mode requires a dedicated account") + set_benchmark(g.quant60_config["index_symbol"]) + commission = ( + float(g.quant60_config["commission_rate"]) + + float(g.quant60_config["transfer_fee_rate"]) + ) + set_order_cost( + OrderCost( + open_tax=0.0, + close_tax=float(g.quant60_config["stamp_duty_rate"]), + open_commission=commission, + close_commission=commission, + close_today_commission=0.0, + min_commission=float( + g.quant60_config["minimum_commission"] + ), + ), + type="stock", + ) + # JoinQuant applies half of PriceRelatedSlippage on each side. The local + # slippage_bps setting is one-sided, hence the explicit factor of two. + set_slippage( + PriceRelatedSlippage( + 2.0 * float(g.quant60_config["slippage_bps"]) / 10000.0 + ), + type="stock", + ) + set_option( + "order_volume_ratio", + float(g.quant60_config["participation_rate"]), + ) + g.quant60_last_plan = None + g.quant60_last_universe = None + g.quant60_last_universe_as_of = None + g.quant60_skipped_orders = [] + g.quant60_pending_first_session = None + g.quant60_last_callback_date = None + run_daily(rebalance, time="open") + + +def _close_history(symbol, count): + raw = attribute_history( + symbol, + count, + unit="1d", + fields=["close"], + skip_paused=False, + df=False, + fq="pre", + ) + values = list(raw.get("close", [])) if hasattr(raw, "get") else [] + if len(values) != int(count): + raise PriceHistoryUnavailable( + "%s needs %d completed daily closes, got %d" + % (symbol, int(count), len(values)) + ) + output = [] + for value in values: + try: + number = float(value) + except (TypeError, ValueError): + raise PriceHistoryUnavailable( + "%s contains a missing/non-numeric close" % symbol + ) + if not math.isfinite(number) or number <= 0.0: + raise PriceHistoryUnavailable( + "%s contains a missing/non-positive close" % symbol + ) + output.append(number) + return output + + +def _close_histories(symbols, count): + raw = history( + int(count), + unit="1d", + field="close", + security_list=list(symbols), + df=False, + skip_paused=False, + fq="pre", + ) + if not hasattr(raw, "get"): + raise PriceHistoryUnavailable("history returned an invalid payload") + output = {} + for symbol in symbols: + values = list(raw.get(symbol, [])) + if len(values) != int(count): + raise PriceHistoryUnavailable( + "%s needs %d completed daily closes, got %d" + % (symbol, int(count), len(values)) + ) + clean = [] + for value in values: + try: + number = float(value) + except (TypeError, ValueError): + raise PriceHistoryUnavailable( + "%s contains a missing/non-numeric close" % symbol + ) + if not math.isfinite(number) or number <= 0.0: + raise PriceHistoryUnavailable( + "%s contains a missing/non-positive close" % symbol + ) + clean.append(number) + output[convert_symbol(symbol, "canonical")] = clean + return output + + +def _decision_universe(context): + config = g.quant60_config + if config.get("universe_mode") == "fixed": + values = list(config["universe"]) + else: + as_of = _as_date( + getattr(context, "previous_date", None), + "previous_date", + ) + values = get_index_stocks( + config["index_symbol"], + date=as_of, + ) + if not values: + raise PriceHistoryUnavailable( + "no PIT index members for %s on %s" + % (config["index_symbol"], as_of.isoformat()) + ) + g.quant60_last_universe_as_of = as_of.isoformat() + normalized = sorted( + set(convert_symbol(symbol, "joinquant") for symbol in values) + ) + if not normalized: + raise PriceHistoryUnavailable("decision universe is empty") + g.quant60_last_universe = list(normalized) + return normalized + + +def _filter_st_universe(context, symbols): + if not g.quant60_config.get("exclude_st", True): + return list(symbols) + as_of = _as_date( + getattr(context, "previous_date", None), + "previous_date", + ) + raw = get_extras( + "is_st", + list(symbols), + end_date=as_of, + count=1, + df=False, + ) + if not hasattr(raw, "get"): + raise PriceHistoryUnavailable( + "get_extras is_st returned an invalid payload" + ) + output = [] + for symbol in symbols: + values = list(raw.get(symbol, [])) + if len(values) != 1: + raise PriceHistoryUnavailable( + "%s needs one PIT is_st value on %s" + % (symbol, as_of.isoformat()) + ) + try: + numeric = float(values[0]) + except (TypeError, ValueError): + raise PriceHistoryUnavailable( + "%s contains a non-boolean PIT is_st value" % symbol + ) + if not math.isfinite(numeric) or numeric not in (0.0, 1.0): + raise PriceHistoryUnavailable( + "%s contains an invalid PIT is_st value" % symbol + ) + if not bool(numeric): + output.append(symbol) + if not output: + raise PriceHistoryUnavailable( + "PIT ST exclusion removed the entire decision universe" + ) + g.quant60_last_universe = list(output) + return output + + +def _position_amount(position, name, default=0): + value = getattr(position, name, default) + return int(value or 0) + + +def _snapshot_portfolio(context, managed): + current = {} + sellable = {} + managed = set(convert_symbol(symbol, "canonical") for symbol in managed) + dynamic = g.quant60_config.get("universe_mode") == "pit_index" + positions = getattr(context.portfolio, "positions", {}) + for symbol, position in positions.items(): + canonical = convert_symbol(str(symbol), "canonical") + quantity = _position_amount(position, "total_amount") + if quantity and canonical not in managed and not dynamic: + raise UnmanagedPositionError( + "account contains unmanaged position %s; use a dedicated " + "strategy account or reconcile ownership before trading" % canonical + ) + if canonical not in managed and not dynamic: + continue + current[canonical] = quantity + sellable[canonical] = _position_amount( + position, "closeable_amount", current[canonical] + ) + equity = float(getattr(context.portfolio, "total_value", 0.0) or 0.0) + return current, sellable, equity + + +def _is_star_board(symbol): + code = str(symbol).split(".", 1)[0] + return code.startswith("688") or code.startswith("689") + + +def _order_request(symbol, delta): + """Build a fail-closed hosted order request from current market data. + + JoinQuant rejects an unprotected market order for STAR Market securities. + A limit at the exchange daily bound supplies equivalent worst-price + protection while keeping the rebalance target expressed in exact shares. + """ + try: + item = get_current_data()[symbol] + except Exception as exc: + g.quant60_skipped_orders.append( + {"symbol": symbol, "reason": "current_data_unavailable", "detail": str(exc)} + ) + return None + if bool(getattr(item, "paused", False)): + g.quant60_skipped_orders.append( + {"symbol": symbol, "reason": "paused"} + ) + return None + if not _is_star_board(symbol): + return {"style": None} + + field = "high_limit" if int(delta) > 0 else "low_limit" + raw_price = getattr(item, field, None) + try: + protection_price = float(raw_price) + except (TypeError, ValueError): + protection_price = float("nan") + if ( + not math.isfinite(protection_price) + or protection_price <= 0.0 + or protection_price >= 10000.0 + ): + g.quant60_skipped_orders.append( + { + "symbol": symbol, + "reason": "star_protection_price_invalid", + "detail": field, + } + ) + return None + try: + style = LimitOrderStyle(protection_price) + except Exception as exc: + g.quant60_skipped_orders.append( + { + "symbol": symbol, + "reason": "star_protection_style_unavailable", + "detail": str(exc), + } + ) + return None + return {"style": style} + + +def compute_plan(context): + """Return the portable plan without submitting orders.""" + config = g.quant60_config + required = int(config["lookback"]) + int(config["skip"]) + 1 + platform_universe = _decision_universe(context) + platform_universe = _filter_st_universe( + context, + platform_universe, + ) + price_history = _close_histories(platform_universe, required) + + current, sellable, equity = _snapshot_portfolio( + context, + platform_universe, + ) + if equity <= 0: + raise ValueError("portfolio.total_value must be positive") + plan = build_rebalance_plan( + price_history=price_history, + current=current, + sellable=sellable, + equity=equity, + lookback=config["lookback"], + skip=config["skip"], + top_n=config["top_n"], + max_weight=config["max_weight"], + gross_target=config["gross_target"], + cash_buffer=config["cash_buffer"], + lot_size=config["lot_size"], + ) + return plan + + +def _execute_rebalance(context): + """Submit the portable plan's exact executable share deltas.""" + plan = compute_plan(context) + order_deltas = plan["orders"] + current, _, unused_equity = _snapshot_portfolio( + context, + g.quant60_last_universe or g.quant60_config["universe"], + ) + del unused_equity + all_symbols = set(current) + all_symbols.update(order_deltas) + + # Exits first. The requested target is current + executable delta, not the + # unconstrained portfolio target; this preserves T+1 sellable caps and odd + # lots from the portable plan. + for canonical in sorted( + all_symbols, + key=lambda item: ( + int(order_deltas.get(item, 0)) > 0, + item, + ), + ): + delta = int(order_deltas.get(canonical, 0)) + if delta == 0: + continue + platform_symbol = convert_symbol(canonical, "joinquant") + request = _order_request(platform_symbol, delta) + if request is None: + continue + target = int(current.get(canonical, 0)) + delta + if request["style"] is None: + order_target(platform_symbol, target) + else: + order_target( + platform_symbol, + target, + style=request["style"], + ) + + g.quant60_last_plan = plan + return plan + + +def _as_date(value, name): + if hasattr(value, "date"): + value = value.date() + if not hasattr(value, "isocalendar"): + raise ClockStateError("%s must be a date/datetime" % name) + return value + + +def rebalance(context): + """Execute on the session immediately after a week's first close. + + A daily state machine is used instead of ``run_weekly(..., 2)`` so a + one-session holiday week still executes on the next available session. + """ + current_date = _as_date(getattr(context, "current_dt", None), "current_dt") + previous_date = _as_date( + getattr(context, "previous_date", None), + "previous_date", + ) + if g.quant60_last_callback_date == current_date: + return None + pending = g.quant60_pending_first_session + is_first_session = ( + previous_date.isocalendar()[:2] + != current_date.isocalendar()[:2] + ) + if pending is not None: + if previous_date != pending: + raise ClockStateError( + "pending first-session close is not the immediately " + "previous trading session" + ) + # Consume the clock token before crossing any hosted order boundary. + # A partial API failure must not make a callback retry duplicate the + # already accepted prefix of the order list. + g.quant60_pending_first_session = ( + current_date if is_first_session else None + ) + g.quant60_last_callback_date = current_date + plan = _execute_rebalance(context) + return plan + if is_first_session: + g.quant60_pending_first_session = current_date + g.quant60_last_callback_date = current_date + return None diff --git a/dist/qmt_builtin_strategy.py b/dist/qmt_builtin_strategy.py new file mode 100644 index 0000000..81a0ea8 --- /dev/null +++ b/dist/qmt_builtin_strategy.py @@ -0,0 +1,843 @@ +# coding: gbk +"""QMT built-in Python 3.6 strategy wrapper. Keep this source ASCII-only.""" + +# __PORTABLE_CORE_BUNDLE_START__ +# Inlined by tools/bundle_platforms.py; do not edit this region. +import math +import re + + +_CANONICAL_EXCHANGES = { + "SH": "XSHG", + "SSE": "XSHG", + "XSHG": "XSHG", + "SZ": "XSHE", + "SZE": "XSHE", + "SZSE": "XSHE", + "XSHE": "XSHE", + "BJ": "XBSE", + "BSE": "XBSE", + "XBSE": "XBSE", +} + +_SHORT_EXCHANGES = { + "XSHG": "SH", + "XSHE": "SZ", + "XBSE": "BJ", +} + + +def _sum_numeric(values): + """Sum numerics without trusting a hosted runtime's global ``sum`` name.""" + + total = 0.0 + for value in values: + total += value + return total + + +def _infer_exchange(code): + if code.startswith(("4", "8")): + return "XBSE" + if code.startswith(("5", "6", "9")): + return "XSHG" + return "XSHE" + + +def _split_symbol(symbol): + if not isinstance(symbol, str): + raise TypeError("symbol must be a string") + value = symbol.strip().upper().replace(" ", "") + if not value: + raise ValueError("symbol must not be empty") + + match = re.match(r"^(\d{6})\.(XSHG|XSHE|XBSE|SH|SZ|BJ)$", value) + if match: + return match.group(1), _CANONICAL_EXCHANGES[match.group(2)] + + match = re.match(r"^(XSHG|XSHE|XBSE|SH|SZ|BJ)[\.:]?(\d{6})$", value) + if match: + return match.group(2), _CANONICAL_EXCHANGES[match.group(1)] + + match = re.match(r"^(\d{6})[\.:](SSE|SZE|SZSE|BSE)$", value) + if match: + return match.group(1), _CANONICAL_EXCHANGES[match.group(2)] + + if re.match(r"^\d{6}$", value): + return value, _infer_exchange(value) + raise ValueError("unsupported A-share symbol: %s" % symbol) + + +def normalize_symbol(symbol, target="canonical"): + """Convert common A-share codes to canonical/JQ, QMT, or Qlib format.""" + + code, exchange = _split_symbol(symbol) + target_name = str(target).strip().lower() + if target_name in ("canonical", "joinquant", "jq"): + return "%s.%s" % (code, exchange) + if target_name in ("qmt", "xtquant"): + return "%s.%s" % (code, _SHORT_EXCHANGES[exchange]) + if target_name == "qlib": + return "%s%s" % (_SHORT_EXCHANGES[exchange], code) + raise ValueError("unsupported target platform: %s" % target) + + +def convert_symbol(symbol, target="canonical"): + """Alias kept as the stable adapter-facing name.""" + + return normalize_symbol(symbol, target) + + +def momentum_score(prices, lookback=20, skip=0): + """Return trailing simple momentum, or ``None`` if history is insufficient. + + ``skip=0`` uses the latest completed decision bar. Set ``skip=1`` only + when intentionally implementing a 20-1 convention. A lookback of N needs + N+skip+1 observations. + """ + + lookback = int(lookback) + skip = int(skip) + if lookback <= 0 or skip < 0: + raise ValueError("lookback must be positive and skip non-negative") + values = list(prices) + if len(values) < lookback + skip + 1: + return None + end_index = len(values) - 1 - skip + start_index = end_index - lookback + start = float(values[start_index]) + end = float(values[end_index]) + if ( + start <= 0.0 + or end <= 0.0 + or not math.isfinite(start) + or not math.isfinite(end) + ): + return None + return end / start - 1.0 + + +def rank_momentum(price_history, lookback=20, skip=0, top_n=None): + """Rank ``{symbol: closes}`` by momentum with deterministic tie-breaking.""" + + ranked = [] + for symbol in sorted(price_history): + score = momentum_score(price_history[symbol], lookback, skip) + if score is not None and math.isfinite(score): + ranked.append((normalize_symbol(symbol), score)) + ranked.sort(key=lambda item: (-item[1], item[0])) + if top_n is not None: + ranked = ranked[: max(0, int(top_n))] + return ranked + + +def capped_target_weights( + scores, max_weight=0.20, gross_target=1.0, min_score=0.0 +): + """Long-only score-proportional weights with iterative cap redistribution.""" + + max_weight = float(max_weight) + gross_target = float(gross_target) + min_score = float(min_score) + if not math.isfinite(max_weight) or not 0.0 < max_weight <= 1.0: + raise ValueError("max_weight must be finite and lie in (0, 1]") + if not math.isfinite(gross_target) or not 0.0 <= gross_target <= 1.0: + raise ValueError("gross_target must be finite and lie in [0, 1]") + if not math.isfinite(min_score): + raise ValueError("min_score must be finite") + + if hasattr(scores, "items"): + items = list(scores.items()) + else: + items = list(scores) + normalized = {} + for symbol, raw_score in items: + name = normalize_symbol(symbol) + score = float(raw_score) + if not math.isfinite(score): + raise ValueError("score must be finite for %s" % name) + normalized[name] = score + + weights = dict((symbol, 0.0) for symbol in normalized) + active = dict( + (symbol, score) + for symbol, score in normalized.items() + if score > min_score and score > 0.0 + ) + remaining = min(gross_target, len(active) * max_weight) + + while active and remaining > 1e-15: + score_sum = _sum_numeric(active.values()) + if score_sum <= 0.0: + break + capped = [] + for symbol in sorted(active): + proposed = remaining * active[symbol] / score_sum + if proposed >= max_weight - 1e-15: + weights[symbol] = max_weight + capped.append(symbol) + if not capped: + for symbol in sorted(active): + weights[symbol] = remaining * active[symbol] / score_sum + remaining = 0.0 + break + for symbol in capped: + active.pop(symbol) + remaining -= max_weight + remaining = max(0.0, remaining) + return weights + + +def round_board_lot(quantity, lot_size=100): + """Round an absolute long position down to a board-lot quantity.""" + + lot_size = int(lot_size) + if lot_size <= 0: + raise ValueError("lot_size must be positive") + if not math.isfinite(float(quantity)): + raise ValueError("quantity must be finite") + quantity = max(0, int(quantity)) + return quantity // lot_size * lot_size + + +def _is_star_market(symbol): + code, exchange = _split_symbol(symbol) + return exchange == "XSHG" and code.startswith(("688", "689")) + + +def target_quantities( + weights, prices, equity, lot_size=100, cash_buffer=0.02 +): + """Convert target weights to affordable board-lot long quantities.""" + + equity = float(equity) + cash_buffer = float(cash_buffer) + if ( + not math.isfinite(equity) + or equity < 0.0 + or not math.isfinite(cash_buffer) + or not 0.0 <= cash_buffer < 1.0 + ): + raise ValueError("invalid equity or cash_buffer") + positive_weights = [] + for raw_symbol, raw_weight in weights.items(): + weight = float(raw_weight) + if not math.isfinite(weight): + raise ValueError( + "weight must be finite for %s" % normalize_symbol(raw_symbol) + ) + positive_weights.append(max(0.0, weight)) + requested_gross = _sum_numeric(positive_weights) + gross_ceiling = 1.0 - cash_buffer + scale = ( + min(1.0, gross_ceiling / requested_gross) + if requested_gross > 0.0 + else 1.0 + ) + result = {} + for raw_symbol in sorted(weights): + symbol = normalize_symbol(raw_symbol) + weight = float(weights[raw_symbol]) + if not math.isfinite(weight): + raise ValueError("weight must be finite for %s" % symbol) + weight = max(0.0, weight) + price = prices.get(raw_symbol) + if price is None: + price = prices.get(symbol) + if price is None: + raise KeyError("missing price for %s" % symbol) + price = float(price) + if price <= 0.0 or not math.isfinite(price): + raise ValueError("invalid price for %s" % symbol) + # Weights are absolute equity weights. cash_buffer is a ceiling, not + # a second multiplicative haircut: gross=.95 and buffer=.02 stays .95. + budget = equity * weight * scale + quantity = round_board_lot(budget / price, lot_size) + # STAR Market auction orders must contain at least 200 shares. Keep + # the portable planner conservative and on the configured 100-share + # grid even though quantities above 200 may increment by one share. + if _is_star_market(symbol) and 0 < quantity < 200: + quantity = 0 + result[symbol] = quantity + return result + + +def order_deltas( + targets, current, sellable=None, lot_size=100 +): + """Return signed board-lot orders; negative sells respect sellable quantity.""" + + canonical_targets = {} + canonical_current = {} + canonical_sellable = {} + for symbol, quantity in targets.items(): + canonical_targets[normalize_symbol(symbol)] = max(0, int(quantity)) + for symbol, quantity in current.items(): + canonical_current[normalize_symbol(symbol)] = max(0, int(quantity)) + if sellable is not None: + for symbol, quantity in sellable.items(): + canonical_sellable[normalize_symbol(symbol)] = max(0, int(quantity)) + + result = {} + symbols = sorted(set(canonical_targets) | set(canonical_current)) + for symbol in symbols: + target = round_board_lot(canonical_targets.get(symbol, 0), lot_size) + if _is_star_market(symbol) and 0 < target < 200: + target = 0 + held = canonical_current.get(symbol, 0) + delta = target - held + if delta > 0: + delta = round_board_lot(delta, lot_size) + if _is_star_market(symbol) and 0 < delta < 200: + delta = 0 + elif delta < 0: + available = held + if sellable is not None: + available = min(held, canonical_sellable.get(symbol, 0)) + requested = min(-delta, available) + # A complete liquidation may submit the entire odd-lot balance. + # This also covers the STAR exception for a remaining balance + # below 200 shares. Partial orders stay on the configured grid. + full_liquidation = ( + target == 0 + and requested == held + and available >= held + ) + if full_liquidation: + delta = -held + else: + delta = -round_board_lot(requested, lot_size) + if _is_star_market(symbol) and 0 < -delta < 200: + delta = 0 + if delta: + result[symbol] = delta + return result + + +def build_rebalance_plan( + price_history, + current, + sellable, + equity, + lookback=20, + skip=0, + top_n=10, + max_weight=0.20, + gross_target=1.0, + cash_buffer=0.02, + lot_size=100, +): + """Build the complete portable signal -> weight -> quantity -> order plan.""" + + ranked = rank_momentum(price_history, lookback, skip, top_n) + scores = dict(ranked) + weights = capped_target_weights(scores, max_weight, gross_target, 0.0) + for raw_symbol in price_history: + symbol = normalize_symbol(raw_symbol) + if symbol not in scores: + scores[symbol] = momentum_score( + price_history[raw_symbol], lookback, skip + ) + if symbol not in weights: + weights[symbol] = 0.0 + prices = {} + for raw_symbol, history in price_history.items(): + if not history: + continue + prices[normalize_symbol(raw_symbol)] = history[-1] + targets = target_quantities( + weights, prices, equity, lot_size, cash_buffer + ) + orders = order_deltas(targets, current, sellable, lot_size) + return { + "scores": scores, + "weights": weights, + "targets": targets, + "orders": orders, + } + + +__all__ = [ + "build_rebalance_plan", + "capped_target_weights", + "convert_symbol", + "momentum_score", + "normalize_symbol", + "order_deltas", + "rank_momentum", + "round_board_lot", + "target_quantities", +] +# __PORTABLE_CORE_BUNDLE_END__ + +import datetime +import math + + +class PriceHistoryUnavailable(RuntimeError): + pass + + +class UnmanagedPositionError(RuntimeError): + pass + + +class UnsupportedStrategyPeriod(RuntimeError): + pass + + +class G(object): + pass + + +# QMT explicitly documents that custom attributes written to ContextInfo can +# roll back before the next handlebar call. Keep all user-owned mutable state +# in a module global and use ContextInfo only for platform-owned fields/APIs. +g = G() + + +# __BASELINE_CONFIG_START__ +# Generated from configs/baseline.json; do not edit this region. +CONFIG = {'account_id': 'test', + 'universe_mode': 'pit_index', + 'index_symbol': '000905.XSHG', + 'qmt_sector_name': '\u4e2d\u8bc1500', + 'dedicated_account_required': True, + 'exclude_st': True, + 'universe': ['600000.SH', '000001.SZ', '300750.SZ', '000333.SZ'], + 'lookback': 20, + 'skip': 0, + 'top_n': 20, + 'max_weight': 0.05, + 'gross_target': 0.95, + 'cash_buffer': 0.02, + 'lot_size': 100, + 'participation_rate': 0.1, + 'slippage_bps': 2.0, + 'commission_rate': 0.0002, + 'minimum_commission': 5.0, + 'stamp_duty_rate': 0.0005, + 'transfer_fee_rate': 1e-05, + 'rebalance_schedule': 'weekly_first_close'} +# __BASELINE_CONFIG_END__ + + +def _param(ContextInfo, name, default=None): + params = getattr(ContextInfo, "_param", {}) + if isinstance(params, dict) and name in params: + return params[name] + return default + + +def init(ContextInfo): + period = str( + getattr(ContextInfo, "period", _param(ContextInfo, "period", "1d")) + ).strip().lower() + if period not in ("1d", "day"): + raise UnsupportedStrategyPeriod( + "quant60 QMT wrapper requires a 1d driving period" + ) + g.config = dict(CONFIG) + for key in CONFIG: + override = _param(ContextInfo, "q60_" + key, None) + if override is not None: + g.config[key] = override + if ( + g.config.get("rebalance_schedule") + != "weekly_first_close" + ): + raise UnsupportedStrategyPeriod( + "unsupported rebalance_schedule" + ) + if g.config.get("universe_mode") not in ("pit_index", "fixed"): + raise UnsupportedStrategyPeriod("unsupported universe_mode") + if ( + g.config.get("universe_mode") == "pit_index" + and not g.config.get("dedicated_account_required") + ): + raise UnsupportedStrategyPeriod( + "pit_index mode requires a dedicated account" + ) + set_commission = getattr(ContextInfo, "set_commission", None) + set_slippage = getattr(ContextInfo, "set_slippage", None) + if not callable(set_commission) or not callable(set_slippage): + raise UnsupportedStrategyPeriod( + "QMT backtest commission/slippage APIs are unavailable" + ) + commission = ( + float(g.config["commission_rate"]) + + float(g.config["transfer_fee_rate"]) + ) + set_commission( + 0, + [ + 0.0, + float(g.config["stamp_duty_rate"]), + commission, + commission, + 0.0, + float(g.config["minimum_commission"]), + ], + ) + set_slippage( + 2, + float(g.config["slippage_bps"]) / 10000.0, + ) + g.last_week = None + g.last_plan = None + g.last_universe = None + g.last_universe_as_of = None + if ( + g.config.get("universe_mode") == "fixed" + and hasattr(ContextInfo, "set_universe") + ): + ContextInfo.set_universe(g.config["universe"]) + + +def _bar_datetime(ContextInfo): + timetag = ContextInfo.get_bar_timetag(ContextInfo.barpos) + return datetime.datetime.fromtimestamp(float(timetag) / 1000.0) + + +def _series_close(frame): + if frame is None: + return [] + if hasattr(frame, "__getitem__"): + try: + values = frame["close"] + if hasattr(values, "tolist"): + values = values.tolist() + return list(values) + except (KeyError, TypeError): + pass + if isinstance(frame, dict): + values = frame.get("close", []) + return list(values) + return [] + + +def _decision_universe(ContextInfo, membership_timetag): + config = g.config + if config.get("universe_mode") == "fixed": + values = list(config["universe"]) + else: + function = getattr( + ContextInfo, + "get_stock_list_in_sector", + None, + ) + if not callable(function): + raise PriceHistoryUnavailable( + "historical sector membership API is unavailable" + ) + values = function( + config["qmt_sector_name"], + int(membership_timetag), + ) + if not values: + raise PriceHistoryUnavailable( + "no PIT index members for %s on %s" + % (config["index_symbol"], membership_timetag) + ) + normalized = sorted( + set(convert_symbol(symbol, "qmt") for symbol in values) + ) + if not normalized: + raise PriceHistoryUnavailable("decision universe is empty") + g.last_universe = list(normalized) + g.last_universe_as_of = int(membership_timetag) + return normalized + + +def _history(ContextInfo, end_time, universe): + config = g.config + count = int(config["lookback"]) + int(config["skip"]) + 1 + raw = ContextInfo.get_market_data_ex( + fields=["close"], + stock_code=universe, + period="1d", + start_time="", + end_time=end_time, + count=count, + dividend_type="front_ratio", + fill_data=False, + subscribe=False, + ) + output = {} + for symbol in universe: + values = _series_close(raw.get(symbol)) + if len(values) != count: + raise PriceHistoryUnavailable( + "%s needs %d completed daily closes, got %d" + % (symbol, count, len(values)) + ) + clean = [] + for value in values: + try: + number = float(value) + except (TypeError, ValueError): + raise PriceHistoryUnavailable( + "%s contains a missing/non-numeric close" % symbol + ) + if not math.isfinite(number) or number <= 0.0: + raise PriceHistoryUnavailable( + "%s contains a missing/non-positive close" % symbol + ) + clean.append(number) + output[convert_symbol(symbol, "canonical")] = clean + return output + + +def _filter_st_universe(ContextInfo, universe, decision_date): + if not g.config.get("exclude_st", True): + return list(universe) + function = getattr(ContextInfo, "get_his_st_data", None) + if not callable(function): + raise PriceHistoryUnavailable( + "historical ST API is unavailable; QMT VIP ST data is required" + ) + output = [] + for symbol in universe: + raw = function(symbol) + if not isinstance(raw, dict): + raise PriceHistoryUnavailable( + "%s historical ST query returned an invalid payload" % symbol + ) + excluded = False + for status in ("ST", "*ST", "PT"): + periods = raw.get(status, []) + if not isinstance(periods, (list, tuple)): + raise PriceHistoryUnavailable( + "%s historical %s periods are invalid" % (symbol, status) + ) + for period in periods: + if ( + not isinstance(period, (list, tuple)) + or len(period) != 2 + ): + raise PriceHistoryUnavailable( + "%s historical %s period is invalid" + % (symbol, status) + ) + start = str(period[0]) + end = str(period[1]) + if ( + len(start) != 8 + or len(end) != 8 + or not start.isdigit() + or not end.isdigit() + or start > end + ): + raise PriceHistoryUnavailable( + "%s historical %s date range is invalid" + % (symbol, status) + ) + if start <= decision_date <= end: + excluded = True + if not excluded: + output.append(symbol) + if not output: + raise PriceHistoryUnavailable( + "PIT ST exclusion removed the entire decision universe" + ) + g.last_universe = list(output) + return output + + +def _get_attr(obj, names, default=0): + for name in names: + if hasattr(obj, name): + value = getattr(obj, name) + if value is not None: + return value + return default + + +def _trade_details(ContextInfo, kind): + account_id = g.config["account_id"] + if hasattr(ContextInfo, "get_trade_detail_data"): + return ContextInfo.get_trade_detail_data(account_id, "STOCK", kind) + fn = globals().get("get_trade_detail_data") + if fn is None: + return [] + return fn(account_id, "STOCK", kind) + + +def _portfolio(ContextInfo, managed): + current = {} + sellable = {} + managed = set( + convert_symbol(symbol, "canonical") + for symbol in managed + ) + dynamic = g.config.get("universe_mode") == "pit_index" + for position in _trade_details(ContextInfo, "position") or []: + code = str( + _get_attr(position, ["stock_code", "m_strInstrumentID"], "") + ) + market = str(_get_attr(position, ["market", "m_strExchangeID"], "")) + if "." not in code and market: + code = code + "." + market + if not code: + continue + canonical = convert_symbol(code, "canonical") + quantity = int( + _get_attr(position, ["volume", "m_nVolume", "total_amount"], 0) + ) + if quantity and canonical not in managed and not dynamic: + raise UnmanagedPositionError( + "account contains unmanaged position %s; use a dedicated " + "strategy account or reconcile ownership before trading" % canonical + ) + if canonical not in managed and not dynamic: + continue + current[canonical] = quantity + sellable[canonical] = int( + _get_attr( + position, + ["can_use_volume", "m_nCanUseVolume", "closeable_amount"], + current[canonical], + ) + ) + + equity = 0.0 + assets = _trade_details(ContextInfo, "account") or [] + if assets: + equity = float( + _get_attr( + assets[0], + ["total_asset", "m_dBalance", "m_dTotalAsset", "asset"], + 0.0, + ) + ) + if equity <= 0: + equity = float(_param(ContextInfo, "asset", 0.0) or 0.0) + return current, sellable, equity + + +def compute_plan(ContextInfo, bar_time=None): + if bar_time is None: + bar_time = _bar_datetime(ContextInfo) + membership_timetag = int( + ContextInfo.get_bar_timetag(ContextInfo.barpos) + ) + end_time = bar_time.strftime("%Y%m%d") + universe = _decision_universe(ContextInfo, membership_timetag) + universe = _filter_st_universe( + ContextInfo, + universe, + end_time, + ) + history = _history(ContextInfo, end_time, universe) + current, sellable, equity = _portfolio(ContextInfo, universe) + if equity <= 0: + raise ValueError("QMT account equity must be positive") + config = g.config + return build_rebalance_plan( + price_history=history, + current=current, + sellable=sellable, + equity=equity, + lookback=config["lookback"], + skip=config["skip"], + top_n=config["top_n"], + max_weight=config["max_weight"], + gross_target=config["gross_target"], + cash_buffer=config["cash_buffer"], + lot_size=config["lot_size"], + ) + + +def _orders_allowed(ContextInfo): + # This hosted wrapper is deliberately backtest-only. Its cached account + # query and in-memory weekly marker cannot provide crash-safe live + # idempotency or a complete fresh reconciliation barrier. Real/shadow + # broker integration belongs behind the external XtTrader adapter. + return _is_backtest(ContextInfo) + + +def _is_backtest(ContextInfo): + if hasattr(ContextInfo, "do_back_test"): + value = getattr(ContextInfo, "do_back_test") + if callable(value): + value = value() + if value is True: + return True + mode = str( + getattr(ContextInfo, "trade_mode", _param(ContextInfo, "trade_mode", "")) + ).strip().lower() + return mode == "backtest" + + +def _submit_delta(ContextInfo, canonical, delta, week_key): + config = g.config + order_code = convert_symbol(canonical, "qmt") + operation = 23 if delta > 0 else 24 + volume = abs(int(delta)) + user_order_id = "q60-%04d%02d-%s" % ( + int(week_key[0]), + int(week_key[1]), + order_code.split(".")[0], + ) + function = globals()["passorder"] + arg_count = getattr(getattr(function, "__code__", None), "co_argcount", 11) + if arg_count <= 8: + # qmttools native-Python compatibility signature. + function( + operation, + 1101, + config["account_id"], + order_code, + 5, + -1, + volume, + ContextInfo, + ) + else: + # QMT built-in hosted signature. + function( + operation, + 1101, + config["account_id"], + order_code, + 5, + -1, + volume, + "quant60", + 0, + user_order_id, + ContextInfo, + ) + + +def handlebar(ContextInfo): + if not _is_backtest(ContextInfo) and hasattr(ContextInfo, "is_last_bar"): + if not ContextInfo.is_last_bar(): + return None + + bar_time = _bar_datetime(ContextInfo) + year, week, unused = bar_time.isocalendar() + del unused + week_key = (year, week) + if g.last_week is None: + # The strategy may be attached to a run in the middle of a week. + # Treat the first observed week as an initialization window; only a + # later ISO-week transition proves that this is the first observed + # trading session of a complete strategy week. + g.last_week = week_key + return None + if g.last_week == week_key: + return None + + plan = compute_plan(ContextInfo, bar_time) + g.last_week = week_key + g.last_plan = plan + if not _orders_allowed(ContextInfo): + return plan + + # Sells before buys. passorder is the QMT-hosted order boundary. + ordered = sorted(plan["orders"].items(), key=lambda item: item[1]) + for canonical, delta in ordered: + if int(delta) != 0: + _submit_delta(ContextInfo, canonical, delta, week_key) + return plan diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..bb579a8 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,274 @@ +# Quant OS architecture and trust boundaries + +Quant OS 是项目；`quant60` 是第一版 A 股 Baseline 内核、CLI namespace +和 artifact 协议前缀。当前形态是模块化单体：共享可移植计算和稳定合同，但把 +数据供应商、hosted 平台、QMT 专有运行时与券商事实放在不同信任边界内。 + +```text +JQData (authorized account) + get_bars(raw OHLCV/money, factor, pre_close, limits, paused) + + daily PIT is_st + daily PIT index membership + | + v + canonical immutable snapshot + exact-set/hash + semantic reconstruction + | + +---------+----------+ + | | + v v + snapshot-backtest snapshot-decision + local event engine Signal -> Target -> Order Delta + | | + +---------+----------+ + | + portable-momentum-v1 + | + +------------+-------------+ + | | | + JoinQuant bundle QMT bundle local contract oracle + hosted backtest built-in/qmttools backtest-only + +synthetic PIT/features -> purged walk-forward -> Ridge challenger + (not deployment-frozen) + +Qlib 0.9.7: + native portable-momentum signal bridge + Alpha158 + LightGBM + Recorder research workflow + (no L3 target / L4 order-intent parity) + +XtTrader: + read-only broker observation -> HMAC account-bound shadow plan + callback/mutation boundary remains uncertified and operator-inaccessible +``` + +箭头表示数据或合同流，不表示平台能力等价。hosted 引擎拥有自己的时钟、行情 +和成交语义；broker snapshot/callback 才是现金、持仓、可卖量、订单和成交的 +事实权威。本地 ledger 不能用预测持仓覆盖券商事实。 + +## Runtime split + +| Layer | Runtime | Rule | +| --- | --- | --- | +| Quant OS / `quant60` | Python ≥3.10 | 本地回测、snapshot pipeline、research、ledger、CLI | +| JQData ingestion | 独立 Python 环境，`jqdatasdk==1.9.8` | 只从授权账户获取数据；凭据不进入快照 | +| Qlib research | CPython 3.12.x，`pyqlib==0.9.7` | 与本地/QMT 环境隔离；provider 必须授权且版本化 | +| Portable strategy core | Python 3.6-compatible source | 无第三方依赖；bundler 内联到 hosted 策略 | +| JoinQuant hosted | 平台管理 | 上传生成的单文件；不能假设安装本地包 | +| QMT built-in | QMT Python 3.6 | 只上传 bundle；模块全局 `g` 持久状态，避免 `ContextInfo` 用户属性回滚 | +| qmttools / XtTrader | 券商分发的 native Python | `xtquant` 专有且 lazy import，不进入仓库 | +| Colab | notebook runtime | 默认只跑无账号流程；JQData/Qlib 是显式可选 cell | + +不要用一个 Python 环境强行兼容所有路径。 + +## Data plane and point-in-time semantics + +JQData adapter 的输入合同是逐交易日历史指数成员和 `get_bars` 日线字段： + +- 未复权 `open/high/low/close`、`volume`、`money`； +- 复权因子 `factor`； +- `pre_close`、`high_limit`、`low_limit`、`paused`； +- 每个 member-date 的严格布尔 PIT `is_st`； +- `skip_paused=False`，保留停牌日语义。 + +快照保存 `bars.jsonl`、`memberships.jsonl`、`quality.json` 和 +`manifest.json`。发布期间存在 incomplete marker；完成后 verifier 要求目录 +是精确 artifact 集、每个文件 hash 正确，并从 JSONL 重建领域对象，重算质量 +汇总、内容 hash 和 `data_version`。这比“下载成功”更强，但在授权真实账户运行 +并保存许可/lineage 前，仍不构成 G1 证据。 + +JQData 日线在 24:00 完成，因此 adapter 拒绝把 Asia/Shanghai retrieval +当日作为 `T_CLOSE_COMPLETE`；请求 `end` 必须早于抓取日期。字段、ST 或成员 +覆盖不完整也不会被当成默认 false/空集合。 + +`snapshot-decision` 只用 `as_of` 当日精确 PIT 成员和截至当日的数据。 +执行/名义价格保持 raw；动量把每个历史 close 乘以当日可见 factor，再归一到 +决策日因子，因此后续公司行动不会改写已经形成的历史决策。held but +untradable 标的不会从 Universe 消失，而会进入 frozen / hold-only / +sell-only 状态。 + +若目标用于下一交易日开盘前 shadow，Signal 时钟仍是 T 日 15:00。snapshot +冻结 JQData `get_trade_days` 与 `get_all_trade_days` 一致性校验后的完整 +交易日序列，并显式保存 end 日后的唯一 `next_trading_session`。decision、 +broker snapshot 和 shadow plan 必须绑定同一个相邻 session，以及 +Asia/Shanghai `[09:00,09:30)` 的 `first_executable_window`；elapsed-day +阈值和 weekday 近似都不是权威。周末、T 日 15:01、错过 09:30、错误 session 或 +迟到多日全部 fail closed，国庆等长假只要下一 session 来自冻结日历即可。 + +账户绑定后还必须保持决策规模有效。shadow 对 `decision.equity` 和当前 +`total_asset` 使用固定的 +`max(1 元, 0.01% × abs(decision equity))` 包含边界容差；缺失、负数、 +非有限值或超差都不能进入 ready，超差原因码为 +`DECISION_EQUITY_DRIFT`。该合同进入 plan hash，verifier 会从原始两个权益值 +重新计算差额、阈值和 blocker。 + +QMT shadow verifier 使用 semantic replay，而不把 plan 当成事实来源。 +`broker_observation.json` 保存 query 后 adapter state、live flag、callback +error/live-authorization history 计数、query 起止、原始记录数、校验结果及 +脱敏资产/持仓/委托/成交事实。原始 decision manifest 是 verify 的必需输入。 +纯 replay 从这两个输入独立重建 account binding、clock gate、资产和现金 +恒等式、open/order-stability、trade/fill identity、完整 blocker 集、 +snapshot-valid、完整 broker snapshot、逐 symbol target/weight/lot、 +board-lot/sellable feasible delta、proposal、portfolio 和 status，再与 plan +exact compare。因而 plan 中同步改写多个派生字段或删除 blocker 不能形成新的 +`SHADOW_READY`；READY 也强制存在可重建 snapshot。 + +普通 artifact hash 之外，planner 使用既有 account HMAC key 和独立 domain +签署 `QMT_SHADOW_EVIDENCE_V1` envelope。envelope 绑定完整 plan、 +broker observation、broker snapshot 的 canonical 内容 hash 和三份实际发布 +bytes 的 SHA-256、原始 decision manifest hash、planner/engine 及 operator +entrypoint `tools/qmt_shadow_plan.py` source hash，以及 semantic version、 +10 秒/1 元 policy ceiling 和 equity drift policy。四个发布 JSON（包括 +不参与自签名的 manifest）还必须逐 byte 符合唯一 deterministic writer +encoding。verifier 从当前真实输入与磁盘实际 bytes 重建 +envelope 并 constant-time 比较 MAC；缺失、错误或轮换后的 key 全部失败。 +这提供的是 Quant OS 本地 evidence publisher authenticity/完整性，不是 +QMT/券商对 query result 的原生签名，也不把 adapter 升级成 broker-certified。 + +策略安全阈值是 canonical policy：query window 上限 10 秒，资产/持仓市值和 +`cash + market_value = total_asset` 的绝对容差上限 1 元。CLI 只能使用等于 +或更严格的值，builder 与 replay 都拒绝更宽值。目标权重总和还必须不超过 +1，作为无报价条件下可独立证明的决策现金预算；实时 buying power（含成交价、 +费用和卖出先后）仍不作证明。 + +持仓 drift 不会复用旧持仓：每次 target diff 都从当前券商 positions 和 +sellable quantity 重算，且 open/变化中订单会阻断。因此数量 residual 不会 +因旧持仓静默失真。由于 shadow 没有实时执行价，现金购买力仍不能被证明； +plan 明示该限制，ready 仅供人工审阅，不能升级为 live submission。 +`trade.amount` 当前会被标准化、认证和重放，但在缺少真实 +terminal/xtquant build 探针时不假设它与 `price × quantity` 的精确误差 +合同；它不参与 target diff，正式券商验收必须记录 build 并冻结字段容差。 + +`snapshot-backtest` 在每个历史决策日重新调用同一 decision path，并使用： + +- 第一实际交易日完整收盘形成计划； +- 紧接着下一实际交易日开盘尝试成交； +- 当日供应商停牌/涨跌停/前收字段； +- 上一交易日成交量参与率； +- T+1、百股交易单位、显式费用和未成交撤销。 + +这是本地 fill model，不是 JoinQuant 或 QMT 的成交事实。 + +## Research and model boundary + +synthetic research pipeline 已实现： + +- PIT table 和未来值拒绝； +- Universe 原因码； +- 透明价量特征与 T+1-open 标签时钟； +- train-only winsor/impute/standardize； +- purge/embargo walk-forward； +- deterministic Ridge、OOS IC/RankIC； +- factor risk、dated cost/capacity 和 constrained target。 + +这些功能证明研究管线可执行。Ridge 尚未冻结成带版本、输入 schema 和推理 +hash 的 deployment bundle，也没有授权真实长样本 OOS 与四引擎推理验证，因此 +不是生产 champion。 + +Qlib 路径固定为 `0.9.7`。CPython 3.12 实际 fixture smoke 已得到： + +- native momentum：24 条 signal； +- Alpha158 + LightGBM：完成 fit、SignalRecord、SigAnaRecord、 + PortAnaRecord 和 model 保存； +- 两条 workflow 都保存表级 hash、行列/时间边界和从 portfolio report + 重算的有限值指标，明确 `investment_value_claim=false`； +- local MLflow file store 使用时，runner 以 + `os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true")` 显式确认。 + +该 fixture 只有两只股票和一个 benchmark，性能没有投资意义。Qlib 的单一 +`limit_threshold` 也不能表达逐日板块/ST 规则，所以它只参与 L1/L2 和诊断 +L6，不声称 L3 target 或 L4 order-intent parity。 + +## Portable strategy and platform boundary + +当前可跨 hosted wrapper 冻结的共同基线是 `portable-momentum-v1`。平台 API +不得进入 symbol normalization、momentum score、target weight、target +quantity 和 order delta 计算。 + +canonical 股票代码采用 JoinQuant 形式： + +| Canonical | QMT/XtQuant | Qlib | +| --- | --- | --- | +| `600000.XSHG` | `600000.SH` | `SH600000` | +| `000001.XSHE` | `000001.SZ` | `SZ000001` | +| `830799.XBSE` | `830799.BJ` | `BJ830799` | + +转换权威是 `quant60.portable_core.normalize_symbol`。当前 XtTrader mutation +边界只接受 `.SH` / `.SZ`，不能从转换覆盖推导北交所实盘覆盖。 + +QMT built-in bundle 和 qmttools runner 都硬限制为 backtest/history。built-in +状态存放在模块全局 `g`，不写入可能在下个 `handlebar` 回滚的 +`ContextInfo` 用户属性。两条路径固定中证 500 benchmark、PIT 历史成分/ST +与 baseline 费率；built-in 的 10% 参与率由 QMT GUI 设置，qmttools +程序化传入。live mutation 只存在于外部 XtTrader adapter，并且默认关闭、 +未认证、没有 live 启动命令；operator 可运行的 shadow CLI 只接受查询边界。 + +## Stable artifacts and release integrity + +最低交换合同是： + +- [`signal.schema.json`](../schemas/signal.schema.json)； +- [`target.schema.json`](../schemas/target.schema.json)； +- [`order_event.schema.json`](../schemas/order_event.schema.json)； +- [`broker_snapshot.schema.json`](../schemas/broker_snapshot.schema.json)。 + +`verify-manifest` 不仅检查 ledger hash chain，还检查 incomplete marker、 +精确 artifact 集、SHA-256、ledger head、report/manifest identity、 +saved/current source aggregate、saved/current schema aggregate 和 payload +schema。`verify-snapshot-decision` 与 data snapshot semantic verifier 承担 +各自边界的同类职责。 + +这些合同已有自动测试；最近一次标准库套件为 211 tests、OK、3 个可选路径 +skip。测试结果不能替代 schema 迁移演练、真实平台导出或券商回调认证。 + +## Clocks and reconciliation + +canonical weekly clock 是： + +```text +ISO 周第一实际交易日：完整 close + -> 使用 as_of 之前实际可得信息形成 signal/target/order intent +紧接着下一实际交易日：open executable window + -> 刷新市场规则与 broker snapshot + -> pre-trade gate + -> backtest/shadow submit 或 fail closed + -> callback append + reconciliation +``` + +“紧接着下一日”是交易日关系，不是固定星期几。每个真实平台 run 必须记录 +`signal_as_of`、`last_feature_bar`、`first_executable_time`、 +price adjustment 和 fill model。 + +完整 reconciliation 要同时比较 cash、positions、sellable quantities、 +open orders 和带时区的新鲜 `as_of/source_time`。legacy/partial snapshot +即使数值碰巧平衡，也不能把 `safe_to_open` 设为 true。 + +## Facts and current claim + +| Fact | Authority | +| --- | --- | +| canonical symbol、portable calculation | versioned source + config | +| snapshot 内容 | semantic verifier 通过的 immutable snapshot + manifest | +| historical provider entitlement | 实际数据账户许可与保存的 lineage 记录 | +| hosted 回测时钟/成交 | 平台真实导出 | +| cash、position、order、fill | broker snapshot/callback | +| 程序化交易权限与报告 | 实际券商书面确认 | +| Gate 状态 | [`gate_scorecard.json`](../gate_scorecard.json) | + +synthetic/fake 数据是合同测试，不是市场证据；Qlib synthetic fixture 的真实 +运行是运行时证据，不是收益证据；mock parity exporter 是 wrapper 合同证据， +不计 G9。 + +## Remaining production gaps + +当前还缺： + +- 授权 JQData 真实快照、基本面公告 PIT、完整公司行动/Security Master； +- Ridge/LightGBM 冻结模型包和真实 OOS 证据； +- 真实数据校准的风险、冲击、容量和 TCA； +- 真实 JoinQuant/QMT 导出； +- 券商状态映射、restart recovery daemon、20 日 shadow 与连续对账； +- 运维监控、kill-switch 演练和程序化交易合规确认。 + +所以 Quant OS 是可执行、可继续填证据的生产候选框架，不是已经达到整体 +60 分的实盘系统。证据政策见 +[`AUDIT_2026-07-25.md`](AUDIT_2026-07-25.md)。 diff --git a/docs/AUDIT_2026-07-25.md b/docs/AUDIT_2026-07-25.md new file mode 100644 index 0000000..095467f --- /dev/null +++ b/docs/AUDIT_2026-07-25.md @@ -0,0 +1,250 @@ +# 2026-07-25 复评：为什么原文还不是“整体 60 分” + +审计对象：`个人 A 股量化系统机构级 60 分 Baseline 技术选型设计：从聚宽研究到 QMT 实盘（2026-07-22）` + +结论先行： + +| 刻度 | 分数 | 准确含义 | +| --- | ---: | --- | +| 设计覆盖度 | 约 70/100 | 原文已经讨论研究、数据、风险、成本、组合、执行、运维与合规的大部分责任边界 | +| 可实施规格度 | 49/100 | 只有约一半内容足够明确，工程师无需继续发明字段、默认值、失败语义和验收口径 | +| 原实现证据 | 6/100 | 截至 2026-07-22，主要证据是文章、架构图、外部资料与验收清单；没有可重放代码、数据、测试或平台运行记录 | +| 硬门 | 0/10 | G1—G10 都缺少完整验收证据；不是“部分通过” | + +这三条分数不能相加，也不能互相替代。设计覆盖 70 不代表系统得 70；代码文件存在也不代表 Gate 通过。依据原文自己的定义，达到 Baseline 60 必须同时满足： + +```text +证据评分 >= 60 +且 G1 ... G10 全部通过 +``` + +当前 Quant OS 代码交付是一个可执行、可测试的工程起点；`quant60` 是其第一版 +A 股 Baseline 内核。它的新增得分必须在授权数据、真实平台运行、QMT 影子盘、 +连续对账和券商合规确认完成后，逐项写回 +[`gate_scorecard.json`](../gate_scorecard.json)。本报告不预支这些分数。 + +截至 2026-07-26 的实现事实快照： + +- 截至 2026-07-26 的最新标准库全量测试为 `Ran 211 tests`、`OK (skipped=3)`； +- JQData raw/factor/pre-close/limits/paused + 每日 PIT `is_st`/指数成员 + 已连接到 semantic snapshot verifier、`snapshot-backtest` 和 + `snapshot-decision`，并拒绝把抓取当日未到 24:00 的日线标成完整收盘， + 但尚未用用户授权账号跑真实快照； +- CPython 3.12 + `pyqlib==0.9.7` native momentum fixture 已真实运行并产生 + 24 条 signal； +- Qlib Alpha158 + LightGBM + Recorder fixture smoke 已真实成功；两条 + Qlib workflow 均保存表级 hash、边界和从 portfolio report 重算的指标， + runner 会为 local MLflow file store 显式设置 + `MLFLOW_ALLOW_FILE_STORE=true`； +- fixture 只有两只 synthetic 股票，任何性能数字都没有投资意义； +- JoinQuant/QMT mock parity exporter 已实现，但 + `real_platform_pass=false` 且不计 G9； +- 2026-07-26 已完成一次 2024 全年真实 JoinQuant hosted smoke，可见日志 + 无 ERROR；它证明 runtime/path，但尚无完整同输入导出和本地 strict peer； +- XtTrader 已有严格只读、HMAC 账户绑定的 observation/snapshot/shadow + plan 与 verifier，但真实 QMT、broker callback/shadow 均未运行， + G1—G10 仍全部 `not_passed`。 + +## 1. 审计方法 + +本次没有按“名词是否出现”评分，而是对原文每个责任域检查六类问题： + +1. 是否明确输入、输出和时钟； +2. 是否给出机器可校验的数据契约； +3. 是否处理异常、恢复和幂等； +4. 是否给出确定默认值或明确标记待确认； +5. 是否有可运行实现和自动化测试； +6. 是否有来自目标平台、真实数据或券商的运行证据。 + +“设计覆盖度”主要看第 1 类及责任域是否齐全；“可实施规格度”要求前 4 类足够清楚；“实现证据”只认可第 5、6 类已保存的产物。外部文档说明某 API 有能力，不等于本系统已经正确调用它。 + +## 2. 原文做对了什么 + +原文比常见的“因子 + 回测净值”方案完整得多，70 分设计覆盖主要来自： + +- 明确拆开 `Forecast → Target → Order → Fill`，避免预测、仓位和成交混为一谈； +- 意识到 PIT、历史指数成分、退市样本、公告可得时间和公司行动； +- 把风险、成本、换手和流动性放到事前组合决策，而不是只在绩效末尾扣滑点； +- 讨论部分成交、撤改单、乱序回报、重启、三账对账和 kill switch； +- 把本地、聚宽、Qlib、QMT 定位为不同角色，而不是追求单一平台全包； +- 给出十道硬门，且明确“目标 66”只有在实现、模拟和券商确认后才成立； +- 对深模型、混合整数优化、逐笔队列与多券商容灾做了合理的 V1 边界控制。 + +因此问题不是方向错误，而是大量段落仍处于“正确的架构意图”，没有下降为可执行合同和证据。 + +## 3. 不足全景 + +| 责任域 | 原文已有 | 阻止整体 60 的主要缺口 | 最低补证 | +| --- | --- | --- | --- | +| 数据/PIT | 分层目录、`available_time`、数据版本思想 | 无真实字段字典、供应商字段到 canonical 的映射、公告时区/盘前盘后规则、历史修订样例、许可边界和 PIT 注入测试 | 固定数据快照、manifest、字段级时点规范、反未来函数测试 | +| Security Master / Universe | PIT 成分、开/持/卖/冻结状态 | 无状态 schema、生效区间合并优先级、代码变更与退市/重组样例；指数成分来源和发布日期语义未冻结 | 历史 fixture 与逐原因码重放 | +| 特征/标签/验证 | LightGBM + Ridge、purge/embargo、walk-forward | 无特征公式和缺失值策略的完整注册表；标签价格与可执行时钟未机器化；无 purge 算法、切分日历和未触碰测试集 | 训练配置、切分测试、冻结 OOS 报告 | +| 风险 | 透明行业/风格风险模型与收缩思想 | 因子定义、估计窗、缺失暴露、协方差清洗、特异风险下限、行业映射版本和压力场景均未定 | 风险快照 schema、数值稳定性测试、压力报告 | +| 成本/容量 | 日期化费用、价差与冲击公式 | 参数没有账户交割单或真实成交校准；最低佣金的非凸性如何进入候选/rounding 未实现；机会成本缺失 | 费率版本、逐笔 TCA、low/base/high 校准 | +| 组合 | 连续优化 + rounding/repair 方向正确 | 目标函数量纲、风险厌恶系数、约束优先级、不可行诊断、冻结仓位与现金联立细节未形成求解器规格 | 可解释优化输出、不可行/rounding 回归测试 | +| 本地回测 | 要求与执行器同源 | 没有事件时钟、公司行动、现金结算、价格基准、成交量切片和可重放输出合同 | 小型确定性 fixture、黄金输出、费用/T+1/涨跌停测试 | +| 聚宽 | 定位为第二引擎 | 无可上传入口、参数表、调度回调、`avoid_future_data` 设置、结果导出和平台限制清单 | 上传包、平台回测记录、canonical 差异报告 | +| QMT 内置 Python | 只笼统写 QMT/XtQuant | 没有处理内置 Python 3.6 与本地现代环境的边界；未说明 GUI 启动方式、回调入口和产物导出 | Python 3.6 薄核心、QMT 内置回测记录 | +| qmttools | 未与其他 QMT 路径清楚分层 | 原文没有说明 `run_strategy_file` 是原生 Python 驱动 QMT 策略文件的独立运行形态，不能等同于 XtTrader OMS | 独立入口、版本/参数记录、回测结果导出 | +| XtTrader | 异步 OMS 方向正确 | 无券商状态码映射、session/account 配置、连接恢复、查询与回调竞态、撤单确认、未知单处置 | 脱敏回调回放、重启/乱序/重复测试、20 日影子盘 | +| Qlib | 工作流选型合理 | “接 Qlib”与“由 Qlib 完成本系统回测”边界模糊；无 provider URI、dataset handler、workflow YAML、canonical 转换和 Records 产物 | Python 3.12 锁定环境、固定数据、Qlib smoke/OOS 报告 | +| 跨引擎 | 提出双引擎一致性 | 未定义到底比较输入、信号、目标、订单还是净值；没有容差、舍入、交易日历和差异原因码 | 见 [`CROSS_ENGINE_CONSISTENCY.md`](CROSS_ENGINE_CONSISTENCY.md) 的逐层合同 | +| Schema/版本迁移 | 列出九类对象 | 原文只有字段清单，不是 JSON Schema；无 `schema_version`、兼容策略、正反例与验证命令 | 版本化 schema、fixture 和 CI 验证 | +| 可复现 | Git/config/data/model/rules 版本思想 | 无 run manifest 格式、环境锁、随机源清单、原子发布、干净机复跑或 hash 容差 | 可保存 manifest、两次重放差异 | +| 运维/安全 | 提到单机、告警、runbook | 无部署命令、目录权限、凭据注入、日志脱敏、磁盘满/时钟偏差/重复调度/损坏账本处置 | [`SECURITY.md`](SECURITY.md) 与故障演练记录 | +| 合规 | 正确设为硬门 | 只链接通用法规；没有实际券商对账户、软件、频率、报告状态的确认 | 券商书面确认索引与“先报告、后交易”记录 | + +## 4. 可实施规格为什么只有 49 + +原文的伪代码和目录树提供了良好边界，但工程实现仍需自行决定很多会改变结果的事实。例如： + +- `AlphaForecast` 写了“分数/预期超额”，却没有决定两者是否都必填、单位是什么、是否允许空值； +- “按公告可得时间进入”没有定义 15:00 后公告应进入哪个交易日、时区和供应商延迟如何处理； +- “Guarded TWAP/POV”没有冻结子单切片、限价、超时、撤单确认和窗口结束残单政策； +- “差异在数值容差内”没有给各层容差，也没有区分浮点误差与平台成交机制差异； +- “QMT adapter”没有区分 QMT 内置 Python、`qmttools.run_strategy_file` 与外部 `XtTrader` 三条生命周期不同的路径； +- “Qlib workflow”没有给 provider、handler、dataset、model、record 和数据转换的可运行组合； +- “三本账”没有定义事件唯一键、追加一致性、崩溃点、重复回调去重和 broker snapshot 的事实优先级。 + +当实现者必须自行补完这些选择时，文章仍是设计说明，不是完整实施规格。新仓库中的四个 JSON Schema 只是把一小部分边界机器化，不能倒推原文在 7 月 22 日已经拥有这些证据。 + +## 5. 原实现证据为什么只有 6 + +6 分不是对文章质量的否定，而是严格区分“说明”和“运行事实”。原文当时可认可的证据主要是： + +- 一份结构完整、明确承认条件性的设计文章； +- 一张架构图和可编辑源图； +- 外部官方资料链接； +- 评分公式、Gate 定义、阶段计划与手工验收清单。 + +缺失的是： + +- 可安装仓库与固定 Python 环境； +- 可运行本地回测和确定性 fixture； +- 聚宽/QMT/Qlib 平台入口； +- 自动化测试、schema 验证、run manifest； +- 数据质量、PIT、walk-forward、风险、成本和 TCA 报告； +- 连续模拟、QMT 影子盘、故障演练和券商确认。 + +因此原实现证据只能落在“文档/原型前期”，不能用文章中的目标分 66 回填。 + +## 6. Quant OS 当前实现的正确能力边界 + +当前已经不只是目录骨架，以下链路可运行、可测试、可扩展： + +- 本地 deterministic event backtest、ledger hash chain、manifest verifier； +- A 股 canonical symbol、T+1 可卖量、百股单位、费用、停牌/涨跌停、 + 部分成交和未成交撤销； +- PIT table、Universe 状态/原因码、透明特征、train-only 预处理、 + purge/embargo walk-forward、deterministic Ridge、IC/RankIC； +- 日期化费用、impact/capacity、因子风险、约束 target 与失败诊断； +- JQData adapter 使用 `get_bars` 获取 raw OHLCV/money、factor、 + `pre_close`、`high_limit`、`low_limit`、`paused`，使用 `get_extras` + 获取 PIT `is_st`，并逐交易日获取历史指数成员； +- canonical snapshot 采用 incomplete marker + exact artifact hash， + verifier 会重建领域对象、重算质量汇总和 `data_version`； +- verified snapshot 可直接进入历史 `snapshot-backtest` 或单日 + `snapshot-decision`，后者可接完整 canonical broker snapshot 并 + fail closed； +- 聚宽/QMT 单文件 bundle、fake harness 与 mock parity exporter； +- Qlib 0.9.7 native momentum CLI，以及 Alpha158/LightGBM/Recorder CLI； +- QMT/XtTrader 严格只读 shadow CLI：账户 HMAC、broker observation、 + T 日 Signal 与 T+1 broker fact 绑定、完整 artifact verifier； +- Colab 默认无账号流程，以及显式可选的 JQData/Qlib cell。 + +最近一次 211-test suite 和两条 Qlib fixture smoke 证明上述代码路径能执行， +但仍明确不证明： + +- synthetic 数据或两股票 Qlib fixture 的收益具有投资价值； +- 当前真实 JQData 账户的数据许可、PIT、质量与保存策略已验收； +- 基本面公告、完整公司行动、Security Master 和退市链已生产化； +- Ridge 或 LightGBM 已冻结为四引擎共同部署模型； +- 聚宽与本地已在真实长样本上对齐； +- QMT 客户端、`xtquant`、数据/账户权限或真实订单回调已经可用； +- QMT 历史 ST 正例、官方 query API 的失败/空列表合同已在真实账号验证； +- OMS 已达到断线恢复、未知单、持续对账和 20 日影子要求； +- 程序化交易报告已经完成。 + +## 7. 平台事实应分开写 + +截至本次复评所依据的官方文档和本地运行记录： + +- QMT 内置策略环境仍说明为 Python 3.6；它适合运行兼容的薄策略文件，不应直接加载要求 Python ≥3.10 的完整 `quant60` 包； +- XtQuant 原生 Python 文档列出 3.6—3.12 的库，并要求运行前启动 MiniQMT；实际可用版本取决于券商分发； +- qmttools 的 `run_strategy_file` 是从原生 Python 驱动策略回测/运行的路径，应与 XtTrader 的查询、报单、撤单和推送生命周期分开； +- `pyqlib==0.9.7` 已有 CPython 3.12 的主流平台 wheel；本项目已经在该组合 + 实际跑通 native momentum fixture（24 signals）和 + Alpha158/LightGBM/Recorder fixture； +- Alpha158 runner 使用 `os.environ.setdefault` 自动确认 + `MLFLOW_ALLOW_FILE_STORE=true`，以兼容本地 MLflow file store； + 该确认只影响本地 tracking backend，不构成数据许可或模型有效性证明。 + +来源：[QMT 内置 Python 快速开始](https://dict.thinktrader.net/innerApi/start_now.html)、[XtQuant 原生 Python 快速开始](https://dict.thinktrader.net/nativeApi/start_now.html)、[原生 Python/qmttools 策略回测](https://dict.thinktrader.net/videos/touyan/ty_native_python.html)、[pyqlib 0.9.7 包与 wheel](https://pypi.org/project/pyqlib/)。 + +Qlib 的运行状态现在可以写成“固定运行时已通过 fixture smoke”，不能写成 +“真实 A 股研究已验收”或“与 portable target/order 一致”。两股票 synthetic +fixture 的 model/portfolio 指标没有投资意义；真实 provider、冻结 OOS 和 +版本化 Recorder artifact 仍要单独补证。 + +对应的最小验证入口： + +```bash +python3.12 -m venv .venv-qlib312 +source .venv-qlib312/bin/activate +python -m pip install -r requirements/research-py312.txt +python tools/build_qlib_tiny_fixture.py artifacts/qlib-fixture --days 80 +PYTHONPATH=src:. python -m platforms.qlib_runner \ + --provider-uri artifacts/qlib-fixture \ + --market csi300 --benchmark SH000300 \ + --start 2024-02-01 --end 2024-04-19 \ + --lookback 20 --topk 1 --n-drop 1 --rebalance weekly \ + --output-json artifacts/qlib-native/result.json +``` + +真实 JQData 验证则必须由用户授权登录： + +```bash +PYTHONPATH=src:. python tools/jqdata_snapshot.py \ + --index 000905.XSHG --start 2024-01-02 --end 2024-12-31 \ + --output data/snapshots/csi500-2024 +PYTHONPATH=src:. python -c \ + "from quant60.data_snapshot import verify_data_snapshot; verify_data_snapshot('data/snapshots/csi500-2024/manifest.json')" +PYTHONPATH=src:. python -m quant60 snapshot-backtest \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json --output artifacts/csi500-2024-backtest +``` + +## 8. 证据更新纪律 + +更新 [`gate_scorecard.json`](../gate_scorecard.json) 时必须遵守： + +1. 先保存不可变证据，再改状态； +2. 每条 evidence 至少包含路径、生成命令、时间、代码版本、配置 hash 和数据版本； +3. `mock`、合成 fixture 和单元测试可证明代码语义，不能证明真实数据、券商或合规； +4. Gate 只有 `passed` 与 `not_passed` 两种对外结论；“完成 80%”仍是未通过； +5. G9 需要真实跨引擎产物，G6/G8 需要券商认证环境或脱敏真实回调，G10 只能由实际券商材料证明； +6. 任何代码变更使旧证据超出适用版本时，必须重新降为 `not_passed`； +7. 总分达到 60 但任一 Gate 未通过，系统仍必须标记 `NOT_BASELINE_60`。 + +## 9. 下一条最短路径 + +按错误成本排序，而不是按模型新颖度排序： + +1. 用户授权登录 JQData，生成不可变真实 snapshot，保存许可/lineage，并通过 + semantic verifier、`snapshot-backtest` 和 `snapshot-decision`； +2. 基于已完成的聚宽 hosted smoke，冻结一个小型 canonical 平台输入并导出 + close arrays/hash、plan、orders/fills； +3. 用本地与聚宽对相同输入逐层比较 L1—L4，而不是先比较净值；mock parity + 只作为回归辅助； +4. 把基本面公告 available-time、完整公司行动/Security Master 加入真实 + PIT data contract； +5. 在授权真实 provider 上运行 Qlib OOS，并把 Ridge/LightGBM 中选定的候选 + 冻结成 versioned model bundle；现有 synthetic Recorder 不能承担这一步； +6. 取得实际券商 QMT client、`xtquant`、数据与账户权限，分别验证 built-in、 + qmttools 与 XtTrader； +7. 完成 60 个交易日模拟、20 个交易日 QMT shadow、故障演练、连续零未知账差； +8. 由实际券商确认程序化交易报告、接口权限、频率和软件要求，再讨论小资金 + canary。 + +在第 7、8 步完成前，正确表述是“可执行、可继续填证据的 Quant OS 生产候选 +框架”，不是“已达到机构级 60 分”。 diff --git a/docs/CROSS_ENGINE_CONSISTENCY.md b/docs/CROSS_ENGINE_CONSISTENCY.md new file mode 100644 index 0000000..82acf5f --- /dev/null +++ b/docs/CROSS_ENGINE_CONSISTENCY.md @@ -0,0 +1,252 @@ +# Quant OS cross-engine consistency contract + +“本地、聚宽、QMT、Qlib 都能跑”不等于四条净值曲线必须逐点相同。跨引擎 +一致性按层比较：相同信息集应产生相同计算；平台特有的日历、价格语义和成交 +差异必须被记录、归因和限制。 + +## 1. Comparison levels + +| Level | Compared artifact | Same canonical input required | Pass rule | +| --- | --- | --- | --- | +| L0 | Source/config manifest | Git state、portable-core hash、strategy/model version、symbol map | Exact | +| L1 | Data/universe | `as_of`、sessions、symbols、PIT membership、raw/factor/declared adjusted prices、tradability | Exact values or explicit provider reason | +| L2 | Signal | symbol、horizon、lookback/skip、score、model bundle | Exact symbol set；numeric tolerance below | +| L3 | Target | target weight/quantity、frozen/sellable state | weight tolerance；exact lot quantity | +| L4 | Order intent | ID、side、quantity、limit、type、TIF、timing、reason | Identical broker snapshot/rules 时 exact | +| L5 | Order/fill/portfolio | status、fills、fees、cash、positions | Semantic invariants；相同 fill fixture 才要求 exact | +| L6 | Performance | return、turnover、drawdown、cost | Diagnostic；不能代替 L1—L5 | + +只比较 L6 不会通过 G9。Alpha158/LightGBM 与 portable momentum 是不同 model +spec，本来就不是 L2 peer；可做研究比较，不能冒充同策略 parity。 + +## 2. Canonical snapshot is the L1 handoff + +JQData ingestion 通过官方 `get_bars` 保存： + +- raw `open/high/low/close/volume/money`； +- `factor`、`pre_close`、`high_limit`、`low_limit`、`paused`； +- 每个 member-date 的 PIT `is_st`； +- 每个交易日的历史指数成员。 + +快照必须保持 `adjustment=none`，front-adjusted feature history 在每个 +decision `as_of` 时由当时已保存的 raw close + factor 构造。这样执行价仍是 +raw，且后来的 factor 不能改写旧决策。 + +L1 verifier 不只比文件 hash，还要： + +1. 拒绝 incomplete marker； +2. 要求目录是 manifest 声明的精确 artifact 集； +3. 重建每个 bar/membership 领域对象； +4. 重算质量统计、内容 hash 与 `data_version`； +5. 拒绝重复键、非法 OHLC、停牌非零量、非法限价、缺失 factor 或非布尔/ + 缺失 `is_st`； +6. 拒绝 retrieval 当日仍未在 Asia/Shanghai 24:00 完成的日线。 + +验证入口： + +```bash +PYTHONPATH=src:. python -c \ + "from quant60.data_snapshot import verify_data_snapshot; verify_data_snapshot('data/snapshots/csi500-2024/manifest.json')" +``` + +`snapshot-backtest` 和 `snapshot-decision` 都先通过该 verifier。它们能证明 +同一个 snapshot 的本地可重放性，但在真实授权 JQData run 和平台导出前， +不能证明 provider entitlement 或 G9。 + +## 3. Required run manifest + +每个 comparison 至少保存： + +```json +{ + "run_id": "unique-id", + "engine": "local|joinquant|qmt_builtin|qmttools|xttrader|qlib", + "mode": "fixture|provider_snapshot|backtest|simulation|shadow|live", + "git_commit": "commit-or-DIRTY", + "portable_core_sha256": "sha256", + "strategy_or_model_version": "immutable-id", + "python_version": "major.minor.patch", + "platform_version": "value-or-unknown", + "config_hash": "sha256", + "data_version": "immutable-id", + "rules_version": "immutable-id", + "timezone": "Asia/Shanghai", + "signal_as_of": "ISO-8601 timestamp", + "last_feature_bar": "ISO-8601 date-or-time", + "first_executable_time": "ISO-8601 timestamp", + "price_adjustment": "raw|pre|post|front_ratio|PIT_FRONT_FROM_RAW_AND_FACTOR", + "fill_model": "declared-name", + "evidence_class": "fake|synthetic|provider-snapshot|real-platform|broker" +} +``` + +unknown platform version 可用于 smoke，但阻止 release certification。 + +本地 execution/provider snapshot run 保存后执行： + +```bash +PYTHONPATH=src:. python -m quant60 verify-manifest \ + artifacts//manifest.json +``` + +它检查 incomplete marker、精确 artifact set/hash、ledger chain/head、 +report/manifest identity、saved/current source/schema aggregate 和 payload +schema。snapshot decision 使用： + +```bash +PYTHONPATH=src:. python -m quant60 verify-snapshot-decision \ + artifacts//manifest.json +``` + +## 4. Numeric and semantic tolerances + +对传入 `portable_core.py` 的同一固定 close arrays： + +| Field | Tolerance | +| --- | --- | +| canonical symbol、rank、selected set | Exact | +| momentum score | absolute error ≤ `1e-12` | +| target weight | absolute error ≤ `1e-10` | +| target quantity | Exact integer | +| order delta | Exact signed integer | + +真实 provider 数据先比较输入数组。不同复权约定、停牌填充、成分股日期或 +feature cutoff 导致的 score 差异是 L1 failure，不是浮点噪声。 + +本地货币 ledger 按配置的 currency quantum 比较。hosted NAV 只有在 bars、 +company actions、fees 和 fill events 都相同时才要求相同；否则必须分类： + +- `input-price difference`； +- `timing difference`； +- `tradability/fill difference`； +- `explicit-fee difference`； +- `residual unexplained difference`。 + +residual 必须为零或低于 release-specific bound，不能静默并入 slippage。 + +## 5. Clock alignment + +canonical weekly clock 是 ISO 周第一实际交易日完整 close，紧接着下一实际 +交易日 open。它不是固定星期几；单交易日长假周会自然跨到下一 ISO 周。 + +comparison key 必须包含： + +```text +(signal_as_of, last_feature_bar, first_executable_time, adjustment_mode) +``` + +任一元素不同就不是 strict peer，只能做诊断 comparison。 + +本地 snapshot backtest 使用： + +- exact daily PIT membership； +- decision date raw close + factor 构造 point-in-time front-adjusted signal； +- next session raw open 执行； +- provider daily pause/limit/pre-close； +- provider daily PIT `is_st`：ST 不新买，已持有 ST 只卖； +- prior-session volume capacity。 + +JoinQuant/QMT 必须导出相同输入或其 hash 才能进入 L2—L4 strict comparison。 + +## 6. Platform interpretation and current facts + +- **Local**：固定 fixture 的 deterministic contract oracle，不自动等于市场 + 真相。 +- **JoinQuant**：以前一交易日查询 PIT index/ST，以批量 `history` 读取完成 + 日线，并显式设置费用、滑点和 10% 参与率；必须导出实际 close + arrays/hash、plan、orders/fills 和平台配置。本地 fake harness 只证明 + wiring。2026-07-26 已完成一次 2024 全年真实 hosted smoke，运行日志无 + ERROR；但尚未导出上述相同输入与分层结果，所以它是 runtime evidence， + 不是 strict peer/G9 证据。 +- **QMT built-in / qmttools**：共享 bundle 但生命周期不同，均硬 + backtest-only；固定中证 500 benchmark，以历史 timetag 查询 PIT 成分/ST。 + built-in 的 10% 最大成交量占比需要 QMT UI 设置并取证，qmttools + 程序化传入。真实 QMT 环境尚未运行。qmttools 把 transfer fee 折入 + commission，minimum commission/fill 行为是声明差异。 +- **XtTrader**：比较 broker snapshot、order intent 与 normalized callback。 + 当前已有只读 HMAC 账户绑定、broker observation/snapshot 和 exact + shadow-plan verifier，但只有 fake broker contract，没有真实 shadow。 +- **Qlib momentum**：`pyqlib==0.9.7` 的 native fixture 已真实运行，产生 + 24 条 signal；结果保存 signal/report hash、表边界和从 report 重算的 + portfolio 指标。它证明 runtime/path 可执行，不证明真实市场或 L3/L4。 +- **Qlib Alpha158/LightGBM**：fixture 上已真实完成 model + Recorder。 + runner 自动以 `setdefault` 设置 `MLFLOW_ALLOW_FILE_STORE=true` 以兼容本地 + MLflow file store，并读取 portfolio report 保存 hash、边界和重算指标。 + 两股票 synthetic 性能没有投资意义，且该模型不是 portable momentum 的 + parity peer。 +- **Mock parity exporter**：当前能逐层比较 JoinQuant/QMT fake wrapper 的 + score、weight、quantity、delta，并校验 source hash。它明确是 + `mock_contract_only`、`real_platform_pass=false`、`gate_credit=[]`。 + +## 7. Difference reason codes + +必须使用枚举 reason code，不能只写“平台差异”： + +```text +CALENDAR +SYMBOL_MAP +UNIVERSE +MISSING_BAR +SUSPENSION_FILL +PRICE_ADJUSTMENT +AS_OF_CLOCK +ROUNDING +FEE_SCHEDULE +PRICE_LIMIT +T_PLUS_ONE +VOLUME_LIMIT +FILL_MODEL +BROKER_STATE +FLOAT_TOLERANCE +UNEXPLAINED +``` + +symbol set、signal、target、order intent、cash 或 position 出现任何 +`UNEXPLAINED` 都阻止 release。 + +## 8. Runnable comparison support + +mock-only exporter： + +```bash +PYTHONPATH=src:. python tools/export_mock_parity.py \ + export --output artifacts/mock-parity/report.json +PYTHONPATH=src:. python tools/export_mock_parity.py \ + verify artifacts/mock-parity/report.json +``` + +Qlib native momentum fixture： + +```bash +python3.12 -m venv .venv-qlib312 +source .venv-qlib312/bin/activate +python -m pip install -r requirements/research-py312.txt +python tools/build_qlib_tiny_fixture.py artifacts/qlib-fixture --days 80 +PYTHONPATH=src:. python -m platforms.qlib_runner \ + --provider-uri artifacts/qlib-fixture \ + --market csi300 --benchmark SH000300 \ + --start 2024-02-01 --end 2024-04-19 \ + --lookback 20 --topk 1 --n-drop 1 --rebalance weekly \ + --output-json artifacts/qlib-native/result.json +``` + +两者都不是 G9 的真实平台证据。 + +## 9. What counts toward G9 + +| Evidence | Counts toward G9? | +| --- | --- | +| symbol/unit tests、fake JoinQuant/QMT harness | Supporting only | +| mock parity exporter | No；contract regression only | +| local snapshot twice with same hash | Supporting G7/L1 only | +| Qlib 24-signal synthetic fixture smoke | Runtime evidence only | +| Qlib Alpha158 synthetic Recorder smoke | Research runtime evidence only | +| real JoinQuant export vs local canonical run | Yes，覆盖的 layers/dates | +| licensed qmttools run with saved version/result | Yes，覆盖的 layers | +| XtTrader shadow using real broker snapshot | Yes，pre-trade/order semantics | +| ≥20 trading days QMT shadow, zero unexplained differences | 文章 G9 必需 | + +真实 JoinQuant smoke 已运行；QMT 和 broker shadow 尚未运行，而且 +JoinQuant 仍缺同输入导出与 local peer，所以 G9 继续为 `not_passed`。 +状态权威是 +[`gate_scorecard.json`](../gate_scorecard.json)。 diff --git a/docs/JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md b/docs/JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md new file mode 100644 index 0000000..a2ef506 --- /dev/null +++ b/docs/JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md @@ -0,0 +1,73 @@ +# JoinQuant hosted backtest evidence — 2026-07-26 + +This is a real hosted-runtime smoke record, not a cross-engine acceptance +report and not an investment-performance claim. + +## Run identity + +- Strategy: `quant_os_portable_momentum_v1` +- Hosted record: + [JoinQuant backtest detail](https://www.joinquant.com/algorithm/backtest/detail?backtestId=62de09f1e0f9e81b260a18e0ee59e953) +- Period: 2024-01-02 through 2024-12-31 +- Initial capital: CNY 1,000,000 +- Frequency/runtime: daily, Python 3 +- Benchmark: CSI 500 (`000905.XSHG`) +- Uploaded bundle: + `dist/joinquant_strategy.py` +- Uploaded bundle SHA-256: + `22ddb0546beb3bd4e4bbedecb271452111324c3903e45be1f4ac3d39f38cce4f` +- Portable core SHA-256: + `af29e4e5e10705cf6aff89874394c1e80d3780f093c19d90ed53ef9d0188a934` + +## Hosted result + +| Metric | Hosted value | +| --- | ---: | +| Strategy return | -19.98% | +| Annualized return | -20.56% | +| Excess return | -24.12% | +| Benchmark return | 5.46% | +| Alpha | -0.259 | +| Beta | 0.830 | +| Sharpe | -0.806 | +| Win rate | 0.474 | +| Profit/loss ratio | 0.816 | +| Maximum drawdown | 32.45% | + +The visible completed-run log contained zero `ERROR`, `Traceback`, or +`订单委托失败` entries. These performance numbers are negative and are recorded +only to prove that the hosted execution path completed; they provide no +investment-value evidence. + +## Defects found by the hosted runtime + +The real platform run found three issues that fake wiring alone did not expose: + +1. JoinQuant's hosted namespace shadowed Python's global `sum`. The portable + core now uses its own deterministic numeric reducer. +2. STAR Market market orders require worst-price protection. The wrapper now + submits a documented `LimitOrderStyle`: daily upper limit for buys and daily + lower limit for sells, and fails closed when that price is absent or invalid. +3. STAR Market auction orders require at least 200 shares, except a complete + sale of the remaining sub-200 balance. The rule now lives in the shared + portable core and local simulator, so JoinQuant and QMT receive the same + legal target/order delta. + +The order API and limit-style contract are documented in the +[official JoinQuant API PDF](https://cdn.joinquant.com/help/img/JoinQuantAPI.pdf). +The 200-share minimum and remaining-balance exception are described by the +[Shanghai Stock Exchange](https://edu.sse.com.cn/tib/). + +## Evidence boundary + +This record proves that one real JoinQuant daily backtest completed with the +uploaded wrapper. It does **not** pass G9 because the following evidence is +still missing: + +- a complete hosted export containing input close arrays or hashes, the + portable plan, orders and fills; +- a local canonical run over the exact same frozen inputs; +- layer-by-layer L1—L4 comparison with enumerated difference reasons; +- a real QMT peer run and at least 20 trading days of broker shadow evidence. + +Accordingly, `gate_scorecard.json` remains `NOT_BASELINE_60`. diff --git a/docs/PLATFORM_MATRIX.md b/docs/PLATFORM_MATRIX.md new file mode 100644 index 0000000..05198f6 --- /dev/null +++ b/docs/PLATFORM_MATRIX.md @@ -0,0 +1,116 @@ +# Quant OS platform capability matrix + +“有入口”与“已在目标平台验证”必须分开。Quant OS 共享的是冻结计算合同， +不是四个平台完全相同的时钟、行情和成交模型。 + +| Path | Intended use | Runtime / prerequisite | Current verified fact | Missing release evidence | +| --- | --- | --- | --- | --- | +| Local synthetic execution | 确定性事件回测、ledger/replay | Python ≥3.10，无账号 | 标准库套件最近记录为 211 tests、OK、3 skip；smoke + exact manifest verifier 可运行 | synthetic 不代表真实市场/收益 | +| Local synthetic research | PIT/features/walk-forward/Ridge/risk/cost/target | Python ≥3.10，无账号 | research smoke + manifest verifier 可运行 | Ridge 未冻结部署，未做授权长样本 OOS | +| JQData ingestion | 授权数据到 canonical immutable snapshot | `jqdatasdk==1.9.8` + 授权账号 | fake-provider 测试覆盖 raw/factor/pre-close/limits/paused/PIT `is_st`/membership、24:00 完整收盘约束和语义 verifier | 尚未登录真实账号保存快照、许可与 lineage | +| Snapshot local backtest | PIT membership 的 provider-data 本地回测 | semantic verifier 通过的 snapshot | `snapshot-backtest` 复用 production decision path，输出普通 run manifest | 尚无授权真实 snapshot；仍是本地 fill model | +| Snapshot decision | 某决策日 Signal/Target/Order Delta | snapshot + equity 或完整 broker snapshot | fail-closed broker state 与 exact decision manifest verifier 已实现 | 未接真实账户 snapshot，未冻结生产模型 | +| Colab | clone、测试、bundle、本地两条 synthetic 链；可选 JQData/Qlib | Colab runtime | notebook 已可本地逐 cell 执行 | 可选账号/依赖 cell 仍需运行时授权；不是平台证据 | +| JoinQuant local harness | wrapper API 合同 | Python ≥3.10，无账号 | fake lifecycle 与 mock parity 可运行 | 不证明 JoinQuant | +| JoinQuant hosted bundle | hosted backtest | JoinQuant 登录/数据权限 | 真实 2024 全年 run 已完成且可见日志 0 ERROR；单文件、PIT index/ST、批量 history、费率/滑点/10% 参与率、科创板保护价/200 股约束已运行 | 尚无完整输入/plan/orders/fills 导出和同输入本地比较 | +| QMT local harness | built-in wrapper 合同 | Python ≥3.10，无账号 | Python 3.6 source、`ContextInfo` 回滚模拟和 mock parity | 不证明 QMT | +| QMT built-in bundle | GUI backtest-only | 授权 QMT 客户端/板块与历史 ST 数据 | bundle 硬限制 backtest；毫秒 timetag PIT 成分/ST；固定中证 500、费率/滑点；模块全局 `g` | 尚无真实 QMT run/export；10% GUI 配置与历史 ST 正例探针待取证 | +| qmttools runner | native-Python QMT backtest/history | 券商分发 `xtquant` + 已登录 terminal | 固定中证 500/10% 参与率、参数、只读 preflight 和 hard backtest/history 合同已测 | 尚无专有运行时实跑 | +| XtTrader read-only shadow | 账户绑定 target-diff 与 broker observation | MiniQMT/QMT、账户查询权限、既有本地 HMAC key | HMAC 账户绑定 + authenticated evidence envelope、decision/observation semantic replay、资产/持仓/委托/成交恒等式、fail-closed exact verifier 与 fake broker 合同 | HMAC 不是 broker attestation；仍缺官方 query 失败/空结果合同、callback/restart、20 日真实 shadow；not live-ready | +| Qlib native momentum | signal/model research | CPython 3.12 + exactly `pyqlib==0.9.7` | 真实本地 fixture 成功，24 signal；保存 signal/report hash、表边界和重算 portfolio 指标 | synthetic 两股票；无真实数据/OOS，无 L3/L4 parity | +| Qlib Alpha158/LightGBM | model fit + Recorder workflow | 同上 + LightGBM/MLflow stack | 真实 fixture 完成 model/Recorder；读取 portfolio report，保存表 hash、边界、重算指标与 artifact path | synthetic 两股票性能无意义；真实 provider/OOS 和冻结模型缺失 | +| Mock parity exporter | JoinQuant/QMT wrapper 的 L2—L4 合同回归 | Python ≥3.10 | `mock_contract_only`、`real_platform_pass=false`、`gate_credit=[]` | 不计 G9；必须换成真实平台导出 | + +G1—G10 当前都未通过。JoinQuant hosted smoke 与 Qlib fixture smoke 是真实 +runtime 证据，但不是市场有效性或跨平台 Gate 通过证据。 + +## Stable strategy boundary + +当前四引擎共同可冻结的是 `portable-momentum-v1`： + +- canonical symbol normalization； +- momentum score/rank； +- capped target weights； +- lot-rounded target quantity； +- T+1/sellable-aware order delta。 +- 科创板 200 股最小申报、余额一次性卖清与 hosted 最坏价保护。 + +本地 Ridge 与 Qlib Alpha158/LightGBM 是研究候选。没有 versioned model +bundle、授权真实 OOS 和跨引擎推理结果前，不能称为已部署 champion。 + +Qlib 的单一 `limit_threshold` 是市场级近似，不能表达逐日板块/ST 限制； +因此它只参加 L1/L2 和诊断 L6。XtTrader mutation 边界目前只接受 `.SH` +和 `.SZ`，北交所 symbol conversion 覆盖不等于北交所实盘覆盖。 + +## Platform terms + +- **QMT built-in**：`init` / `handlebar` 在 QMT 管理的 Python 3.6 环境执行。 + 用户状态放在模块全局 `g`，避免 `ContextInfo` 用户属性在下个 + `handlebar` 回滚。 +- **qmttools**：native Python 通过 `run_strategy_file` 驱动策略文件。 + supplied runner 固定 backtest/history，不能等同于 XtTrader OMS。 +- **XtTrader**：连接 MiniQMT 的查询、报单/撤单与异步推送 API。存在 guarded + mutation path 不代表券商认证或 live-ready；本项目对 operator 暴露的 + shadow CLI 严格只读，没有报单/撤单命令。 +- **fake harness**：函数名和生命周期匹配的本地对象，只用于合同回归。 +- **Qlib Recorder**：研究 artifact 记录机制。两股票 synthetic Recorder + 成功不等于模型有投资价值。 + +## Minimum verification commands + +从 `quant-os` 项目根目录运行： + +```bash +# 核心和 Colab 默认流程 +make local + +# JQData 快照与 semantic verifier（会交互式无回显读取密码） +PYTHONPATH=src:. python tools/jqdata_snapshot.py \ + --index 000905.XSHG --start 2024-01-02 --end 2024-12-31 \ + --output data/snapshots/csi500-2024 +PYTHONPATH=src:. python -c \ + "from quant60.data_snapshot import verify_data_snapshot; verify_data_snapshot('data/snapshots/csi500-2024/manifest.json')" + +# 快照回测和决策 +PYTHONPATH=src:. python -m quant60 snapshot-backtest \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json --output artifacts/csi500-2024-backtest +PYTHONPATH=src:. python -m quant60 snapshot-decision \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json --as-of 2024-12-31 \ + --equity 1000000 --output artifacts/csi500-2024-decision + +# hosted bundle 和 mock-only parity +python tools/bundle_platforms.py +PYTHONPATH=src:. python tools/export_mock_parity.py \ + export --output artifacts/mock-parity/report.json +PYTHONPATH=src:. python tools/export_mock_parity.py \ + verify artifacts/mock-parity/report.json + +# Colab notebook 本地合同/执行 +python tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb +python tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb --execute + +# QMT 环境就绪后：先运行未绑定账户的只读 bootstrap（预期 exit 2），再用 +# 输出的非 null broker_snapshot.json 重建同一 Signal 日 decision； +# broker observation 本身仍来自唯一下一交易日盘前窗口。完整步骤见 runbook +PYTHONPATH=src:. python tools/qmt_shadow_plan.py plan \ + --decision artifacts/latest-decision/manifest.json \ + --output artifacts/qmt-shadow-bootstrap +``` + +后续 `verify` 必须同时提供原始 decision，并读取 plan 时使用的既有 +`QMT_ACCOUNT_HASH_KEY_FILE`；verify 不会补建丢失 key。manifest HMAC +envelope 绑定 plan/observation/snapshot 的内容与发布 byte hash、 +decision/source（含 operator CLI）/policy；四个 JSON 必须保持唯一 writer +encoding。但信任主体是本地 Quant OS key holder，不是 QMT/券商。 + +Qlib 和真实平台的完整命令见 +[`runbooks/PLATFORM_DEPLOYMENT.md`](../runbooks/PLATFORM_DEPLOYMENT.md)。 + +## Version and evidence policy + +每个真实 run 都要记录 JoinQuant environment、QMT client/build、 +`xtquant`、broker plugin、Qlib/Python、bundle hash、data version、 +config/rules hash 和实际日历。文档说明 API 存在不证明当前账号有权限； +mock/fake/synthetic 成功不证明真实平台；Qlib synthetic 成功不证明模型收益。 diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..9d00b5d --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,201 @@ +# Quant OS data, credential and live-trading safety + +Quant OS 仓库需要能够被 review、clone 和推送，但不能携带可复用凭据、账户 +身份、受许可限制的数据或券商专有运行时。Obsidian 只保存项目说明，也不能 +成为密码、cookie 或行情文件的旁路存储。 + +## Never commit or copy into documentation + +- JoinQuant/JQData username、password、cookie、token、session export； +- QMT/MiniQMT account ID、terminal profile、userdata directory、session + file、device fingerprint； +- 从持牌 QMT 安装复制出的 `xtquant` binary； +- broker statement、raw callback 或含账户/个人信息的截图； +- private key、API key、`.env`、subscription URL、带凭据的 clone URL； +- provider licence 不允许再分发的 raw/derived market data； +- Colab notebook output 中的 secret、完整账户值或授权数据； +- MLflow/Recorder artifact 中意外缓存的环境变量、machine path 或身份信息。 + +`.gitignore` 不是安全边界。每次提交前必须检查 staged diff 和所有 generated +artifacts。 + +## Credential injection + +提交到 Git 的配置只能含 placeholder。推荐本地变量： + +```text +JQDATA_USERNAME +JQDATA_PASSWORD +QMT_ACCOUNT_ID +QMT_USERDATA_PATH +QMT_SESSION_ID +QMT_ACCOUNT_HASH_KEY_FILE +``` + +JQData adapter 优先读取本地环境变量，也支持交互式读取；密码提示无回显。 +不得把密码作为 CLI 参数，因为 shell history/process list 会泄漏。 + +Colab 可选 JQData cell 必须在 runtime 内通过无回显 prompt/Colab secret +注入。禁止： + +- 把 secret 写在 notebook source 或共享链接里； +- `print(os.environ)`、输出 auth object 或异常中的 credential； +- 把带 secret 的 notebook output 下载、同步或提交； +- 运行结束后继续复用包含授权数据和 secret 的共享 runtime。 + +用完后清除 runtime。Colab secret 只是注入渠道，不授予数据再分发权。 + +QMT 账号不应作为 CLI 参数。`.env.example` 只列变量名和安全默认值； +真实 `.env` 必须保留在 operator-controlled local secret store。 + +不得在 `--help`、exception、run manifest、test snapshot 或 OB 文档打印上述 +值。低熵账户号的普通 SHA-256 可枚举恢复；对外证据应使用 keyed one-way +identifier，并把 key 保存在独立 secret store。 + +`QMT_ACCOUNT_HASH_KEY_FILE` 只指向本地 HMAC key，不是 key 本身。 +`qmt_shadow_plan.py plan` 在文件不存在时生成至少 32 byte 的安全随机 key；Unix +要求文件 mode 0600，并且不会为了创建 key 而放宽或改写一个已经存在的父目录 +权限。`verify` 只读取既有 key，绝不会在 key 丢失时静默生成替代品。该 key +必须稳定备份和最小权限访问：轮换后账户 hash 会变化，历史 broker snapshot +与 decision 会按设计失配，历史 authenticated evidence 也无法再验签。key +内容不得进入 Git、OB、notebook、artifact、stdout、异常或聊天。 + +shadow manifest 保存的是 `QMT_SHADOW_EVIDENCE_V1` HMAC，不是 secret。 +它用与账户标识不同的 domain 签署 canonical evidence envelope，覆盖完整 +plan、broker observation、broker snapshot 的内容 hash 和三份实际发布文件 +原始 byte SHA-256、原始 decision manifest hash、planner/engine 以及 +`tools/qmt_shadow_plan.py` operator entrypoint source hash，并绑定固定 +semantic/policy ID 和阈值。四个发布 JSON（含不自签的 manifest）必须符合 +唯一 deterministic writer bytes；minify、key 重排和空白变化不是等价发布物。 +verifier 必须同时拿到原始 decision 与既有 key，并读取磁盘实际 bytes 重建 +envelope 后用 +constant-time compare 验证。普通 SHA-256 重新封装、替换 decision、改写 +observation 或同步修改 plan 派生字段都不能在不知道 key 时伪造该 MAC。 + +这个 HMAC 的信任主体是“持有本地 key 的 Quant OS evidence publisher”。 +它证明发布后没有被无 key 的第三方改写；它**不是** QMT/券商对 API 返回值的 +签名，不证明终端、账号或 callback 未在采集前被攻陷。key 泄漏后攻击者可以 +伪造本地 envelope，应立即按 credential incident 轮换、隔离旧证据并重新做 +可信采集。 + +## JQData and provider-data licence + +`jqdatasdk==1.9.8` adapter 可获取 raw OHLCV/money、factor、pre-close、 +daily limits/paused、PIT `is_st` 和历史指数成员。可获取不等于可再分发。 + +首次真实 snapshot 前记录： + +- provider、账户 entitlement 和允许用途； +- local/Colab 是否允许持久保存、保存期限； +- raw 与 derived data 是否可共享； +- API/field semantic version； +- query、retrieval time、event/effective/available time； +- partition/data hash、retention 和 deletion policy。 + +snapshot manifest 不含密码，但 snapshot 本身仍可能是受许可数据。默认放在 +被 Git 忽略的 local storage，不上传代码仓库、OB、公开 object storage 或 +公开 Colab drive。 + +synthetic fixture 可共享但不是市场证据。真实 JQData snapshot 只有在许可、 +lineage 和语义 verifier 都完成后，才可能成为 G1 的候选证据。 + +## QMT proprietary and licensed boundary + +`xtquant` 由持牌 QMT/MiniQMT 与券商分发。Quant OS： + +- 不 vendoring、不上传、不从非官方 PyPI 模仿包安装； +- 仅在 QMT-specific path 内 lazy import； +- 没有专有运行时也能跑核心/fake tests； +- 只使用实际券商支持的 Python 和 client/plugin 组合。 + +QMT 并非只提供用户名密码即可运行。还需要已授权 QMT/MiniQMT client、 +匹配的 `xtquant`、userdata path、行情/历史数据 entitlement 和登录会话。 +复制其他用户安装不能建立许可或兼容性。 + +平台导出必须先脱敏：raw account、broker order ID、userdata path、device +信息和专有异常字符串不进入 Git。 + +只读 shadow 的 `broker_observation.json` 只保存 keyed account hash、query +安全 envelope 和通过标准化的资产、持仓、委托、成交事实；不保存原始 +account ID。manifest 通过 authenticated evidence HMAC 将 observation、 +broker snapshot、plan、decision、source 和 canonical policy 绑定。即便如此， +它仍可能包含敏感持仓/交易信息，只能放在被 Git 忽略且访问受控的证据目录。 + +## Qlib, MLflow and Colab artifacts + +Qlib 固定为 CPython 3.12 + `pyqlib==0.9.7`。Alpha158 runner 为本地 +MLflow file store 使用： + +```text +MLFLOW_ALLOW_FILE_STORE=true +``` + +这是对 local backend 的显式兼容确认，不是安全授权，也不会替代访问控制。 +Recorder 目录可能包含 model、label、prediction、code cache/status 和 +provider-derived output；提交前必须按 provider licence 与 secret scan +检查。真实模型 artifact 应存放在访问受控、可审计的 artifact store。 + +不要在公开 Colab runtime 上加载券商文件。JQData/Qlib 可选流程只处理研究 +数据；QMT/XtTrader 应留在受控且持牌的本地/Windows 环境。 + +## Default live posture + +所有 broker-facing 路径默认只能是： + +```text +BACKTEST +SIMULATION +SHADOW +``` + +QMT built-in 和 qmttools supplied runner 硬限制 backtest/history。XtTrader +`allow_live_orders` 默认 false，仓库不提供 live launch command。 + +未来任何 live mutation 至少还要求： + +1. broker read-only preflight 成功； +2. market data 与完整 broker snapshot 新鲜且带时区； +3. account allow-list 匹配； +4. 无 unknown open order 或 reconciliation difference； +5. notional/order/rate limit 已配置； +6. operator approval 与 kill switch 已演练； +7. 程序化交易报告和软件/频率要求由实际券商确认； +8. shadow、重启恢复和连续日终对账达到 release 条件。 + +环境变量本身不构成批准。mock guard、fake broker 和单元测试也不构成券商认证。 + +## Safe evidence and redaction + +可安全导出的候选内容： + +- source/config/data/rules/model hash； +- canonical symbols 和非受限聚合； +- order state/reason code； +- keyed account hash； +- 在不需要精确值时脱敏/取整的 operational metric。 + +需要删除或保护： + +- raw account 和可关联 broker order ID； +- 用户 machine path、用户名、device/profile 信息； +- proprietary client exception； +- notebook cell output 中的 token/credential； +- provider licence 不允许公开的数据行。 + +审计所需原始版本应留在加密、访问受控的本地/企业存储，而不是为了方便直接 +提交 Git。 + +## Incident response + +如果 credential、账户标识或授权数据进入 Git/OB/Colab output： + +1. 停止新订单和相关自动任务； +2. 通过 provider/broker 撤销或轮换 credential； +3. 在仓库外保存 incident timeline； +4. 从当前树移除，并在明确批准后清理历史和 mirror； +5. 检查 CI、artifact、OB sync、Colab/Drive 和 downstream clone； +6. 评估数据许可/个人信息泄漏范围并完成必要通知； +7. 记录恢复和复发预防。 + +删除可见文件不等于 credential rotation。交易与数据故障流程见 +[`runbooks/INCIDENTS.md`](../runbooks/INCIDENTS.md)。 diff --git a/gate_scorecard.json b/gate_scorecard.json new file mode 100644 index 0000000..314a4cb --- /dev/null +++ b/gate_scorecard.json @@ -0,0 +1,177 @@ +{ + "schema_version": "1.0", + "system": "Quant OS / quant60 baseline kernel", + "audit_as_of": "2026-07-26", + "claim_policy": { + "baseline_threshold": 60, + "all_gates_required": true, + "rule": "A score is earned only by linked, reproducible evidence. Source files, mocks, fixtures and successful unit tests do not prove real-platform, live-broker or compliance gates.", + "current_claim": "NOT_BASELINE_60" + }, + "article_reassessment": { + "design_coverage": { + "score": 70, + "out_of": 100, + "scope": "How much of the intended research-to-production responsibility chain the 2026-07-22 article discusses.", + "status": "estimated" + }, + "implementable_specification": { + "score": 49, + "out_of": 100, + "scope": "How much of the article was sufficiently specified to implement without inventing contracts, data semantics, defaults or operational decisions.", + "status": "estimated" + }, + "original_implementation_evidence": { + "score": 6, + "out_of": 100, + "scope": "Evidence present before the first code delivery; this historical score is not the score of the current Quant OS repository.", + "status": "historical" + }, + "gates_passed": 0, + "gates_total": 10 + }, + "current_delivery": { + "score": null, + "status": "not_scored", + "reason": "Quant OS now has executable local pipelines, real Qlib fixture smokes and one completed real JoinQuant hosted smoke, but it has not produced the same-input JoinQuant comparison, real QMT, broker-shadow, continuous reconciliation or compliance evidence required to score Baseline 60.", + "verified_supporting_facts": [ + "The latest standard-library suite completed 211 tests with OK and 3 optional-runtime skips; the CPython 3.12 pyqlib environment completed the same 211 tests with no skips.", + "The JQData adapter maps raw bars, adjustment factor, previous close, daily limits, pause state, PIT is_st and daily PIT index membership into a semantically revalidated snapshot; it rejects an end date that has not reached the provider's complete daily-bar boundary, and only fake-provider tests have been run so far.", + "The local snapshot can drive snapshot-backtest and snapshot-decision with manifest verification; it freezes a cross-checked provider trading calendar, and a T-close signal can bind only to a broker observation in the unique next session's Asia/Shanghai [09:00,09:30) window without collapsing the two clocks.", + "CPython 3.12 with pyqlib 0.9.7 completed a native momentum fixture smoke with 24 signals and saved audited portfolio-table hashes, bounds and recomputed metrics.", + "Qlib Alpha158, LightGBM and Recorder completed a real local fixture smoke with audited portfolio output; the deterministic two-stock synthetic fixture has no investment-value meaning.", + "A real JoinQuant daily hosted backtest completed for 2024-01-02 through 2024-12-31 with CNY 1,000,000; the visible completed-run log had zero ERROR/Traceback/order-rejection entries. This proves hosted runtime execution only: exact input/plan/order/fill export and a local same-input peer are still missing.", + "The QMT shadow CLI is strictly read-only and produces HMAC account-bound broker observation, broker snapshot and target-diff artifacts. QMT_SHADOW_EVIDENCE_V1 authenticates semantic content, deterministic published bytes, the original decision, planner/engine/operator source and fixed policy; semantic replay verifies exact derived state. Independent local QA completed 58 deterministic scenarios, 500 malformed fuzz cases and 300 legal-observation fuzz cases with no P0/P1. This is Quant OS publisher integrity, not broker attestation, and it still has only fake-broker evidence.", + "JoinQuant/QMT mock parity is mock_contract_only, real_platform_pass is false and it earns no G9 gate credit." + ], + "external_evidence_missing": [ + "authorized JQData market snapshot and licence/lineage record", + "complete JoinQuant hosted input/plan/order/fill export and same-input local comparison", + "real QMT built-in and qmttools backtest exports", + "real QMT historical-ST positive control, terminal/xtquant build record, broker query contract and trade-amount semantic probes, callback/restart and at least 20 trading days of shadow evidence", + "continuous reconciliation, recovery drills and broker compliance confirmation" + ] + }, + "gates": [ + { + "id": "G1", + "name": "数据时点", + "status": "not_passed", + "acceptance": "历史预测可重放当时实际可得数据；财报按 available_time 连接；未来公告注入测试被拒绝。", + "required_evidence": [ + "PIT 数据 manifest 与字段可得时间规范", + "未来公告注入测试报告", + "授权真实数据的历史决策日抽样重放报告" + ], + "evidence": [] + }, + { + "id": "G2", + "name": "样本外验证", + "status": "not_passed", + "acceptance": "时间有序且带 purge/embargo 的 walk-forward；测试集隔离；成本后多状态结果已保存。", + "required_evidence": [ + "切分代码与测试", + "真实授权长样本上的冻结 walk-forward 报告", + "未触碰测试集声明和冻结模型包" + ], + "evidence": [] + }, + { + "id": "G3", + "name": "A 股现实", + "status": "not_passed", + "acceptance": "停复牌、涨跌停、交易单位、T+1、费用、公司行动和日期化规则均被回放验证。", + "required_evidence": [ + "每类规则的回归夹具", + "日期化规则版本", + "真实平台跨引擎差异报告" + ], + "evidence": [] + }, + { + "id": "G4", + "name": "稳定合约", + "status": "not_passed", + "acceptance": "Signal、Target、OrderEvent、BrokerSnapshot 分层落盘并通过版本化 JSON Schema 验证。", + "required_evidence": [ + "四类 schema 的发布验证报告", + "正反例 fixture", + "迁移兼容性策略和演练" + ], + "evidence": [] + }, + { + "id": "G5", + "name": "事前风险", + "status": "not_passed", + "acceptance": "目标生成前检查集中度、行业/风格、组合波动、流动性、换手与不可卖仓位。", + "required_evidence": [ + "真实数据校准的风险模型版本", + "约束解释与不可行降级测试", + "压力测试和容量报告" + ], + "evidence": [] + }, + { + "id": "G6", + "name": "订单状态", + "status": "not_passed", + "acceptance": "真实或券商认证环境覆盖受理、部分成交、全成、撤单中、已撤、拒单、乱序/重复与未知状态。", + "required_evidence": [ + "状态机测试报告", + "券商回调脱敏回放", + "重启幂等演练" + ], + "evidence": [] + }, + { + "id": "G7", + "name": "可复现", + "status": "not_passed", + "acceptance": "每个 run 绑定代码、Python/依赖、配置 hash、数据、模型、规则与随机种子；固定输入可重放。", + "required_evidence": [ + "发布级 run manifest", + "干净环境重建记录", + "版本冻结后的确定性重放差异报告" + ], + "evidence": [] + }, + { + "id": "G8", + "name": "对账与停机", + "status": "not_passed", + "acceptance": "启动、运行中、日终对账；不明差异阻止新单；kill switch 与恢复手册经过演练。", + "required_evidence": [ + "连续对账报告", + "故障注入/恢复演练", + "kill switch 测试" + ], + "evidence": [] + }, + { + "id": "G9", + "name": "跨引擎一致性", + "status": "not_passed", + "acceptance": "同一 canonical fixture/config 在本地、聚宽与可用 QMT/Qlib 路径上的语义差异处于声明容差且有原因码。", + "required_evidence": [ + "真实平台跨引擎 manifest", + "逐层 signal/target/order 差异", + "QMT 至少 20 个交易日影子记录" + ], + "evidence": [] + }, + { + "id": "G10", + "name": "合规", + "status": "not_passed", + "acceptance": "实际券商书面确认接口权限、频率、模拟/实盘差异及程序化交易报告要求，完成先报告后交易。", + "required_evidence": [ + "券商确认材料索引", + "程序化交易报告状态", + "账户权限与限额检查记录" + ], + "evidence": [] + } + ] +} diff --git a/notebooks/quant_os_colab.ipynb b/notebooks/quant_os_colab.ipynb new file mode 100644 index 0000000..e103b70 --- /dev/null +++ b/notebooks/quant_os_colab.ipynb @@ -0,0 +1,235 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Quant OS / quant60 Baseline：Colab 与本地可复现实验\n", + "\n", + "这份 notebook 不保存账号、密码或 token。先在下面设置可选运行开关；默认流程不需要账号。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "optional" + ] + }, + "outputs": [], + "source": [ + "RUN_JQDATA = False\n", + "JQDATA_START = \"2024-01-02\"\n", + "JQDATA_AS_OF = \"2024-06-28\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 默认工程验证\n", + "\n", + "这份 notebook 不保存账号、密码或 token。它会定位已有源码，或在 Colab 的空运行时从 `git.gomars.fun` 克隆代码，然后运行与 CI 相同的测试、bundle、合成回测和证据校验。合成数据只验证工程语义，不证明投资收益。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import json\n", + "import os\n", + "import shutil\n", + "import subprocess\n", + "import sys\n", + "\n", + "REPO_URL = \"https://git.gomars.fun/boat/quants-strategies.git\"\n", + "REPO_BRANCH = \"master\"\n", + "\n", + "def run(command, *, env=None):\n", + " print(\"+\", \" \".join(str(item) for item in command))\n", + " subprocess.run(command, check=True, env=env)\n", + "\n", + "cwd = Path.cwd().resolve()\n", + "if cwd.name == \"quant-os\" and (cwd / \"pyproject.toml\").is_file():\n", + " project = cwd\n", + "elif (cwd / \"quant-os\" / \"pyproject.toml\").is_file():\n", + " project = cwd / \"quant-os\"\n", + "else:\n", + " checkout = Path(\"/content/quants-strategies\")\n", + " if not checkout.exists():\n", + " run([\"git\", \"clone\", \"--depth\", \"1\", \"--branch\", REPO_BRANCH, REPO_URL, str(checkout)])\n", + " project = checkout / \"quant-os\"\n", + "\n", + "if not (project / \"pyproject.toml\").is_file():\n", + " raise RuntimeError(f\"Quant OS project not found at {project}\")\n", + "os.chdir(project)\n", + "print({\"project\": str(project), \"python\": sys.version})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pythonpath = os.pathsep.join([\"src\", \".\"])\n", + "child_env = dict(os.environ, PYTHONPATH=pythonpath)\n", + "smoke_dir = Path(\"artifacts/colab-smoke\")\n", + "research_dir = Path(\"artifacts/colab-research\")\n", + "for generated_dir in (smoke_dir, research_dir):\n", + " if generated_dir.is_symlink():\n", + " raise RuntimeError(f\"refusing to replace symlinked output: {generated_dir}\")\n", + " if generated_dir.exists():\n", + " shutil.rmtree(generated_dir)\n", + "run([sys.executable, \"-m\", \"unittest\", \"discover\", \"-s\", \"tests\", \"-p\", \"*.py\", \"-q\"], env=child_env)\n", + "run([sys.executable, \"tools/bundle_platforms.py\"])\n", + "run([sys.executable, \"-m\", \"quant60\", \"smoke\", \"--config\", \"configs/baseline.json\", \"--output\", str(smoke_dir)], env=child_env)\n", + "run([sys.executable, \"-m\", \"quant60\", \"verify-manifest\", str(smoke_dir / \"manifest.json\")], env=child_env)\n", + "run([sys.executable, \"-m\", \"quant60\", \"research-smoke\", \"--output\", str(research_dir)], env=child_env)\n", + "run([sys.executable, \"-m\", \"quant60\", \"verify-research-manifest\", str(research_dir / \"manifest.json\")], env=child_env)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "report = json.loads(Path(\"artifacts/colab-smoke/report.json\").read_text(encoding=\"utf-8\"))\n", + "summary = {key: report[key] for key in [\"run_id\", \"trading_days\", \"order_count\", \"fill_count\", \"total_return\", \"max_drawdown\", \"investment_value_claim\"]}\n", + "print(json.dumps(summary, ensure_ascii=False, indent=2))\n", + "assert summary[\"investment_value_claim\"] is False\n", + "assert summary[\"fill_count\"] > 0\n", + "research_report = json.loads(Path(\"artifacts/colab-research/report.json\").read_text(encoding=\"utf-8\"))\n", + "research_summary = {key: research_report[key] for key in [\"run_id\", \"sample_count\", \"fold_count\", \"mean_oos_rank_ic\", \"target_gross_weight\", \"investment_value_claim\"]}\n", + "print(json.dumps(research_summary, ensure_ascii=False, indent=2))\n", + "assert research_summary[\"investment_value_claim\"] is False\n", + "assert research_summary[\"fold_count\"] > 0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "equity = json.loads(Path(\"artifacts/colab-smoke/equity.json\").read_text(encoding=\"utf-8\"))\n", + "try:\n", + " import matplotlib.pyplot as plt\n", + "except ImportError:\n", + " print(\"matplotlib is optional; equity artifact is still available\")\n", + "else:\n", + " plt.figure(figsize=(10, 4))\n", + " plt.plot([row[\"date\"] for row in equity], [row[\"equity\"] for row in equity])\n", + " plt.title(\"Quant60 synthetic engineering smoke (not an investment result)\")\n", + " plt.xticks(rotation=45)\n", + " plt.tight_layout()\n", + " chart_path = Path(\"artifacts/colab-smoke/equity.png\")\n", + " plt.savefig(chart_path, dpi=140)\n", + " plt.close()\n", + " print({\"equity_chart\": str(chart_path.resolve())})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 可选：JQData 授权快照 → PIT 回测与决策\n", + "\n", + "把首个代码 cell 的 `RUN_JQDATA` 改为 `True` 后，账号和密码只通过无回显交互输入，不写入 cell、artifact 或 Git。`JQDATA_AS_OF` 必须是区间内的真实交易日。该路径生成 provider snapshot、同一 `portable-momentum-v1` 的本地历史回测和单日信号/目标，并逐层验证；它不是聚宽 hosted 回测。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "optional" + ] + }, + "outputs": [], + "source": [ + "if RUN_JQDATA:\n", + " run([sys.executable, \"-m\", \"pip\", \"install\", \"-r\", \"requirements/jqdata.txt\"])\n", + " jq_snapshot = Path(\"artifacts/jqdata-csi500\")\n", + " jq_decision = Path(\"artifacts/jqdata-csi500-decision\")\n", + " jq_backtest = Path(\"artifacts/jqdata-csi500-backtest\")\n", + " run([sys.executable, \"tools/jqdata_snapshot.py\", \"--index\", \"000905.XSHG\", \"--start\", JQDATA_START, \"--end\", JQDATA_AS_OF, \"--output\", str(jq_snapshot)], env=child_env)\n", + " run([sys.executable, \"-m\", \"quant60\", \"snapshot-backtest\", \"--snapshot\", str(jq_snapshot / \"manifest.json\"), \"--config\", \"configs/baseline.json\", \"--output\", str(jq_backtest)], env=child_env)\n", + " run([sys.executable, \"-m\", \"quant60\", \"verify-manifest\", str(jq_backtest / \"manifest.json\")], env=child_env)\n", + " run([sys.executable, \"-m\", \"quant60\", \"snapshot-decision\", \"--snapshot\", str(jq_snapshot / \"manifest.json\"), \"--config\", \"configs/baseline.json\", \"--as-of\", JQDATA_AS_OF, \"--equity\", \"1000000\", \"--output\", str(jq_decision)], env=child_env)\n", + " run([sys.executable, \"-m\", \"quant60\", \"verify-snapshot-decision\", str(jq_decision / \"manifest.json\")], env=child_env)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 可选：Qlib 0.9.7 原生 smoke\n", + "\n", + "只在 Python 3.12 环境把下面的 `RUN_QLIB` 改为 `True`。安装和运行结果必须单独保存；未执行时不得写成已验证。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "optional" + ] + }, + "outputs": [], + "source": [ + "RUN_QLIB = False\n", + "if RUN_QLIB:\n", + " if sys.version_info[:2] != (3, 12):\n", + " raise RuntimeError(\"Qlib adapter is pinned to CPython 3.12\")\n", + " run([sys.executable, \"-m\", \"pip\", \"install\", \"-r\", \"requirements/research-py312.txt\"])\n", + " fixture = Path(\"artifacts/qlib-tiny-provider\")\n", + " run([sys.executable, \"tools/build_qlib_tiny_fixture.py\", str(fixture), \"--days\", \"80\"])\n", + " run([sys.executable, \"-m\", \"platforms.qlib_runner\", \"--provider-uri\", str(fixture), \"--market\", \"csi300\", \"--benchmark\", \"SH000300\", \"--start\", \"2024-02-01\", \"--end\", \"2024-04-19\", \"--lookback\", \"20\", \"--topk\", \"1\", \"--n-drop\", \"1\", \"--rebalance\", \"weekly\", \"--output-json\", \"artifacts/qlib-native/result.json\"], env=child_env)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "smoke_archive = shutil.make_archive(\"artifacts/quant-os-colab-smoke\", \"zip\", \"artifacts/colab-smoke\")\n", + "research_archive = shutil.make_archive(\"artifacts/quant-os-colab-research\", \"zip\", \"artifacts/colab-research\")\n", + "download_artifacts = [smoke_archive, research_archive]\n", + "if Path(\"artifacts/jqdata-csi500\").is_dir():\n", + " download_artifacts.append(shutil.make_archive(\"artifacts/quant-os-jqdata-snapshot\", \"zip\", \"artifacts/jqdata-csi500\"))\n", + "if Path(\"artifacts/jqdata-csi500-decision\").is_dir():\n", + " download_artifacts.append(shutil.make_archive(\"artifacts/quant-os-jqdata-decision\", \"zip\", \"artifacts/jqdata-csi500-decision\"))\n", + "if Path(\"artifacts/jqdata-csi500-backtest\").is_dir():\n", + " download_artifacts.append(shutil.make_archive(\"artifacts/quant-os-jqdata-backtest\", \"zip\", \"artifacts/jqdata-csi500-backtest\"))\n", + "if Path(\"artifacts/qlib-native\").is_dir():\n", + " download_artifacts.append(shutil.make_archive(\"artifacts/quant-os-qlib-native\", \"zip\", \"artifacts/qlib-native\"))\n", + "print({\"download_artifacts\": download_artifacts})" + ] + } + ], + "metadata": { + "colab": { + "name": "quant_os_colab.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/platforms/__init__.py b/platforms/__init__.py new file mode 100644 index 0000000..f2fe28a --- /dev/null +++ b/platforms/__init__.py @@ -0,0 +1 @@ +"""Thin wrappers for platform-hosted and native backtest runtimes.""" diff --git a/platforms/fake_joinquant_harness.py b/platforms/fake_joinquant_harness.py new file mode 100644 index 0000000..86993c5 --- /dev/null +++ b/platforms/fake_joinquant_harness.py @@ -0,0 +1,189 @@ +"""Small deterministic JoinQuant API harness for local contract tests.""" + +import datetime +from types import SimpleNamespace + + +class FakePosition(object): + def __init__(self, total_amount=0, closeable_amount=None): + self.total_amount = int(total_amount) + self.closeable_amount = int( + total_amount if closeable_amount is None else closeable_amount + ) + + +class FakeJoinQuantHarness(object): + """Install fake hosted functions into a strategy module.""" + + def __init__( + self, + histories, + total_value=1_000_000.0, + positions=None, + st_symbols=None, + ): + self.histories = { + symbol: [float(value) for value in values] + for symbol, values in histories.items() + } + self.options = {} + self.benchmark = None + self.order_cost = None + self.slippage = None + self.schedules = [] + self.orders = [] + self.order_styles = [] + self.last_index_request = None + self.last_history_request = None + self.last_extras_request = None + self.st_symbols = set(st_symbols or []) + self.current_data = { + symbol: SimpleNamespace( + paused=False, + is_st=False, + high_limit=9999.0, + low_limit=0.01, + ) + for symbol in histories + } + self.context = SimpleNamespace( + current_dt=datetime.datetime(2026, 7, 20, 9, 30), + previous_date=datetime.date(2026, 7, 17), + portfolio=SimpleNamespace( + total_value=float(total_value), + positions=dict(positions or {}), + ) + ) + self.g = SimpleNamespace() + + def install(self, module): + module.g = self.g + module.set_option = self.set_option + module.run_daily = self.run_daily + module.attribute_history = self.attribute_history + module.history = self.history + module.get_index_stocks = self.get_index_stocks + module.get_extras = self.get_extras + module.set_benchmark = self.set_benchmark + module.set_order_cost = self.set_order_cost + module.set_slippage = self.set_slippage + module.OrderCost = self.OrderCost + module.PriceRelatedSlippage = self.PriceRelatedSlippage + module.LimitOrderStyle = self.LimitOrderStyle + module.order_target = self.order_target + module.get_current_data = self.get_current_data + return self + + def set_option(self, key, value): + self.options[key] = value + + def set_benchmark(self, symbol): + self.benchmark = symbol + + @staticmethod + def OrderCost(**kwargs): + return SimpleNamespace(**kwargs) + + @staticmethod + def PriceRelatedSlippage(value): + return SimpleNamespace(value=float(value)) + + @staticmethod + def LimitOrderStyle(limit_price): + return SimpleNamespace( + kind="limit", + limit_price=float(limit_price), + ) + + def set_order_cost(self, cost, type=None): + self.order_cost = {"cost": cost, "type": type} + + def set_slippage(self, slippage, type=None): + self.slippage = {"slippage": slippage, "type": type} + + def run_daily(self, callback, time="open"): + self.schedules.append((callback, time)) + + def attribute_history( + self, + symbol, + count, + unit="1d", + fields=None, + skip_paused=False, + df=False, + fq="pre", + ): + del unit, fields, skip_paused, df, fq + return {"close": self.histories[symbol][-int(count) :]} + + def history( + self, + count, + unit="1d", + field="close", + security_list=None, + df=False, + skip_paused=False, + fq="pre", + ): + del unit, field, df, skip_paused, fq + symbols = list(security_list or self.histories) + self.last_history_request = { + "count": int(count), + "security_list": symbols, + } + return { + symbol: self.histories[symbol][-int(count) :] + for symbol in symbols + } + + def get_index_stocks(self, index_symbol, date=None): + self.last_index_request = { + "index_symbol": index_symbol, + "date": date, + } + return sorted(self.histories) + + def get_extras( + self, + info, + security_list, + end_date=None, + count=1, + df=False, + ): + self.last_extras_request = { + "info": info, + "security_list": list(security_list), + "end_date": end_date, + "count": count, + "df": df, + } + return { + symbol: [symbol in self.st_symbols] + for symbol in security_list + } + + def order_target(self, symbol, quantity, style=None): + self.orders.append((symbol, int(quantity))) + self.order_styles.append((symbol, style)) + return SimpleNamespace(security=symbol, target_quantity=int(quantity)) + + def get_current_data(self): + return self.current_data + + def initialize(self, module): + self.install(module) + module.initialize(self.context) + return self + + def run_scheduled(self, index=0): + callback = self.schedules[index][0] + return callback(self.context) + + def advance_session(self, days=1): + previous = self.context.current_dt.date() + self.context.previous_date = previous + self.context.current_dt += datetime.timedelta(days=int(days)) + return self diff --git a/platforms/fake_qmt_harness.py b/platforms/fake_qmt_harness.py new file mode 100644 index 0000000..e7eee28 --- /dev/null +++ b/platforms/fake_qmt_harness.py @@ -0,0 +1,158 @@ +"""Deterministic fake QMT ContextInfo and passorder recorder.""" + +import datetime +from types import SimpleNamespace + + +class FakeFrame(object): + def __init__(self, closes): + self._closes = list(closes) + + def __getitem__(self, key): + if key != "close": + raise KeyError(key) + return list(self._closes) + + +class FakeQmtContext(object): + def __init__( + self, + histories, + asset=1_000_000.0, + positions=None, + bar_time=None, + trade_mode="backtest", + params=None, + st_periods=None, + ): + self.histories = { + symbol: [float(value) for value in values] + for symbol, values in histories.items() + } + self.trade_mode = trade_mode + self.do_back_test = trade_mode == "backtest" + self._param = {"asset": float(asset), "trade_mode": trade_mode} + self._param.update(params or {}) + self.barpos = 0 + self._bar_time = bar_time or datetime.datetime(2026, 7, 20, 15, 0) + self._positions = list(positions or []) + self._asset = float(asset) + self.universe = [] + self.last_market_request = None + self.last_sector_request = None + self.commission = None + self.slippage = None + self.st_periods = dict(st_periods or {}) + + def set_universe(self, symbols): + self.universe = list(symbols) + + def set_commission(self, commission_type, commission_list): + self.commission = { + "type": int(commission_type), + "values": list(commission_list), + } + + def set_slippage(self, slippage_type, slippage): + self.slippage = { + "type": int(slippage_type), + "value": float(slippage), + } + + def get_bar_timetag(self, barpos): + del barpos + return int(self._bar_time.timestamp() * 1000) + + def is_last_bar(self): + return True + + def get_market_data_ex( + self, + fields, + stock_code, + period, + start_time, + end_time, + count, + dividend_type, + fill_data, + subscribe, + ): + self.last_market_request = { + "fields": fields, + "stock_code": stock_code, + "period": period, + "start_time": start_time, + "end_time": end_time, + "count": count, + "dividend_type": dividend_type, + "fill_data": fill_data, + "subscribe": subscribe, + } + return { + symbol: FakeFrame(self.histories[symbol][-int(count) :]) + for symbol in stock_code + } + + def get_stock_list_in_sector(self, sector_name, timetag=None): + self.last_sector_request = { + "sector_name": sector_name, + "timetag": timetag, + } + return sorted(self.histories) + + def get_his_st_data(self, symbol): + return dict(self.st_periods.get(symbol, {})) + + def get_trade_detail_data(self, account_id, account_type, kind): + del account_id, account_type + if str(kind).lower() == "position": + return list(self._positions) + if str(kind).lower() == "account": + return [SimpleNamespace(total_asset=self._asset)] + return [] + + +class FakeQmtHarness(object): + def __init__(self, context): + self.context = context + self.orders = [] + + def install(self, module): + module.passorder = self.passorder + return self + + def passorder( + self, + operation, + order_type, + account_id, + order_code, + price_type, + price, + volume, + strategy_name, + quick_trade, + user_order_id, + context, + ): + self.orders.append( + { + "operation": operation, + "order_type": order_type, + "account_id": account_id, + "order_code": order_code, + "price_type": price_type, + "price": price, + "volume": volume, + "strategy_name": strategy_name, + "quick_trade": quick_trade, + "user_order_id": user_order_id, + "context": context, + } + ) + + def run(self, module): + self.install(module) + module.init(self.context) + return module.handlebar(self.context) diff --git a/platforms/joinquant_strategy.py b/platforms/joinquant_strategy.py new file mode 100644 index 0000000..3b9bfc3 --- /dev/null +++ b/platforms/joinquant_strategy.py @@ -0,0 +1,465 @@ +# coding: utf-8 +"""JoinQuant hosted wrapper. + +Upload the generated bundle, not this source file, to JoinQuant. The bundle +tool replaces the marked import block with the exact portable core source. +""" + +# __PORTABLE_CORE_BUNDLE_START__ +from quant60.portable_core import build_rebalance_plan, convert_symbol +# __PORTABLE_CORE_BUNDLE_END__ + +import math + + +class PriceHistoryUnavailable(RuntimeError): + pass + + +class UnmanagedPositionError(RuntimeError): + pass + + +class ClockStateError(RuntimeError): + pass + + +# __BASELINE_CONFIG_START__ +CONFIG = { + "universe_mode": "pit_index", + "index_symbol": "000905.XSHG", + "qmt_sector_name": "中证500", + "dedicated_account_required": True, + "exclude_st": True, + "universe": [ + "600000.XSHG", + "000001.XSHE", + "300750.XSHE", + "000333.XSHE", + ], + "lookback": 20, + # The first session close supplies the signal; the second session open + # supplies the hosted execution boundary, matching the QMT daily wrapper. + "skip": 0, + "top_n": 20, + "max_weight": 0.05, + "gross_target": 0.95, + "cash_buffer": 0.02, + "lot_size": 100, + "participation_rate": 0.1, + "slippage_bps": 2.0, + "commission_rate": 0.0002, + "minimum_commission": 5.0, + "stamp_duty_rate": 0.0005, + "transfer_fee_rate": 0.00001, + "rebalance_schedule": "weekly_first_close", +} +# __BASELINE_CONFIG_END__ + + +def initialize(context): + """Configure a daily-open clock that executes a first-week-close signal.""" + set_option("avoid_future_data", True) + set_option("use_real_price", True) + g.quant60_config = dict(CONFIG) + if g.quant60_config.get("rebalance_schedule") != "weekly_first_close": + raise ValueError("unsupported rebalance_schedule") + if g.quant60_config.get("universe_mode") not in ("pit_index", "fixed"): + raise ValueError("unsupported universe_mode") + if ( + g.quant60_config.get("universe_mode") == "pit_index" + and not g.quant60_config.get("dedicated_account_required") + ): + raise ValueError("pit_index mode requires a dedicated account") + set_benchmark(g.quant60_config["index_symbol"]) + commission = ( + float(g.quant60_config["commission_rate"]) + + float(g.quant60_config["transfer_fee_rate"]) + ) + set_order_cost( + OrderCost( + open_tax=0.0, + close_tax=float(g.quant60_config["stamp_duty_rate"]), + open_commission=commission, + close_commission=commission, + close_today_commission=0.0, + min_commission=float( + g.quant60_config["minimum_commission"] + ), + ), + type="stock", + ) + # JoinQuant applies half of PriceRelatedSlippage on each side. The local + # slippage_bps setting is one-sided, hence the explicit factor of two. + set_slippage( + PriceRelatedSlippage( + 2.0 * float(g.quant60_config["slippage_bps"]) / 10000.0 + ), + type="stock", + ) + set_option( + "order_volume_ratio", + float(g.quant60_config["participation_rate"]), + ) + g.quant60_last_plan = None + g.quant60_last_universe = None + g.quant60_last_universe_as_of = None + g.quant60_skipped_orders = [] + g.quant60_pending_first_session = None + g.quant60_last_callback_date = None + run_daily(rebalance, time="open") + + +def _close_history(symbol, count): + raw = attribute_history( + symbol, + count, + unit="1d", + fields=["close"], + skip_paused=False, + df=False, + fq="pre", + ) + values = list(raw.get("close", [])) if hasattr(raw, "get") else [] + if len(values) != int(count): + raise PriceHistoryUnavailable( + "%s needs %d completed daily closes, got %d" + % (symbol, int(count), len(values)) + ) + output = [] + for value in values: + try: + number = float(value) + except (TypeError, ValueError): + raise PriceHistoryUnavailable( + "%s contains a missing/non-numeric close" % symbol + ) + if not math.isfinite(number) or number <= 0.0: + raise PriceHistoryUnavailable( + "%s contains a missing/non-positive close" % symbol + ) + output.append(number) + return output + + +def _close_histories(symbols, count): + raw = history( + int(count), + unit="1d", + field="close", + security_list=list(symbols), + df=False, + skip_paused=False, + fq="pre", + ) + if not hasattr(raw, "get"): + raise PriceHistoryUnavailable("history returned an invalid payload") + output = {} + for symbol in symbols: + values = list(raw.get(symbol, [])) + if len(values) != int(count): + raise PriceHistoryUnavailable( + "%s needs %d completed daily closes, got %d" + % (symbol, int(count), len(values)) + ) + clean = [] + for value in values: + try: + number = float(value) + except (TypeError, ValueError): + raise PriceHistoryUnavailable( + "%s contains a missing/non-numeric close" % symbol + ) + if not math.isfinite(number) or number <= 0.0: + raise PriceHistoryUnavailable( + "%s contains a missing/non-positive close" % symbol + ) + clean.append(number) + output[convert_symbol(symbol, "canonical")] = clean + return output + + +def _decision_universe(context): + config = g.quant60_config + if config.get("universe_mode") == "fixed": + values = list(config["universe"]) + else: + as_of = _as_date( + getattr(context, "previous_date", None), + "previous_date", + ) + values = get_index_stocks( + config["index_symbol"], + date=as_of, + ) + if not values: + raise PriceHistoryUnavailable( + "no PIT index members for %s on %s" + % (config["index_symbol"], as_of.isoformat()) + ) + g.quant60_last_universe_as_of = as_of.isoformat() + normalized = sorted( + set(convert_symbol(symbol, "joinquant") for symbol in values) + ) + if not normalized: + raise PriceHistoryUnavailable("decision universe is empty") + g.quant60_last_universe = list(normalized) + return normalized + + +def _filter_st_universe(context, symbols): + if not g.quant60_config.get("exclude_st", True): + return list(symbols) + as_of = _as_date( + getattr(context, "previous_date", None), + "previous_date", + ) + raw = get_extras( + "is_st", + list(symbols), + end_date=as_of, + count=1, + df=False, + ) + if not hasattr(raw, "get"): + raise PriceHistoryUnavailable( + "get_extras is_st returned an invalid payload" + ) + output = [] + for symbol in symbols: + values = list(raw.get(symbol, [])) + if len(values) != 1: + raise PriceHistoryUnavailable( + "%s needs one PIT is_st value on %s" + % (symbol, as_of.isoformat()) + ) + try: + numeric = float(values[0]) + except (TypeError, ValueError): + raise PriceHistoryUnavailable( + "%s contains a non-boolean PIT is_st value" % symbol + ) + if not math.isfinite(numeric) or numeric not in (0.0, 1.0): + raise PriceHistoryUnavailable( + "%s contains an invalid PIT is_st value" % symbol + ) + if not bool(numeric): + output.append(symbol) + if not output: + raise PriceHistoryUnavailable( + "PIT ST exclusion removed the entire decision universe" + ) + g.quant60_last_universe = list(output) + return output + + +def _position_amount(position, name, default=0): + value = getattr(position, name, default) + return int(value or 0) + + +def _snapshot_portfolio(context, managed): + current = {} + sellable = {} + managed = set(convert_symbol(symbol, "canonical") for symbol in managed) + dynamic = g.quant60_config.get("universe_mode") == "pit_index" + positions = getattr(context.portfolio, "positions", {}) + for symbol, position in positions.items(): + canonical = convert_symbol(str(symbol), "canonical") + quantity = _position_amount(position, "total_amount") + if quantity and canonical not in managed and not dynamic: + raise UnmanagedPositionError( + "account contains unmanaged position %s; use a dedicated " + "strategy account or reconcile ownership before trading" % canonical + ) + if canonical not in managed and not dynamic: + continue + current[canonical] = quantity + sellable[canonical] = _position_amount( + position, "closeable_amount", current[canonical] + ) + equity = float(getattr(context.portfolio, "total_value", 0.0) or 0.0) + return current, sellable, equity + + +def _is_star_board(symbol): + code = str(symbol).split(".", 1)[0] + return code.startswith("688") or code.startswith("689") + + +def _order_request(symbol, delta): + """Build a fail-closed hosted order request from current market data. + + JoinQuant rejects an unprotected market order for STAR Market securities. + A limit at the exchange daily bound supplies equivalent worst-price + protection while keeping the rebalance target expressed in exact shares. + """ + try: + item = get_current_data()[symbol] + except Exception as exc: + g.quant60_skipped_orders.append( + {"symbol": symbol, "reason": "current_data_unavailable", "detail": str(exc)} + ) + return None + if bool(getattr(item, "paused", False)): + g.quant60_skipped_orders.append( + {"symbol": symbol, "reason": "paused"} + ) + return None + if not _is_star_board(symbol): + return {"style": None} + + field = "high_limit" if int(delta) > 0 else "low_limit" + raw_price = getattr(item, field, None) + try: + protection_price = float(raw_price) + except (TypeError, ValueError): + protection_price = float("nan") + if ( + not math.isfinite(protection_price) + or protection_price <= 0.0 + or protection_price >= 10000.0 + ): + g.quant60_skipped_orders.append( + { + "symbol": symbol, + "reason": "star_protection_price_invalid", + "detail": field, + } + ) + return None + try: + style = LimitOrderStyle(protection_price) + except Exception as exc: + g.quant60_skipped_orders.append( + { + "symbol": symbol, + "reason": "star_protection_style_unavailable", + "detail": str(exc), + } + ) + return None + return {"style": style} + + +def compute_plan(context): + """Return the portable plan without submitting orders.""" + config = g.quant60_config + required = int(config["lookback"]) + int(config["skip"]) + 1 + platform_universe = _decision_universe(context) + platform_universe = _filter_st_universe( + context, + platform_universe, + ) + price_history = _close_histories(platform_universe, required) + + current, sellable, equity = _snapshot_portfolio( + context, + platform_universe, + ) + if equity <= 0: + raise ValueError("portfolio.total_value must be positive") + plan = build_rebalance_plan( + price_history=price_history, + current=current, + sellable=sellable, + equity=equity, + lookback=config["lookback"], + skip=config["skip"], + top_n=config["top_n"], + max_weight=config["max_weight"], + gross_target=config["gross_target"], + cash_buffer=config["cash_buffer"], + lot_size=config["lot_size"], + ) + return plan + + +def _execute_rebalance(context): + """Submit the portable plan's exact executable share deltas.""" + plan = compute_plan(context) + order_deltas = plan["orders"] + current, _, unused_equity = _snapshot_portfolio( + context, + g.quant60_last_universe or g.quant60_config["universe"], + ) + del unused_equity + all_symbols = set(current) + all_symbols.update(order_deltas) + + # Exits first. The requested target is current + executable delta, not the + # unconstrained portfolio target; this preserves T+1 sellable caps and odd + # lots from the portable plan. + for canonical in sorted( + all_symbols, + key=lambda item: ( + int(order_deltas.get(item, 0)) > 0, + item, + ), + ): + delta = int(order_deltas.get(canonical, 0)) + if delta == 0: + continue + platform_symbol = convert_symbol(canonical, "joinquant") + request = _order_request(platform_symbol, delta) + if request is None: + continue + target = int(current.get(canonical, 0)) + delta + if request["style"] is None: + order_target(platform_symbol, target) + else: + order_target( + platform_symbol, + target, + style=request["style"], + ) + + g.quant60_last_plan = plan + return plan + + +def _as_date(value, name): + if hasattr(value, "date"): + value = value.date() + if not hasattr(value, "isocalendar"): + raise ClockStateError("%s must be a date/datetime" % name) + return value + + +def rebalance(context): + """Execute on the session immediately after a week's first close. + + A daily state machine is used instead of ``run_weekly(..., 2)`` so a + one-session holiday week still executes on the next available session. + """ + current_date = _as_date(getattr(context, "current_dt", None), "current_dt") + previous_date = _as_date( + getattr(context, "previous_date", None), + "previous_date", + ) + if g.quant60_last_callback_date == current_date: + return None + pending = g.quant60_pending_first_session + is_first_session = ( + previous_date.isocalendar()[:2] + != current_date.isocalendar()[:2] + ) + if pending is not None: + if previous_date != pending: + raise ClockStateError( + "pending first-session close is not the immediately " + "previous trading session" + ) + # Consume the clock token before crossing any hosted order boundary. + # A partial API failure must not make a callback retry duplicate the + # already accepted prefix of the order list. + g.quant60_pending_first_session = ( + current_date if is_first_session else None + ) + g.quant60_last_callback_date = current_date + plan = _execute_rebalance(context) + return plan + if is_first_session: + g.quant60_pending_first_session = current_date + g.quant60_last_callback_date = current_date + return None diff --git a/platforms/qlib_runner.py b/platforms/qlib_runner.py new file mode 100644 index 0000000..b49bb2b --- /dev/null +++ b/platforms/qlib_runner.py @@ -0,0 +1,866 @@ +"""Qlib 0.9.7 native momentum backtest and Alpha158/LightGBM workflow. + +This module never downloads data. A real, user-authorized ``provider_uri`` is +required. Qlib, pandas and LightGBM are imported only inside execution paths. +""" + +from __future__ import annotations + +import argparse +from collections import deque +import datetime as dt +import hashlib +import importlib +import importlib.util +import json +import math +import os +from pathlib import Path +import tempfile +from typing import Any, Dict, Iterable, Mapping, Optional, Sequence + +from quant60.portable_core import momentum_score + + +class QlibUnavailableError(RuntimeError): + """Raised when Qlib or its configured provider cannot be used.""" + + +def _table_audit(table: Any) -> Optional[Dict[str, Any]]: + """Return compact deterministic evidence for a pandas Series/DataFrame.""" + + to_json = getattr(table, "to_json", None) + if not callable(to_json): + return None + encoded = to_json( + orient="split", + date_format="iso", + date_unit="ns", + double_precision=15, + ).encode("utf-8") + columns = [ + str(value) + for value in getattr(table, "columns", [getattr(table, "name", "value")]) + ] + audit: Dict[str, Any] = { + "row_count": int(len(table)), + "columns": columns, + "content_sha256": hashlib.sha256(encoded).hexdigest(), + } + index = getattr(table, "index", None) + if index is not None and len(index): + audit["period_start"] = str(index[0]) + audit["period_end"] = str(index[-1]) + + def finite_values(name: str) -> list[float]: + if name not in columns: + return [] + try: + raw = table[name].dropna().tolist() + values = [float(value) for value in raw] + except (KeyError, TypeError, ValueError, AttributeError): + return [] + return [value for value in values if math.isfinite(value)] + + returns = finite_values("return") + benchmark = finite_values("bench") + costs = finite_values("cost") + turnover = finite_values("turnover") + if returns: + wealth = 1.0 + peak = 1.0 + max_drawdown = 0.0 + for value in returns: + wealth *= 1.0 + value + peak = max(peak, wealth) + max_drawdown = min(max_drawdown, wealth / peak - 1.0) + audit["derived_performance"] = { + "cumulative_return": wealth - 1.0, + "max_drawdown": max_drawdown, + "benchmark_cumulative_return": ( + math.prod(1.0 + value for value in benchmark) - 1.0 + if benchmark + else None + ), + "total_cost": sum(costs) if costs else None, + "total_turnover": sum(turnover) if turnover else None, + } + return audit + + +def _mapping_table_audits(values: Any) -> Dict[str, Any]: + if not isinstance(values, Mapping): + return {} + output: Dict[str, Any] = {} + for frequency, item in sorted(values.items(), key=lambda pair: str(pair[0])): + table = item[0] if isinstance(item, (tuple, list)) and item else item + audit = _table_audit(table) + if audit is not None: + output[str(frequency)] = audit + return output + + +def _version_tuple(value: str) -> tuple[int, ...]: + parts = [] + for chunk in str(value).split("."): + digits = "".join(character for character in chunk if character.isdigit()) + if not digits: + break + parts.append(int(digits)) + return tuple(parts) + + +def _load_qlib() -> Dict[str, Any]: + if importlib.util.find_spec("qlib") is None: + raise QlibUnavailableError( + "pyqlib 0.9.7 is not installed in this research environment" + ) + try: + qlib = importlib.import_module("qlib") + data_module = importlib.import_module("qlib.data") + executor_module = importlib.import_module("qlib.backtest.executor") + backtest_module = importlib.import_module("qlib.backtest") + strategy_module = importlib.import_module( + "qlib.contrib.strategy.signal_strategy" + ) + except Exception as exc: + raise QlibUnavailableError(f"Qlib import failed: {exc}") from exc + version = str(getattr(qlib, "__version__", "unknown")) + if version != "unknown" and _version_tuple(version) != (0, 9, 7): + raise QlibUnavailableError( + f"Qlib exactly 0.9.7 is required by this pinned adapter; found {version}" + ) + return { + "qlib": qlib, + "D": data_module.D, + "SimulatorExecutor": executor_module.SimulatorExecutor, + "backtest": backtest_module.backtest, + "TopkDropoutStrategy": strategy_module.TopkDropoutStrategy, + "version": version, + } + + +def _validate_provider(provider_uri: str) -> str: + value = str(provider_uri or "").strip() + if not value: + raise ValueError("provider_uri is required; no public data is downloaded") + if "://" not in value: + path = Path(value).expanduser().resolve() + if not path.is_dir(): + raise FileNotFoundError( + f"Qlib provider_uri does not exist: {path}" + ) + return str(path) + return value + + +def _infer_columns(frame: Any) -> tuple[str, str, str]: + columns = [str(column) for column in frame.columns] + instrument = next( + ( + name + for name in ("instrument", "level_0", "level_1") + if name in columns + ), + None, + ) + datetime_column = next( + ( + name + for name in ("datetime", "date", "level_1", "level_0") + if name in columns and name != instrument + ), + None, + ) + close = next( + (name for name in ("$close", "close") if name in columns), + None, + ) + if close is None: + non_index = [ + name + for name in columns + if name not in {instrument, datetime_column} + ] + close = non_index[0] if len(non_index) == 1 else None + if instrument is None or datetime_column is None or close is None: + raise ValueError( + f"unexpected D.features columns/index after reset: {columns}" + ) + + # Unnamed MultiIndexes may reset as level_0/level_1. Infer their order + # from the first non-null value instead of assuming a Qlib build detail. + if instrument.startswith("level_") and datetime_column.startswith("level_"): + sample = frame[[instrument, datetime_column]].dropna().head(1) + if not sample.empty: + first = sample.iloc[0][instrument] + if not isinstance(first, str) or not ( + first.upper().startswith(("SH", "SZ", "BJ")) + or first[:1].isdigit() + ): + instrument, datetime_column = datetime_column, instrument + return instrument, datetime_column, close + + +def generate_portable_momentum_signal( + D: Any, + instruments: Any, + *, + start_time: str, + end_time: str, + feature_start_time: Optional[str] = None, + lookback: int = 20, + skip: int = 0, + rebalance: str = "weekly", +) -> Any: + """Use ``D.features`` and portable momentum to build a Qlib score Series. + + The result is indexed ``(datetime, instrument)``. Qlib's + ``BaseSignalStrategy`` reads the prior-step score (``shift=1``), so a score + computed on completed bar T is executed on the next engine step. + """ + pandas = importlib.import_module("pandas") + if feature_start_time is None: + start = dt.datetime.fromisoformat(str(start_time)[:10]) + padding_days = max(14, 3 * (int(lookback) + int(skip) + 2)) + feature_start_time = (start - dt.timedelta(days=padding_days)).date().isoformat() + if rebalance not in {"daily", "weekly"}: + raise ValueError("rebalance must be daily or weekly") + instrument_query = instruments + if isinstance(instruments, str): + instrument_query = D.instruments(market=instruments) + raw = D.features( + instrument_query, + ["$close"], + start_time=feature_start_time, + end_time=end_time, + freq="day", + ) + if raw is None or len(raw) == 0: + raise QlibUnavailableError("D.features returned no close data") + records = raw.reset_index() + instrument_col, datetime_col, close_col = _infer_columns(records) + records[datetime_col] = pandas.to_datetime(records[datetime_col]) + records = records.sort_values([instrument_col, datetime_col]) + + rows = [] + start_bound = pandas.Timestamp(start_time) + end_bound = pandas.Timestamp(end_time) + for instrument, group in records.groupby(instrument_col, sort=True): + prices = deque(maxlen=int(lookback) + int(skip) + 1) + for _, row in group.iterrows(): + value = row[close_col] + if value is None or ( + isinstance(value, float) and not math.isfinite(value) + ): + prices.append(float("nan")) + continue + prices.append(float(value)) + score = momentum_score(prices, lookback=lookback, skip=skip) + when = pandas.Timestamp(row[datetime_col]) + if score is None or when < start_bound or when > end_bound: + continue + rows.append( + { + "datetime": when, + "instrument": str(instrument), + "score": float(score), + } + ) + if not rows: + raise QlibUnavailableError( + "no momentum signal was produced; provider history may be shorter " + "than lookback + skip + 1" + ) + signal = pandas.DataFrame(rows) + if rebalance == "weekly": + # Select the true first provider session of each week, rather than + # the first date on which a warmed-up signal happens to exist. + calendar = records[[datetime_col]].drop_duplicates().copy() + calendar = calendar[ + (calendar[datetime_col] >= start_bound) + & (calendar[datetime_col] <= end_bound) + ] + calendar["_week"] = calendar[datetime_col].dt.to_period("W") + first_dates = set( + calendar.groupby("_week")[datetime_col].min().tolist() + ) + signal = signal[signal["datetime"].isin(first_dates)] + if signal.empty: + raise QlibUnavailableError( + "no signal exists on a first weekly provider session" + ) + return signal.set_index(["datetime", "instrument"]).sort_index()["score"] + + +def run_native_momentum_backtest( + *, + provider_uri: str, + market: str, + benchmark: str, + start_time: str, + end_time: str, + feature_start_time: Optional[str] = None, + lookback: int = 20, + skip: int = 0, + rebalance: str = "weekly", + topk: int = 50, + n_drop: int = 5, + account: float = 10_000_000.0, + deal_price: str = "open", + open_cost: float = 0.0003, + close_cost: float = 0.0008, + min_cost: float = 5.0, + limit_threshold: float = 0.095, +) -> Dict[str, Any]: + """Run the official SimulatorExecutor + TopkDropoutStrategy stack.""" + provider = _validate_provider(provider_uri) + api = _load_qlib() + constant = importlib.import_module("qlib.constant") + api["qlib"].init(provider_uri=provider, region=constant.REG_CN) + signal = generate_portable_momentum_signal( + api["D"], + market, + start_time=start_time, + end_time=end_time, + feature_start_time=feature_start_time, + lookback=lookback, + skip=skip, + rebalance=rebalance, + ) + strategy = api["TopkDropoutStrategy"]( + signal=signal, + topk=int(topk), + n_drop=int(n_drop), + hold_thresh=1, + only_tradable=True, + forbid_all_trade_at_limit=False, + ) + executor = api["SimulatorExecutor"]( + time_per_step="day", + generate_portfolio_metrics=True, + ) + portfolio_metrics, indicators = api["backtest"]( + start_time=start_time, + end_time=end_time, + strategy=strategy, + executor=executor, + account=float(account), + benchmark=benchmark, + exchange_kwargs={ + "freq": "day", + "limit_threshold": float(limit_threshold), + "deal_price": deal_price, + "open_cost": float(open_cost), + "close_cost": float(close_cost), + "min_cost": float(min_cost), + }, + ) + return { + "qlib_version": api["version"], + "signal": signal, + "portfolio_metrics": portfolio_metrics, + "indicators": indicators, + "clock_contract": ( + "weekly_first_provider_session_close_to_next_engine_step" + if rebalance == "weekly" + else "T_close_signal_to_T_plus_1_engine_step" + ), + "strategy_fidelity": ( + "portable_signal_only_research_bridge_not_quant60_" + "target_or_order_parity" + ), + "a_share_rule_fidelity": ( + "uniform_limit_threshold_approximation_not_point_in_time_" + "board_or_ST_rules" + ), + } + + +def build_alpha158_lightgbm_task( + *, + market: str, + train: Sequence[str], + valid: Sequence[str], + test: Sequence[str], + topk: int = 50, + n_drop: int = 5, + benchmark: str = "SH000300", + account: float = 10_000_000.0, +) -> Dict[str, Any]: + """Return a Qlib 0.9.7 task plus official portfolio analysis config.""" + if not (len(train) == len(valid) == len(test) == 2): + raise ValueError("train/valid/test must each contain start and end") + handler = { + "start_time": train[0], + "end_time": test[1], + "fit_start_time": train[0], + "fit_end_time": train[1], + "instruments": market, + } + task = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + "kwargs": { + "loss": "mse", + "learning_rate": 0.05, + "num_leaves": 64, + "max_depth": 8, + "subsample": 0.9, + "colsample_bytree": 0.9, + "lambda_l1": 1.0, + "lambda_l2": 1.0, + "num_threads": 4, + }, + }, + "dataset": { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": handler, + }, + "segments": { + "train": tuple(train), + "valid": tuple(valid), + "test": tuple(test), + }, + }, + }, + } + portfolio_analysis = { + "executor": { + "class": "SimulatorExecutor", + "module_path": "qlib.backtest.executor", + "kwargs": { + "time_per_step": "day", + "generate_portfolio_metrics": True, + }, + }, + "strategy": { + "class": "TopkDropoutStrategy", + "module_path": "qlib.contrib.strategy.signal_strategy", + "kwargs": { + # The runner replaces this with the actual (model, dataset). + "signal": None, + "topk": int(topk), + "n_drop": int(n_drop), + }, + }, + "backtest": { + "start_time": test[0], + "end_time": test[1], + "account": float(account), + "benchmark": benchmark, + "exchange_kwargs": { + "freq": "day", + "limit_threshold": 0.095, + "deal_price": "open", + "open_cost": 0.0003, + "close_cost": 0.0008, + "min_cost": 5.0, + }, + }, + } + return {"task": task, "portfolio_analysis": portfolio_analysis} + + +def run_alpha158_lightgbm_workflow( + *, + provider_uri: str, + market: str, + benchmark: str, + train: Sequence[str], + valid: Sequence[str], + test: Sequence[str], + experiment_name: str = "quant60_alpha158_lightgbm", + topk: int = 50, + n_drop: int = 5, +) -> Dict[str, Any]: + """Fit, record signals and run PortAnaRecord with official Qlib APIs.""" + provider = _validate_provider(provider_uri) + # Qlib 0.9.7 defaults to a local MLflow file store. MLflow 3.14 requires + # an explicit acknowledgement before it will open that backend. Keep the + # acknowledgement local to this process; a caller-provided tracking + # configuration still owns the actual destination. + os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true") + api = _load_qlib() + constant = importlib.import_module("qlib.constant") + utils = importlib.import_module("qlib.utils") + workflow = importlib.import_module("qlib.workflow") + records = importlib.import_module("qlib.workflow.record_temp") + api["qlib"].init(provider_uri=provider, region=constant.REG_CN) + + config = build_alpha158_lightgbm_task( + market=market, + benchmark=benchmark, + train=train, + valid=valid, + test=test, + topk=topk, + n_drop=n_drop, + ) + model = utils.init_instance_by_config(config["task"]["model"]) + dataset = utils.init_instance_by_config(config["task"]["dataset"]) + config["portfolio_analysis"]["strategy"]["kwargs"]["signal"] = (model, dataset) + + with workflow.R.start(experiment_name=experiment_name): + model.fit(dataset) + workflow.R.save_objects(**{"trained_model.pkl": model}) + recorder = workflow.R.get_recorder() + records.SignalRecord(model, dataset, recorder).generate() + records.SigAnaRecord(recorder).generate() + portfolio_record = records.PortAnaRecord( + recorder, + config["portfolio_analysis"], + "day", + ) + portfolio_record.generate() + try: + report = recorder.load_object( + "portfolio_analysis/report_normal_1day.pkl" + ) + except Exception as exc: + raise QlibUnavailableError( + "PortAnaRecord completed without a readable 1day report" + ) from exc + report_audit = _table_audit(report) + if report_audit is None: + raise QlibUnavailableError( + "Qlib portfolio report is not a pandas-compatible table" + ) + recorded_metrics = { + str(key): float(value) + for key, value in sorted(recorder.list_metrics().items()) + if math.isfinite(float(value)) + } + artifact_paths = sorted( + str(value) + for value in recorder.list_artifacts("portfolio_analysis") + ) + recorder_id = getattr(recorder, "id", None) + return { + "qlib_version": api["version"], + "experiment_name": experiment_name, + "recorder_id": recorder_id, + "mlflow_local_file_store_explicitly_allowed": ( + os.environ.get("MLFLOW_ALLOW_FILE_STORE") == "true" + ), + "task": config["task"], + "portfolio_analysis": config["portfolio_analysis"], + "recorder_evidence": { + "portfolio_report": report_audit, + "recorded_metrics": recorded_metrics, + "portfolio_artifact_paths": artifact_paths, + }, + } + + +def _canonical_cli_date(value: Optional[str], *, name: str) -> str: + raw = str(value or "").strip() + if not raw: + raise ValueError(f"{name} is required") + try: + parsed = dt.date.fromisoformat(raw) + except ValueError as exc: + raise ValueError(f"{name} must be an ISO date (YYYY-MM-DD)") from exc + if parsed.isoformat() != raw: + raise ValueError(f"{name} must be an ISO date (YYYY-MM-DD)") + return raw + + +def _validate_nonempty(value: str, *, name: str) -> str: + normalized = str(value or "").strip() + if not normalized: + raise ValueError(f"{name} must not be empty") + return normalized + + +def _validate_common_cli_args(args: argparse.Namespace) -> None: + args.provider_uri = _validate_nonempty( + args.provider_uri, + name="provider-uri", + ) + args.market = _validate_nonempty(args.market, name="market") + args.benchmark = _validate_nonempty(args.benchmark, name="benchmark") + if args.topk <= 0: + raise ValueError("topk must be greater than zero") + if args.n_drop < 0: + raise ValueError("n-drop must be zero or greater") + if args.n_drop > args.topk: + raise ValueError("n-drop must not exceed topk") + if args.output_json is not None: + args.output_json = _validate_nonempty( + args.output_json, + name="output-json", + ) + + +def _validated_momentum_dates(args: argparse.Namespace) -> tuple[str, str, Optional[str]]: + start = _canonical_cli_date(args.start, name="start") + end = _canonical_cli_date(args.end, name="end") + if start > end: + raise ValueError("start must be on or before end") + feature_start = None + if args.feature_start is not None: + feature_start = _canonical_cli_date( + args.feature_start, + name="feature-start", + ) + if feature_start > start: + raise ValueError("feature-start must be on or before start") + if args.lookback <= 0: + raise ValueError("lookback must be greater than zero") + if args.skip < 0: + raise ValueError("skip must be zero or greater") + return start, end, feature_start + + +def _validated_alpha158_segments( + args: argparse.Namespace, +) -> tuple[tuple[str, str], tuple[str, str], tuple[str, str]]: + train = ( + _canonical_cli_date(args.train_start, name="train-start"), + _canonical_cli_date(args.train_end, name="train-end"), + ) + valid = ( + _canonical_cli_date(args.valid_start, name="valid-start"), + _canonical_cli_date(args.valid_end, name="valid-end"), + ) + test = ( + _canonical_cli_date(args.test_start, name="test-start"), + _canonical_cli_date(args.test_end, name="test-end"), + ) + for name, segment in (("train", train), ("valid", valid), ("test", test)): + if segment[0] > segment[1]: + raise ValueError(f"{name}-start must be on or before {name}-end") + if train[1] >= valid[0]: + raise ValueError("train and valid segments must be ordered and non-overlapping") + if valid[1] >= test[0]: + raise ValueError("valid and test segments must be ordered and non-overlapping") + args.experiment_name = _validate_nonempty( + args.experiment_name, + name="experiment-name", + ) + return train, valid, test + + +def _portfolio_frequencies(portfolio_metrics: Any) -> list[str]: + if isinstance(portfolio_metrics, Mapping): + values = portfolio_metrics.keys() + elif isinstance(portfolio_metrics, (list, tuple, set, frozenset)): + values = portfolio_metrics + else: + values = () + return sorted(str(value) for value in values) + + +def _momentum_cli_summary(result: Mapping[str, Any]) -> Dict[str, Any]: + signal = result.get("signal") + signal_table = ( + signal.to_frame(name="score") + if callable(getattr(signal, "to_frame", None)) + else None + ) + return { + "workflow": "momentum", + "qlib_version": str(result.get("qlib_version", "unknown")), + "signal_rows": len(signal) if signal is not None else 0, + "portfolio_frequencies": _portfolio_frequencies( + result.get("portfolio_metrics") + ), + "signal_evidence": _table_audit(signal_table), + "portfolio_report_evidence": _mapping_table_audits( + result.get("portfolio_metrics") + ), + "indicator_evidence": _mapping_table_audits( + result.get("indicators") + ), + "clock_contract": result.get("clock_contract"), + "strategy_fidelity": result.get("strategy_fidelity"), + "a_share_rule_fidelity": result.get("a_share_rule_fidelity"), + "investment_value_claim": False, + } + + +def _alpha158_cli_summary( + result: Mapping[str, Any], + *, + market: str, + benchmark: str, + train: Sequence[str], + valid: Sequence[str], + test: Sequence[str], +) -> Dict[str, Any]: + recorder_id = result.get("recorder_id") + return { + "workflow": "alpha158", + "qlib_version": str(result.get("qlib_version", "unknown")), + "experiment_name": str(result.get("experiment_name", "")), + "recorder_id": None if recorder_id is None else str(recorder_id), + "mlflow_local_file_store_explicitly_allowed": bool( + result.get("mlflow_local_file_store_explicitly_allowed", False) + ), + "market": market, + "benchmark": benchmark, + "segments": { + "train": list(train), + "valid": list(valid), + "test": list(test), + }, + "recorder_evidence": result.get("recorder_evidence"), + "investment_value_claim": False, + } + + +def _write_cli_json(path: str, payload: Mapping[str, Any]) -> None: + destination = Path(path).expanduser().resolve() + if destination.exists() and destination.is_dir(): + raise ValueError(f"output-json points to a directory: {destination}") + destination.parent.mkdir(parents=True, exist_ok=True) + encoded = ( + json.dumps( + dict(payload), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + descriptor, temporary = tempfile.mkstemp( + prefix=f".{destination.name}.", + suffix=".tmp", + dir=str(destination.parent), + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, destination) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def _main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--workflow", + choices=("momentum", "alpha158"), + default="momentum", + help="default momentum preserves the original CLI behavior", + ) + parser.add_argument("--provider-uri", required=True) + parser.add_argument("--market", default="csi300") + parser.add_argument("--benchmark", default="SH000300") + parser.add_argument("--start") + parser.add_argument("--end") + parser.add_argument("--feature-start") + parser.add_argument("--lookback", type=int, default=20) + parser.add_argument("--skip", type=int, default=0) + parser.add_argument("--topk", type=int, default=50) + parser.add_argument("--n-drop", type=int, default=5) + parser.add_argument("--rebalance", choices=("daily", "weekly"), default="weekly") + parser.add_argument("--train-start") + parser.add_argument("--train-end") + parser.add_argument("--valid-start") + parser.add_argument("--valid-end") + parser.add_argument("--test-start") + parser.add_argument("--test-end") + parser.add_argument( + "--experiment-name", + default="quant60_alpha158_lightgbm", + ) + parser.add_argument( + "--output-json", + "--result-json", + dest="output_json", + help="optionally save the same JSON summary printed to stdout", + ) + args = parser.parse_args(argv) + try: + _validate_common_cli_args(args) + if args.workflow == "momentum": + if any( + value is not None + for value in ( + args.train_start, + args.train_end, + args.valid_start, + args.valid_end, + args.test_start, + args.test_end, + ) + ): + raise ValueError( + "train/valid/test segment arguments are only valid for " + "workflow=alpha158" + ) + start, end, feature_start = _validated_momentum_dates(args) + result = run_native_momentum_backtest( + provider_uri=args.provider_uri, + market=args.market, + benchmark=args.benchmark, + start_time=start, + end_time=end, + feature_start_time=feature_start, + lookback=args.lookback, + skip=args.skip, + topk=args.topk, + n_drop=args.n_drop, + rebalance=args.rebalance, + ) + summary = _momentum_cli_summary(result) + else: + if any( + value is not None + for value in (args.start, args.end, args.feature_start) + ): + raise ValueError( + "start/end/feature-start are only valid for " + "workflow=momentum" + ) + train, valid, test = _validated_alpha158_segments(args) + result = run_alpha158_lightgbm_workflow( + provider_uri=args.provider_uri, + market=args.market, + benchmark=args.benchmark, + train=train, + valid=valid, + test=test, + experiment_name=args.experiment_name, + topk=args.topk, + n_drop=args.n_drop, + ) + summary = _alpha158_cli_summary( + result, + market=args.market, + benchmark=args.benchmark, + train=train, + valid=valid, + test=test, + ) + if args.output_json: + _write_cli_json(args.output_json, summary) + except (OSError, ValueError) as exc: + parser.error(str(exc)) + print( + json.dumps( + summary, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/platforms/qmt_builtin_strategy.py b/platforms/qmt_builtin_strategy.py new file mode 100644 index 0000000..4123706 --- /dev/null +++ b/platforms/qmt_builtin_strategy.py @@ -0,0 +1,485 @@ +# coding: gbk +"""QMT built-in Python 3.6 strategy wrapper. Keep this source ASCII-only.""" + +# __PORTABLE_CORE_BUNDLE_START__ +from quant60.portable_core import build_rebalance_plan, convert_symbol +# __PORTABLE_CORE_BUNDLE_END__ + +import datetime +import math + + +class PriceHistoryUnavailable(RuntimeError): + pass + + +class UnmanagedPositionError(RuntimeError): + pass + + +class UnsupportedStrategyPeriod(RuntimeError): + pass + + +class G(object): + pass + + +# QMT explicitly documents that custom attributes written to ContextInfo can +# roll back before the next handlebar call. Keep all user-owned mutable state +# in a module global and use ContextInfo only for platform-owned fields/APIs. +g = G() + + +# __BASELINE_CONFIG_START__ +CONFIG = { + "account_id": "test", + "universe_mode": "pit_index", + "index_symbol": "000905.XSHG", + "qmt_sector_name": "\u4e2d\u8bc1500", + "dedicated_account_required": True, + "exclude_st": True, + "universe": [ + "600000.SH", + "000001.SZ", + "300750.SZ", + "000333.SZ", + ], + "lookback": 20, + # T close is the signal clock; quickTrade=0 executes on the next step. + "skip": 0, + "top_n": 20, + "max_weight": 0.05, + "gross_target": 0.95, + "cash_buffer": 0.02, + "lot_size": 100, + "participation_rate": 0.1, + "slippage_bps": 2.0, + "commission_rate": 0.0002, + "minimum_commission": 5.0, + "stamp_duty_rate": 0.0005, + "transfer_fee_rate": 0.00001, + "rebalance_schedule": "weekly_first_close", +} +# __BASELINE_CONFIG_END__ + + +def _param(ContextInfo, name, default=None): + params = getattr(ContextInfo, "_param", {}) + if isinstance(params, dict) and name in params: + return params[name] + return default + + +def init(ContextInfo): + period = str( + getattr(ContextInfo, "period", _param(ContextInfo, "period", "1d")) + ).strip().lower() + if period not in ("1d", "day"): + raise UnsupportedStrategyPeriod( + "quant60 QMT wrapper requires a 1d driving period" + ) + g.config = dict(CONFIG) + for key in CONFIG: + override = _param(ContextInfo, "q60_" + key, None) + if override is not None: + g.config[key] = override + if ( + g.config.get("rebalance_schedule") + != "weekly_first_close" + ): + raise UnsupportedStrategyPeriod( + "unsupported rebalance_schedule" + ) + if g.config.get("universe_mode") not in ("pit_index", "fixed"): + raise UnsupportedStrategyPeriod("unsupported universe_mode") + if ( + g.config.get("universe_mode") == "pit_index" + and not g.config.get("dedicated_account_required") + ): + raise UnsupportedStrategyPeriod( + "pit_index mode requires a dedicated account" + ) + set_commission = getattr(ContextInfo, "set_commission", None) + set_slippage = getattr(ContextInfo, "set_slippage", None) + if not callable(set_commission) or not callable(set_slippage): + raise UnsupportedStrategyPeriod( + "QMT backtest commission/slippage APIs are unavailable" + ) + commission = ( + float(g.config["commission_rate"]) + + float(g.config["transfer_fee_rate"]) + ) + set_commission( + 0, + [ + 0.0, + float(g.config["stamp_duty_rate"]), + commission, + commission, + 0.0, + float(g.config["minimum_commission"]), + ], + ) + set_slippage( + 2, + float(g.config["slippage_bps"]) / 10000.0, + ) + g.last_week = None + g.last_plan = None + g.last_universe = None + g.last_universe_as_of = None + if ( + g.config.get("universe_mode") == "fixed" + and hasattr(ContextInfo, "set_universe") + ): + ContextInfo.set_universe(g.config["universe"]) + + +def _bar_datetime(ContextInfo): + timetag = ContextInfo.get_bar_timetag(ContextInfo.barpos) + return datetime.datetime.fromtimestamp(float(timetag) / 1000.0) + + +def _series_close(frame): + if frame is None: + return [] + if hasattr(frame, "__getitem__"): + try: + values = frame["close"] + if hasattr(values, "tolist"): + values = values.tolist() + return list(values) + except (KeyError, TypeError): + pass + if isinstance(frame, dict): + values = frame.get("close", []) + return list(values) + return [] + + +def _decision_universe(ContextInfo, membership_timetag): + config = g.config + if config.get("universe_mode") == "fixed": + values = list(config["universe"]) + else: + function = getattr( + ContextInfo, + "get_stock_list_in_sector", + None, + ) + if not callable(function): + raise PriceHistoryUnavailable( + "historical sector membership API is unavailable" + ) + values = function( + config["qmt_sector_name"], + int(membership_timetag), + ) + if not values: + raise PriceHistoryUnavailable( + "no PIT index members for %s on %s" + % (config["index_symbol"], membership_timetag) + ) + normalized = sorted( + set(convert_symbol(symbol, "qmt") for symbol in values) + ) + if not normalized: + raise PriceHistoryUnavailable("decision universe is empty") + g.last_universe = list(normalized) + g.last_universe_as_of = int(membership_timetag) + return normalized + + +def _history(ContextInfo, end_time, universe): + config = g.config + count = int(config["lookback"]) + int(config["skip"]) + 1 + raw = ContextInfo.get_market_data_ex( + fields=["close"], + stock_code=universe, + period="1d", + start_time="", + end_time=end_time, + count=count, + dividend_type="front_ratio", + fill_data=False, + subscribe=False, + ) + output = {} + for symbol in universe: + values = _series_close(raw.get(symbol)) + if len(values) != count: + raise PriceHistoryUnavailable( + "%s needs %d completed daily closes, got %d" + % (symbol, count, len(values)) + ) + clean = [] + for value in values: + try: + number = float(value) + except (TypeError, ValueError): + raise PriceHistoryUnavailable( + "%s contains a missing/non-numeric close" % symbol + ) + if not math.isfinite(number) or number <= 0.0: + raise PriceHistoryUnavailable( + "%s contains a missing/non-positive close" % symbol + ) + clean.append(number) + output[convert_symbol(symbol, "canonical")] = clean + return output + + +def _filter_st_universe(ContextInfo, universe, decision_date): + if not g.config.get("exclude_st", True): + return list(universe) + function = getattr(ContextInfo, "get_his_st_data", None) + if not callable(function): + raise PriceHistoryUnavailable( + "historical ST API is unavailable; QMT VIP ST data is required" + ) + output = [] + for symbol in universe: + raw = function(symbol) + if not isinstance(raw, dict): + raise PriceHistoryUnavailable( + "%s historical ST query returned an invalid payload" % symbol + ) + excluded = False + for status in ("ST", "*ST", "PT"): + periods = raw.get(status, []) + if not isinstance(periods, (list, tuple)): + raise PriceHistoryUnavailable( + "%s historical %s periods are invalid" % (symbol, status) + ) + for period in periods: + if ( + not isinstance(period, (list, tuple)) + or len(period) != 2 + ): + raise PriceHistoryUnavailable( + "%s historical %s period is invalid" + % (symbol, status) + ) + start = str(period[0]) + end = str(period[1]) + if ( + len(start) != 8 + or len(end) != 8 + or not start.isdigit() + or not end.isdigit() + or start > end + ): + raise PriceHistoryUnavailable( + "%s historical %s date range is invalid" + % (symbol, status) + ) + if start <= decision_date <= end: + excluded = True + if not excluded: + output.append(symbol) + if not output: + raise PriceHistoryUnavailable( + "PIT ST exclusion removed the entire decision universe" + ) + g.last_universe = list(output) + return output + + +def _get_attr(obj, names, default=0): + for name in names: + if hasattr(obj, name): + value = getattr(obj, name) + if value is not None: + return value + return default + + +def _trade_details(ContextInfo, kind): + account_id = g.config["account_id"] + if hasattr(ContextInfo, "get_trade_detail_data"): + return ContextInfo.get_trade_detail_data(account_id, "STOCK", kind) + fn = globals().get("get_trade_detail_data") + if fn is None: + return [] + return fn(account_id, "STOCK", kind) + + +def _portfolio(ContextInfo, managed): + current = {} + sellable = {} + managed = set( + convert_symbol(symbol, "canonical") + for symbol in managed + ) + dynamic = g.config.get("universe_mode") == "pit_index" + for position in _trade_details(ContextInfo, "position") or []: + code = str( + _get_attr(position, ["stock_code", "m_strInstrumentID"], "") + ) + market = str(_get_attr(position, ["market", "m_strExchangeID"], "")) + if "." not in code and market: + code = code + "." + market + if not code: + continue + canonical = convert_symbol(code, "canonical") + quantity = int( + _get_attr(position, ["volume", "m_nVolume", "total_amount"], 0) + ) + if quantity and canonical not in managed and not dynamic: + raise UnmanagedPositionError( + "account contains unmanaged position %s; use a dedicated " + "strategy account or reconcile ownership before trading" % canonical + ) + if canonical not in managed and not dynamic: + continue + current[canonical] = quantity + sellable[canonical] = int( + _get_attr( + position, + ["can_use_volume", "m_nCanUseVolume", "closeable_amount"], + current[canonical], + ) + ) + + equity = 0.0 + assets = _trade_details(ContextInfo, "account") or [] + if assets: + equity = float( + _get_attr( + assets[0], + ["total_asset", "m_dBalance", "m_dTotalAsset", "asset"], + 0.0, + ) + ) + if equity <= 0: + equity = float(_param(ContextInfo, "asset", 0.0) or 0.0) + return current, sellable, equity + + +def compute_plan(ContextInfo, bar_time=None): + if bar_time is None: + bar_time = _bar_datetime(ContextInfo) + membership_timetag = int( + ContextInfo.get_bar_timetag(ContextInfo.barpos) + ) + end_time = bar_time.strftime("%Y%m%d") + universe = _decision_universe(ContextInfo, membership_timetag) + universe = _filter_st_universe( + ContextInfo, + universe, + end_time, + ) + history = _history(ContextInfo, end_time, universe) + current, sellable, equity = _portfolio(ContextInfo, universe) + if equity <= 0: + raise ValueError("QMT account equity must be positive") + config = g.config + return build_rebalance_plan( + price_history=history, + current=current, + sellable=sellable, + equity=equity, + lookback=config["lookback"], + skip=config["skip"], + top_n=config["top_n"], + max_weight=config["max_weight"], + gross_target=config["gross_target"], + cash_buffer=config["cash_buffer"], + lot_size=config["lot_size"], + ) + + +def _orders_allowed(ContextInfo): + # This hosted wrapper is deliberately backtest-only. Its cached account + # query and in-memory weekly marker cannot provide crash-safe live + # idempotency or a complete fresh reconciliation barrier. Real/shadow + # broker integration belongs behind the external XtTrader adapter. + return _is_backtest(ContextInfo) + + +def _is_backtest(ContextInfo): + if hasattr(ContextInfo, "do_back_test"): + value = getattr(ContextInfo, "do_back_test") + if callable(value): + value = value() + if value is True: + return True + mode = str( + getattr(ContextInfo, "trade_mode", _param(ContextInfo, "trade_mode", "")) + ).strip().lower() + return mode == "backtest" + + +def _submit_delta(ContextInfo, canonical, delta, week_key): + config = g.config + order_code = convert_symbol(canonical, "qmt") + operation = 23 if delta > 0 else 24 + volume = abs(int(delta)) + user_order_id = "q60-%04d%02d-%s" % ( + int(week_key[0]), + int(week_key[1]), + order_code.split(".")[0], + ) + function = globals()["passorder"] + arg_count = getattr(getattr(function, "__code__", None), "co_argcount", 11) + if arg_count <= 8: + # qmttools native-Python compatibility signature. + function( + operation, + 1101, + config["account_id"], + order_code, + 5, + -1, + volume, + ContextInfo, + ) + else: + # QMT built-in hosted signature. + function( + operation, + 1101, + config["account_id"], + order_code, + 5, + -1, + volume, + "quant60", + 0, + user_order_id, + ContextInfo, + ) + + +def handlebar(ContextInfo): + if not _is_backtest(ContextInfo) and hasattr(ContextInfo, "is_last_bar"): + if not ContextInfo.is_last_bar(): + return None + + bar_time = _bar_datetime(ContextInfo) + year, week, unused = bar_time.isocalendar() + del unused + week_key = (year, week) + if g.last_week is None: + # The strategy may be attached to a run in the middle of a week. + # Treat the first observed week as an initialization window; only a + # later ISO-week transition proves that this is the first observed + # trading session of a complete strategy week. + g.last_week = week_key + return None + if g.last_week == week_key: + return None + + plan = compute_plan(ContextInfo, bar_time) + g.last_week = week_key + g.last_plan = plan + if not _orders_allowed(ContextInfo): + return plan + + # Sells before buys. passorder is the QMT-hosted order boundary. + ordered = sorted(plan["orders"].items(), key=lambda item: item[1]) + for canonical, delta in ordered: + if int(delta) != 0: + _submit_delta(ContextInfo, canonical, delta, week_key) + return plan diff --git a/platforms/qmt_research_runner.py b/platforms/qmt_research_runner.py new file mode 100644 index 0000000..d810f05 --- /dev/null +++ b/platforms/qmt_research_runner.py @@ -0,0 +1,344 @@ +"""Native-Python runner for QMT research backtests. + +All xtquant imports are deliberately lazy. Importing this module is safe on a +machine that does not have QMT, while ``run_qmt_backtest`` fails before the +strategy is started if the local client or data permission is unavailable. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import importlib +import importlib.util +import json +import math +import os +import re +from pathlib import Path +from typing import Any, Dict, Optional + + +class QmtUnavailableError(RuntimeError): + """Raised when the native QMT runtime cannot pass its read-only preflight.""" + + +def _baseline_execution_defaults() -> Dict[str, Any]: + path = Path(__file__).resolve().parents[1] / "configs" / "baseline.json" + raw_bytes = path.read_bytes() + raw = json.loads(raw_bytes.decode("utf-8")) + fees = raw["fees"] + execution = raw["execution"] + strategy = raw["strategy"] + universe = raw["universe"] + index_symbol = str(universe["index_symbol"]) + benchmark = ( + index_symbol.replace(".XSHG", ".SH") + .replace(".XSHE", ".SZ") + .replace(".XBSE", ".BJ") + ) + return { + "slippage_type": 2, + "slippage": float(execution["slippage_bps"]) / 10_000.0, + "max_vol_rate": float(execution["participation_rate"]), + "open_tax": 0.0, + "close_tax": float(fees["stamp_duty_rate"]), + "min_commission": float(fees["minimum_commission"]), + # QMT has no separate transfer-fee parameter in this backtest + # contract, so the proportional transfer fee is folded into the + # commission rate. The minimum-fee edge remains a declared engine + # difference rather than an exact local-fill claim. + "open_commission": ( + float(fees["commission_rate"]) + + float(fees["transfer_fee_rate"]) + ), + "close_commission": ( + float(fees["commission_rate"]) + + float(fees["transfer_fee_rate"]) + ), + "q60_baseline_config_sha256": hashlib.sha256(raw_bytes).hexdigest(), + "q60_rebalance_schedule": str( + strategy["rebalance_schedule"] + ), + "q60_index_symbol": index_symbol, + "q60_qmt_sector_name": str(universe["qmt_sector_name"]), + "q60_exclude_st": bool(universe["exclude_st"]), + "benchmark": benchmark, + } + + +def _normalize_qmt_time(value: str, name: str, end_of_day: bool = False) -> str: + """Accept compact/ISO inputs and emit qmttools' documented timestamp.""" + text = str(value).strip() + parsed: Optional[dt.datetime] = None + formats = ( + "%Y%m%d", + "%Y-%m-%d", + "%Y%m%d%H%M%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S", + ) + used_date_only = False + for fmt in formats: + try: + parsed = dt.datetime.strptime(text, fmt) + used_date_only = fmt in {"%Y%m%d", "%Y-%m-%d"} + break + except ValueError: + continue + if parsed is None: + raise ValueError( + f"{name} must use YYYYMMDD, YYYY-MM-DD, or an ISO timestamp" + ) + if used_date_only and end_of_day: + parsed = parsed.replace(hour=23, minute=59, second=59) + return parsed.strftime("%Y-%m-%d %H:%M:%S") + + +def _compact_time(value: str) -> str: + return dt.datetime.strptime(value, "%Y-%m-%d %H:%M:%S").strftime( + "%Y%m%d%H%M%S" + ) + + +def build_qmt_parameters( + *, + stock_code: str, + start_time: str, + end_time: str, + period: str = "1d", + asset: float = 1_000_000.0, + account_id: str = "test", + overrides: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Build the documented minimum history/backtest parameter set.""" + normalized_start = _normalize_qmt_time(start_time, "start_time") + normalized_end = _normalize_qmt_time(end_time, "end_time", end_of_day=True) + if normalized_start > normalized_end: + raise ValueError("start_time must not be after end_time") + normalized_stock = str(stock_code).strip().upper() + if not re.fullmatch(r"\d{6}\.(SH|SZ|BJ)", normalized_stock): + raise ValueError("stock_code must use QMT code.market form") + if period != "1d": + raise ValueError( + "quant60 QMT strategy uses completed daily bars; period must be 1d" + ) + numeric_asset = float(asset) + if not math.isfinite(numeric_asset) or numeric_asset <= 0: + raise ValueError("asset must be finite and positive") + normalized_account = str(account_id).strip() + if not normalized_account: + raise ValueError("account_id must be non-empty") + + params: Dict[str, Any] = { + "stock_code": normalized_stock, + "period": period, + "start_time": normalized_start, + "end_time": normalized_end, + "trade_mode": "backtest", + "quote_mode": "history", + "dividend_type": "front_ratio", + "asset": numeric_asset, + "account_id": normalized_account, + "q60_account_id": normalized_account, + } + params.update(_baseline_execution_defaults()) + if overrides: + forbidden = { + "stock_code", + "period", + "start_time", + "end_time", + "trade_mode", + "quote_mode", + "asset", + "account_id", + "q60_account_id", + "benchmark", + } + invalid = forbidden.intersection(overrides) + if invalid: + raise ValueError( + "backtest runner does not allow overrides for " + + ", ".join(sorted(invalid)) + ) + params.update(overrides) + bounded_rates = ( + "max_vol_rate", + "open_tax", + "close_tax", + "open_commission", + "close_commission", + ) + for name in bounded_rates: + value = float(params[name]) + if not math.isfinite(value) or not (0 <= value <= 1): + raise ValueError(f"{name} must be finite and lie in [0, 1]") + params[name] = value + for name in ("slippage", "min_commission"): + value = float(params[name]) + if not math.isfinite(value) or value < 0: + raise ValueError(f"{name} must be finite and non-negative") + params[name] = value + return params + + +def _load_xtquant() -> tuple[Any, Any, str]: + if importlib.util.find_spec("xtquant") is None: + raise QmtUnavailableError( + "xtquant is not installed; use the Python shipped/supported by the " + "licensed QMT research terminal" + ) + try: + xtquant = importlib.import_module("xtquant") + qmttools = importlib.import_module("xtquant.qmttools") + xtdata = importlib.import_module("xtquant.xtdata") + except Exception as exc: + raise QmtUnavailableError(f"xtquant import failed: {exc}") from exc + version = str( + getattr(xtquant, "__version__", None) + or getattr(xtquant, "version", None) + or "unknown" + ) + return qmttools, xtdata, version + + +def preflight_qmt( + strategy_file: os.PathLike[str] | str, + params: Dict[str, Any], + *, + check_data_access: bool = True, +) -> Dict[str, Any]: + """Validate file, runtime and one read-only bar before starting a run.""" + path = Path(strategy_file).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(path) + required = { + "stock_code", + "period", + "start_time", + "end_time", + "trade_mode", + "quote_mode", + } + missing = required.difference(params) + if missing: + raise ValueError("missing QMT parameters: " + ", ".join(sorted(missing))) + if params["trade_mode"] != "backtest" or params["quote_mode"] != "history": + raise ValueError("this runner is backtest/history only") + if params["period"] != "1d": + raise ValueError("this runner requires period=1d") + expected_benchmark = _baseline_execution_defaults()["benchmark"] + if params.get("benchmark") != expected_benchmark: + raise ValueError( + "QMT benchmark must match the configured PIT index " + f"{expected_benchmark}" + ) + if not re.fullmatch( + r"\d{6}\.(SH|SZ|BJ)", + str(params["stock_code"]).strip().upper(), + ): + raise ValueError("invalid QMT stock_code in params") + asset = float(params.get("asset", 0.0)) + if not math.isfinite(asset) or asset <= 0: + raise ValueError("params.asset must be finite and positive") + + qmttools, xtdata, version = _load_xtquant() + run_strategy_file = getattr(qmttools, "run_strategy_file", None) + if not callable(run_strategy_file): + raise QmtUnavailableError("xtquant.qmttools.run_strategy_file is unavailable") + + report: Dict[str, Any] = { + "ok": True, + "strategy_file": str(path), + "xtquant_version": version, + "data_access_checked": bool(check_data_access), + "data_access_ok": None, + } + if check_data_access: + try: + sample = xtdata.get_market_data_ex( + field_list=["time"], + stock_list=[params["stock_code"]], + period=params["period"], + start_time=_compact_time(params["start_time"]), + end_time=_compact_time(params["end_time"]), + count=1, + fill_data=False, + ) + except Exception as exc: + raise QmtUnavailableError( + "QMT read-only data preflight failed; check terminal login, " + f"research entitlement and local data: {exc}" + ) from exc + frame = sample.get(params["stock_code"]) if isinstance(sample, dict) else None + if frame is None or len(frame) == 0: + raise QmtUnavailableError( + "QMT returned no preflight bar; verify entitlement and download " + "history for the requested symbol/date" + ) + report["data_access_ok"] = True + return report + + +def run_qmt_backtest( + strategy_file: os.PathLike[str] | str, + params: Dict[str, Any], + *, + check_data_access: bool = True, +) -> Any: + """Run ``xtquant.qmttools.run_strategy_file`` after fail-fast checks.""" + preflight_qmt( + strategy_file, + params, + check_data_access=check_data_access, + ) + qmttools, _, _ = _load_xtquant() + result = qmttools.run_strategy_file( + str(Path(strategy_file).expanduser().resolve()), + param=dict(params), + ) + if result is None: + raise RuntimeError("QMT strategy returned no result") + return result + + +def _main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("strategy_file") + parser.add_argument("--stock-code", required=True) + parser.add_argument("--start", required=True) + parser.add_argument("--end", required=True) + parser.add_argument("--period", default="1d") + parser.add_argument("--asset", type=float, default=1_000_000.0) + parser.add_argument("--account-id", default="test") + parser.add_argument( + "--skip-data-preflight", + action="store_true", + help="Only for offline test doubles; real runs should keep preflight enabled.", + ) + args = parser.parse_args(argv) + params = build_qmt_parameters( + stock_code=args.stock_code, + start_time=args.start, + end_time=args.end, + period=args.period, + asset=args.asset, + account_id=args.account_id, + ) + result = run_qmt_backtest( + args.strategy_file, + params, + check_data_access=not args.skip_data_preflight, + ) + summary = { + "result_type": type(result).__name__, + "has_backtest_index": callable(getattr(result, "get_backtest_index", None)), + } + print(json.dumps(summary, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3f96904 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "quant60" +version = "0.1.0" +description = "Portable, auditable A-share quantitative trading baseline" +requires-python = ">=3.10" +dependencies = [] + +[project.scripts] +quant60 = "quant60.cli:main" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.unittest] +start-directory = "tests" diff --git a/requirements/base.txt b/requirements/base.txt new file mode 100644 index 0000000..bbe5eb6 --- /dev/null +++ b/requirements/base.txt @@ -0,0 +1,3 @@ +# The portable quant60 core intentionally has no third-party runtime dependency. +# Install the local package itself from the repository root: +-e . diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 0000000..bbb8891 --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,4 @@ +-r base.txt + +jsonschema==4.25.0 +pytest==8.4.1 diff --git a/requirements/jqdata.txt b/requirements/jqdata.txt new file mode 100644 index 0000000..c82eca2 --- /dev/null +++ b/requirements/jqdata.txt @@ -0,0 +1,4 @@ +-r base.txt + +# Official PyPI release verified 2026-07-26. +jqdatasdk==1.9.8 diff --git a/requirements/qmt.txt b/requirements/qmt.txt new file mode 100644 index 0000000..c94b979 --- /dev/null +++ b/requirements/qmt.txt @@ -0,0 +1,8 @@ +# Do not add xtquant/qmttools here. +# +# XtQuant is proprietary software distributed with the user's licensed +# MiniQMT/QMT installation. It must not be downloaded from an unofficial +# package index, copied from another account, or vendored into this repository. +# +# The external XtTrader/qmttools environment may install the portable project: +-r base.txt diff --git a/requirements/research-py312.txt b/requirements/research-py312.txt new file mode 100644 index 0000000..2023eee --- /dev/null +++ b/requirements/research-py312.txt @@ -0,0 +1,14 @@ +# Research environment contract: CPython 3.12.x only. +# Verified together on macOS ARM64 with CPython 3.12.13 on 2026-07-26. +# Pin the numerical/workflow stack because pyqlib's own requirements are +# intentionally broad and otherwise resolve differently over time. +# Verify with: python3.12 -c "import sys; assert sys.version_info[:2] == (3, 12)" +-r base.txt + +numpy==2.5.1 +pandas==2.3.3 +scipy==1.18.0 +lightgbm==4.7.0 +cvxpy==1.9.2 +mlflow==3.14.0 +pyqlib==0.9.7 diff --git a/runbooks/INCIDENTS.md b/runbooks/INCIDENTS.md new file mode 100644 index 0000000..55e8ef1 --- /dev/null +++ b/runbooks/INCIDENTS.md @@ -0,0 +1,209 @@ +# Incident and safe-mode runbook + +This runbook describes operator actions. The current scaffold does not implement a production supervisor, automatic broker kill switch or durable reconciliation service. Do not mistake these procedures for automated guarantees. + +## Universal first response + +For any unexplained data, order, cash or position state: + +1. stop generating and submitting **new opening orders**; +2. do not blindly cancel or resubmit—first query the broker for current orders and fills; +3. capture local time, broker/source time, platform version, run/config/data identifiers and the last known ledger hash; +4. export a fresh read-only broker snapshot; +5. preserve logs and raw callbacks locally with credentials/account identifiers protected; +6. classify whether existing risk-reduction orders may continue; if uncertain, require human approval; +7. recover only after the exit condition in this runbook is met. + +“Restart until it works” is not a recovery method. A restart can duplicate orders when submission succeeded but the acknowledgement was lost. + +## Severity + +| Severity | Examples | Posture | +| --- | --- | --- | +| S0 | Credential exposure, unexplained live position/cash, suspected duplicate live order, wrong account | Stop all strategy submissions; broker/operator intervention | +| S1 | Broker disconnect during active orders, unknown order state, reconciliation difference, stale live data | Block new orders; query and reconcile | +| S2 | Platform backtest failure, Qlib dependency/model failure, synthetic test regression | No production release; live system unaffected if isolated | +| S3 | Documentation, optional report or non-critical metric failure | Record and repair before next release | + +## Data stale, missing or inconsistent + +Symptoms: + +- latest bar/session absent; +- provider and canonical hashes unexpectedly differ; +- timestamps are in the future or timezone is unknown; +- suspended sessions were forward-filled without an explicit rule. + +Actions: + +1. block target publication; +2. identify the last complete immutable data version; +3. compare provider fetch time, event time and available time; +4. rerun quality checks without mutating the prior snapshot; +5. if using an older snapshot for risk reduction, mark the decision `STALE_DATA_RISK_REDUCTION_ONLY`. + +Exit: a new immutable snapshot passes quality checks and the same input replays the same signal/target. Never patch a historical snapshot in place. + +## QMT/MiniQMT disconnect + +Symptoms: + +- native API import works but query calls fail; +- callbacks stop; +- QMT terminal logs out or session reconnects; +- order request timed out without an acknowledgement. + +Actions: + +1. block new submissions; +2. keep the client/terminal state and logs; do not immediately allocate a new session ID and resend; +3. restore terminal login/connectivity; +4. query assets, positions, orders and fills for the full affected window; +5. match by local idempotency key, broker order ID, symbol/side/quantity/time; +6. classify every pending local order as broker-known terminal/open or `UNKNOWN`; +7. escalate any `UNKNOWN` to a human/broker before resubmission. + +Exit: no unknown order, position/cash reconcile, callback stream is fresh, and a read-only preflight passes. + +## Lost acknowledgement or possible duplicate order + +Actions: + +1. never assume a timeout means rejection; +2. query broker orders and fills first; +3. search using client strategy/order labels plus symbol, side, quantity and time; +4. if one broker order matches, bind its broker ID and continue from broker state; +5. if multiple match, stop the affected symbol/account and escalate; +6. resubmit only after proving no live/open/filled broker order represents the intent. + +Exit: a one-to-one mapping exists or the operator explicitly retires the intent. + +## Callback duplicate, out of order or unknown state + +Actions: + +1. preserve the raw callback and its receive/source timestamps; +2. deduplicate by broker event identity or a documented content hash; +3. rebuild state from broker query plus the complete event stream; +4. do not force a terminal local order backwards to an earlier state; +5. map unmapped broker codes to `UNKNOWN`, not the nearest convenient status. + +Exit: normalized event history is monotonic/idempotent and equals the broker's current query. + +## Reconciliation difference + +Compare four sets independently: + +```text +cash +positions and sellable quantities +orders +fills and fees +``` + +Actions: + +1. block new opening orders; +2. freeze the local ledger and snapshot it; +3. determine whether the difference is timing, fee, corporate action, manual trade, callback loss or wrong account; +4. append an explicit correction/reconciliation event; never edit prior ledger lines; +5. require human review for manual trades, unexplained fees or position differences. + +Exit: zero unexplained difference and a signed reconciliation report. A balancing “miscellaneous P&L” line is not an explanation. + +## Price limit, suspension or zero liquidity + +Actions: + +1. treat the position as held/frozen; never mark it sold because the target is zero; +2. stop repeated cancel/resubmit loops; +3. retain exposure in risk and cash calculations; +4. schedule a new decision only when a valid quote and broker sellable quantity exist; +5. record opportunity cost separately from explicit transaction cost. + +Exit: valid tradability returns or the target is deliberately withdrawn. + +## Insufficient cash or T+1 rejection + +Actions: + +1. refresh broker cash, positions, sellable quantity, open orders and fills; +2. include reserved cash/open buy orders and minimum commission; +3. do not sell today's purchase to fund another order; +4. reduce/cancel unsubmitted buys before modifying risk-reduction sells; +5. record the target-to-order shortfall reason. + +Exit: pre-trade calculation based on the fresh broker snapshot passes. + +## Ledger integrity or disk failure + +Symptoms: + +- hash-chain verification fails; +- JSONL is truncated; +- disk is full; +- atomic replacement did not complete. + +Actions: + +1. stop submission; +2. preserve the corrupted file and filesystem diagnostics; +3. verify the last known good immutable copy; +4. query broker truth for the entire uncertain interval; +5. reconstruct a new ledger by replay, retaining a link to the incident and old head hash; +6. never delete or hand-edit the corrupt original. + +Exit: reconstructed ledger verifies and reconciles to broker truth; storage monitoring and free space are restored. + +## Clock or timezone drift + +Actions: + +1. block time-sensitive target/order publication; +2. record OS, platform and broker times; +3. restore trusted time synchronization; +4. recompute `as_of` and executable-window eligibility; +5. discard—not rename—any decision that used future or ambiguous data, retaining it as failed evidence. + +Exit: clocks are within the release tolerance and a fresh decision is computed. + +## Model, optimizer or Qlib failure + +Actions: + +1. do not silently substitute a different model or equal weight; +2. mark the decision `NO_TRADE` or, when separately approved, `RISK_REDUCTION_ONLY`; +3. preserve model/config/data versions and traceback; +4. reproduce in the isolated Python 3.12 research environment; +5. require the normal champion promotion process before restoring. + +Exit: the approved model/run reproduces and its outputs pass schema/risk gates. + +## Credential exposure + +Treat as S0. Stop order automation, rotate/revoke the credential, inspect Git history/artifacts/logs and notify the provider/broker when appropriate. Removing a file without rotation is insufficient. Follow [`docs/SECURITY.md`](../docs/SECURITY.md). + +## Evidence template + +Save an incident record outside credential-bearing logs: + +```text +incident_id: +severity: +opened_at / closed_at: +account_hash: +engine/mode/version: +git_commit: +config_hash / data_version / rules_version: +last_good_ledger_hash: +symptom: +orders/fills at risk: +containment: +broker queries performed: +root cause: +recovery evidence: +operator: +follow-up tests: +``` + +An incident remains open while any affected broker order or position is `UNKNOWN`. diff --git a/runbooks/PLATFORM_DEPLOYMENT.md b/runbooks/PLATFORM_DEPLOYMENT.md new file mode 100644 index 0000000..3ad5aa7 --- /dev/null +++ b/runbooks/PLATFORM_DEPLOYMENT.md @@ -0,0 +1,551 @@ +# Quant OS platform deployment and verification runbook + +从仓库中的 `quant-os` 项目根目录运行命令。本文没有任何命令会启用真实订单。 +Quant OS 是项目名；`quant60` 是第一版 A 股 Baseline 内核和 CLI namespace。 + +当前结论是 `NOT_BASELINE_60`。最近一次标准库套件为 211 tests、OK、 +3 个可选路径 skip。CPython 3.12 + `pyqlib==0.9.7` 的 native momentum +和 Alpha158/LightGBM/Recorder fixture 已真实跑通；JoinQuant、QMT 与 broker +中，JoinQuant 已有一次 2024 全年真实 hosted smoke，QMT 与 broker 尚无 +真实运行证据；聚宽仍缺完整导出与同输入 peer。 + +## 0. One-command local verification + +要求 Python ≥3.10。核心路径无第三方运行依赖： + +```bash +make local +``` + +等价入口： + +```bash +bash scripts/validate_all.sh +``` + +流程包括： + +1. standard-library unit/contract suite； +2. JoinQuant/QMT hosted bundle； +3. synthetic execution smoke + exact manifest verifier； +4. synthetic research-to-target + research manifest verifier； +5. JoinQuant/QMT mock parity export + verifier； +6. capability probe； +7. Colab notebook syntax check + local cell execution。 + +`make local` 成功是本地代码证据，不是平台或 Gate 证据。 + +## 1. Build hosted single-file artifacts + +JoinQuant 和 QMT built-in 不能假设已安装本地 `quant60` 包。生成内联 +Python 3.6-compatible portable core 的单文件： + +```bash +python3 tools/bundle_platforms.py +``` + +输出： + +```text +dist/joinquant_strategy.py +dist/qmt_builtin_strategy.py +dist/bundle_manifest.json +``` + +manifest 把两个 bundle 绑定到相同 `portable_core.py` SHA-256。QMT bundle +另做 legacy source compatibility 检查。 + +静态 capability： + +```bash +PYTHONPATH=src:. python3 tools/capability_probe.py +``` + +只有在明确允许导入 proprietary/optional modules 的环境中才用： + +```bash +PYTHONPATH=src:. python3 tools/capability_probe.py --deep +``` + +probe 不执行平台回测，也不发送 broker mutation。 + +## 2. JQData authorized snapshot to local PIT backtest + +### Prerequisites + +- 用户授权的 JQData 账号和对应数据权益； +- provider licence 允许的本地/Colab 保存方式； +- CPython 3.12 独立环境。 + +凭据仅由 environment 或无回显 prompt 注入，不能写入 Git、OB、notebook +source/output 或 snapshot。 + +```bash +python3.12 -m venv .venv-jqdata +source .venv-jqdata/bin/activate +python -m pip install -r requirements/jqdata.txt +``` + +下载中证 500 示例： + +```bash +PYTHONPATH=src:. python tools/jqdata_snapshot.py \ + --index 000905.XSHG \ + --start 2024-01-02 \ + --end 2024-12-31 \ + --output data/snapshots/csi500-2024 +``` + +adapter 使用 `get_bars` 保存未复权 OHLCV/money、factor、`pre_close`、 +`high_limit`、`low_limit`、`paused`，通过 `get_extras("is_st", ...)` +保存 PIT ST 状态，并逐交易日获取历史指数成员。`skip_paused=False` 保留 +停牌日。请求的 `end` 必须早于 Asia/Shanghai 的 retrieval date，确保日线在 +官方 24:00 完成后才被标为 `T_CLOSE_COMPLETE`；字段、日期、ST 或成员覆盖 +不完整即 fail closed。manifest 记录 query、provider version、retrieval +time、内容 hash 与 `data_version`，不记录 credential。 + +显式运行 semantic verifier： + +```bash +PYTHONPATH=src:. python -c \ + "from quant60.data_snapshot import verify_data_snapshot; verify_data_snapshot('data/snapshots/csi500-2024/manifest.json')" +``` + +verifier 要求： + +- 无 incomplete marker； +- 目录是精确 artifact set； +- 文件与 canonical row hash 正确； +- bar/membership 可以重建为合法领域对象； +- quality summary、query semantic 和 `data_version` 可重算。 +- `is_st` 是严格布尔值并覆盖所有 member-date；缺失不能当作非 ST。 + +运行动态 PIT 成分本地事件回测并验证 release artifacts： + +```bash +PYTHONPATH=src:. python -m quant60 snapshot-backtest \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json \ + --output artifacts/csi500-2024-backtest + +PYTHONPATH=src:. python -m quant60 verify-manifest \ + artifacts/csi500-2024-backtest/manifest.json +``` + +生成单日 Signal/Target/Order Delta： + +```bash +PYTHONPATH=src:. python -m quant60 snapshot-decision \ + --snapshot data/snapshots/csi500-2024/manifest.json \ + --config configs/baseline.json \ + --as-of 2024-12-31 \ + --equity 1000000 \ + --output artifacts/csi500-2024-decision + +PYTHONPATH=src:. python -m quant60 verify-snapshot-decision \ + artifacts/csi500-2024-decision/manifest.json +``` + +生产账户输入应把 `--equity` 替换成 `--broker-snapshot`。broker snapshot +必须满足 schema、日期/新鲜度、持仓/可卖量和 no-unreconciled-open-order +要求；否则 fail closed。 + +这个 backtest 用 raw price 执行，以每个 decision `as_of` 的 raw close + +factor 构造 point-in-time front-adjusted momentum；使用 provider daily +paused/limits/pre-close/is_st、prior-session volume、T+1、百股单位和显式 +费用。ST 不进入新目标；已持有 ST 只允许卖出，停牌持仓冻结。 +它仍是本地 fill model，不是 JoinQuant/QMT 真实成交证据。 + +## 3. Colab + +打开 `notebooks/quant_os_colab.ipynb`，默认执行全部 cell。默认流程 clone +代码后运行测试、bundle、两条 synthetic pipeline 和 verifier，不需要账号。 + +可选： + +- `RUN_JQDATA=True`：无回显获取 credential，生成 snapshot、PIT backtest + 和 decision； +- `RUN_QLIB=True`：仅在 CPython 3.12 安装固定 Qlib stack 并跑 native + fixture。 + +禁止把 password/token/credential clone URL 写进 cell。不要保存带 secret、 +授权数据或账户信息的 notebook output；完成后清除 runtime。 + +本地验证 notebook： + +```bash +python3 tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb +python3 tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb --execute +``` + +## 4. JoinQuant hosted backtest + +### Prerequisites + +- 用户已登录 JoinQuant； +- project/data entitlement； +- 已记录 backtest dates、capital、benchmark 和预期 evidence path。 + +步骤： + +1. 运行 `python3 tools/bundle_platforms.py`； +2. 新建 JoinQuant 策略； +3. 把 `dist/joinquant_strategy.py` 完整上传/粘贴； +4. 保留 `set_option("avoid_future_data", True)` 和 + `set_option("use_real_price", True)`； +5. 先选 backtest，不直接提升 simulation； +6. 记录 runtime/data version、dates、capital、benchmark、 + fee/slippage 和 bundle manifest hash； +7. 导出 `g.quant60_last_plan`、orders/fills、logs 和完整结果。 + +canonical weekly clock 是 ISO 周第一实际交易日完整 close，紧接着下一实际 +交易日 open。wrapper 以 daily-open state machine 保存 close token，因此 +单交易日长假周会跨到下一周执行。它以 `context.previous_date` 查询 PIT +指数成员和 `get_extras(..., "is_st")`，用批量 +`history(..., fq="pre")` 读取已完成日线；卖单先行，以 `order_target` +提交 `current + executable delta`，并在提交前刷新 current data gate。 +手续费、滑点和 10% 最大成交量占比均在策略内显式设置。 +科创板订单使用当日价格上下限作为 `LimitOrderStyle` 最坏价保护；共享核心 +不会生成不足 200 股的普通申报，并保留不足 200 股余额一次性卖清的例外。 + +2026-07-26 的已完成参考 run 见 +[`JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md`](../docs/JOINQUANT_HOSTED_EVIDENCE_2026-07-26.md)。 +它可以用于验证上传与运行步骤，但不能替代步骤 7 的完整导出。 + +JoinQuant 拥有 hosted fill semantics。要做 G9，必须再保存和本地可比较的 +close arrays 或 hash；“平台显示回测成功”本身不够。 + +rollback：停止 hosted simulation，保留失败 run/export，重新部署上一个已 +review 且 manifest hash 已知的 bundle。不要直接编辑 generated file；修改 +wrapper/core 后重新测试和 bundle。 + +## 5. QMT built-in Python backtest + +### Prerequisites + +- 券商授权的 QMT research terminal 和历史数据； +- QMT Python editor/backtest； +- 已记录 client/build、broker plugin 和 data version。 + +步骤： + +1. 运行 `python3 tools/bundle_platforms.py`； +2. 在 QMT 导入 `dist/qmt_builtin_strategy.py`； +3. 选择 `1d` 和 **backtest**； +4. 主图/benchmark 选择 `000905.SH`，在 UI 设置 capital/date range； +5. 在回测设置中把最大成交量占比设为 **10%**； +6. 确认“中证 500”历史板块成员数据以及历史 ST 数据已下载且账号有权限； +7. 用一只已知历史 ST 股票做 `get_his_st_data` 正例探针； +8. 运行并导出结果、日志、UI 配置截图和 `bundle_manifest.json`。 + +QMT built-in 文档环境是 Python 3.6，所以上传 bundle，不安装 Python ≥3.10 +本地包。策略状态存放在模块全局 `g`，避免 `ContextInfo` 用户属性在下个 +`handlebar` 回滚。bundle 硬 backtest-only；参数、账号或环境变量都不能 +开启 live `passorder`。 + +wrapper 用原始毫秒 timetag 查询决策日历史“中证 500”成分，用 +`ContextInfo.get_his_st_data` 排除 ST/*ST/PT。接口缺失、返回格式异常或 +覆盖不足会 fail closed。QMT 可能用空结果同时表示“无数据”或“无 ST”，所以 +正例探针和版本/权限记录是正式证据的硬前置。 + +## 6. qmttools native-Python backtest + +### Prerequisites + +- Windows 上由券商/QMT 分发且匹配 Python 的 `xtquant`； +- 已登录 QMT terminal； +- driver/universe 历史数据已下载且有 entitlement。 + +```powershell +$env:PYTHONPATH = "src;." +python tools/bundle_platforms.py +python -m platforms.qmt_research_runner ` + dist/qmt_builtin_strategy.py ` + --stock-code 000905.SH ` + --start 20240102 ` + --end 20241231 ` + --period 1d ` + --asset 1000000 ` + --account-id test +``` + +runner 固定 `trade_mode=backtest`、`quote_mode=history`，从 baseline +配置固定 benchmark `000905.SH` 与 10% `max_vol_rate`，并在 +`run_strategy_file` 前做只读 one-bar preflight。真实证据禁止 +`--skip-data-preflight`。 + +slippage、volume participation、stamp duty、commission 和 minimum +commission 来自 `configs/baseline.json` 并绑定 hash。QMT 没有独立 transfer +fee 参数，所以 transfer fee 折入 open/close commission；minimum commission +和 fill behavior 仍是必须单列的 engine difference。 + +当前 CLI 只输出摘要，真实 evidence 还必须在授权环境保存完整平台结果。 + +## 7. XtTrader read-only preflight and shadow boundary + +### Prerequisites + +- logged-in MiniQMT/QMT 与 broker permission； +- 本地 `QMT_USERDATA_PATH`、`QMT_SESSION_ID`、`QMT_ACCOUNT_ID`； +- 本地 `QMT_ACCOUNT_HASH_KEY_FILE`（文件内容是独立 secret）； +- broker-distributed `xtquant`； +- 不与其他策略冲突的 numeric session ID。 + +read-only preflight： + +```powershell +$env:PYTHONPATH = "src;." +@' +import os +from adapters.xttrader_live import XtTraderLiveAdapter + +adapter = XtTraderLiveAdapter.from_xtquant( + userdata_path=os.environ["QMT_USERDATA_PATH"], + session_id=int(os.environ["QMT_SESSION_ID"]), + account_id=os.environ["QMT_ACCOUNT_ID"], +) +try: + adapter.connect() + asset = adapter.query_asset() + asset.pop("account_id", None) + print({ + "state": adapter.state, + "asset": asset, + "position_count": len(adapter.query_positions()), + "order_count": len(adapter.query_orders()), + "trade_count": len(adapter.query_trades()), + }) +finally: + adapter.close() +'@ | python - +``` + +以上原始查询只用于诊断；正式只读 shadow 使用下面的账户绑定流程。工具始终 +以 `allow_live_orders=False` 创建 adapter，代码中没有报单或撤单入口。 + +先设置本机 secret/path： + +```powershell +$env:PYTHONPATH = "src;." +$env:QMT_USERDATA_PATH = "C:\local\qmt\userdata_mini" +$env:QMT_ACCOUNT_ID = "" +$env:QMT_SESSION_ID = "" +$env:QMT_ACCOUNT_HASH_KEY_FILE = "C:\local\quant-os\account-hash.key" +$env:DECISION_DATE = "" +``` + +首次只能在 decision 所声明的唯一下一交易日 +Asia/Shanghai `[09:00,09:30)` 盘前窗口内运行： + +```powershell +python tools/qmt_shadow_plan.py plan ` + --decision artifacts\latest-decision\manifest.json ` + --output artifacts\qmt-shadow-bootstrap +``` + +若账户/资产/持仓/委托/成交查询均可信，首次仍会以 +`DECISION_NOT_ACCOUNT_BOUND` 返回预期退出码 2，同时写出： + +```text +broker_observation.json +broker_snapshot.json +manifest.json +shadow_plan.json +``` + +`broker_snapshot.json` 若为 JSON `null`，表示查询事实不可信，禁止继续。 +否则用该 snapshot 在**同一个 `$env:DECISION_DATE`** 重新生成账户绑定决策： + +```powershell +python -m quant60 snapshot-decision ` + --snapshot data\snapshots\latest\manifest.json ` + --config configs\baseline.json ` + --as-of $env:DECISION_DATE ` + --broker-snapshot artifacts\qmt-shadow-bootstrap\broker_snapshot.json ` + --output artifacts\qmt-bound-decision + +python tools/qmt_shadow_plan.py plan ` + --decision artifacts\qmt-bound-decision\manifest.json ` + --output artifacts\qmt-shadow + +python tools/qmt_shadow_plan.py verify ` + --manifest artifacts\qmt-shadow\manifest.json ` + --decision artifacts\qmt-bound-decision\manifest.json +``` + +planner 会检查账户 HMAC、query duration、连接/live-mode 二次状态、账户归属、 +资产恒等式、持仓市值、委托/成交唯一性与数量恒等式；所有观察事实会进入脱敏 +`broker_observation.json` 并被 manifest/verifier 绑定。任一 blocker 会把 +proposed delta 强制为零。 + +`verify` 的 `--decision` 是必需参数，必须指向生成 shadow 时的原始、未改写 +decision manifest；`QMT_ACCOUNT_HASH_KEY_FILE` 必须指向 plan 时使用的既有 +至少 32-byte key。verify 不会创建新 key，缺失、错误或已轮换 key 都失败。 +query duration 的 canonical 上限是 10 秒，资产/持仓市值 +及 `cash + market_value = total_asset` 的绝对容差上限是 1 元； +`--max-query-duration-seconds` 和 `--market-value-tolerance` 只能收紧， +不能放宽。不要为“让本次通过”改大参数。 + +verifier 会用原始 decision 和 read-only observation 做 semantic replay， +独立重建 blocker、status、snapshot-valid、完整 snapshot、cash/portfolio、 +原始目标绑定、board-lot/sellable feasible delta 和 proposed delta，然后 +exact compare。必须把 blockers 被删/新增、target quantity/weight/lot 被改、 +before/after orders 不一致、snapshot 为 `null` 或 observation/plan 派生字段 +不同步视为 artifact 验证失败，而不是人工豁免。READY 必须有 replay 可重建的 +非空 broker snapshot。 + +manifest 中的 `QMT_SHADOW_EVIDENCE_V1` HMAC envelope 还绑定完整 +plan/observation/snapshot 的 canonical 内容与三份磁盘发布文件 byte +SHA-256、原始 decision、planner/engine/`tools/qmt_shadow_plan.py` source 和 +canonical policy。四个发布 JSON（含 manifest）必须保持唯一 writer bytes； +不要 minify、重排 key 或手工“美化”。它阻止不知道 key 的人靠重算普通 hash +重新封装证据。它只证明持 key +的 Quant OS 本地发布与后续完整性，**不是券商对 query 原始事实的 attestation**； +仍必须完成官方 API 失败语义、终端安全、callback 和真实对账认证。 + +它还会对比账户绑定 decision 的 `equity` 与当前 `total_asset`。固定容差是 +`max(1 元, 0.01% × abs(decision equity))`（包含边界），不可通过 CLI +放宽。超差时出现 `DECISION_EQUITY_DRIFT`；应使用本次可信 +`broker_snapshot.json` 重建 decision，禁止继续采用旧绝对目标股数。缺失、 +负数或非有限权益同样失败关闭。 + +这里的 Signal `as_of` 始终是 T 日 15:00 完整收盘。snapshot 会冻结 +JQData 两套交易日历 API 一致性校验后的 session 序列；decision、 +broker snapshot 和 shadow plan 都保存相同的 `next_trading_session` 与 +`first_executable_window`。broker query 的开始和完成必须都落在唯一下一 +session 的 `[09:00,09:30)` 内；周末、同日 15:01、09:30、错误 session +及迟到多日均失败关闭。长假按冻结日历处理，不按 elapsed-day 或 weekday +近似推算。 + +positions 和 sellable quantity 始终以当前 broker query 为准重新做 target +diff；逐 symbol target quantity/weight/lot size 必须 exact 绑定原始 +decision，未完成委托会阻断；这使持仓变化不会复用旧 residual。当前 cash、 +market value、total asset 与 position market value 由 replay 独立核算。 +但无实时执行价就 +不能做可靠购买力检查，所以 `NO_BUYING_POWER_CHECK_WITHOUT_EXECUTION_PRICE` +必须保留，`SHADOW_READY` 仅表示可人工复核，不表示可提交。 + +HMAC key 在 plan 首次运行时以安全随机数生成；Unix 要求 0600。verify 只读 +已有 key。备份时保持访问控制，绝不提交 Git、OB、notebook、日志或聊天。 +更换 key 会使旧账户绑定决策失配并使历史 evidence 无法验签。 + +这些是 local contract，不是 broker certification。当前 adapter +`uncertified / not live-ready`，本文故意没有 live-order command。正式 +QMT 账号还要先探测官方 query API 在“失败”与“空列表”时的实际合同，并完成 +terminal/xtquant build 记录以及 `XtTrade.traded_amount` 对 +`price × quantity` 的真实容差探针。当前 amount 是被认证的 informational +fact，不参与 target/delta。之后还需脱敏 callback replay、restart recovery、 +完整对账、至少 20 trading days shadow 和 G10。当前 `.BJ` mutation 不支持。 + +## 8. Qlib 0.9.7 research + +创建隔离环境： + +```bash +python3.12 -m venv .venv-qlib312 +source .venv-qlib312/bin/activate +python -c "import sys; assert sys.version_info[:2] == (3, 12)" +python -m pip install -r requirements/research-py312.txt +python -c "import qlib; assert qlib.__version__ == '0.9.7'" +``` + +### Native momentum fixture + +```bash +python tools/build_qlib_tiny_fixture.py artifacts/qlib-fixture --days 80 +PYTHONPATH=src:. python -m platforms.qlib_runner \ + --provider-uri artifacts/qlib-fixture \ + --market csi300 \ + --benchmark SH000300 \ + --start 2024-02-01 \ + --end 2024-04-19 \ + --lookback 20 \ + --topk 1 \ + --n-drop 1 \ + --rebalance weekly \ + --output-json artifacts/qlib-native/result.json +``` + +该固定运行时已实际得到 24 条 signal，使用 +`SimulatorExecutor + TopkDropoutStrategy`。结果 JSON 保存 signal/report +稳定 hash、表行列与时间边界，以及从 report 重算的累计收益、最大回撤、基准 +收益、成本和换手，并明确 `investment_value_claim=false`。Qlib 单一 +`limit_threshold` 是 A 股逐日板块/ST 限制的近似，所以只参加 L1/L2 和 +诊断 L6。 + +### Alpha158 + LightGBM + Recorder fixture + +```bash +python tools/build_qlib_tiny_fixture.py \ + artifacts/qlib-fixture-alpha --days 400 +PYTHONPATH=src:. python -m platforms.qlib_runner \ + --workflow alpha158 \ + --provider-uri artifacts/qlib-fixture-alpha \ + --market csi300 \ + --benchmark SH000300 \ + --train-start 2024-04-01 \ + --train-end 2024-10-31 \ + --valid-start 2024-11-01 \ + --valid-end 2025-01-31 \ + --test-start 2025-02-03 \ + --test-end 2025-05-16 \ + --topk 1 \ + --n-drop 1 \ + --experiment-name quant-os-alpha158-fixture \ + --output-json artifacts/qlib-alpha158/result.json +``` + +本项目已经实际跑通该类 fixture，完成 model fit、SignalRecord、 +SigAnaRecord、PortAnaRecord 和 Recorder。runner 使用 +`os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true")` 自动确认 +local MLflow file store；不需要人工先设置该变量。它还读取 +`portfolio_analysis/report_normal_1day.pkl`，保存表级 hash、行列/时间边界、 +重算指标和 artifact path。 + +fixture 只有两只 synthetic 股票和一个 benchmark，所有 performance/IC +都没有投资意义。真实研究必须换成授权、immutable provider，保存 data/model/ +Recorder version，并重新做 OOS；Alpha158 不是 portable-momentum 的 L2 peer， +也不声称 L3/L4 parity。 + +## 9. Mock parity and real G9 + +生成/验证 wrapper mock contract： + +```bash +PYTHONPATH=src:. python tools/export_mock_parity.py \ + export --output artifacts/mock-parity/report.json +PYTHONPATH=src:. python tools/export_mock_parity.py \ + verify artifacts/mock-parity/report.json +``` + +报告逐层比较 score、weight、target quantity 和 order delta，并固定 source +hash。其声明必须保持： + +```text +evidence_class = mock_contract_only +real_platform_pass = false +gate_credit = [] +``` + +因此 mock parity 不计 G9。真实 G9 仍要求真实 JoinQuant/QMT export、同输入 +的 L1—L5 difference report 和至少 20 trading days QMT/broker shadow。 + +## 10. Evidence checklist + +每个真实平台 run 保存： + +- Git commit 或 `DIRTY`； +- `bundle_manifest.json`（适用时）； +- Python/platform/client/broker plugin version； +- config/data/rules/model hash； +- dates、capital、universe、adjustment、fee/fill setting； +- stdout、platform export 和所有 failure； +- evidence class：`synthetic`、`fake`、`provider-snapshot`、 + `real-platform` 或 `broker`； +- compared layer、tolerance 和 reason code； +- 数据许可、脱敏与 artifact retention 记录。 + +真实 JoinQuant smoke 已运行；QMT、broker 以及相同输入跨引擎比较仍未 +运行，G1—G10 全部 `not_passed`。完整层级见 +[`CROSS_ENGINE_CONSISTENCY.md`](../docs/CROSS_ENGINE_CONSISTENCY.md)； +安全规则见 [`SECURITY.md`](../docs/SECURITY.md)。 diff --git a/schemas/broker_snapshot.schema.json b/schemas/broker_snapshot.schema.json new file mode 100644 index 0000000..865307b --- /dev/null +++ b/schemas/broker_snapshot.schema.json @@ -0,0 +1,236 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://boat.local/quant60/schemas/broker_snapshot.schema.json", + "title": "Quant60 broker snapshot", + "description": "A point-in-time broker fact used for pre-trade and reconciliation.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "snapshot_id", + "signal_as_of", + "next_trading_session", + "first_executable_window", + "observation_as_of", + "as_of", + "source_time", + "broker", + "account_hash", + "cash", + "total_asset", + "positions", + "open_orders" + ], + "properties": { + "schema_version": { + "const": "1.0" + }, + "snapshot_id": { + "type": "string", + "minLength": 1 + }, + "as_of": { + "type": "string", + "format": "date-time", + "description": "Broker observation completion time. This may be later than the signal close when the snapshot is queried on the next executable session." + }, + "observation_as_of": { + "type": "string", + "format": "date-time", + "description": "Explicit observation-clock alias for as_of. Account-bound decision paths require the two instants to be equal." + }, + "signal_as_of": { + "type": "string", + "format": "date-time", + "description": "Required completed signal-close clock this snapshot is intended to bind." + }, + "next_trading_session": { + "type": "string", + "format": "date", + "description": "The unique adjacent trading session from the verified decision calendar." + }, + "first_executable_window": { + "$ref": "#/$defs/first_executable_window" + }, + "broker": { + "type": "string", + "minLength": 1 + }, + "account_hash": { + "type": "string", + "minLength": 8, + "description": "One-way account identifier; never store the raw account number." + }, + "cash": { + "type": "number", + "minimum": 0 + }, + "total_asset": { + "type": "number", + "minimum": 0 + }, + "market_value": { + "type": "number", + "minimum": 0 + }, + "positions": { + "type": "array", + "items": { + "$ref": "#/$defs/position" + } + }, + "open_orders": { + "type": "array", + "items": { + "$ref": "#/$defs/open_order" + } + }, + "source_time": { + "type": "string", + "format": "date-time", + "description": "Timestamp assigned to the broker facts; it must not predate the bound signal and must remain fresh relative to the observation clock." + }, + "metadata": { + "type": "object" + } + }, + "$defs": { + "first_executable_window": { + "type": "object", + "additionalProperties": false, + "required": [ + "session", + "timezone", + "start", + "end", + "boundary", + "purpose" + ], + "properties": { + "session": { + "type": "string", + "format": "date" + }, + "timezone": { + "const": "Asia/Shanghai" + }, + "start": { + "type": "string", + "format": "date-time" + }, + "end": { + "type": "string", + "format": "date-time" + }, + "boundary": { + "const": "[start,end)" + }, + "purpose": { + "const": "PRE_OPEN_BROKER_OBSERVATION" + } + } + }, + "canonical_symbol": { + "type": "string", + "pattern": "^[0-9]{6}\\.(XSHG|XSHE|XBSE)$" + }, + "position": { + "type": "object", + "additionalProperties": false, + "required": [ + "symbol", + "quantity", + "sellable_quantity" + ], + "properties": { + "symbol": { + "$ref": "#/$defs/canonical_symbol" + }, + "quantity": { + "type": "integer", + "minimum": 0 + }, + "sellable_quantity": { + "type": "integer", + "minimum": 0 + }, + "average_cost": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "market_value": { + "type": [ + "number", + "null" + ], + "minimum": 0 + } + } + }, + "open_order": { + "type": "object", + "additionalProperties": false, + "required": [ + "order_id", + "broker_order_id", + "symbol", + "side", + "quantity", + "filled_quantity", + "status", + "limit_price", + "order_type", + "time_in_force" + ], + "properties": { + "order_id": { + "type": "string", + "minLength": 1 + }, + "broker_order_id": { + "type": "string", + "minLength": 1 + }, + "symbol": { + "$ref": "#/$defs/canonical_symbol" + }, + "side": { + "enum": [ + "BUY", + "SELL" + ] + }, + "quantity": { + "type": "integer", + "minimum": 1 + }, + "filled_quantity": { + "type": "integer", + "minimum": 0 + }, + "status": { + "enum": [ + "NEW", + "ACCEPTED", + "PARTIALLY_FILLED", + "CANCEL_PENDING", + "UNKNOWN" + ] + }, + "limit_price": { + "type": "number", + "exclusiveMinimum": 0 + }, + "order_type": { + "const": "LIMIT" + }, + "time_in_force": { + "const": "DAY" + } + } + } + } +} diff --git a/schemas/order_event.schema.json b/schemas/order_event.schema.json new file mode 100644 index 0000000..bc952a0 --- /dev/null +++ b/schemas/order_event.schema.json @@ -0,0 +1,259 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://boat.local/quant60/schemas/order_event.schema.json", + "title": "Quant60 normalized ledger replay event", + "description": "Canonical JSON envelope consumed by LedgerProjector for ORDER_STATUS and FILL events. It is the LedgerRecord.as_dict shape emitted by SimBroker; previous_hash and hash are optional only for explicitly ordered, unhashed replay fixtures.", + "$comment": "The canonical payload names are quantity and filled_quantity. Python compatibility aliases accepted by LedgerProjector are intentionally not part of this wire contract.", + "type": "object", + "additionalProperties": false, + "required": [ + "sequence", + "timestamp", + "event_id", + "event_type", + "payload" + ], + "properties": { + "sequence": { + "type": "integer", + "minimum": 1 + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "event_id": { + "type": "string", + "minLength": 1 + }, + "event_type": { + "type": "string", + "enum": [ + "ORDER_STATUS", + "FILL" + ] + }, + "payload": { + "type": "object" + }, + "previous_hash": { + "$ref": "#/$defs/sha256" + }, + "hash": { + "$ref": "#/$defs/sha256" + } + }, + "dependentRequired": { + "previous_hash": [ + "hash" + ], + "hash": [ + "previous_hash" + ] + }, + "oneOf": [ + { + "title": "ORDER_STATUS ledger envelope", + "required": [ + "event_type", + "payload" + ], + "properties": { + "event_type": { + "const": "ORDER_STATUS" + }, + "payload": { + "$ref": "#/$defs/order_status_payload" + } + } + }, + { + "title": "FILL ledger envelope", + "required": [ + "event_type", + "payload" + ], + "properties": { + "event_type": { + "const": "FILL" + }, + "payload": { + "$ref": "#/$defs/fill_payload" + } + } + } + ], + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "canonical_symbol": { + "type": "string", + "pattern": "^[0-9]{6}\\.(XSHG|XSHE|XBSE)$" + }, + "side": { + "type": "string", + "enum": [ + "BUY", + "SELL" + ] + }, + "status": { + "type": "string", + "enum": [ + "NEW", + "ACCEPTED", + "PARTIALLY_FILLED", + "FILLED", + "CANCEL_PENDING", + "CANCELLED", + "REJECTED", + "UNKNOWN" + ] + }, + "decimal": { + "oneOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "^-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$" + } + ] + }, + "nonnegative_decimal": { + "oneOf": [ + { + "type": "number", + "minimum": 0 + }, + { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$" + } + ] + }, + "positive_decimal": { + "oneOf": [ + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "string", + "pattern": "^(?:[1-9][0-9]*(?:\\.[0-9]+)?|0\\.[0-9]*[1-9][0-9]*)$" + } + ] + }, + "order_status_payload": { + "type": "object", + "additionalProperties": false, + "required": [ + "order_id", + "symbol", + "side", + "quantity", + "filled_quantity", + "status" + ], + "properties": { + "order_id": { + "type": "string", + "minLength": 1 + }, + "broker_order_id": { + "type": [ + "string", + "null" + ] + }, + "symbol": { + "$ref": "#/$defs/canonical_symbol" + }, + "side": { + "$ref": "#/$defs/side" + }, + "quantity": { + "type": "integer", + "minimum": 1 + }, + "filled_quantity": { + "type": "integer", + "minimum": 0 + }, + "average_fill_price": { + "$ref": "#/$defs/nonnegative_decimal" + }, + "status": { + "$ref": "#/$defs/status" + }, + "reason": { + "type": [ + "string", + "null" + ] + }, + "submitted_at": { + "type": "string", + "format": "date" + }, + "metadata": { + "type": "object" + } + } + }, + "fill_payload": { + "type": "object", + "additionalProperties": false, + "required": [ + "fill_id", + "order_id", + "symbol", + "side", + "quantity", + "price" + ], + "properties": { + "fill_id": { + "type": "string", + "minLength": 1 + }, + "order_id": { + "type": "string", + "minLength": 1 + }, + "trading_date": { + "type": "string", + "format": "date" + }, + "symbol": { + "$ref": "#/$defs/canonical_symbol" + }, + "side": { + "$ref": "#/$defs/side" + }, + "quantity": { + "type": "integer", + "minimum": 1 + }, + "price": { + "$ref": "#/$defs/positive_decimal" + }, + "commission": { + "$ref": "#/$defs/nonnegative_decimal" + }, + "stamp_duty": { + "$ref": "#/$defs/nonnegative_decimal" + }, + "transfer_fee": { + "$ref": "#/$defs/nonnegative_decimal" + }, + "cash_delta": { + "$ref": "#/$defs/decimal" + } + } + } + } +} diff --git a/schemas/signal.schema.json b/schemas/signal.schema.json new file mode 100644 index 0000000..3e9d081 --- /dev/null +++ b/schemas/signal.schema.json @@ -0,0 +1,88 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://boat.local/quant60/schemas/signal.schema.json", + "title": "Quant60 signal record", + "description": "A point-in-time model output. It is not a target position or an order.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "run_id", + "signal_id", + "as_of", + "symbol", + "horizon_trading_days", + "score", + "model_version", + "data_version" + ], + "properties": { + "schema_version": { + "const": "1.0" + }, + "run_id": { + "type": "string", + "minLength": 1 + }, + "signal_id": { + "type": "string", + "minLength": 1 + }, + "as_of": { + "type": "string", + "format": "date-time" + }, + "symbol": { + "$ref": "#/$defs/canonical_symbol" + }, + "horizon_trading_days": { + "type": "integer", + "minimum": 1 + }, + "score": { + "type": "number" + }, + "expected_excess_return": { + "type": [ + "number", + "null" + ] + }, + "confidence": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "model_version": { + "type": "string", + "minLength": 1 + }, + "data_version": { + "type": "string", + "minLength": 1 + }, + "feature_version": { + "type": [ + "string", + "null" + ] + }, + "metadata": { + "type": "object" + } + }, + "$defs": { + "canonical_symbol": { + "type": "string", + "pattern": "^[0-9]{6}\\.(XSHG|XSHE|XBSE)$", + "examples": [ + "600000.XSHG", + "000001.XSHE", + "830799.XBSE" + ] + } + } +} diff --git a/schemas/target.schema.json b/schemas/target.schema.json new file mode 100644 index 0000000..8922829 --- /dev/null +++ b/schemas/target.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://boat.local/quant60/schemas/target.schema.json", + "title": "Quant60 target-position record", + "description": "A portfolio decision produced from signals. It is not proof of an order or fill.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "run_id", + "decision_id", + "as_of", + "symbol", + "target_weight", + "target_quantity", + "config_hash" + ], + "properties": { + "schema_version": { + "const": "1.0" + }, + "run_id": { + "type": "string", + "minLength": 1 + }, + "decision_id": { + "type": "string", + "minLength": 1 + }, + "as_of": { + "type": "string", + "format": "date-time" + }, + "symbol": { + "$ref": "#/$defs/canonical_symbol" + }, + "target_weight": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "target_quantity": { + "type": "integer", + "minimum": 0 + }, + "lot_size": { + "type": "integer", + "minimum": 1, + "default": 100 + }, + "signal_id": { + "type": [ + "string", + "null" + ] + }, + "constraint_state": { + "type": "string", + "enum": [ + "FEASIBLE", + "REPAIRED", + "RISK_REDUCTION_ONLY", + "NO_TRADE" + ] + }, + "config_hash": { + "type": "string", + "minLength": 1 + }, + "metadata": { + "type": "object" + } + }, + "$defs": { + "canonical_symbol": { + "type": "string", + "pattern": "^[0-9]{6}\\.(XSHG|XSHE|XBSE)$" + } + } +} diff --git a/scripts/validate_all.sh b/scripts/validate_all.sh new file mode 100755 index 0000000..8cdb0c7 --- /dev/null +++ b/scripts/validate_all.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${project_root}" + +python_bin="${PYTHON_BIN:-python3}" + +"${python_bin}" -c \ + 'import sys; assert sys.version_info >= (3, 10), sys.version' +PYTHONPATH=src:. "${python_bin}" -m unittest discover -s tests -p '*.py' -q +"${python_bin}" tools/bundle_platforms.py +PYTHONPATH=src:. "${python_bin}" -m quant60 smoke \ + --config configs/baseline.json \ + --output artifacts/local-smoke +PYTHONPATH=src:. "${python_bin}" -m quant60 verify-manifest \ + artifacts/local-smoke/manifest.json +PYTHONPATH=src:. "${python_bin}" -m quant60 research-smoke \ + --output artifacts/research-smoke +PYTHONPATH=src:. "${python_bin}" -m quant60 verify-research-manifest \ + artifacts/research-smoke/manifest.json +PYTHONPATH=src:. "${python_bin}" tools/export_mock_parity.py \ + export \ + --output artifacts/mock-parity/report.json +PYTHONPATH=src:. "${python_bin}" tools/export_mock_parity.py \ + verify \ + artifacts/mock-parity/report.json +PYTHONPATH=src:. "${python_bin}" tools/capability_probe.py +"${python_bin}" tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb +"${python_bin}" tools/run_colab_notebook.py \ + notebooks/quant_os_colab.ipynb \ + --execute diff --git a/src/quant60/__init__.py b/src/quant60/__init__.py new file mode 100644 index 0000000..1e6f78d --- /dev/null +++ b/src/quant60/__init__.py @@ -0,0 +1,23 @@ +"""Quant60: a dependency-free, auditable A-share research baseline.""" + +from .backtest import BacktestEngine, BacktestResult +from .config import BacktestConfig, load_config +from .domain import Bar, Fill, Order, OrderStatus, Portfolio, Position, Side +from .synthetic import generate_synthetic_bars + +__all__ = [ + "BacktestConfig", + "BacktestEngine", + "BacktestResult", + "Bar", + "Fill", + "Order", + "OrderStatus", + "Portfolio", + "Position", + "Side", + "generate_synthetic_bars", + "load_config", +] + +__version__ = "0.1.0" diff --git a/src/quant60/__main__.py b/src/quant60/__main__.py new file mode 100644 index 0000000..a049ad7 --- /dev/null +++ b/src/quant60/__main__.py @@ -0,0 +1,5 @@ +from .cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/quant60/backtest.py b/src/quant60/backtest.py new file mode 100644 index 0000000..bc1513f --- /dev/null +++ b/src/quant60/backtest.py @@ -0,0 +1,672 @@ +"""Deterministic event-driven local backtest and evidence artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import statistics +import tempfile +from dataclasses import dataclass +from datetime import date +from decimal import Decimal +from pathlib import Path +from typing import Any, Iterable + +from .broker import SimBroker +from .config import BacktestConfig +from .contracts import schema_directory, validate_records +from .domain import Bar, OrderStatus, Portfolio, Side, money +from .ledger import HashChainLedger, canonical_json, jsonable +from .portable_core import build_rebalance_plan +from .risk import check_target_weights, weights_from_positions +from .synthetic import group_bars_by_date + + +def _sha256(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def _atomic_write_bytes(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=str(path.parent), + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_name, path) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise + + +def _engine_source_manifest() -> tuple[str, dict[str, str]]: + package = Path(__file__).resolve().parent + sources = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(package.glob("*.py"), key=lambda item: item.name) + } + return _sha256(sources), sources + + +def _contract_manifest() -> tuple[str, dict[str, str]]: + directory = schema_directory() + names = ( + "signal.schema.json", + "target.schema.json", + "order_event.schema.json", + "broker_snapshot.schema.json", + ) + contracts = { + name: hashlib.sha256((directory / name).read_bytes()).hexdigest() + for name in names + } + return _sha256(contracts), contracts + + +def _strict_json_file(path: Path) -> Any: + def reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant {value}") + + return json.loads( + path.read_text(encoding="utf-8"), + parse_constant=reject_constant, + ) + + +def verify_artifact_manifest( + manifest_path: str | Path, + *, + require_current_source: bool = True, +) -> dict[str, Any]: + """Verify a completed local artifact set and its current source identity.""" + path = Path(manifest_path) + root = path.parent + marker = root / ".quant60-incomplete" + if marker.exists(): + raise ValueError(f"artifact publication is incomplete: {marker}") + manifest = _strict_json_file(path) + if not isinstance(manifest, dict): + raise ValueError("manifest root must be an object") + expected = manifest.get("artifact_sha256") + if not isinstance(expected, dict): + raise ValueError("manifest.artifact_sha256 must be an object") + engine_sources = manifest.get("engine_sources") + if not isinstance(engine_sources, dict) or not engine_sources: + raise ValueError("manifest.engine_sources must be a non-empty object") + if _sha256(engine_sources) != manifest.get("engine_source_sha256"): + raise ValueError("engine_sources aggregate hash mismatch") + if ( + engine_sources.get("portable_core.py") + != manifest.get("portable_core_sha256") + ): + raise ValueError("portable_core source hash mismatch") + contract_schemas = manifest.get("contract_schemas") + if not isinstance(contract_schemas, dict) or not contract_schemas: + raise ValueError( + "manifest.contract_schemas must be a non-empty object" + ) + if _sha256(contract_schemas) != manifest.get("contracts_sha256"): + raise ValueError("contract_schemas aggregate hash mismatch") + filenames = { + "signal": "signal.json", + "target": "target.json", + "equity": "equity.json", + "report": "report.json", + "events": "events.jsonl", + } + if set(expected) != set(filenames): + raise ValueError("manifest artifact set is incomplete or unknown") + allowed_entries = set(filenames.values()) | {path.name} + actual_entries = {entry.name for entry in root.iterdir()} + unexpected_entries = sorted(actual_entries - allowed_entries) + missing_entries = sorted(allowed_entries - actual_entries) + if unexpected_entries or missing_entries: + raise ValueError( + "artifact directory is not an exact completed set: " + f"unexpected={unexpected_entries}, missing={missing_entries}" + ) + verified: dict[str, str] = {} + for name, filename in filenames.items(): + artifact = root / filename + actual = hashlib.sha256(artifact.read_bytes()).hexdigest() + if actual != expected[name]: + raise ValueError(f"{name} artifact hash mismatch") + verified[name] = actual + + ledger = HashChainLedger.read_jsonl(root / filenames["events"]) + ledger.verify() + if ledger.head_hash != manifest.get("ledger_head_hash"): + raise ValueError("ledger head does not match manifest") + signals = _strict_json_file(root / filenames["signal"]) + targets = _strict_json_file(root / filenames["target"]) + if not isinstance(signals, list) or not isinstance(targets, list): + raise ValueError("signal and target artifacts must be arrays") + validate_records(signals, "signal.schema.json") + validate_records(targets, "target.schema.json") + validate_records( + ( + record.as_dict() + for record in ledger.records + if record.event_type in {"ORDER_STATUS", "FILL"} + ), + "order_event.schema.json", + ) + + report = _strict_json_file(root / filenames["report"]) + for field in ( + "run_id", + "config_hash", + "data_version", + "engine_source_sha256", + "rules_sha256", + "strategy_sha256", + "contracts_sha256", + "ledger_head_hash", + ): + if report.get(field) != manifest.get(field): + raise ValueError(f"report/manifest {field} mismatch") + + current_source_sha256, unused_sources = _engine_source_manifest() + del unused_sources + source_matches = ( + current_source_sha256 == manifest.get("engine_source_sha256") + ) + if require_current_source and not source_matches: + raise ValueError("current quant60 source does not match run manifest") + current_contracts_sha256, unused_contracts = _contract_manifest() + del unused_contracts + contracts_match = ( + current_contracts_sha256 == manifest.get("contracts_sha256") + ) + if require_current_source and not contracts_match: + raise ValueError("current quant60 schemas do not match run manifest") + return { + "ok": True, + "run_id": manifest.get("run_id"), + "artifacts": verified, + "ledger_records": len(ledger.records), + "ledger_head_hash": ledger.head_hash, + "current_source_matches": source_matches, + "current_contracts_match": contracts_match, + } + + +def _bar_payload(bar: Bar) -> dict[str, Any]: + return { + "date": bar.trading_date.isoformat(), + "symbol": bar.symbol, + "open": format(bar.open, "f"), + "high": format(bar.high, "f"), + "low": format(bar.low, "f"), + "close": format(bar.close, "f"), + "volume": bar.volume, + "previous_close": ( + format(bar.previous_close, "f") + if bar.previous_close is not None + else None + ), + "limit_up": format(bar.limit_up, "f") if bar.limit_up is not None else None, + "limit_down": ( + format(bar.limit_down, "f") if bar.limit_down is not None else None + ), + "suspended": bar.suspended, + } + + +def _metrics( + equity_curve: list[dict[str, Any]], + broker: SimBroker, + initial_cash: float, +) -> dict[str, Any]: + equities = [float(item["equity"]) for item in equity_curve] + returns = [ + equities[index] / equities[index - 1] - 1.0 + for index in range(1, len(equities)) + if equities[index - 1] > 0 + ] + total_return = equities[-1] / initial_cash - 1.0 if equities else 0.0 + annualized_return = ( + (1.0 + total_return) ** (252.0 / max(1, len(returns))) - 1.0 + if 1.0 + total_return > 0 and returns + else total_return + ) + daily_volatility = statistics.stdev(returns) if len(returns) > 1 else 0.0 + annualized_volatility = daily_volatility * math.sqrt(252.0) + mean_return = statistics.fmean(returns) if returns else 0.0 + sharpe = ( + mean_return / daily_volatility * math.sqrt(252.0) + if daily_volatility > 0 + else 0.0 + ) + peak = 0.0 + max_drawdown = 0.0 + for value in equities: + peak = max(peak, value) + if peak > 0: + max_drawdown = min(max_drawdown, value / peak - 1.0) + + filled_quantity = sum(fill.quantity for fill in broker.fills) + submitted_quantity = sum(order.quantity for order in broker.orders.values()) + traded_notional = sum(float(fill.notional) for fill in broker.fills) + total_fees = sum(float(fill.total_fees) for fill in broker.fills) + average_equity = statistics.fmean(equities) if equities else initial_cash + blocked: dict[str, int] = {} + for record in broker.ledger.records: + if record.event_type == "ORDER_BLOCKED": + reason = str(record.payload.get("reason", "UNKNOWN")) + blocked[reason] = blocked.get(reason, 0) + 1 + statuses: dict[str, int] = {} + for order in broker.orders.values(): + statuses[order.status.value] = statuses.get(order.status.value, 0) + 1 + + return { + "trading_days": len(equity_curve), + "initial_cash": initial_cash, + "final_equity": equities[-1] if equities else initial_cash, + "total_return": total_return, + "annualized_return": annualized_return, + "annualized_volatility": annualized_volatility, + "sharpe_zero_rate": sharpe, + "max_drawdown": max_drawdown, + "order_count": len(broker.orders), + "fill_count": len(broker.fills), + "submitted_quantity": submitted_quantity, + "filled_quantity": filled_quantity, + "fill_rate": ( + filled_quantity / submitted_quantity if submitted_quantity else 0.0 + ), + "turnover_on_average_equity": ( + traded_notional / average_equity if average_equity > 0 else 0.0 + ), + "total_fees": total_fees, + "blocked_events": dict(sorted(blocked.items())), + "terminal_order_statuses": dict(sorted(statuses.items())), + } + + +@dataclass(slots=True) +class BacktestResult: + run_id: str + config_hash: str + data_version: str + signals: list[dict[str, Any]] + targets: list[dict[str, Any]] + equity_curve: list[dict[str, Any]] + report: dict[str, Any] + manifest: dict[str, Any] + ledger: HashChainLedger + + def write_artifacts(self, output_dir: str | Path) -> dict[str, Path]: + root = Path(output_dir) + root.mkdir(parents=True, exist_ok=True) + validate_records(self.signals, "signal.schema.json") + validate_records(self.targets, "target.schema.json") + validate_records( + ( + record.as_dict() + for record in self.ledger.records + if record.event_type in {"ORDER_STATUS", "FILL"} + ), + "order_event.schema.json", + ) + marker = root / ".quant60-incomplete" + _atomic_write_bytes(marker, (self.run_id + "\n").encode("ascii")) + payloads = { + "signal": self.signals, + "target": self.targets, + "equity": self.equity_curve, + "report": self.report, + } + paths: dict[str, Path] = {} + for name, payload in payloads.items(): + path = root / f"{name}.json" + encoded = ( + json.dumps( + jsonable(payload), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + _atomic_write_bytes(path, encoded) + paths[name] = path + paths["events"] = self.ledger.write_jsonl(root / "events.jsonl") + + manifest = dict(self.manifest) + artifact_hashes: dict[str, str] = {} + for name, path in sorted(paths.items()): + artifact_hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest() + manifest["artifact_sha256"] = artifact_hashes + manifest["ledger_head_hash"] = self.ledger.head_hash + manifest_path = root / "manifest.json" + manifest_encoded = ( + json.dumps( + jsonable(manifest), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + _atomic_write_bytes(manifest_path, manifest_encoded) + marker.unlink() + paths["manifest"] = manifest_path + return paths + + +class BacktestEngine: + """Reference clock: T completed close decides, T+1 open attempts fills.""" + + def __init__(self, config: BacktestConfig): + config.validate() + self.config = config + + def run(self, bars: Iterable[Bar]) -> BacktestResult: + all_bars = list(bars) + sessions = group_bars_by_date(all_bars) + if not sessions: + raise ValueError("backtest requires at least one bar") + expected_symbols = set(self.config.symbols) + seen_symbols = {bar.symbol for bar in all_bars} + missing = expected_symbols - seen_symbols + if missing: + raise ValueError(f"bars are missing configured symbols: {sorted(missing)}") + for trading_date, session_bars in sessions.items(): + missing_for_session = expected_symbols - set(session_bars) + if missing_for_session: + raise ValueError( + "incomplete market-data session " + f"{trading_date.isoformat()}: {sorted(missing_for_session)}" + ) + + config_payload = self.config.as_dict() + config_hash = _sha256(config_payload) + data_version = _sha256([_bar_payload(bar) for bar in all_bars]) + engine_source_sha256, engine_sources = _engine_source_manifest() + contracts_sha256, contract_schemas = _contract_manifest() + portable_path = Path(__file__).with_name("portable_core.py") + portable_core_sha256 = hashlib.sha256( + portable_path.read_bytes() + ).hexdigest() + strategy_sha256 = _sha256( + { + "version": "portable-momentum-v1", + "parameters": config_payload["strategy"], + "portable_core_sha256": portable_core_sha256, + } + ) + rules_sha256 = _sha256( + { + "version": "synthetic-cn-a-v2", + "fees": config_payload["fees"], + "execution": config_payload["execution"], + "clock": "weekly_first_close_to_next_open", + } + ) + run_id = ( + "q60-" + + _sha256( + { + "config": config_hash, + "data": data_version, + "engine_source": engine_source_sha256, + "strategy": strategy_sha256, + "rules": rules_sha256, + "contracts": contracts_sha256, + } + )[:16] + ) + portfolio = Portfolio(Decimal(str(self.config.initial_cash))) + ledger = HashChainLedger() + broker = SimBroker( + portfolio, + self.config.execution, + self.config.fees, + ledger, + ) + histories: dict[str, list[float]] = { + symbol: [] for symbol in self.config.symbols + } + signals: list[dict[str, Any]] = [] + targets: list[dict[str, Any]] = [] + equity_curve: list[dict[str, Any]] = [] + strategy = self.config.strategy + required_history = strategy.lookback + strategy.skip + 1 + + session_items = list(sessions.items()) + for session_index, (trading_date, session_bars) in enumerate(session_items): + # Purchases filled yesterday become sellable before today's open. + portfolio.start_session() + broker.execute_session(session_bars) + if self.config.execution.cancel_open_orders_at_end: + # The reference execution window is one daily bar. Any + # unfilled residual from yesterday is explicitly cancelled + # before a new decision can be published. + broker.cancel_open_orders( + trading_date, + "DAILY_WINDOW_END", + phase="post_open", + ) + + portfolio.mark(session_bars.values()) + for symbol in self.config.symbols: + bar = session_bars.get(symbol) + if bar is not None: + histories[symbol].append(float(bar.close)) + + equity_curve.append( + { + "date": trading_date.isoformat(), + "cash": float(portfolio.cash), + "market_value": float(portfolio.market_value), + "equity": float(portfolio.equity), + "positions": portfolio.quantities(), + } + ) + ledger.append( + "PORTFOLIO_SNAPSHOT", + equity_curve[-1], + timestamp=f"{trading_date.isoformat()}T15:00:00+08:00", + event_id=f"portfolio:{trading_date.isoformat()}", + ) + + ready = all(len(values) >= required_history for values in histories.values()) + # Canonical cross-engine clock: + # first trading session of each ISO week, after its close; + # execution is the next available session's open. + # Hosted adapters implement the same relation with a daily + # state machine; no fixed weekday is assumed. + first_session_of_week = ( + session_index == 0 + or session_items[session_index - 1][0].isocalendar()[:2] + != trading_date.isocalendar()[:2] + ) + rebalance = ( + ready + and first_session_of_week + and session_index < len(session_items) - 1 + ) + if not rebalance: + continue + + prices = { + symbol: float(session_bars[symbol].close) + for symbol in self.config.symbols + if symbol in session_bars + } + current_weights = weights_from_positions( + portfolio.quantities(), + prices, + float(portfolio.equity), + ) + plan = build_rebalance_plan( + price_history=histories, + current=portfolio.quantities(), + sellable=portfolio.sellable_quantities(), + equity=float(portfolio.equity), + lookback=strategy.lookback, + skip=strategy.skip, + top_n=strategy.top_n, + max_weight=strategy.max_weight, + gross_target=strategy.gross_target, + cash_buffer=strategy.cash_buffer, + lot_size=self.config.execution.lot_size, + ) + risk = check_target_weights( + plan["weights"], + current_weights, + max_single_weight=strategy.max_weight, + max_gross_weight=min( + strategy.gross_target, 1.0 - strategy.cash_buffer + ), + # The smoke permits a full initial deployment. Production + # turnover is a separate, account-specific authorization. + max_one_way_turnover=1.0, + ) + risk.require_approved() + decision_id = f"D-{trading_date.strftime('%Y%m%d')}" + as_of = f"{trading_date.isoformat()}T15:00:00+08:00" + + for symbol in sorted(plan["scores"]): + score = plan["scores"][symbol] + if score is None or not math.isfinite(float(score)): + continue + signals.append( + { + "schema_version": "1.0", + "run_id": run_id, + "signal_id": f"S-{trading_date.strftime('%Y%m%d')}-{symbol}", + "as_of": as_of, + "symbol": symbol, + "horizon_trading_days": strategy.rebalance_every, + "score": float(score), + "expected_excess_return": None, + "confidence": None, + "model_version": "portable-momentum-v1", + "data_version": data_version, + "feature_version": ( + f"close_momentum_{strategy.lookback}_skip_{strategy.skip}" + ), + "metadata": {"selected": plan["weights"].get(symbol, 0.0) > 0}, + } + ) + for symbol in sorted(plan["targets"]): + targets.append( + { + "schema_version": "1.0", + "run_id": run_id, + "decision_id": decision_id, + "as_of": as_of, + "symbol": symbol, + "target_weight": float(plan["weights"].get(symbol, 0.0)), + "target_quantity": int(plan["targets"][symbol]), + "lot_size": self.config.execution.lot_size, + "signal_id": f"S-{trading_date.strftime('%Y%m%d')}-{symbol}", + "constraint_state": "FEASIBLE", + "config_hash": config_hash, + "metadata": { + "gross_weight": risk.gross_weight, + "one_way_turnover": risk.one_way_turnover, + }, + } + ) + ledger.append( + "DECISION", + { + "run_id": run_id, + "decision_id": decision_id, + "as_of": as_of, + "weights": plan["weights"], + "targets": plan["targets"], + "orders": plan["orders"], + "risk_approved": risk.approved, + }, + timestamp=as_of, + event_id=f"decision:{decision_id}", + ) + for symbol, delta in sorted( + plan["orders"].items(), key=lambda item: (item[1] > 0, item[0]) + ): + if not delta: + continue + broker.submit( + symbol, + Side.BUY if delta > 0 else Side.SELL, + abs(int(delta)), + trading_date, + metadata={"run_id": run_id, "decision_id": decision_id}, + ) + + last_date = session_items[-1][0] + broker.cancel_open_orders(last_date, "BACKTEST_END") + ledger.verify() + report = _metrics( + equity_curve, + broker, + float(self.config.initial_cash), + ) + report.update( + { + "run_id": run_id, + "config_hash": config_hash, + "data_version": data_version, + "engine_source_sha256": engine_source_sha256, + "rules_sha256": rules_sha256, + "strategy_sha256": strategy_sha256, + "contracts_sha256": contracts_sha256, + "final_cash": float(portfolio.cash), + "final_positions": portfolio.quantities(), + "ledger_records": len(ledger.records), + "ledger_head_hash": ledger.head_hash, + "evidence_class": "synthetic-local", + "investment_value_claim": False, + } + ) + manifest = { + "schema_version": "1.0", + "run_id": run_id, + "engine": "local", + "mode": "fixture", + "evidence_class": "synthetic", + "config_hash": config_hash, + "data_version": data_version, + "engine_source_sha256": engine_source_sha256, + "engine_sources": engine_sources, + "portable_core_sha256": portable_core_sha256, + "rules_sha256": rules_sha256, + "strategy_sha256": strategy_sha256, + "contracts_sha256": contracts_sha256, + "contract_schemas": contract_schemas, + "strategy_version": "portable-momentum-v1", + "rules_version": "synthetic-cn-a-v2", + "timezone": "Asia/Shanghai", + "signal_clock": "T_CLOSE_COMPLETE", + "first_executable_clock": "T_PLUS_1_OPEN", + "price_adjustment": "synthetic_raw", + "fill_model": "prior_day_volume_capped_open_slippage_v2", + "deterministic": True, + } + return BacktestResult( + run_id=run_id, + config_hash=config_hash, + data_version=data_version, + signals=signals, + targets=targets, + equity_curve=equity_curve, + report=report, + manifest=manifest, + ledger=ledger, + ) diff --git a/src/quant60/broker.py b/src/quant60/broker.py new file mode 100644 index 0000000..398435a --- /dev/null +++ b/src/quant60/broker.py @@ -0,0 +1,368 @@ +"""Deterministic daily-bar execution simulator for A-share constraints.""" + +from __future__ import annotations + +from datetime import date +from decimal import Decimal, ROUND_HALF_UP + +from .config import ExecutionConfig, FeeConfig +from .domain import ( + PRICE_QUANT, + Bar, + Fill, + Order, + OrderStatus, + Portfolio, + Side, + decimal, + money, +) +from .ledger import HashChainLedger + + +class SimBroker: + def __init__( + self, + portfolio: Portfolio, + execution: ExecutionConfig, + fees: FeeConfig, + ledger: HashChainLedger, + ) -> None: + self.portfolio = portfolio + self.execution = execution + self.fees = fees + self.ledger = ledger + self.orders: dict[str, Order] = {} + self.fills: list[Fill] = [] + self._order_sequence = 0 + self._fill_sequence = 0 + # A T+1 open fill may only use liquidity observed by T close. + # Current-day full-session volume/high/low are future information at + # 09:30 and are deliberately excluded from the execution decision. + self._prior_session_volume: dict[str, int] = {} + + def _timestamp(self, trading_date: date, phase: str) -> str: + times = { + "open": "09:30:00", + "post_open": "09:31:00", + "close": "15:00:00", + "cancel": "15:01:00", + } + return f"{trading_date.isoformat()}T{times[phase]}+08:00" + + def submit( + self, + symbol: str, + side: Side, + quantity: int, + submitted_at: date, + metadata: dict[str, object] | None = None, + ) -> Order: + self._order_sequence += 1 + order_id = f"O{self._order_sequence:08d}" + order = Order( + order_id=order_id, + symbol=symbol, + side=side, + quantity=int(quantity), + submitted_at=submitted_at, + metadata=dict(metadata or {}), + ) + self.orders[order_id] = order + position = self.portfolio.position(order.symbol) + full_balance_sell = ( + side == Side.SELL + and int(quantity) == position.quantity + and int(quantity) == position.sellable_quantity + ) + code = order.symbol.split(".", 1)[0] + is_star_market = ( + order.symbol.endswith(".XSHG") + and code.startswith(("688", "689")) + ) + if is_star_market and int(quantity) < 200 and not full_balance_sell: + order.transition( + OrderStatus.REJECTED, + "BELOW_MARKET_MINIMUM", + ) + elif quantity % self.execution.lot_size and not full_balance_sell: + order.transition(OrderStatus.REJECTED, "NOT_BOARD_LOT") + else: + order.transition(OrderStatus.ACCEPTED) + self.ledger.append( + "ORDER_STATUS", + self._order_payload(order), + timestamp=self._timestamp(submitted_at, "close"), + event_id=f"order:{order_id}:{order.status.value}", + ) + return order + + @staticmethod + def _order_payload(order: Order) -> dict[str, object]: + return { + "order_id": order.order_id, + "symbol": order.symbol, + "side": order.side.value, + "quantity": order.quantity, + "filled_quantity": order.filled_quantity, + "average_fill_price": format(order.average_fill_price, "f"), + "status": order.status.value, + "reason": order.status_reason, + "submitted_at": order.submitted_at.isoformat(), + "metadata": order.metadata, + } + + def _fees( + self, symbol: str, side: Side, quantity: int, price: Decimal + ) -> tuple[Decimal, Decimal, Decimal]: + notional = price * quantity + commission = max( + decimal(self.fees.minimum_commission), + notional * decimal(self.fees.commission_rate), + ) + stamp = ( + notional * decimal(self.fees.stamp_duty_rate) + if side == Side.SELL + else Decimal("0") + ) + transfer = notional * decimal(self.fees.transfer_fee_rate) + return money(commission), money(stamp), money(transfer) + + def _fill_price(self, order: Order, bar: Bar) -> Decimal: + slip = decimal(self.execution.slippage_bps) / Decimal("10000") + if order.side == Side.BUY: + price = bar.open * (Decimal("1") + slip) + if bar.limit_up is not None: + price = min(price, bar.limit_up) + else: + price = bar.open * (Decimal("1") - slip) + if bar.limit_down is not None: + price = max(price, bar.limit_down) + return price.quantize(PRICE_QUANT, rounding=ROUND_HALF_UP) + + def _affordable_buy_quantity( + self, order: Order, desired: int, price: Decimal + ) -> int: + lot = self.execution.lot_size + approximate = int(self.portfolio.cash / price) // lot * lot + quantity = min(desired, approximate) + while quantity > 0: + commission, stamp, transfer = self._fees( + order.symbol, order.side, quantity, price + ) + required = price * quantity + commission + stamp + transfer + if required <= self.portfolio.cash: + return quantity + quantity -= lot + return 0 + + def _blocked_reason(self, order: Order, bar: Bar) -> str | None: + if bar.suspended: + return "SUSPENDED" + # At the opening boundary only the open and today's published price + # limits are known. A buy at limit-up / sell at limit-down is + # conservatively rejected without consulting the future daily range. + if ( + order.side == Side.BUY + and bar.limit_up is not None + and bar.open >= bar.limit_up + ): + return "LOCKED_LIMIT_UP" + if ( + order.side == Side.SELL + and bar.limit_down is not None + and bar.open <= bar.limit_down + ): + return "LOCKED_LIMIT_DOWN" + return None + + def _record_block( + self, order: Order, bar: Bar, reason: str + ) -> None: + self.ledger.append( + "ORDER_BLOCKED", + { + "order_id": order.order_id, + "symbol": order.symbol, + "reason": reason, + "remaining_quantity": order.remaining_quantity, + }, + timestamp=self._timestamp(bar.trading_date, "open"), + event_id=( + f"block:{order.order_id}:{bar.trading_date.isoformat()}:{reason}" + ), + ) + + def execute_session(self, bars: dict[str, Bar]) -> list[Fill]: + if not bars: + return [] + dates = {bar.trading_date for bar in bars.values()} + if len(dates) != 1: + raise ValueError("all bars in a session must have the same date") + trading_date = next(iter(dates)) + volume_left = { + symbol: ( + int( + self._prior_session_volume.get(symbol, 0) + * self.execution.participation_rate + ) + // self.execution.lot_size + * self.execution.lot_size + ) + for symbol in bars + } + session_fills: list[Fill] = [] + active = [ + order + for order in self.orders.values() + if order.status + in {OrderStatus.ACCEPTED, OrderStatus.PARTIALLY_FILLED} + and order.submitted_at < trading_date + ] + active.sort( + key=lambda order: ( + 0 if order.side == Side.SELL else 1, + order.order_id, + ) + ) + for order in active: + bar = bars.get(order.symbol) + if bar is None: + self._record_block( + order, + Bar( + trading_date=trading_date, + symbol=order.symbol, + open=Decimal("1"), + high=Decimal("1"), + low=Decimal("1"), + close=Decimal("1"), + volume=0, + suspended=True, + ), + "NO_MARKET_DATA", + ) + continue + reason = self._blocked_reason(order, bar) + if reason is not None: + self._record_block(order, bar, reason) + continue + capacity = volume_left.get(order.symbol, 0) + desired = min(order.remaining_quantity, capacity) + if order.side == Side.SELL: + position = self.portfolio.position(order.symbol) + sellable = position.sellable_quantity + desired = min(desired, sellable) + full_balance_sell = ( + desired == order.remaining_quantity + and desired == position.quantity + and desired == sellable + ) + if not full_balance_sell: + desired = ( + desired + // self.execution.lot_size + * self.execution.lot_size + ) + if desired <= 0: + self._record_block(order, bar, "T_PLUS_ONE") + continue + else: + desired = ( + desired + // self.execution.lot_size + * self.execution.lot_size + ) + price = self._fill_price(order, bar) + if order.side == Side.BUY: + desired = self._affordable_buy_quantity( + order, desired, price + ) + if desired <= 0: + self._record_block(order, bar, "INSUFFICIENT_CASH") + continue + if desired <= 0: + self._record_block(order, bar, "PARTICIPATION_LIMIT") + continue + commission, stamp, transfer = self._fees( + order.symbol, order.side, desired, price + ) + self._fill_sequence += 1 + fill = Fill( + fill_id=f"F{self._fill_sequence:08d}", + order_id=order.order_id, + trading_date=trading_date, + symbol=order.symbol, + side=order.side, + quantity=desired, + price=price, + commission=commission, + stamp_duty=stamp, + transfer_fee=transfer, + ) + self.portfolio.apply_fill(fill) + order.apply_fill(desired, price) + volume_left[order.symbol] -= desired + self.fills.append(fill) + session_fills.append(fill) + self.ledger.append( + "FILL", + { + "fill_id": fill.fill_id, + "order_id": fill.order_id, + "trading_date": fill.trading_date.isoformat(), + "symbol": fill.symbol, + "side": fill.side.value, + "quantity": fill.quantity, + "price": format(fill.price, "f"), + "commission": format(fill.commission, "f"), + "stamp_duty": format(fill.stamp_duty, "f"), + "transfer_fee": format(fill.transfer_fee, "f"), + "cash_delta": format(fill.cash_delta, "f"), + }, + timestamp=self._timestamp(trading_date, "open"), + event_id=f"fill:{fill.fill_id}", + ) + self.ledger.append( + "ORDER_STATUS", + self._order_payload(order), + timestamp=self._timestamp(trading_date, "open"), + event_id=( + f"order:{order.order_id}:{order.status.value}:" + f"{order.filled_quantity}" + ), + ) + self._prior_session_volume = { + symbol: int(bar.volume) for symbol, bar in bars.items() + } + return session_fills + + def cancel_open_orders( + self, + trading_date: date, + reason: str = "EXECUTION_WINDOW_END", + *, + phase: str = "cancel", + ) -> None: + if phase not in {"post_open", "cancel"}: + raise ValueError("cancel phase must be post_open or cancel") + for order in sorted(self.orders.values(), key=lambda item: item.order_id): + if order.status in { + OrderStatus.ACCEPTED, + OrderStatus.PARTIALLY_FILLED, + OrderStatus.UNKNOWN, + }: + order.transition(OrderStatus.CANCEL_PENDING, reason) + self.ledger.append( + "ORDER_STATUS", + self._order_payload(order), + timestamp=self._timestamp(trading_date, phase), + event_id=f"order:{order.order_id}:CANCEL_PENDING", + ) + order.transition(OrderStatus.CANCELLED, reason) + self.ledger.append( + "ORDER_STATUS", + self._order_payload(order), + timestamp=self._timestamp(trading_date, phase), + event_id=f"order:{order.order_id}:CANCELLED", + ) diff --git a/src/quant60/cli.py b/src/quant60/cli.py new file mode 100644 index 0000000..67280e1 --- /dev/null +++ b/src/quant60/cli.py @@ -0,0 +1,299 @@ +"""Command-line entry point for the dependency-free local baseline.""" + +from __future__ import annotations + +import argparse +from datetime import date +import importlib.util +import json +import sys +from pathlib import Path +from typing import Sequence + +from . import __version__ +from .backtest import BacktestEngine, verify_artifact_manifest +from .config import BacktestConfig, load_config +from .experiment import ( + run_research_smoke, + verify_research_manifest, + write_research_artifacts, +) +from .ledger import HashChainLedger, jsonable +from .snapshot_pipeline import ( + build_snapshot_decision, + verify_snapshot_decision, + write_snapshot_decision, +) +from .snapshot_backtest import run_snapshot_backtest +from .synthetic import generate_synthetic_bars + + +def _project_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _default_config_path() -> Path: + return _project_root() / "configs" / "baseline.json" + + +def _load_or_default(path: str | None) -> BacktestConfig: + return load_config(path) if path else BacktestConfig() + + +def _run_backtest(config: BacktestConfig, output: str | Path) -> dict: + bars = generate_synthetic_bars( + config.symbols, + start_date=config.synthetic.start_date, + trading_days_count=config.synthetic.trading_days, + seed=config.synthetic.seed, + base_volume=config.synthetic.base_volume, + ) + result = BacktestEngine(config).run(bars) + paths = result.write_artifacts(output) + return { + "ok": True, + "run_id": result.run_id, + "report": result.report, + "artifacts": {name: str(path.resolve()) for name, path in paths.items()}, + } + + +def _preflight() -> dict: + project = _project_root() + return { + "local": { + "available": True, + "python": sys.version.split()[0], + }, + "qlib": { + "available": importlib.util.find_spec("qlib") is not None, + "required_runtime": "CPython 3.12 + pyqlib==0.9.7", + }, + "jqdatasdk": { + "available": importlib.util.find_spec("jqdatasdk") is not None, + "required_runtime": "authorized JQData account + jqdatasdk==1.9.8", + }, + "cvxpy": { + "available": importlib.util.find_spec("cvxpy") is not None, + "fallback": "dependency-free deterministic optimizer", + }, + "xtquant": { + "available": importlib.util.find_spec("xtquant") is not None, + "required_runtime": "licensed QMT/MiniQMT environment", + }, + "joinquant_bundle": { + "available": (project / "dist" / "joinquant_strategy.py").is_file(), + }, + "qmt_bundle": { + "available": (project / "dist" / "qmt_builtin_strategy.py").is_file(), + }, + "colab_notebook": { + "available": ( + project / "notebooks" / "quant_os_colab.ipynb" + ).is_file(), + }, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="quant60", description=__doc__) + parser.add_argument("--version", action="version", version=__version__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate = subparsers.add_parser( + "validate-config", help="validate a baseline JSON configuration" + ) + validate.add_argument( + "--config", default=str(_default_config_path()), help="configuration path" + ) + + smoke = subparsers.add_parser( + "smoke", help="run the deterministic synthetic local backtest" + ) + smoke.add_argument( + "--config", default=str(_default_config_path()), help="configuration path" + ) + smoke.add_argument("--output", required=True, help="artifact directory") + + backtest = subparsers.add_parser( + "backtest", + help="alias of smoke: run the deterministic synthetic local backtest", + ) + backtest.add_argument( + "--config", default=str(_default_config_path()), help="configuration path" + ) + backtest.add_argument("--output", required=True, help="artifact directory") + + verify = subparsers.add_parser( + "verify-ledger", help="verify an events.jsonl hash chain" + ) + verify.add_argument("path") + + verify_manifest = subparsers.add_parser( + "verify-manifest", + help="verify a completed local artifact manifest and current source", + ) + verify_manifest.add_argument("path") + + research = subparsers.add_parser( + "research-smoke", + help="run the deterministic research-to-target walk-forward experiment", + ) + research.add_argument("--output", required=True, help="artifact directory") + research.add_argument("--trading-days", type=int, default=320) + research.add_argument("--seed", type=int, default=60) + + verify_research = subparsers.add_parser( + "verify-research-manifest", + help="verify a completed research experiment artifact set", + ) + verify_research.add_argument("path") + + snapshot_decision = subparsers.add_parser( + "snapshot-decision", + help=( + "build a PIT portable signal/target from a verified provider " + "snapshot" + ), + ) + snapshot_decision.add_argument("--snapshot", required=True) + snapshot_decision.add_argument( + "--config", + default=str(_default_config_path()), + help="strategy/execution configuration path", + ) + snapshot_decision.add_argument( + "--as-of", + required=True, + type=date.fromisoformat, + ) + funding = snapshot_decision.add_mutually_exclusive_group(required=True) + funding.add_argument("--equity", type=float) + funding.add_argument( + "--broker-snapshot", + help="canonical broker snapshot JSON; open orders must be empty", + ) + snapshot_decision.add_argument("--output", required=True) + + snapshot_backtest = subparsers.add_parser( + "snapshot-backtest", + help=( + "run the local event backtest with PIT membership, raw prices, " + "adjustment factors and provider daily trading fields" + ), + ) + snapshot_backtest.add_argument("--snapshot", required=True) + snapshot_backtest.add_argument( + "--config", + default=str(_default_config_path()), + ) + snapshot_backtest.add_argument("--output", required=True) + + verify_snapshot = subparsers.add_parser( + "verify-snapshot-decision", + help="verify a provider-snapshot decision artifact set", + ) + verify_snapshot.add_argument("path") + + subparsers.add_parser( + "preflight", help="report local and optional platform capabilities" + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + if args.command == "validate-config": + config = load_config(args.config) + payload = {"ok": True, "config": config.as_dict()} + elif args.command in {"smoke", "backtest"}: + payload = _run_backtest(load_config(args.config), args.output) + elif args.command == "verify-ledger": + ledger = HashChainLedger.read_jsonl(args.path) + payload = { + "ok": ledger.verify(), + "records": len(ledger.records), + "head_hash": ledger.head_hash, + } + elif args.command == "verify-manifest": + payload = verify_artifact_manifest(args.path) + elif args.command == "research-smoke": + result = run_research_smoke( + trading_days=args.trading_days, + seed=args.seed, + ) + paths = write_research_artifacts(result, args.output) + payload = { + "ok": True, + "run_id": result["manifest"]["run_id"], + "report": result["report"], + "artifacts": paths, + } + elif args.command == "verify-research-manifest": + payload = verify_research_manifest(args.path) + elif args.command == "snapshot-decision": + broker_snapshot = None + if args.broker_snapshot: + broker_snapshot = json.loads( + Path(args.broker_snapshot).read_text(encoding="utf-8") + ) + result = build_snapshot_decision( + snapshot_manifest=args.snapshot, + config=load_config(args.config), + as_of=args.as_of, + equity=args.equity, + broker_snapshot=broker_snapshot, + ) + paths = write_snapshot_decision(result, args.output) + payload = { + "ok": True, + "run_id": result["manifest"]["run_id"], + "decision": result["decision"], + "artifacts": paths, + } + elif args.command == "verify-snapshot-decision": + payload = verify_snapshot_decision(args.path) + elif args.command == "snapshot-backtest": + result = run_snapshot_backtest( + snapshot_manifest=args.snapshot, + config=load_config(args.config), + ) + paths = result.write_artifacts(args.output) + payload = { + "ok": True, + "run_id": result.run_id, + "report": result.report, + "artifacts": { + name: str(path.resolve()) + for name, path in paths.items() + }, + } + elif args.command == "preflight": + payload = {"ok": True, "capabilities": _preflight()} + else: + raise ValueError(f"unsupported command {args.command}") + except Exception as exc: + print( + json.dumps( + {"ok": False, "error": type(exc).__name__, "message": str(exc)}, + ensure_ascii=False, + sort_keys=True, + ), + file=sys.stderr, + ) + return 2 + print( + json.dumps( + jsonable(payload), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/quant60/config.py b/src/quant60/config.py new file mode 100644 index 0000000..f5072c2 --- /dev/null +++ b/src/quant60/config.py @@ -0,0 +1,263 @@ +"""Validated JSON configuration for local and platform-neutral runs.""" + +from __future__ import annotations + +import json +import math +from dataclasses import asdict, dataclass, field +from datetime import date +from pathlib import Path +from typing import Any + +from .portable_core import normalize_symbol + + +def _finite_number(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be a finite number") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be a finite number") + return result + + +def _exact_int(value: Any, name: str) -> int: + if type(value) is not int: + raise ValueError(f"{name} must be an integer") + return value + + +@dataclass(frozen=True, slots=True) +class FeeConfig: + commission_rate: float = 0.0002 + minimum_commission: float = 5.0 + stamp_duty_rate: float = 0.0005 + transfer_fee_rate: float = 0.00001 + + def validate(self) -> None: + values = { + "commission_rate": self.commission_rate, + "minimum_commission": self.minimum_commission, + "stamp_duty_rate": self.stamp_duty_rate, + "transfer_fee_rate": self.transfer_fee_rate, + } + for name, value in values.items(): + numeric = _finite_number(value, name) + if numeric < 0: + raise ValueError(f"{name} must be finite and non-negative") + + +@dataclass(frozen=True, slots=True) +class ExecutionConfig: + lot_size: int = 100 + participation_rate: float = 0.10 + slippage_bps: float = 2.0 + cancel_open_orders_at_end: bool = True + + def validate(self) -> None: + if _exact_int(self.lot_size, "lot_size") <= 0: + raise ValueError("lot_size must be positive") + participation_rate = _finite_number( + self.participation_rate, + "participation_rate", + ) + if not (0 < participation_rate <= 1): + raise ValueError("participation_rate must lie in (0, 1]") + slippage_bps = _finite_number(self.slippage_bps, "slippage_bps") + if not (0 <= slippage_bps < 10_000): + raise ValueError("slippage_bps must lie in [0, 10000)") + if type(self.cancel_open_orders_at_end) is not bool: + raise ValueError("cancel_open_orders_at_end must be a bool") + + +@dataclass(frozen=True, slots=True) +class UniverseConfig: + mode: str = "pit_index" + index_symbol: str = "000905.XSHG" + qmt_sector_name: str = "中证500" + dedicated_account_required: bool = True + exclude_st: bool = True + + def validate(self) -> None: + if self.mode not in {"pit_index", "fixed"}: + raise ValueError("universe mode must be pit_index or fixed") + normalize_symbol(self.index_symbol) + if not str(self.qmt_sector_name).strip(): + raise ValueError("qmt_sector_name must be non-empty") + if type(self.dedicated_account_required) is not bool: + raise ValueError("dedicated_account_required must be a bool") + if type(self.exclude_st) is not bool: + raise ValueError("exclude_st must be a bool") + if self.mode == "pit_index" and not self.dedicated_account_required: + raise ValueError( + "pit_index mode requires a dedicated strategy account" + ) + + +@dataclass(frozen=True, slots=True) +class StrategyConfig: + lookback: int = 20 + skip: int = 0 + top_n: int = 5 + max_weight: float = 0.20 + gross_target: float = 0.95 + cash_buffer: float = 0.02 + rebalance_every: int = 5 + rebalance_schedule: str = "weekly_first_close" + + def validate(self) -> None: + lookback = _exact_int(self.lookback, "lookback") + skip = _exact_int(self.skip, "skip") + top_n = _exact_int(self.top_n, "top_n") + if lookback <= 0 or skip < 0 or top_n <= 0: + raise ValueError("invalid momentum window/top_n") + max_weight = _finite_number(self.max_weight, "max_weight") + gross_target = _finite_number(self.gross_target, "gross_target") + cash_buffer = _finite_number(self.cash_buffer, "cash_buffer") + if not (0 < max_weight <= 1): + raise ValueError("max_weight must lie in (0, 1]") + if not (0 <= gross_target <= 1): + raise ValueError("gross_target must lie in [0, 1]") + if not (0 <= cash_buffer < 1): + raise ValueError("cash_buffer must lie in [0, 1)") + if _exact_int(self.rebalance_every, "rebalance_every") <= 0: + raise ValueError("rebalance_every must be positive") + if ( + type(self.rebalance_schedule) is not str + or self.rebalance_schedule != "weekly_first_close" + ): + raise ValueError( + "rebalance_schedule must be weekly_first_close" + ) + + +@dataclass(frozen=True, slots=True) +class SyntheticConfig: + start_date: str = "2024-01-02" + trading_days: int = 90 + seed: int = 60 + base_volume: int = 100_000 + + def validate(self) -> None: + if type(self.start_date) is not str: + raise ValueError("start_date must be an ISO date string") + date.fromisoformat(self.start_date) + trading_days = _exact_int(self.trading_days, "trading_days") + _exact_int(self.seed, "seed") + base_volume = _exact_int(self.base_volume, "base_volume") + if trading_days <= 1 or base_volume < 100: + raise ValueError("synthetic data needs >1 day and volume >=100") + + +@dataclass(frozen=True, slots=True) +class BacktestConfig: + initial_cash: float = 1_000_000.0 + symbols: tuple[str, ...] = ( + "600000.XSHG", + "000001.XSHE", + "300750.XSHE", + "830799.XBSE", + ) + universe: UniverseConfig = field(default_factory=UniverseConfig) + fees: FeeConfig = field(default_factory=FeeConfig) + execution: ExecutionConfig = field(default_factory=ExecutionConfig) + strategy: StrategyConfig = field(default_factory=StrategyConfig) + synthetic: SyntheticConfig = field(default_factory=SyntheticConfig) + + def validate(self) -> None: + initial_cash = _finite_number(self.initial_cash, "initial_cash") + if initial_cash <= 0: + raise ValueError("initial_cash must be finite and positive") + normalized = tuple(normalize_symbol(symbol) for symbol in self.symbols) + if not normalized or len(normalized) != len(set(normalized)): + raise ValueError("symbols must be non-empty and unique") + self.universe.validate() + self.fees.validate() + self.execution.validate() + self.strategy.validate() + self.synthetic.validate() + + def as_dict(self) -> dict[str, Any]: + result = asdict(self) + result["symbols"] = list(self.symbols) + return result + + +def _only_known(mapping: dict[str, Any], allowed: set[str], name: str) -> None: + unknown = set(mapping) - allowed + if unknown: + raise ValueError(f"unknown {name} keys: {sorted(unknown)}") + + +def config_from_dict(raw: dict[str, Any]) -> BacktestConfig: + defaults = BacktestConfig() + _only_known( + raw, + { + "initial_cash", + "symbols", + "universe", + "fees", + "execution", + "strategy", + "synthetic", + }, + "config", + ) + nested = { + name: raw.get(name, {}) + for name in ( + "universe", + "fees", + "execution", + "strategy", + "synthetic", + ) + } + for name, value in nested.items(): + if not isinstance(value, dict): + raise ValueError(f"{name} must be an object") + universe_raw = dict(nested["universe"]) + fee_raw = dict(nested["fees"]) + execution_raw = dict(nested["execution"]) + strategy_raw = dict(nested["strategy"]) + synthetic_raw = dict(nested["synthetic"]) + _only_known( + universe_raw, + set(UniverseConfig.__dataclass_fields__), + "universe", + ) + _only_known(fee_raw, set(FeeConfig.__dataclass_fields__), "fees") + _only_known( + execution_raw, set(ExecutionConfig.__dataclass_fields__), "execution" + ) + _only_known( + strategy_raw, set(StrategyConfig.__dataclass_fields__), "strategy" + ) + _only_known( + synthetic_raw, set(SyntheticConfig.__dataclass_fields__), "synthetic" + ) + symbols_raw = raw.get("symbols", defaults.symbols) + if not isinstance(symbols_raw, (list, tuple)) or any( + type(symbol) is not str for symbol in symbols_raw + ): + raise ValueError("symbols must be an array of strings") + config = BacktestConfig( + initial_cash=raw.get("initial_cash", defaults.initial_cash), + symbols=tuple(normalize_symbol(symbol) for symbol in symbols_raw), + universe=UniverseConfig(**universe_raw), + fees=FeeConfig(**fee_raw), + execution=ExecutionConfig(**execution_raw), + strategy=StrategyConfig(**strategy_raw), + synthetic=SyntheticConfig(**synthetic_raw), + ) + config.validate() + return config + + +def load_config(path: str | Path) -> BacktestConfig: + with Path(path).open("r", encoding="utf-8") as handle: + raw = json.load(handle) + if not isinstance(raw, dict): + raise ValueError("configuration root must be an object") + return config_from_dict(raw) diff --git a/src/quant60/contracts.py b/src/quant60/contracts.py new file mode 100644 index 0000000..2bd9166 --- /dev/null +++ b/src/quant60/contracts.py @@ -0,0 +1,223 @@ +"""Dependency-free validation for the repository's JSON contract subset.""" + +from __future__ import annotations + +import json +import math +import re +from datetime import date, datetime +from pathlib import Path +from typing import Any, Iterable, Mapping + + +class ContractValidationError(ValueError): + """A value does not satisfy its versioned JSON contract.""" + + +def _matches_type(value: Any, name: str) -> bool: + if name == "null": + return value is None + if name == "object": + return isinstance(value, dict) + if name == "array": + return isinstance(value, list) + if name == "string": + return isinstance(value, str) + if name == "boolean": + return isinstance(value, bool) + if name == "integer": + return type(value) is int + if name == "number": + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + ) + raise ContractValidationError(f"unsupported schema type {name!r}") + + +def validate_contract( + value: Any, + schema: Mapping[str, Any], + *, + root: Mapping[str, Any] | None = None, + path: str = "$", +) -> None: + """Validate the JSON-Schema 2020-12 subset used by quant60.""" + root_schema = schema if root is None else root + reference = schema.get("$ref") + if reference is not None: + if not str(reference).startswith("#/"): + raise ContractValidationError( + f"{path}: only local schema references are supported" + ) + target: Any = root_schema + for component in str(reference)[2:].split("/"): + key = component.replace("~1", "/").replace("~0", "~") + target = target[key] + validate_contract(value, target, root=root_schema, path=path) + return + + expected_type = schema.get("type") + if expected_type is not None: + names = ( + list(expected_type) + if isinstance(expected_type, list) + else [expected_type] + ) + if not any(_matches_type(value, str(name)) for name in names): + raise ContractValidationError( + f"{path}: expected {names}, got {type(value).__name__}" + ) + if "const" in schema and value != schema["const"]: + raise ContractValidationError( + f"{path}: expected constant {schema['const']!r}" + ) + if "enum" in schema and value not in schema["enum"]: + raise ContractValidationError(f"{path}: value is outside enum") + if ( + value is not None + and "minLength" in schema + and len(value) < int(schema["minLength"]) + ): + raise ContractValidationError(f"{path}: string is too short") + if ( + value is not None + and "pattern" in schema + and re.search(str(schema["pattern"]), value) is None + ): + raise ContractValidationError(f"{path}: string does not match pattern") + if value is not None and "minimum" in schema and value < schema["minimum"]: + raise ContractValidationError(f"{path}: value is below minimum") + if value is not None and "maximum" in schema and value > schema["maximum"]: + raise ContractValidationError(f"{path}: value is above maximum") + if ( + value is not None + and "exclusiveMinimum" in schema + and value <= schema["exclusiveMinimum"] + ): + raise ContractValidationError( + f"{path}: value is not above exclusive minimum" + ) + + value_format = schema.get("format") + if value_format == "date": + try: + date.fromisoformat(value) + except (TypeError, ValueError) as exc: + raise ContractValidationError( + f"{path}: value is not an ISO date" + ) from exc + elif value_format == "date-time" and value is not None: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except (AttributeError, ValueError) as exc: + raise ContractValidationError( + f"{path}: value is not an ISO date-time" + ) from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ContractValidationError( + f"{path}: date-time has no timezone offset" + ) + + if isinstance(value, dict): + missing = [ + key for key in schema.get("required", []) if key not in value + ] + if missing: + raise ContractValidationError( + f"{path}: missing required fields {missing}" + ) + properties = schema.get("properties", {}) + if schema.get("additionalProperties") is False: + unknown = sorted(set(value) - set(properties)) + if unknown: + raise ContractValidationError( + f"{path}: unsupported fields {unknown}" + ) + for key, child_schema in properties.items(): + if key in value: + validate_contract( + value[key], + child_schema, + root=root_schema, + path=f"{path}.{key}", + ) + for trigger, dependencies in schema.get( + "dependentRequired", + {}, + ).items(): + if trigger in value: + absent = [key for key in dependencies if key not in value] + if absent: + raise ContractValidationError( + f"{path}: {trigger} requires {absent}" + ) + + if isinstance(value, list) and "items" in schema: + for index, item in enumerate(value): + validate_contract( + item, + schema["items"], + root=root_schema, + path=f"{path}[{index}]", + ) + + if "oneOf" in schema: + matches = 0 + failures = [] + for candidate in schema["oneOf"]: + try: + validate_contract( + value, + candidate, + root=root_schema, + path=path, + ) + matches += 1 + except ContractValidationError as exc: + failures.append(str(exc)) + if matches != 1: + raise ContractValidationError( + f"{path}: expected one schema branch, got {matches}: {failures}" + ) + + +def schema_directory() -> Path: + directory = Path(__file__).resolve().parents[2] / "schemas" + if not directory.is_dir(): + raise FileNotFoundError(f"quant60 schema directory not found: {directory}") + return directory + + +def load_schema(filename: str) -> dict[str, Any]: + name = Path(filename).name + if name != filename or not name.endswith(".schema.json"): + raise ValueError("schema filename must be a local *.schema.json name") + raw = json.loads( + (schema_directory() / name).read_text(encoding="utf-8"), + parse_constant=lambda constant: (_ for _ in ()).throw( + ValueError(f"non-finite JSON constant {constant}") + ), + ) + if not isinstance(raw, dict): + raise ContractValidationError("schema root must be an object") + return raw + + +def validate_records( + records: Iterable[Mapping[str, Any]], + schema_filename: str, +) -> int: + schema = load_schema(schema_filename) + count = 0 + for count, record in enumerate(records, 1): + validate_contract(record, schema, path=f"$[{count - 1}]") + return count + + +def validate_broker_snapshot(snapshot: Mapping[str, Any]) -> None: + validate_contract( + dict(snapshot), + load_schema("broker_snapshot.schema.json"), + ) diff --git a/src/quant60/costs.py b/src/quant60/costs.py new file mode 100644 index 0000000..8f8e3b0 --- /dev/null +++ b/src/quant60/costs.py @@ -0,0 +1,153 @@ +"""Versioned A-share fee rules and transparent spread/impact forecasts.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from datetime import date + +from .config import FeeConfig +from .domain import Side +from .portable_core import normalize_symbol + + +@dataclass(frozen=True, slots=True) +class DatedFeeRule: + effective_from: date + effective_to: date | None + fees: FeeConfig + version: str + + def __post_init__(self) -> None: + if self.effective_to is not None and self.effective_to < self.effective_from: + raise ValueError("fee effective_to precedes effective_from") + if not str(self.version).strip(): + raise ValueError("fee rule version is required") + self.fees.validate() + + def contains(self, trading_date: date) -> bool: + return self.effective_from <= trading_date and ( + self.effective_to is None or trading_date <= self.effective_to + ) + + +class VersionedFeeSchedule: + def __init__(self, rules: list[DatedFeeRule]) -> None: + if not rules: + raise ValueError("at least one fee rule is required") + ordered = sorted(rules, key=lambda rule: rule.effective_from) + for previous, current in zip(ordered, ordered[1:]): + if previous.effective_to is None: + raise ValueError("open-ended fee rule must be last") + if current.effective_from <= previous.effective_to: + raise ValueError("fee rule effective intervals overlap") + self.rules = tuple(ordered) + + def at(self, trading_date: date) -> DatedFeeRule: + matches = [rule for rule in self.rules if rule.contains(trading_date)] + if len(matches) != 1: + raise ValueError( + f"expected one fee rule for {trading_date.isoformat()}, " + f"found {len(matches)}" + ) + return matches[0] + + +@dataclass(frozen=True, slots=True) +class CostModelParameters: + half_spread_bps: float = 2.0 + impact_coefficient: float = 0.10 + max_participation: float = 0.10 + + def validate(self) -> None: + for name in ("half_spread_bps", "impact_coefficient"): + value = float(getattr(self, name)) + if not math.isfinite(value) or value < 0: + raise ValueError(f"{name} must be finite and non-negative") + participation = float(self.max_participation) + if not math.isfinite(participation) or not (0 < participation <= 1): + raise ValueError("max_participation must lie in (0, 1]") + + +@dataclass(frozen=True, slots=True) +class CostForecast: + symbol: str + side: Side + quantity: int + notional: float + explicit_fees: float + spread_cost: float + impact_cost: float + low: float + base: float + high: float + base_bps: float + participation: float + capacity_breached: bool + fee_version: str + + +def forecast_trade_cost( + *, + symbol: str, + side: Side, + quantity: int, + price: float, + adv_quantity: float, + daily_volatility: float, + fee_rule: DatedFeeRule, + parameters: CostModelParameters = CostModelParameters(), +) -> CostForecast: + parameters.validate() + if not isinstance(side, Side): + raise ValueError("side must be a Side") + if type(quantity) is not int or quantity <= 0: + raise ValueError("quantity must be a positive integer") + numeric = { + "price": float(price), + "adv_quantity": float(adv_quantity), + "daily_volatility": float(daily_volatility), + } + for name, value in numeric.items(): + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive") + notional = numeric["price"] * quantity + fees = fee_rule.fees + commission = max( + float(fees.minimum_commission), + notional * float(fees.commission_rate), + ) + stamp = ( + notional * float(fees.stamp_duty_rate) + if side == Side.SELL + else 0.0 + ) + transfer = notional * float(fees.transfer_fee_rate) + explicit = commission + stamp + transfer + spread = notional * float(parameters.half_spread_bps) / 10_000.0 + participation = quantity / numeric["adv_quantity"] + impact_rate = ( + float(parameters.impact_coefficient) + * numeric["daily_volatility"] + * math.sqrt(participation) + ) + impact = notional * impact_rate + low = explicit + spread * 0.5 + impact * 0.5 + base = explicit + spread + impact + high = explicit + spread * 1.5 + impact * 2.0 + return CostForecast( + symbol=normalize_symbol(symbol), + side=side, + quantity=quantity, + notional=notional, + explicit_fees=explicit, + spread_cost=spread, + impact_cost=impact, + low=low, + base=base, + high=high, + base_bps=base / notional * 10_000.0, + participation=participation, + capacity_breached=participation > parameters.max_participation, + fee_version=fee_rule.version, + ) diff --git a/src/quant60/data_snapshot.py b/src/quant60/data_snapshot.py new file mode 100644 index 0000000..e0b1e0d --- /dev/null +++ b/src/quant60/data_snapshot.py @@ -0,0 +1,467 @@ +"""Canonical local market-data snapshots with quality and hash manifests.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import tempfile +from dataclasses import dataclass +from datetime import date, datetime +from pathlib import Path +from typing import Any, Iterable + +from .ledger import canonical_json, jsonable +from .portable_core import normalize_symbol + + +class DataQualityError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class CanonicalBarRecord: + trading_date: date + symbol: str + open: float + high: float + low: float + close: float + volume: int + money: float + paused: bool + source: str + is_st: bool = False + adjustment: str = "none" + adjustment_factor: float | None = None + previous_close: float | None = None + limit_up: float | None = None + limit_down: float | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "symbol", normalize_symbol(self.symbol)) + if not isinstance(self.trading_date, date): + raise DataQualityError("trading_date must be a date") + for name in ("open", "high", "low", "close"): + value = float(getattr(self, name)) + if not math.isfinite(value) or value <= 0: + raise DataQualityError(f"{name} must be finite and positive") + if not (self.low <= self.open <= self.high): + raise DataQualityError("open must lie within [low, high]") + if not (self.low <= self.close <= self.high): + raise DataQualityError("close must lie within [low, high]") + if type(self.volume) is not int or self.volume < 0: + raise DataQualityError("volume must be a non-negative integer") + if not math.isfinite(float(self.money)) or self.money < 0: + raise DataQualityError("money must be finite and non-negative") + if type(self.paused) is not bool: + raise DataQualityError("paused must be a bool") + if type(self.is_st) is not bool: + raise DataQualityError("is_st must be a bool") + if not str(self.source).strip(): + raise DataQualityError("bar source is required") + if self.adjustment not in {"none", "front", "back"}: + raise DataQualityError("unsupported adjustment") + if self.paused and self.volume != 0: + raise DataQualityError("paused bar must have zero volume") + for name in ( + "adjustment_factor", + "previous_close", + "limit_up", + "limit_down", + ): + value = getattr(self, name) + if value is not None and ( + not math.isfinite(float(value)) or float(value) <= 0 + ): + raise DataQualityError( + f"{name} must be finite and positive when present" + ) + if ( + self.limit_up is not None + and self.limit_down is not None + and float(self.limit_down) > float(self.limit_up) + ): + raise DataQualityError("limit_down must not exceed limit_up") + + def as_dict(self) -> dict[str, Any]: + return jsonable(self) + + +@dataclass(frozen=True, slots=True) +class IndexMembershipRecord: + effective_date: date + index_symbol: str + member_symbol: str + source: str + + def __post_init__(self) -> None: + if not isinstance(self.effective_date, date): + raise DataQualityError("effective_date must be a date") + object.__setattr__( + self, + "index_symbol", + normalize_symbol(self.index_symbol), + ) + object.__setattr__( + self, + "member_symbol", + normalize_symbol(self.member_symbol), + ) + if not str(self.source).strip(): + raise DataQualityError("membership source is required") + + def as_dict(self) -> dict[str, Any]: + return jsonable(self) + + +def _hash_rows(rows: Iterable[Any]) -> str: + return hashlib.sha256(canonical_json(list(rows)).encode("utf-8")).hexdigest() + + +def build_snapshot_payload( + *, + bars: Iterable[CanonicalBarRecord], + memberships: Iterable[IndexMembershipRecord], + provider: str, + provider_version: str, + retrieved_at: datetime, + query: dict[str, Any], + next_trading_session: date, +) -> dict[str, Any]: + if retrieved_at.tzinfo is None or retrieved_at.utcoffset() is None: + raise DataQualityError("retrieved_at must include a timezone") + provider_name = str(provider).strip() + version = str(provider_version).strip() + if not provider_name or not version: + raise DataQualityError("provider and provider_version are required") + if not isinstance(next_trading_session, date): + raise DataQualityError("next_trading_session must be a date") + ordered_bars = sorted( + tuple(bars), + key=lambda item: (item.trading_date, item.symbol), + ) + ordered_memberships = sorted( + tuple(memberships), + key=lambda item: ( + item.effective_date, + item.index_symbol, + item.member_symbol, + ), + ) + if not ordered_bars or not ordered_memberships: + raise DataQualityError("snapshot bars and memberships must be non-empty") + bar_keys: set[tuple[date, str]] = set() + for bar in ordered_bars: + key = (bar.trading_date, bar.symbol) + if key in bar_keys: + raise DataQualityError( + f"duplicate bar {bar.trading_date} {bar.symbol}" + ) + bar_keys.add(key) + membership_keys: set[tuple[date, str, str]] = set() + for membership in ordered_memberships: + key = ( + membership.effective_date, + membership.index_symbol, + membership.member_symbol, + ) + if key in membership_keys: + raise DataQualityError( + "duplicate membership " + f"{membership.effective_date} {membership.member_symbol}" + ) + membership_keys.add(key) + if (membership.effective_date, membership.member_symbol) not in bar_keys: + raise DataQualityError( + "index membership has no same-session canonical bar: " + f"{membership.effective_date} {membership.member_symbol}" + ) + query_start = query.get("start_date") + query_end = query.get("end_date") + if query_start is not None and query_end is not None: + start_date = date.fromisoformat(str(query_start)) + end_date = date.fromisoformat(str(query_end)) + if start_date > end_date: + raise DataQualityError("query start_date must not follow end_date") + for bar in ordered_bars: + if not start_date <= bar.trading_date <= end_date: + raise DataQualityError( + "canonical bar falls outside the declared query range" + ) + for membership in ordered_memberships: + if not start_date <= membership.effective_date <= end_date: + raise DataQualityError( + "index membership falls outside the declared query range" + ) + if next_trading_session <= end_date: + raise DataQualityError( + "next_trading_session must follow query end_date" + ) + trading_sessions_raw = query.get("trading_sessions") + if not isinstance(trading_sessions_raw, list): + raise DataQualityError( + "query.trading_sessions must freeze the provider calendar" + ) + try: + trading_sessions = [ + date.fromisoformat(str(item)) for item in trading_sessions_raw + ] + except (TypeError, ValueError) as exc: + raise DataQualityError( + "query.trading_sessions contains an invalid date" + ) from exc + if ( + len(trading_sessions) < 2 + or trading_sessions != sorted(set(trading_sessions)) + ): + raise DataQualityError( + "query.trading_sessions must be unique and strictly increasing" + ) + if trading_sessions[-1] != next_trading_session: + raise DataQualityError( + "query.trading_sessions must end at next_trading_session" + ) + observed_sessions = { + item.trading_date for item in ordered_bars + } | { + item.effective_date for item in ordered_memberships + } + if set(trading_sessions[:-1]) != observed_sessions: + raise DataQualityError( + "query.trading_sessions does not exactly match snapshot sessions" + ) + bar_rows = [bar.as_dict() for bar in ordered_bars] + membership_rows = [item.as_dict() for item in ordered_memberships] + quality = { + "ok": True, + "bar_count": len(bar_rows), + "membership_count": len(membership_rows), + "symbol_count": len({bar.symbol for bar in ordered_bars}), + "session_count": len({bar.trading_date for bar in ordered_bars}), + "paused_bar_count": sum(1 for bar in ordered_bars if bar.paused), + "membership_bar_coverage_ok": True, + "start_date": ordered_bars[0].trading_date.isoformat(), + "end_date": ordered_bars[-1].trading_date.isoformat(), + } + manifest_body = { + "schema_version": "1.0", + "provider": provider_name, + "provider_version": version, + "retrieved_at": retrieved_at.isoformat(), + "next_trading_session": next_trading_session.isoformat(), + "query": jsonable(query), + "bars_sha256": _hash_rows(bar_rows), + "memberships_sha256": _hash_rows(membership_rows), + "quality": quality, + } + manifest_body["data_version"] = _hash_rows([manifest_body]) + return { + "bars": bar_rows, + "memberships": membership_rows, + "quality": quality, + "manifest": manifest_body, + } + + +def _atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=str(path.parent), + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def _json_bytes(payload: Any) -> bytes: + return ( + json.dumps( + jsonable(payload), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + + +def _jsonl_bytes(rows: list[dict[str, Any]]) -> bytes: + return ( + "".join(canonical_json(row) + "\n" for row in rows) + ).encode("utf-8") + + +def write_data_snapshot( + payload: dict[str, Any], + output_dir: str | Path, +) -> dict[str, str]: + root = Path(output_dir) + root.mkdir(parents=True, exist_ok=True) + marker = root / ".quant60-incomplete" + _atomic_write( + marker, + (payload["manifest"]["data_version"] + "\n").encode("ascii"), + ) + bars_path = root / "bars.jsonl" + memberships_path = root / "memberships.jsonl" + quality_path = root / "quality.json" + _atomic_write(bars_path, _jsonl_bytes(payload["bars"])) + _atomic_write(memberships_path, _jsonl_bytes(payload["memberships"])) + _atomic_write(quality_path, _json_bytes(payload["quality"])) + manifest = dict(payload["manifest"]) + manifest["artifacts"] = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in (bars_path, memberships_path, quality_path) + } + manifest_path = root / "manifest.json" + _atomic_write(manifest_path, _json_bytes(manifest)) + marker.unlink() + return { + "bars": str(bars_path.resolve()), + "memberships": str(memberships_path.resolve()), + "quality": str(quality_path.resolve()), + "manifest": str(manifest_path.resolve()), + } + + +def verify_data_snapshot(manifest_file: str | Path) -> dict[str, Any]: + path = Path(manifest_file) + root = path.parent + if (root / ".quant60-incomplete").exists(): + raise DataQualityError("data snapshot publication is incomplete") + manifest = json.loads(path.read_text(encoding="utf-8")) + expected = set(manifest.get("artifacts", {})) | {path.name} + actual = {entry.name for entry in root.iterdir()} + if actual != expected: + raise DataQualityError("data snapshot directory is not an exact set") + for name, expected_hash in manifest["artifacts"].items(): + digest = hashlib.sha256((root / name).read_bytes()).hexdigest() + if digest != expected_hash: + raise DataQualityError(f"data snapshot artifact hash mismatch: {name}") + bars = [ + json.loads(line) + for line in (root / "bars.jsonl").read_text(encoding="utf-8").splitlines() + if line + ] + memberships = [ + json.loads(line) + for line in (root / "memberships.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line + ] + if _hash_rows(bars) != manifest["bars_sha256"]: + raise DataQualityError("canonical bars content hash mismatch") + if _hash_rows(memberships) != manifest["memberships_sha256"]: + raise DataQualityError("membership content hash mismatch") + try: + bar_records = [ + CanonicalBarRecord( + trading_date=date.fromisoformat(row["trading_date"]), + symbol=row["symbol"], + open=row["open"], + high=row["high"], + low=row["low"], + close=row["close"], + volume=row["volume"], + money=row["money"], + paused=row["paused"], + source=row["source"], + is_st=row["is_st"], + adjustment=row.get("adjustment", "none"), + adjustment_factor=row.get("adjustment_factor"), + previous_close=row.get("previous_close"), + limit_up=row.get("limit_up"), + limit_down=row.get("limit_down"), + ) + for row in bars + ] + membership_records = [ + IndexMembershipRecord( + effective_date=date.fromisoformat(row["effective_date"]), + index_symbol=row["index_symbol"], + member_symbol=row["member_symbol"], + source=row["source"], + ) + for row in memberships + ] + rebuilt = build_snapshot_payload( + bars=bar_records, + memberships=membership_records, + provider=manifest["provider"], + provider_version=manifest["provider_version"], + retrieved_at=datetime.fromisoformat(manifest["retrieved_at"]), + query=manifest["query"], + next_trading_session=date.fromisoformat( + manifest["next_trading_session"] + ), + ) + except (KeyError, TypeError, ValueError) as exc: + raise DataQualityError( + f"data snapshot semantic validation failed: {exc}" + ) from exc + for field in ( + "schema_version", + "provider", + "provider_version", + "retrieved_at", + "next_trading_session", + "query", + "bars_sha256", + "memberships_sha256", + "quality", + "data_version", + ): + if rebuilt["manifest"][field] != manifest.get(field): + raise DataQualityError( + f"data snapshot manifest semantic mismatch: {field}" + ) + return { + "ok": True, + "data_version": manifest["data_version"], + "bar_count": len(bars), + "membership_count": len(memberships), + } + + +def load_verified_data_snapshot( + manifest_file: str | Path, +) -> dict[str, Any]: + """Return a verified snapshot; no caller can bypass its quality gate.""" + + path = Path(manifest_file).expanduser().resolve() + verification = verify_data_snapshot(path) + root = path.parent + return { + "verification": verification, + "manifest": json.loads(path.read_text(encoding="utf-8")), + "bars": [ + json.loads(line) + for line in (root / "bars.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line + ], + "memberships": [ + json.loads(line) + for line in (root / "memberships.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line + ], + "manifest_file": str(path), + } diff --git a/src/quant60/domain.py b/src/quant60/domain.py new file mode 100644 index 0000000..de1ccec --- /dev/null +++ b/src/quant60/domain.py @@ -0,0 +1,357 @@ +"""Modern domain objects and invariants for the local Python >=3.10 runtime.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date +from decimal import Decimal, ROUND_HALF_UP +from enum import Enum +from typing import Any, Iterable + +from .portable_core import normalize_symbol + + +MONEY_QUANT = Decimal("0.000001") +PRICE_QUANT = Decimal("0.0001") + + +def decimal(value: Decimal | str | int | float) -> Decimal: + if isinstance(value, Decimal): + return value + return Decimal(str(value)) + + +def money(value: Decimal | str | int | float) -> Decimal: + return decimal(value).quantize(MONEY_QUANT, rounding=ROUND_HALF_UP) + + +class Side(str, Enum): + BUY = "BUY" + SELL = "SELL" + + +class OrderStatus(str, Enum): + NEW = "NEW" + ACCEPTED = "ACCEPTED" + PARTIALLY_FILLED = "PARTIALLY_FILLED" + FILLED = "FILLED" + CANCEL_PENDING = "CANCEL_PENDING" + REJECTED = "REJECTED" + CANCELLED = "CANCELLED" + UNKNOWN = "UNKNOWN" + + +TERMINAL_ORDER_STATUSES = { + OrderStatus.FILLED, + OrderStatus.REJECTED, + OrderStatus.CANCELLED, +} + +ORDER_TRANSITIONS: dict[OrderStatus, set[OrderStatus]] = { + OrderStatus.NEW: { + OrderStatus.ACCEPTED, + OrderStatus.REJECTED, + OrderStatus.CANCELLED, + OrderStatus.UNKNOWN, + }, + OrderStatus.ACCEPTED: { + OrderStatus.PARTIALLY_FILLED, + OrderStatus.FILLED, + OrderStatus.CANCEL_PENDING, + OrderStatus.CANCELLED, + OrderStatus.REJECTED, + OrderStatus.UNKNOWN, + }, + OrderStatus.PARTIALLY_FILLED: { + OrderStatus.PARTIALLY_FILLED, + OrderStatus.FILLED, + OrderStatus.CANCEL_PENDING, + OrderStatus.CANCELLED, + OrderStatus.UNKNOWN, + }, + OrderStatus.CANCEL_PENDING: { + OrderStatus.PARTIALLY_FILLED, + OrderStatus.FILLED, + OrderStatus.CANCELLED, + OrderStatus.UNKNOWN, + }, + OrderStatus.FILLED: set(), + OrderStatus.REJECTED: set(), + OrderStatus.CANCELLED: set(), + OrderStatus.UNKNOWN: { + OrderStatus.ACCEPTED, + OrderStatus.PARTIALLY_FILLED, + OrderStatus.FILLED, + OrderStatus.CANCEL_PENDING, + OrderStatus.REJECTED, + OrderStatus.CANCELLED, + }, +} + + +@dataclass(frozen=True, slots=True) +class Bar: + trading_date: date + symbol: str + open: Decimal + high: Decimal + low: Decimal + close: Decimal + volume: int + previous_close: Decimal | None = None + limit_up: Decimal | None = None + limit_down: Decimal | None = None + suspended: bool = False + + def __post_init__(self) -> None: + object.__setattr__(self, "symbol", normalize_symbol(self.symbol)) + for name in ("open", "high", "low", "close"): + value = decimal(getattr(self, name)).quantize(PRICE_QUANT) + if value <= 0: + raise ValueError(f"{name} must be positive") + object.__setattr__(self, name, value) + for name in ("previous_close", "limit_up", "limit_down"): + value = getattr(self, name) + if value is not None: + object.__setattr__( + self, name, decimal(value).quantize(PRICE_QUANT) + ) + if self.volume < 0: + raise ValueError("volume must be non-negative") + if not (self.low <= self.open <= self.high): + raise ValueError("open must lie within [low, high]") + if not (self.low <= self.close <= self.high): + raise ValueError("close must lie within [low, high]") + + @property + def locked_limit_up(self) -> bool: + return ( + self.limit_up is not None + and self.low >= self.limit_up + and self.high >= self.limit_up + ) + + @property + def locked_limit_down(self) -> bool: + return ( + self.limit_down is not None + and self.high <= self.limit_down + and self.low <= self.limit_down + ) + + +@dataclass(slots=True) +class Order: + order_id: str + symbol: str + side: Side + quantity: int + submitted_at: date + status: OrderStatus = OrderStatus.NEW + filled_quantity: int = 0 + average_fill_price: Decimal = Decimal("0") + status_reason: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + self.symbol = normalize_symbol(self.symbol) + if self.quantity <= 0: + raise ValueError("order quantity must be positive") + if self.filled_quantity < 0 or self.filled_quantity > self.quantity: + raise ValueError("invalid filled quantity") + + @property + def remaining_quantity(self) -> int: + return self.quantity - self.filled_quantity + + @property + def is_terminal(self) -> bool: + return self.status in TERMINAL_ORDER_STATUSES + + def transition( + self, next_status: OrderStatus, reason: str | None = None + ) -> None: + # Broker callbacks are at-least-once. Repeating the same observed + # state must not make a replay fail or create a second accounting + # effect. + if next_status == self.status: + if reason is not None: + self.status_reason = reason + return + if next_status not in ORDER_TRANSITIONS[self.status]: + raise ValueError( + f"illegal order transition {self.status.value}" + f" -> {next_status.value}" + ) + self.status = next_status + self.status_reason = reason + + def apply_fill(self, quantity: int, price: Decimal) -> None: + if self.status not in { + OrderStatus.ACCEPTED, + OrderStatus.PARTIALLY_FILLED, + OrderStatus.CANCEL_PENDING, + }: + raise ValueError(f"cannot fill an order in {self.status.value}") + if quantity <= 0 or quantity > self.remaining_quantity: + raise ValueError("invalid fill quantity") + price = decimal(price) + total_notional = ( + self.average_fill_price * self.filled_quantity + price * quantity + ) + self.filled_quantity += quantity + self.average_fill_price = ( + total_notional / self.filled_quantity + ).quantize(PRICE_QUANT, rounding=ROUND_HALF_UP) + next_status = ( + OrderStatus.FILLED + if self.filled_quantity == self.quantity + else OrderStatus.PARTIALLY_FILLED + ) + self.transition(next_status) + + +@dataclass(frozen=True, slots=True) +class Fill: + fill_id: str + order_id: str + trading_date: date + symbol: str + side: Side + quantity: int + price: Decimal + commission: Decimal + stamp_duty: Decimal + transfer_fee: Decimal + + def __post_init__(self) -> None: + object.__setattr__(self, "symbol", normalize_symbol(self.symbol)) + object.__setattr__(self, "price", decimal(self.price)) + object.__setattr__(self, "commission", money(self.commission)) + object.__setattr__(self, "stamp_duty", money(self.stamp_duty)) + object.__setattr__(self, "transfer_fee", money(self.transfer_fee)) + if self.quantity <= 0: + raise ValueError("fill quantity must be positive") + + @property + def notional(self) -> Decimal: + return money(self.price * self.quantity) + + @property + def total_fees(self) -> Decimal: + return money(self.commission + self.stamp_duty + self.transfer_fee) + + @property + def cash_delta(self) -> Decimal: + if self.side == Side.BUY: + return money(-self.notional - self.total_fees) + return money(self.notional - self.total_fees) + + +@dataclass(slots=True) +class Position: + symbol: str + quantity: int = 0 + sellable_quantity: int = 0 + today_bought: int = 0 + average_cost: Decimal = Decimal("0") + + def __post_init__(self) -> None: + self.symbol = normalize_symbol(self.symbol) + self.average_cost = decimal(self.average_cost) + self._validate() + + def _validate(self) -> None: + if min(self.quantity, self.sellable_quantity, self.today_bought) < 0: + raise ValueError("position quantities must be non-negative") + if self.sellable_quantity + self.today_bought != self.quantity: + raise ValueError("sellable + today_bought must equal total quantity") + + def release_t_plus_one(self) -> None: + self.sellable_quantity += self.today_bought + self.today_bought = 0 + self._validate() + + def buy(self, quantity: int, price: Decimal, fees: Decimal) -> None: + if quantity <= 0: + raise ValueError("buy quantity must be positive") + prior_cost = self.average_cost * self.quantity + total_cost = prior_cost + decimal(price) * quantity + decimal(fees) + self.quantity += quantity + self.today_bought += quantity + self.average_cost = ( + total_cost / self.quantity + ).quantize(PRICE_QUANT, rounding=ROUND_HALF_UP) + self._validate() + + def sell(self, quantity: int) -> None: + if quantity <= 0 or quantity > self.sellable_quantity: + raise ValueError("sell exceeds T+1 sellable quantity") + self.quantity -= quantity + self.sellable_quantity -= quantity + if self.quantity == 0: + self.average_cost = Decimal("0") + self._validate() + + +@dataclass(slots=True) +class Portfolio: + cash: Decimal + positions: dict[str, Position] = field(default_factory=dict) + last_prices: dict[str, Decimal] = field(default_factory=dict) + + def __post_init__(self) -> None: + self.cash = money(self.cash) + + def position(self, symbol: str) -> Position: + canonical = normalize_symbol(symbol) + if canonical not in self.positions: + self.positions[canonical] = Position(canonical) + return self.positions[canonical] + + def start_session(self) -> None: + for position in self.positions.values(): + position.release_t_plus_one() + + def mark(self, bars: Iterable[Bar]) -> None: + for bar in bars: + self.last_prices[bar.symbol] = bar.close + + @property + def market_value(self) -> Decimal: + total = Decimal("0") + for symbol, position in self.positions.items(): + price = self.last_prices.get(symbol, position.average_cost) + total += price * position.quantity + return money(total) + + @property + def equity(self) -> Decimal: + return money(self.cash + self.market_value) + + def apply_fill(self, fill: Fill) -> None: + position = self.position(fill.symbol) + if fill.side == Side.BUY: + required = -fill.cash_delta + if required > self.cash: + raise ValueError("insufficient cash") + self.cash = money(self.cash + fill.cash_delta) + position.buy(fill.quantity, fill.price, fill.total_fees) + else: + position.sell(fill.quantity) + self.cash = money(self.cash + fill.cash_delta) + self.last_prices[fill.symbol] = fill.price + + def quantities(self) -> dict[str, int]: + return { + symbol: position.quantity + for symbol, position in self.positions.items() + if position.quantity + } + + def sellable_quantities(self) -> dict[str, int]: + return { + symbol: position.sellable_quantity + for symbol, position in self.positions.items() + if position.sellable_quantity + } diff --git a/src/quant60/execution.py b/src/quant60/execution.py new file mode 100644 index 0000000..af2f386 --- /dev/null +++ b/src/quant60/execution.py @@ -0,0 +1,237 @@ +"""Deterministic parent/child planning for a guarded TWAP/POV executor.""" + +from __future__ import annotations + +import hashlib +import math +from dataclasses import dataclass +from datetime import datetime, timedelta +from enum import Enum + +from .domain import Side +from .portable_core import normalize_symbol + + +class ResidualPolicy(str, Enum): + DEFER = "DEFER" + CANCEL = "CANCEL" + + +def _aware(value: datetime, name: str) -> datetime: + if not isinstance(value, datetime): + raise ValueError(f"{name} must be a datetime") + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{name} must include a timezone") + return value + + +@dataclass(frozen=True, slots=True) +class ExecutionBucket: + scheduled_at: datetime + expected_market_volume: int + + def __post_init__(self) -> None: + _aware(self.scheduled_at, "scheduled_at") + if ( + type(self.expected_market_volume) is not int + or self.expected_market_volume < 0 + ): + raise ValueError( + "expected_market_volume must be a non-negative integer" + ) + + +@dataclass(frozen=True, slots=True) +class ParentOrderIntent: + decision_id: str + symbol: str + side: Side + quantity: int + limit_price: float + window_start: datetime + window_end: datetime + market_data_as_of: datetime + lot_size: int = 100 + max_participation: float = 0.10 + max_child_quantity: int | None = None + max_child_notional: float | None = None + residual_policy: ResidualPolicy = ResidualPolicy.DEFER + + def __post_init__(self) -> None: + decision = str(self.decision_id).strip() + if not decision: + raise ValueError("decision_id is required") + object.__setattr__(self, "decision_id", decision) + object.__setattr__(self, "symbol", normalize_symbol(self.symbol)) + if not isinstance(self.side, Side): + raise ValueError("side must be a Side") + if type(self.quantity) is not int or self.quantity <= 0: + raise ValueError("quantity must be a positive integer") + if type(self.lot_size) is not int or self.lot_size <= 0: + raise ValueError("lot_size must be positive") + if self.quantity % self.lot_size: + raise ValueError("parent quantity must be a complete board lot") + if ( + not math.isfinite(float(self.limit_price)) + or self.limit_price <= 0 + ): + raise ValueError("limit_price must be finite and positive") + _aware(self.window_start, "window_start") + _aware(self.window_end, "window_end") + _aware(self.market_data_as_of, "market_data_as_of") + if self.window_end <= self.window_start: + raise ValueError("execution window must have positive duration") + if not math.isfinite(float(self.max_participation)) or not ( + 0 < self.max_participation <= 1 + ): + raise ValueError("max_participation must lie in (0, 1]") + if self.max_child_quantity is not None and ( + type(self.max_child_quantity) is not int + or self.max_child_quantity < self.lot_size + ): + raise ValueError("max_child_quantity is invalid") + if self.max_child_notional is not None and ( + not math.isfinite(float(self.max_child_notional)) + or self.max_child_notional <= 0 + ): + raise ValueError("max_child_notional must be finite and positive") + if not isinstance(self.residual_policy, ResidualPolicy): + raise ValueError("residual_policy must be ResidualPolicy") + + @property + def idempotency_key(self) -> str: + return ( + f"{self.decision_id}:{self.symbol}:{self.side.value}" + ) + + +@dataclass(frozen=True, slots=True) +class ChildOrderIntent: + child_id: str + parent_key: str + sequence: int + symbol: str + side: Side + quantity: int + limit_price: float + scheduled_at: datetime + expected_market_volume: int + participation_cap_quantity: int + + +@dataclass(frozen=True, slots=True) +class ExecutionPlan: + parent_key: str + children: tuple[ChildOrderIntent, ...] + planned_quantity: int + residual_quantity: int + residual_policy: ResidualPolicy + diagnostics: dict[str, object] + + +def market_data_is_fresh( + *, + as_of: datetime, + now: datetime, + max_age: timedelta = timedelta(seconds=5), + max_future: timedelta = timedelta(seconds=1), +) -> bool: + _aware(as_of, "as_of") + _aware(now, "now") + if max_age.total_seconds() < 0 or max_future.total_seconds() < 0: + raise ValueError("freshness tolerances must be non-negative") + age = now - as_of + return -max_future <= age <= max_age + + +def plan_guarded_twap_pov( + parent: ParentOrderIntent, + buckets: list[ExecutionBucket], +) -> ExecutionPlan: + if not buckets: + raise ValueError("execution buckets are required") + ordered = sorted(buckets, key=lambda item: item.scheduled_at) + if len({bucket.scheduled_at for bucket in ordered}) != len(ordered): + raise ValueError("execution bucket timestamps must be unique") + for bucket in ordered: + if not ( + parent.window_start <= bucket.scheduled_at <= parent.window_end + ): + raise ValueError("execution bucket lies outside parent window") + if parent.market_data_as_of > parent.window_start: + raise ValueError( + "parent market_data_as_of cannot follow execution window start" + ) + + remaining = parent.quantity + children: list[ChildOrderIntent] = [] + skipped_zero_capacity = 0 + for bucket_index, bucket in enumerate(ordered, 1): + if remaining <= 0: + break + buckets_left = len(ordered) - bucket_index + 1 + twap_quantity = ( + math.ceil(remaining / buckets_left / parent.lot_size) + * parent.lot_size + ) + pov_cap = ( + int(bucket.expected_market_volume * parent.max_participation) + // parent.lot_size + * parent.lot_size + ) + caps = [remaining, twap_quantity, pov_cap] + if parent.max_child_quantity is not None: + caps.append( + parent.max_child_quantity + // parent.lot_size + * parent.lot_size + ) + if parent.max_child_notional is not None: + caps.append( + int(parent.max_child_notional / parent.limit_price) + // parent.lot_size + * parent.lot_size + ) + quantity = min(caps) + quantity = quantity // parent.lot_size * parent.lot_size + if quantity <= 0: + skipped_zero_capacity += 1 + continue + sequence = len(children) + 1 + digest = hashlib.sha256( + ( + f"{parent.idempotency_key}:" + f"{bucket.scheduled_at.isoformat()}:{sequence}" + ).encode("utf-8") + ).hexdigest()[:16] + children.append( + ChildOrderIntent( + child_id=f"QOS-{digest}", + parent_key=parent.idempotency_key, + sequence=sequence, + symbol=parent.symbol, + side=parent.side, + quantity=quantity, + limit_price=float(parent.limit_price), + scheduled_at=bucket.scheduled_at, + expected_market_volume=bucket.expected_market_volume, + participation_cap_quantity=pov_cap, + ) + ) + remaining -= quantity + planned = sum(child.quantity for child in children) + if planned + remaining != parent.quantity: + raise AssertionError("execution quantity conservation failed") + return ExecutionPlan( + parent_key=parent.idempotency_key, + children=tuple(children), + planned_quantity=planned, + residual_quantity=remaining, + residual_policy=parent.residual_policy, + diagnostics={ + "bucket_count": len(ordered), + "child_count": len(children), + "skipped_zero_capacity": skipped_zero_capacity, + "fully_planned": remaining == 0, + }, + ) diff --git a/src/quant60/experiment.py b/src/quant60/experiment.py new file mode 100644 index 0000000..beabcdf --- /dev/null +++ b/src/quant60/experiment.py @@ -0,0 +1,623 @@ +"""Deterministic research-to-target experiment for local and Colab execution.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import statistics +import tempfile +from collections import defaultdict +from datetime import date +from pathlib import Path +from typing import Any + +from .contracts import validate_records +from .domain import Bar +from .features import ( + TrainOnlyFeaturePreprocessor, + build_executable_label, + compute_price_features, +) +from .ledger import canonical_json, jsonable +from .optimizer import ( + PortfolioCandidate, + PortfolioConstraints, + optimize_deterministic, +) +from .portable_core import target_quantities +from .research import ( + DeterministicRidge, + pearson_correlation, + purged_walk_forward, + spearman_correlation, +) +from .synthetic import generate_synthetic_bars +from .universe import SecurityEligibility, classify_security + + +RESEARCH_SYMBOLS = ( + "600000.XSHG", + "600519.XSHG", + "601318.XSHG", + "601012.XSHG", + "000001.XSHE", + "000333.XSHE", + "000651.XSHE", + "002415.XSHE", + "002594.XSHE", + "300059.XSHE", + "300750.XSHE", + "688981.XSHG", +) + +INDUSTRY_BY_SYMBOL = { + symbol: ("industry_%d" % (index % 4)) + for index, symbol in enumerate(RESEARCH_SYMBOLS) +} + + +def _sha256(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def _atomic_json(path: Path, payload: Any) -> None: + encoded = ( + json.dumps( + jsonable(payload), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=str(path.parent), + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def _source_identity() -> tuple[str, dict[str, str]]: + directory = Path(__file__).resolve().parent + sources = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(directory.glob("*.py"), key=lambda item: item.name) + } + return _sha256(sources), sources + + +def _bar_data_version(bars: list[Bar]) -> str: + payload = [ + { + "date": bar.trading_date.isoformat(), + "symbol": bar.symbol, + "open": format(bar.open, "f"), + "high": format(bar.high, "f"), + "low": format(bar.low, "f"), + "close": format(bar.close, "f"), + "volume": bar.volume, + "suspended": bar.suspended, + } + for bar in bars + ] + return _sha256(payload) + + +def _first_sessions_by_week(dates: list[date]) -> list[date]: + result: list[date] = [] + prior_week = None + for session in dates: + week = session.isocalendar()[:2] + if week != prior_week: + result.append(session) + prior_week = week + return result + + +def _matrix( + rows: list[dict[str, Any]], + processor: TrainOnlyFeaturePreprocessor, +) -> list[list[float]]: + transformed = processor.transform([row["features"] for row in rows]) + names = processor.output_names + return [[item[name] for name in names] for item in transformed] + + +def _daily_correlations( + rows: list[dict[str, Any]], + predictions: list[float], +) -> list[dict[str, Any]]: + grouped: dict[str, list[tuple[float, float]]] = defaultdict(list) + for row, prediction in zip(rows, predictions): + grouped[row["decision_date"]].append((prediction, row["label"])) + output = [] + for decision_date, values in sorted(grouped.items()): + predicted = [value[0] for value in values] + observed = [value[1] for value in values] + output.append( + { + "decision_date": decision_date, + "count": len(values), + "ic": pearson_correlation(predicted, observed), + "rank_ic": spearman_correlation(predicted, observed), + } + ) + return output + + +def _research_rows( + bars: list[Bar], + *, + warmup_sessions: int, + horizon_sessions: int, +) -> tuple[list[dict[str, Any]], list[date], dict[str, list[Bar]]]: + by_symbol: dict[str, list[Bar]] = defaultdict(list) + for bar in bars: + by_symbol[bar.symbol].append(bar) + sessions = sorted({bar.trading_date for bar in bars}) + session_position = {session: index for index, session in enumerate(sessions)} + weekly = [ + session + for session in _first_sessions_by_week(sessions) + if session_position[session] >= warmup_sessions + and session_position[session] + horizon_sessions + 1 < len(sessions) + ] + rows: list[dict[str, Any]] = [] + for decision_date in weekly: + date_rows: list[dict[str, Any]] = [] + labels = [] + for symbol in RESEARCH_SYMBOLS: + snapshot = compute_price_features( + by_symbol[symbol], + as_of=decision_date, + ) + label = build_executable_label( + by_symbol[symbol], + decision_date=decision_date, + horizon_sessions=horizon_sessions, + ) + if label.value is None: + continue + labels.append(label.value) + date_rows.append( + { + "sample_id": ( + f"{decision_date.strftime('%Y%m%d')}-{symbol}" + ), + "decision_date": decision_date.isoformat(), + "symbol": symbol, + "features": snapshot.values, + "raw_label": label.value, + "label_status": label.status, + "entry_date": label.entry_date.isoformat(), + "exit_date": label.exit_date.isoformat(), + } + ) + if not date_rows: + continue + cross_section_mean = statistics.fmean(labels) + for row in date_rows: + row["label"] = row.pop("raw_label") - cross_section_mean + rows.append(row) + used_dates = sorted({date.fromisoformat(row["decision_date"]) for row in rows}) + return rows, used_dates, by_symbol + + +def run_research_smoke( + *, + trading_days: int = 320, + seed: int = 60, + warmup_sessions: int = 130, + horizon_sessions: int = 5, +) -> dict[str, Any]: + """Run a chronological Ridge experiment and generate a final target.""" + + if trading_days < 220: + raise ValueError("research smoke requires at least 220 trading days") + bars = generate_synthetic_bars( + RESEARCH_SYMBOLS, + start_date="2023-01-03", + trading_days_count=trading_days, + seed=seed, + base_volume=500_000, + include_market_events=True, + ) + rows, decision_dates, by_symbol = _research_rows( + bars, + warmup_sessions=warmup_sessions, + horizon_sessions=horizon_sessions, + ) + if len(decision_dates) < 22: + raise ValueError("research smoke did not produce enough weekly dates") + rows_by_date: dict[date, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + rows_by_date[date.fromisoformat(row["decision_date"])].append(row) + + train_size = 16 + test_size = 4 + folds = list( + purged_walk_forward( + len(decision_dates), + train_size=train_size, + test_size=test_size, + purge=1, + embargo=1, + ) + ) + if not folds: + raise ValueError("research smoke produced no walk-forward fold") + forecasts: list[dict[str, Any]] = [] + fold_reports = [] + all_daily_metrics = [] + for fold_number, fold in enumerate(folds, 1): + train_rows = [ + row + for index in fold.train + for row in rows_by_date[decision_dates[index]] + ] + test_rows = [ + row + for index in fold.test + for row in rows_by_date[decision_dates[index]] + ] + processor = TrainOnlyFeaturePreprocessor().fit( + [row["features"] for row in train_rows] + ) + model = DeterministicRidge(alpha=5.0).fit( + _matrix(train_rows, processor), + [row["label"] for row in train_rows], + ) + predictions = model.predict(_matrix(test_rows, processor)) + daily_metrics = _daily_correlations(test_rows, predictions) + all_daily_metrics.extend(daily_metrics) + fold_reports.append( + { + "fold": fold_number, + "train_start": decision_dates[min(fold.train)].isoformat(), + "train_end": decision_dates[max(fold.train)].isoformat(), + "test_start": decision_dates[min(fold.test)].isoformat(), + "test_end": decision_dates[max(fold.test)].isoformat(), + "purged_dates": [ + decision_dates[index].isoformat() for index in fold.purged + ], + "embargoed_dates": [ + decision_dates[index].isoformat() for index in fold.embargoed + ], + "train_samples": len(train_rows), + "test_samples": len(test_rows), + "mean_ic": statistics.fmean( + item["ic"] for item in daily_metrics + ), + "mean_rank_ic": statistics.fmean( + item["rank_ic"] for item in daily_metrics + ), + } + ) + for row, prediction in zip(test_rows, predictions): + forecasts.append( + { + "decision_date": row["decision_date"], + "symbol": row["symbol"], + "prediction": prediction, + "realized_excess_proxy": row["label"], + "fold": fold_number, + } + ) + + # Train strictly before the final decision and emit a point-in-time target. + target_date = decision_dates[-1] + final_train = [ + row + for row in rows + if date.fromisoformat(row["decision_date"]) < target_date + ] + final_rows = rows_by_date[target_date] + final_processor = TrainOnlyFeaturePreprocessor().fit( + [row["features"] for row in final_train] + ) + final_model = DeterministicRidge(alpha=5.0).fit( + _matrix(final_train, final_processor), + [row["label"] for row in final_train], + ) + final_predictions = final_model.predict(_matrix(final_rows, final_processor)) + centered = statistics.fmean(final_predictions) + scores = { + row["symbol"]: prediction - centered + for row, prediction in zip(final_rows, final_predictions) + } + candidates = [] + current_weights = {symbol: 0.0 for symbol in RESEARCH_SYMBOLS} + latest_prices = {} + for symbol in RESEARCH_SYMBOLS: + completed = [ + bar for bar in by_symbol[symbol] if bar.trading_date <= target_date + ] + latest = completed[-1] + latest_prices[symbol] = float(latest.close) + returns = [ + float(completed[index].close) + / float(completed[index - 1].close) + - 1.0 + for index in range(max(1, len(completed) - 60), len(completed)) + ] + variance = ( + statistics.variance(returns) if len(returns) > 1 else 1e-4 + ) + feature_row = next( + row for row in final_rows if row["symbol"] == symbol + ) + turnover = feature_row["features"]["median_turnover_20"] or 0.0 + eligibility = SecurityEligibility( + symbol=symbol, + index_member=True, + listing_sessions=500, + median_turnover=turnover, + minimum_turnover=0.0, + held_quantity=0, + sellable_quantity=0, + ) + state = classify_security(eligibility).state + candidates.append( + PortfolioCandidate( + symbol=symbol, + score=scores[symbol], + variance=max(1e-6, variance), + current_weight=current_weights[symbol], + cost_bps=15.0, + max_adv_weight=max( + 0.0, + min(0.10, turnover * 0.03 / 1_000_000.0), + ), + industry=INDUSTRY_BY_SYMBOL[symbol], + state=state, + ) + ) + solution = optimize_deterministic( + candidates, + PortfolioConstraints( + gross_target=0.90, + single_name_cap=0.10, + industry_cap=0.30, + one_way_turnover_cap=0.45, + risk_aversion=1.0, + cost_aversion=1.0, + ), + ) + quantities = target_quantities( + solution.weights, + latest_prices, + 1_000_000.0, + lot_size=100, + cash_buffer=0.05, + ) + + data_version = _bar_data_version(bars) + source_sha256, sources = _source_identity() + config = { + "trading_days": trading_days, + "seed": seed, + "warmup_sessions": warmup_sessions, + "horizon_sessions": horizon_sessions, + "symbols": list(RESEARCH_SYMBOLS), + "feature_version": "transparent-price-v1", + "model": "deterministic-ridge-alpha-5", + "label": "t_plus_1_open_to_t_plus_6_open_cross_section_excess_proxy", + "validation": { + "train_dates": train_size, + "test_dates": test_size, + "purge_dates": 1, + "embargo_dates": 1, + }, + } + config_hash = _sha256(config) + run_id = "q60-research-" + _sha256( + { + "data": data_version, + "source": source_sha256, + "config": config_hash, + } + )[:16] + signal_records = [ + { + "schema_version": "1.0", + "run_id": run_id, + "signal_id": f"RS-{target_date.strftime('%Y%m%d')}-{symbol}", + "as_of": f"{target_date.isoformat()}T15:00:00+08:00", + "symbol": symbol, + "horizon_trading_days": horizon_sessions, + "score": scores[symbol], + "expected_excess_return": scores[symbol], + "confidence": None, + "model_version": "deterministic-ridge-alpha-5", + "data_version": data_version, + "feature_version": "transparent-price-v1", + "metadata": { + "evidence_class": "synthetic-research", + "label_proxy": config["label"], + }, + } + for symbol in sorted(scores) + ] + target_records = [ + { + "schema_version": "1.0", + "run_id": run_id, + "decision_id": f"RD-{target_date.strftime('%Y%m%d')}", + "as_of": f"{target_date.isoformat()}T15:00:00+08:00", + "symbol": symbol, + "target_weight": solution.weights[symbol], + "target_quantity": quantities[symbol], + "lot_size": 100, + "signal_id": f"RS-{target_date.strftime('%Y%m%d')}-{symbol}", + "constraint_state": "FEASIBLE", + "config_hash": config_hash, + "metadata": { + "solver": solution.solver, + "industry": INDUSTRY_BY_SYMBOL[symbol], + }, + } + for symbol in sorted(solution.weights) + ] + validate_records(signal_records, "signal.schema.json") + validate_records(target_records, "target.schema.json") + rank_ics = [item["rank_ic"] for item in all_daily_metrics] + rank_ic_std = statistics.stdev(rank_ics) if len(rank_ics) > 1 else 0.0 + report = { + "run_id": run_id, + "evidence_class": "synthetic-research", + "investment_value_claim": False, + "config_hash": config_hash, + "data_version": data_version, + "engine_source_sha256": source_sha256, + "sample_count": len(rows), + "decision_date_count": len(decision_dates), + "fold_count": len(folds), + "folds": fold_reports, + "daily_oos_metrics": all_daily_metrics, + "mean_oos_ic": statistics.fmean( + item["ic"] for item in all_daily_metrics + ), + "mean_oos_rank_ic": statistics.fmean(rank_ics), + "oos_rank_icir": ( + statistics.fmean(rank_ics) / rank_ic_std + if rank_ic_std > 0 + else 0.0 + ), + "target_date": target_date.isoformat(), + "target_solver": solution.solver, + "target_gross_weight": solution.gross_weight, + "target_cash_weight": solution.cash_weight, + "target_one_way_turnover": solution.one_way_turnover, + "target_predicted_variance": solution.predicted_variance, + "label_warning": ( + "Synthetic daily-open cross-sectional excess proxy; not the " + "production CSI500 T+1/T+6 VWAP label." + ), + } + sample_index = [ + { + "sample_id": row["sample_id"], + "decision_date": row["decision_date"], + "symbol": row["symbol"], + "entry_date": row["entry_date"], + "exit_date": row["exit_date"], + "label_status": row["label_status"], + "feature_sha256": _sha256(row["features"]), + "label": row["label"], + } + for row in rows + ] + manifest = { + "schema_version": "1.0", + "run_id": run_id, + "evidence_class": "synthetic-research", + "config": config, + "config_hash": config_hash, + "data_version": data_version, + "engine_source_sha256": source_sha256, + "engine_sources": sources, + "deterministic": True, + "artifacts": {}, + } + return { + "manifest": manifest, + "report": report, + "signals": signal_records, + "targets": target_records, + "forecasts": forecasts, + "samples": sample_index, + } + + +RESEARCH_ARTIFACTS = { + "report": "report.json", + "signals": "signals.json", + "targets": "targets.json", + "forecasts": "forecasts.json", + "samples": "samples.json", +} + + +def write_research_artifacts( + result: dict[str, Any], + output_dir: str | Path, +) -> dict[str, str]: + root = Path(output_dir) + root.mkdir(parents=True, exist_ok=True) + marker = root / ".quant60-incomplete" + marker.write_text(str(result["manifest"]["run_id"]) + "\n", encoding="ascii") + paths: dict[str, Path] = {} + for name, filename in RESEARCH_ARTIFACTS.items(): + path = root / filename + _atomic_json(path, result[name]) + paths[name] = path + manifest = dict(result["manifest"]) + manifest["artifacts"] = { + name: { + "file": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + for name, path in sorted(paths.items()) + } + manifest_path = root / "manifest.json" + _atomic_json(manifest_path, manifest) + marker.unlink() + paths["manifest"] = manifest_path + return {name: str(path.resolve()) for name, path in paths.items()} + + +def verify_research_manifest(path: str | Path) -> dict[str, Any]: + manifest_path = Path(path) + root = manifest_path.parent + if (root / ".quant60-incomplete").exists(): + raise ValueError("research artifact publication is incomplete") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + expected_files = { + entry["file"] for entry in manifest.get("artifacts", {}).values() + } | {manifest_path.name} + actual_files = {entry.name for entry in root.iterdir()} + if actual_files != expected_files: + raise ValueError("research artifact directory is not an exact set") + verified = {} + for name, entry in manifest["artifacts"].items(): + artifact = root / entry["file"] + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + if digest != entry["sha256"]: + raise ValueError(f"research artifact hash mismatch: {name}") + verified[name] = digest + source_sha256, sources = _source_identity() + if _sha256(manifest["engine_sources"]) != manifest["engine_source_sha256"]: + raise ValueError("saved research source aggregate mismatch") + if sources != manifest["engine_sources"] or source_sha256 != manifest[ + "engine_source_sha256" + ]: + raise ValueError("current research source does not match manifest") + signals = json.loads((root / RESEARCH_ARTIFACTS["signals"]).read_text()) + targets = json.loads((root / RESEARCH_ARTIFACTS["targets"]).read_text()) + validate_records(signals, "signal.schema.json") + validate_records(targets, "target.schema.json") + report = json.loads((root / RESEARCH_ARTIFACTS["report"]).read_text()) + if report["run_id"] != manifest["run_id"]: + raise ValueError("research report/manifest run_id mismatch") + return { + "ok": True, + "run_id": manifest["run_id"], + "artifacts": verified, + "source_matches": True, + } diff --git a/src/quant60/factor_risk.py b/src/quant60/factor_risk.py new file mode 100644 index 0000000..4d5fe0c --- /dev/null +++ b/src/quant60/factor_risk.py @@ -0,0 +1,276 @@ +"""Transparent industry/style factor risk reference implementation.""" + +from __future__ import annotations + +import math +import statistics +from dataclasses import dataclass +from typing import Mapping, Sequence + +from .portable_core import normalize_symbol +from .research import DeterministicRidge + + +class RiskModelError(ValueError): + pass + + +def _finite(value: float, name: str) -> float: + numeric = float(value) + if not math.isfinite(numeric): + raise RiskModelError(f"{name} must be finite") + return numeric + + +@dataclass(frozen=True, slots=True) +class FactorExposureSnapshot: + as_of: str + exposures: dict[str, dict[str, float]] + + def __post_init__(self) -> None: + if not str(self.as_of).strip(): + raise RiskModelError("exposure as_of is required") + if not self.exposures: + raise RiskModelError("factor exposures are empty") + normalized: dict[str, dict[str, float]] = {} + factor_set = None + for raw_symbol, raw_factors in self.exposures.items(): + symbol = normalize_symbol(raw_symbol) + factors = { + str(name): _finite(value, f"{symbol}.{name}") + for name, value in raw_factors.items() + } + if not factors: + raise RiskModelError(f"factor exposures are empty for {symbol}") + if factor_set is None: + factor_set = set(factors) + elif set(factors) != factor_set: + raise RiskModelError("factor exposure columns are inconsistent") + normalized[symbol] = dict(sorted(factors.items())) + object.__setattr__(self, "exposures", dict(sorted(normalized.items()))) + + @property + def factor_names(self) -> tuple[str, ...]: + first = next(iter(self.exposures.values())) + return tuple(sorted(first)) + + +@dataclass(frozen=True, slots=True) +class CrossSectionalFactorFit: + factor_returns: dict[str, float] + residuals: dict[str, float] + observation_count: int + + +def estimate_factor_returns( + *, + returns: Mapping[str, float], + exposure: FactorExposureSnapshot, + ridge_alpha: float = 1e-4, +) -> CrossSectionalFactorFit: + normalized_returns = { + normalize_symbol(symbol): _finite(value, f"return.{symbol}") + for symbol, value in returns.items() + } + symbols = sorted(set(normalized_returns) & set(exposure.exposures)) + factors = exposure.factor_names + if len(symbols) <= len(factors): + raise RiskModelError( + "cross-sectional regression needs more securities than factors" + ) + matrix = [ + [exposure.exposures[symbol][factor] for factor in factors] + for symbol in symbols + ] + target = [normalized_returns[symbol] for symbol in symbols] + model = DeterministicRidge( + alpha=ridge_alpha, + fit_intercept=False, + ).fit(matrix, target) + predictions = model.predict(matrix) + assert model.coef_ is not None + return CrossSectionalFactorFit( + factor_returns={ + factor: model.coef_[index] + for index, factor in enumerate(factors) + }, + residuals={ + symbol: observed - predicted + for symbol, observed, predicted in zip( + symbols, + target, + predictions, + ) + }, + observation_count=len(symbols), + ) + + +@dataclass(frozen=True, slots=True) +class FactorRiskForecast: + factor_names: tuple[str, ...] + factor_covariance: dict[str, dict[str, float]] + specific_variance: dict[str, float] + half_life: float + shrinkage: float + variance_floor: float + + +def _ewma_weights(count: int, half_life: float) -> list[float]: + if count <= 0: + raise RiskModelError("EWMA history is empty") + if not math.isfinite(float(half_life)) or half_life <= 0: + raise RiskModelError("half_life must be finite and positive") + decay = math.exp(math.log(0.5) / float(half_life)) + raw = [decay ** (count - 1 - index) for index in range(count)] + total = sum(raw) + return [value / total for value in raw] + + +def build_factor_risk_forecast( + *, + factor_return_history: Sequence[Mapping[str, float]], + residual_history: Mapping[str, Sequence[float]], + half_life: float = 20.0, + shrinkage: float = 0.20, + variance_floor: float = 1e-8, +) -> FactorRiskForecast: + if not factor_return_history: + raise RiskModelError("factor return history is empty") + if not math.isfinite(float(shrinkage)) or not (0 <= shrinkage <= 1): + raise RiskModelError("shrinkage must lie in [0, 1]") + if not math.isfinite(float(variance_floor)) or variance_floor <= 0: + raise RiskModelError("variance_floor must be finite and positive") + factors = tuple(sorted(factor_return_history[0])) + if not factors: + raise RiskModelError("factor return columns are empty") + rows = [] + for raw_row in factor_return_history: + if set(raw_row) != set(factors): + raise RiskModelError("factor return columns are inconsistent") + rows.append( + { + factor: _finite(raw_row[factor], f"factor_return.{factor}") + for factor in factors + } + ) + weights = _ewma_weights(len(rows), half_life) + means = { + factor: sum( + weight * row[factor] for weight, row in zip(weights, rows) + ) + for factor in factors + } + covariance: dict[str, dict[str, float]] = {} + for left in factors: + covariance[left] = {} + for right in factors: + sample = sum( + weight + * (row[left] - means[left]) + * (row[right] - means[right]) + for weight, row in zip(weights, rows) + ) + if left == right: + value = max(variance_floor, sample) + else: + value = (1.0 - shrinkage) * sample + covariance[left][right] = value + + specific: dict[str, float] = {} + for raw_symbol, raw_values in residual_history.items(): + symbol = normalize_symbol(raw_symbol) + values = [ + _finite(value, f"specific_residual.{symbol}") + for value in raw_values + ] + if len(values) < 2: + specific[symbol] = variance_floor + else: + specific[symbol] = max( + variance_floor, + statistics.variance(values), + ) + if not specific: + raise RiskModelError("specific residual history is empty") + return FactorRiskForecast( + factor_names=factors, + factor_covariance=covariance, + specific_variance=dict(sorted(specific.items())), + half_life=float(half_life), + shrinkage=float(shrinkage), + variance_floor=float(variance_floor), + ) + + +def portfolio_factor_variance( + *, + portfolio_weights: Mapping[str, float], + exposure: FactorExposureSnapshot, + forecast: FactorRiskForecast, +) -> float: + weights = { + normalize_symbol(symbol): _finite(value, f"weight.{symbol}") + for symbol, value in portfolio_weights.items() + } + if any(value < 0 for value in weights.values()): + raise RiskModelError("portfolio weights must be non-negative") + missing_exposure = set(weights) - set(exposure.exposures) + missing_specific = set(weights) - set(forecast.specific_variance) + if missing_exposure or missing_specific: + raise RiskModelError( + "risk inputs missing for " + f"exposure={sorted(missing_exposure)}, " + f"specific={sorted(missing_specific)}" + ) + if set(exposure.factor_names) != set(forecast.factor_names): + raise RiskModelError("exposure/forecast factors differ") + factor_weight = { + factor: sum( + weight * exposure.exposures[symbol][factor] + for symbol, weight in weights.items() + ) + for factor in forecast.factor_names + } + factor_variance = sum( + factor_weight[left] + * forecast.factor_covariance[left][right] + * factor_weight[right] + for left in forecast.factor_names + for right in forecast.factor_names + ) + specific_variance = sum( + weight * weight * forecast.specific_variance[symbol] + for symbol, weight in weights.items() + ) + result = factor_variance + specific_variance + if not math.isfinite(result) or result < -1e-12: + raise RiskModelError("portfolio variance is invalid") + return max(0.0, result) + + +def factor_stress_loss( + *, + portfolio_weights: Mapping[str, float], + exposure: FactorExposureSnapshot, + factor_shocks: Mapping[str, float], +) -> float: + shocks = { + str(factor): _finite(value, f"shock.{factor}") + for factor, value in factor_shocks.items() + } + unknown = set(shocks) - set(exposure.factor_names) + if unknown: + raise RiskModelError(f"unknown stress factors: {sorted(unknown)}") + loss = 0.0 + for raw_symbol, raw_weight in portfolio_weights.items(): + symbol = normalize_symbol(raw_symbol) + if symbol not in exposure.exposures: + raise RiskModelError(f"missing exposure for {symbol}") + weight = _finite(raw_weight, f"weight.{symbol}") + return_shock = sum( + exposure.exposures[symbol][factor] * shock + for factor, shock in shocks.items() + ) + loss -= weight * return_shock + return loss diff --git a/src/quant60/features.py b/src/quant60/features.py new file mode 100644 index 0000000..d8d41ba --- /dev/null +++ b/src/quant60/features.py @@ -0,0 +1,338 @@ +"""Transparent daily-bar features and train-only preprocessing.""" + +from __future__ import annotations + +import math +import statistics +from dataclasses import dataclass +from datetime import date +from typing import Iterable, Mapping, Sequence + +from .domain import Bar +from .portable_core import normalize_symbol + + +class FeatureIntegrityError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class FeatureSnapshot: + symbol: str + as_of: date + values: dict[str, float | None] + history_start: date + history_end: date + feature_version: str = "transparent-price-v1" + + +@dataclass(frozen=True, slots=True) +class ExecutableLabel: + symbol: str + decision_date: date + entry_date: date | None + exit_date: date | None + value: float | None + status: str + horizon_sessions: int + + +def _ordered_symbol_bars(bars: Iterable[Bar]) -> list[Bar]: + ordered = sorted(tuple(bars), key=lambda item: item.trading_date) + if not ordered: + raise FeatureIntegrityError("feature history is empty") + symbol = ordered[0].symbol + seen: set[date] = set() + for bar in ordered: + if bar.symbol != symbol: + raise FeatureIntegrityError("feature history must contain one symbol") + if bar.trading_date in seen: + raise FeatureIntegrityError("duplicate feature history date") + seen.add(bar.trading_date) + return ordered + + +def _returns(closes: Sequence[float]) -> list[float]: + return [ + closes[index] / closes[index - 1] - 1.0 + for index in range(1, len(closes)) + if closes[index - 1] > 0 + ] + + +def _sample_std(values: Sequence[float]) -> float: + return statistics.stdev(values) if len(values) > 1 else 0.0 + + +def compute_price_features( + bars: Iterable[Bar], + *, + as_of: date | None = None, +) -> FeatureSnapshot: + """Compute only from bars completed by ``as_of``. + + Missing long-window values remain explicit ``None`` and are handled by a + train-fitted preprocessor rather than by silently dropping the security. + """ + + ordered = _ordered_symbol_bars(bars) + decision_date = ordered[-1].trading_date if as_of is None else as_of + history = [bar for bar in ordered if bar.trading_date <= decision_date] + if not history: + raise FeatureIntegrityError("no completed bar exists at feature as_of") + closes = [float(bar.close) for bar in history] + daily_returns = _returns(closes) + + def momentum(window: int) -> float | None: + if len(closes) < window + 1: + return None + return closes[-1] / closes[-1 - window] - 1.0 + + def volatility(window: int, *, downside: bool = False) -> float | None: + if len(daily_returns) < window: + return None + values = daily_returns[-window:] + if downside: + values = [min(0.0, value) for value in values] + return _sample_std(values) * math.sqrt(252.0) + + def median_turnover(window: int) -> float | None: + if len(history) < window: + return None + return statistics.median( + float(bar.close) * int(bar.volume) for bar in history[-window:] + ) + + def amihud(window: int) -> float | None: + if len(history) < window + 1: + return None + recent_bars = history[-window:] + recent_returns = daily_returns[-window:] + values = [] + for bar, daily_return in zip(recent_bars, recent_returns): + turnover = float(bar.close) * int(bar.volume) + if turnover > 0: + values.append(abs(daily_return) / turnover) + return statistics.fmean(values) if values else None + + def distance_to_high(window: int) -> float | None: + if len(closes) < window: + return None + high = max(closes[-window:]) + return closes[-1] / high - 1.0 if high > 0 else None + + gap_values = [] + for bar in history[-20:]: + if bar.previous_close is not None and float(bar.previous_close) > 0: + gap_values.append(float(bar.open) / float(bar.previous_close) - 1.0) + + values = { + "momentum_5": momentum(5), + "momentum_20": momentum(20), + "momentum_60": momentum(60), + "momentum_120": momentum(120), + "reversal_5": ( + -momentum(5) if momentum(5) is not None else None + ), + "volatility_20": volatility(20), + "volatility_60": volatility(60), + "downside_volatility_20": volatility(20, downside=True), + "distance_to_high_60": distance_to_high(60), + "median_turnover_20": median_turnover(20), + "amihud_20": amihud(20), + "mean_open_gap_20": ( + statistics.fmean(gap_values) if gap_values else None + ), + } + for name, value in values.items(): + if value is not None and not math.isfinite(float(value)): + raise FeatureIntegrityError(f"non-finite feature {name}") + return FeatureSnapshot( + symbol=normalize_symbol(history[-1].symbol), + as_of=decision_date, + values=values, + history_start=history[0].trading_date, + history_end=history[-1].trading_date, + ) + + +def build_executable_label( + bars: Iterable[Bar], + *, + decision_date: date, + horizon_sessions: int = 5, + benchmark_bars: Iterable[Bar] | None = None, +) -> ExecutableLabel: + """Build a T+1-open to T+1+horizon-open research label. + + Daily open is an explicit proxy for the article's VWAP label. Suspended or + zero-volume endpoints are marked censored instead of silently deleted. + """ + + if type(horizon_sessions) is not int or horizon_sessions <= 0: + raise ValueError("horizon_sessions must be a positive integer") + ordered = _ordered_symbol_bars(bars) + future = [bar for bar in ordered if bar.trading_date > decision_date] + if len(future) <= horizon_sessions: + return ExecutableLabel( + symbol=ordered[0].symbol, + decision_date=decision_date, + entry_date=None, + exit_date=None, + value=None, + status="INSUFFICIENT_FUTURE", + horizon_sessions=horizon_sessions, + ) + entry = future[0] + exit_bar = future[horizon_sessions] + if ( + entry.suspended + or exit_bar.suspended + or entry.volume <= 0 + or exit_bar.volume <= 0 + ): + return ExecutableLabel( + symbol=ordered[0].symbol, + decision_date=decision_date, + entry_date=entry.trading_date, + exit_date=exit_bar.trading_date, + value=None, + status="NON_EXECUTABLE_CENSORED", + horizon_sessions=horizon_sessions, + ) + value = float(exit_bar.open) / float(entry.open) - 1.0 + if benchmark_bars is not None: + benchmark = build_executable_label( + benchmark_bars, + decision_date=decision_date, + horizon_sessions=horizon_sessions, + ) + if benchmark.value is None: + return ExecutableLabel( + symbol=ordered[0].symbol, + decision_date=decision_date, + entry_date=entry.trading_date, + exit_date=exit_bar.trading_date, + value=None, + status="BENCHMARK_CENSORED", + horizon_sessions=horizon_sessions, + ) + value -= benchmark.value + if not math.isfinite(value): + raise FeatureIntegrityError("label is non-finite") + return ExecutableLabel( + symbol=ordered[0].symbol, + decision_date=decision_date, + entry_date=entry.trading_date, + exit_date=exit_bar.trading_date, + value=value, + status="OK", + horizon_sessions=horizon_sessions, + ) + + +@dataclass(frozen=True, slots=True) +class FeatureTransform: + lower: float + upper: float + median: float + mean: float + scale: float + + +def _quantile(values: Sequence[float], fraction: float) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("quantile requires observations") + position = (len(ordered) - 1) * fraction + lower = int(math.floor(position)) + upper = int(math.ceil(position)) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +class TrainOnlyFeaturePreprocessor: + """Winsorize, impute and standardize from training observations only.""" + + def __init__(self, winsor_fraction: float = 0.01) -> None: + if not math.isfinite(float(winsor_fraction)) or not ( + 0 <= winsor_fraction < 0.5 + ): + raise ValueError("winsor_fraction must lie in [0, 0.5)") + self.winsor_fraction = float(winsor_fraction) + self.transforms: dict[str, FeatureTransform] | None = None + + def fit( + self, + rows: Sequence[Mapping[str, float | None]], + ) -> "TrainOnlyFeaturePreprocessor": + if not rows: + raise ValueError("training feature rows are empty") + names = sorted({name for row in rows for name in row}) + if not names: + raise ValueError("training features are empty") + transforms: dict[str, FeatureTransform] = {} + for name in names: + values = [ + float(row[name]) + for row in rows + if row.get(name) is not None + ] + if any(not math.isfinite(value) for value in values): + raise ValueError(f"training feature {name} is non-finite") + if not values: + raise ValueError(f"training feature {name} is entirely missing") + lower = _quantile(values, self.winsor_fraction) + upper = _quantile(values, 1.0 - self.winsor_fraction) + median = statistics.median(values) + clipped = [min(upper, max(lower, value)) for value in values] + mean = statistics.fmean(clipped) + scale = _sample_std(clipped) + transforms[name] = FeatureTransform( + lower=lower, + upper=upper, + median=median, + mean=mean, + scale=scale if scale > 1e-12 else 1.0, + ) + self.transforms = transforms + return self + + @property + def output_names(self) -> tuple[str, ...]: + if self.transforms is None: + raise ValueError("preprocessor is not fitted") + return tuple( + item + for name in sorted(self.transforms) + for item in (name, f"{name}__missing") + ) + + def transform_one( + self, + row: Mapping[str, float | None], + ) -> dict[str, float]: + if self.transforms is None: + raise ValueError("preprocessor is not fitted") + unknown = set(row) - set(self.transforms) + if unknown: + raise ValueError(f"unknown feature fields: {sorted(unknown)}") + output: dict[str, float] = {} + for name, spec in sorted(self.transforms.items()): + raw = row.get(name) + missing = raw is None + value = spec.median if missing else float(raw) + if not math.isfinite(value): + raise ValueError(f"feature {name} is non-finite") + clipped = min(spec.upper, max(spec.lower, value)) + output[name] = (clipped - spec.mean) / spec.scale + output[f"{name}__missing"] = 1.0 if missing else 0.0 + return output + + def transform( + self, + rows: Sequence[Mapping[str, float | None]], + ) -> list[dict[str, float]]: + return [self.transform_one(row) for row in rows] diff --git a/src/quant60/ledger.py b/src/quant60/ledger.py new file mode 100644 index 0000000..168596f --- /dev/null +++ b/src/quant60/ledger.py @@ -0,0 +1,225 @@ +"""Append-only hash-chain ledger with deterministic JSONL serialization.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from dataclasses import asdict, dataclass, is_dataclass +from datetime import date, datetime +from decimal import Decimal +from enum import Enum +from pathlib import Path +from typing import Any, Iterable + + +GENESIS_HASH = "0" * 64 + + +class LedgerIntegrityError(ValueError): + pass + + +class DuplicateEventError(ValueError): + pass + + +def jsonable(value: Any) -> Any: + if is_dataclass(value): + return jsonable(asdict(value)) + if isinstance(value, dict): + return {str(key): jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [jsonable(item) for item in value] + if isinstance(value, (date, datetime)): + return value.isoformat() + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, Enum): + return value.value + if isinstance(value, Path): + return str(value) + if value is None or isinstance(value, (str, int, float, bool)): + return value + raise TypeError(f"value is not JSON serializable: {type(value).__name__}") + + +def canonical_json(value: Any) -> str: + return json.dumps( + jsonable(value), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + +@dataclass(frozen=True, slots=True) +class LedgerRecord: + sequence: int + timestamp: str + event_id: str + event_type: str + payload: dict[str, Any] + previous_hash: str + hash: str + + def as_dict(self) -> dict[str, Any]: + return jsonable(asdict(self)) + + +class HashChainLedger: + def __init__(self, records: Iterable[LedgerRecord] = ()) -> None: + self.records: list[LedgerRecord] = [] + self._by_event_id: dict[str, LedgerRecord] = {} + for record in records: + self._accept_loaded(record) + self.verify() + + @property + def head_hash(self) -> str: + return self.records[-1].hash if self.records else GENESIS_HASH + + def _body( + self, + sequence: int, + timestamp: str, + event_id: str, + event_type: str, + payload: dict[str, Any], + previous_hash: str, + ) -> dict[str, Any]: + return { + "sequence": sequence, + "timestamp": timestamp, + "event_id": event_id, + "event_type": event_type, + "payload": jsonable(payload), + "previous_hash": previous_hash, + } + + @staticmethod + def _digest(body: dict[str, Any]) -> str: + return hashlib.sha256(canonical_json(body).encode("utf-8")).hexdigest() + + def append( + self, + event_type: str, + payload: dict[str, Any], + *, + timestamp: str, + event_id: str, + ) -> LedgerRecord: + if not event_type or not event_id or not timestamp: + raise ValueError("event_type, event_id and timestamp are required") + normalized_payload = jsonable(payload) + prior = self._by_event_id.get(event_id) + if prior is not None: + if ( + prior.event_type == event_type + and prior.payload == normalized_payload + and prior.timestamp == timestamp + ): + return prior + raise DuplicateEventError( + f"event_id {event_id!r} already exists with different content" + ) + sequence = len(self.records) + 1 + previous_hash = self.head_hash + body = self._body( + sequence, + timestamp, + event_id, + event_type, + normalized_payload, + previous_hash, + ) + record = LedgerRecord( + **body, + hash=self._digest(body), + ) + self.records.append(record) + self._by_event_id[event_id] = record + return record + + def _accept_loaded(self, record: LedgerRecord) -> None: + if record.event_id in self._by_event_id: + raise LedgerIntegrityError( + f"duplicate event_id in ledger: {record.event_id}" + ) + self.records.append(record) + self._by_event_id[record.event_id] = record + + def verify(self) -> bool: + previous_hash = GENESIS_HASH + seen: set[str] = set() + for expected_sequence, record in enumerate(self.records, 1): + if record.sequence != expected_sequence: + raise LedgerIntegrityError( + f"sequence gap at {expected_sequence}" + ) + if record.previous_hash != previous_hash: + raise LedgerIntegrityError( + f"previous hash mismatch at {expected_sequence}" + ) + if record.event_id in seen: + raise LedgerIntegrityError( + f"duplicate event_id at {expected_sequence}" + ) + seen.add(record.event_id) + body = self._body( + record.sequence, + record.timestamp, + record.event_id, + record.event_type, + record.payload, + record.previous_hash, + ) + expected_hash = self._digest(body) + if record.hash != expected_hash: + raise LedgerIntegrityError( + f"hash mismatch at sequence {expected_sequence}" + ) + previous_hash = record.hash + return True + + def write_jsonl(self, path: str | Path) -> Path: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{destination.name}.", + suffix=".tmp", + dir=str(destination.parent), + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + for record in self.records: + handle.write(canonical_json(record.as_dict())) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_name, destination) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise + return destination + + @classmethod + def read_jsonl(cls, path: str | Path) -> HashChainLedger: + records: list[LedgerRecord] = [] + with Path(path).open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + continue + try: + raw = json.loads(line) + records.append(LedgerRecord(**raw)) + except (TypeError, json.JSONDecodeError) as error: + raise LedgerIntegrityError( + f"invalid ledger line {line_number}: {error}" + ) from error + return cls(records) diff --git a/src/quant60/optimizer.py b/src/quant60/optimizer.py new file mode 100644 index 0000000..c15e7f0 --- /dev/null +++ b/src/quant60/optimizer.py @@ -0,0 +1,425 @@ +"""Long-only portfolio construction with deterministic and CVXPY paths.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Any, Iterable + +from .portable_core import normalize_symbol +from .universe import UniverseState + + +class OptimizationError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class PortfolioCandidate: + symbol: str + score: float + variance: float + current_weight: float + cost_bps: float + max_adv_weight: float + industry: str + state: UniverseState = UniverseState.OPENABLE + + def __post_init__(self) -> None: + object.__setattr__(self, "symbol", normalize_symbol(self.symbol)) + if not isinstance(self.state, UniverseState): + raise OptimizationError("candidate state must be UniverseState") + if not str(self.industry).strip(): + raise OptimizationError("industry is required") + for name in ( + "score", + "variance", + "current_weight", + "cost_bps", + "max_adv_weight", + ): + value = float(getattr(self, name)) + if not math.isfinite(value): + raise OptimizationError(f"{name} must be finite") + if self.variance <= 0: + raise OptimizationError("variance must be positive") + if self.current_weight < 0 or self.max_adv_weight < 0: + raise OptimizationError("weights/capacity must be non-negative") + if self.cost_bps < 0: + raise OptimizationError("cost_bps must be non-negative") + + +@dataclass(frozen=True, slots=True) +class PortfolioConstraints: + gross_target: float = 0.95 + single_name_cap: float = 0.03 + industry_cap: float = 0.25 + one_way_turnover_cap: float = 0.20 + risk_aversion: float = 1.0 + cost_aversion: float = 1.0 + turnover_aversion: float = 0.01 + variance_floor: float = 1e-8 + + def validate(self) -> None: + fractions = ( + "gross_target", + "single_name_cap", + "industry_cap", + "one_way_turnover_cap", + ) + for name in fractions: + value = float(getattr(self, name)) + if not math.isfinite(value) or not (0 <= value <= 1): + raise OptimizationError(f"{name} must lie in [0, 1]") + if self.single_name_cap <= 0 or self.industry_cap <= 0: + raise OptimizationError("single-name and industry caps must be positive") + for name in ( + "risk_aversion", + "cost_aversion", + "turnover_aversion", + ): + value = float(getattr(self, name)) + if not math.isfinite(value) or value < 0: + raise OptimizationError(f"{name} must be non-negative") + if ( + not math.isfinite(float(self.variance_floor)) + or self.variance_floor <= 0 + ): + raise OptimizationError("variance_floor must be positive") + + +@dataclass(frozen=True, slots=True) +class PortfolioSolution: + weights: dict[str, float] + cash_weight: float + gross_weight: float + one_way_turnover: float + predicted_variance: float + expected_score: float + estimated_cost_bps_weighted: float + solver: str + status: str + diagnostics: dict[str, Any] + + +def _validate_candidates( + candidates: Iterable[PortfolioCandidate], +) -> tuple[PortfolioCandidate, ...]: + ordered = tuple(sorted(candidates, key=lambda item: item.symbol)) + if not ordered: + raise OptimizationError("candidate set is empty") + seen: set[str] = set() + for item in ordered: + if item.symbol in seen: + raise OptimizationError(f"duplicate candidate {item.symbol}") + seen.add(item.symbol) + if item.state == UniverseState.EXCLUDED and item.current_weight > 1e-12: + raise OptimizationError( + f"held position {item.symbol} cannot disappear as EXCLUDED" + ) + return ordered + + +def _solution( + candidates: tuple[PortfolioCandidate, ...], + weights: dict[str, float], + *, + solver: str, + status: str, + diagnostics: dict[str, Any], +) -> PortfolioSolution: + gross = sum(weights.values()) + current = {item.symbol: item.current_weight for item in candidates} + turnover = 0.5 * sum( + abs(weights.get(symbol, 0.0) - current.get(symbol, 0.0)) + for symbol in set(weights) | set(current) + ) + predicted_variance = sum( + item.variance * weights[item.symbol] ** 2 for item in candidates + ) + expected_score = sum( + item.score * weights[item.symbol] for item in candidates + ) + weighted_cost = sum( + item.cost_bps + * abs(weights[item.symbol] - item.current_weight) + for item in candidates + ) + return PortfolioSolution( + weights=dict(sorted(weights.items())), + cash_weight=max(0.0, 1.0 - gross), + gross_weight=gross, + one_way_turnover=turnover, + predicted_variance=predicted_variance, + expected_score=expected_score, + estimated_cost_bps_weighted=weighted_cost, + solver=solver, + status=status, + diagnostics=diagnostics, + ) + + +def _post_check( + candidates: tuple[PortfolioCandidate, ...], + constraints: PortfolioConstraints, + weights: dict[str, float], +) -> None: + tolerance = 1e-8 + by_symbol = {item.symbol: item for item in candidates} + for symbol, weight in weights.items(): + item = by_symbol[symbol] + if not math.isfinite(weight) or weight < -tolerance: + raise OptimizationError(f"invalid target weight for {symbol}") + cap = min(constraints.single_name_cap, item.max_adv_weight) + if item.state == UniverseState.OPENABLE and weight > cap + tolerance: + raise OptimizationError(f"target exceeds cap for {symbol}") + if item.state == UniverseState.FROZEN and ( + abs(weight - item.current_weight) > tolerance + ): + raise OptimizationError(f"frozen weight changed for {symbol}") + if item.state == UniverseState.HOLD_ONLY and ( + weight > item.current_weight + tolerance + ): + raise OptimizationError(f"hold-only weight increased for {symbol}") + if item.state in {UniverseState.SELL_ONLY, UniverseState.EXCLUDED} and ( + weight > item.current_weight + tolerance + ): + raise OptimizationError(f"exit-only weight increased for {symbol}") + industries: dict[str, float] = {} + for item in candidates: + industries[item.industry] = ( + industries.get(item.industry, 0.0) + weights[item.symbol] + ) + if any( + weight > constraints.industry_cap + tolerance + for weight in industries.values() + ): + raise OptimizationError("industry cap violated") + gross = sum(weights.values()) + current_gross = sum(item.current_weight for item in candidates) + # A pre-existing breach may only decline; it cannot always be removed in a + # single turnover-constrained rebalance. + allowed_gross = max(constraints.gross_target, current_gross) + if gross > allowed_gross + tolerance: + raise OptimizationError("gross cap violated") + turnover = 0.5 * sum( + abs(weights[item.symbol] - item.current_weight) for item in candidates + ) + if turnover > constraints.one_way_turnover_cap + tolerance: + raise OptimizationError("turnover cap violated") + + +def optimize_deterministic( + candidates: Iterable[PortfolioCandidate], + constraints: PortfolioConstraints = PortfolioConstraints(), +) -> PortfolioSolution: + """Deterministic risk/cost-adjusted allocation with conservative fallbacks.""" + + constraints.validate() + ordered = _validate_candidates(candidates) + weights = {item.symbol: 0.0 for item in ordered} + + # Frozen and hold-only positions survive the decision. Sell-only positions + # are eligible to reduce toward zero; excluded unheld names stay zero. + for item in ordered: + if item.state in {UniverseState.FROZEN, UniverseState.HOLD_ONLY}: + weights[item.symbol] = item.current_weight + + fixed_gross = sum(weights.values()) + available = max(0.0, constraints.gross_target - fixed_gross) + active: dict[str, float] = {} + caps: dict[str, float] = {} + by_symbol = {item.symbol: item for item in ordered} + for item in ordered: + if item.state != UniverseState.OPENABLE: + continue + net_score = item.score - ( + constraints.cost_aversion * item.cost_bps / 10_000.0 + ) + utility = max(0.0, net_score) / max( + constraints.variance_floor, + item.variance * max(constraints.risk_aversion, 1e-12), + ) + if utility <= 0: + continue + active[item.symbol] = utility + caps[item.symbol] = min( + constraints.single_name_cap, + item.max_adv_weight, + ) + + remaining = available + while active and remaining > 1e-12: + utility_sum = sum(active.values()) + if utility_sum <= 0: + break + capped: list[str] = [] + for symbol in sorted(active): + proposal = remaining * active[symbol] / utility_sum + room = max(0.0, caps[symbol] - weights[symbol]) + if proposal >= room - 1e-12: + weights[symbol] += room + remaining -= room + capped.append(symbol) + if not capped: + for symbol in sorted(active): + weights[symbol] += remaining * active[symbol] / utility_sum + remaining = 0.0 + break + for symbol in capped: + active.pop(symbol) + remaining = max(0.0, remaining) + + # Industry limits are hard post-allocation constraints. Scaling leaves the + # unused amount as cash instead of inventing a second optimizer pass. + industry_totals: dict[str, float] = {} + for item in ordered: + industry_totals[item.industry] = ( + industry_totals.get(item.industry, 0.0) + weights[item.symbol] + ) + for industry, total in sorted(industry_totals.items()): + if total <= constraints.industry_cap + 1e-12: + continue + fixed = sum( + weights[item.symbol] + for item in ordered + if item.industry == industry + and item.state in {UniverseState.FROZEN, UniverseState.HOLD_ONLY} + ) + if fixed > constraints.industry_cap + 1e-12: + raise OptimizationError( + f"fixed {industry} exposure exceeds the industry cap" + ) + scalable = total - fixed + factor = ( + (constraints.industry_cap - fixed) / scalable + if scalable > 0 + else 0.0 + ) + for item in ordered: + if ( + item.industry == industry + and item.state not in {UniverseState.FROZEN, UniverseState.HOLD_ONLY} + ): + weights[item.symbol] *= max(0.0, factor) + + raw_turnover = 0.5 * sum( + abs(weights[item.symbol] - item.current_weight) for item in ordered + ) + turnover_scaled = False + if raw_turnover > constraints.one_way_turnover_cap + 1e-12: + scale = constraints.one_way_turnover_cap / raw_turnover + for item in ordered: + delta = weights[item.symbol] - item.current_weight + weights[item.symbol] = item.current_weight + delta * scale + turnover_scaled = True + + _post_check(ordered, constraints, weights) + return _solution( + ordered, + weights, + solver="deterministic_reference", + status="OPTIMAL_REFERENCE", + diagnostics={ + "unallocated_cash_from_caps": remaining, + "raw_one_way_turnover": raw_turnover, + "turnover_scaled": turnover_scaled, + "candidate_count": len(ordered), + }, + ) + + +def optimize_cvxpy( + candidates: Iterable[PortfolioCandidate], + constraints: PortfolioConstraints = PortfolioConstraints(), + *, + solver: str | None = None, +) -> PortfolioSolution: + """Solve the declared convex surrogate when CVXPY is available.""" + + constraints.validate() + ordered = _validate_candidates(candidates) + try: + import cvxpy as cp + except ImportError as exc: + raise OptimizationError( + "CVXPY is unavailable; install the full research environment " + "or use optimize_deterministic" + ) from exc + + count = len(ordered) + weight = cp.Variable(count) + current = [item.current_weight for item in ordered] + alpha = [item.score for item in ordered] + variance = [item.variance for item in ordered] + cost_rate = [item.cost_bps / 10_000.0 for item in ordered] + delta = weight - current + objective = cp.Maximize( + alpha @ weight + - constraints.risk_aversion + * cp.sum(cp.multiply(variance, cp.square(weight))) + - constraints.cost_aversion + * cp.sum(cp.multiply(cost_rate, cp.abs(delta))) + - constraints.turnover_aversion * cp.norm1(delta) + ) + limits = [ + weight >= 0, + cp.sum(weight) <= constraints.gross_target, + 0.5 * cp.norm1(delta) <= constraints.one_way_turnover_cap, + ] + for index, item in enumerate(ordered): + if item.state == UniverseState.OPENABLE: + limits.append( + weight[index] + <= min(constraints.single_name_cap, item.max_adv_weight) + ) + elif item.state == UniverseState.FROZEN: + limits.append(weight[index] == item.current_weight) + elif item.state == UniverseState.HOLD_ONLY: + limits.append(weight[index] <= item.current_weight) + elif item.state in {UniverseState.SELL_ONLY, UniverseState.EXCLUDED}: + limits.append(weight[index] <= item.current_weight) + for industry in sorted({item.industry for item in ordered}): + indexes = [ + index + for index, item in enumerate(ordered) + if item.industry == industry + ] + limits.append(cp.sum(weight[indexes]) <= constraints.industry_cap) + problem = cp.Problem(objective, limits) + selected_solver = solver + if selected_solver is None: + installed = set(cp.installed_solvers()) + selected_solver = next( + ( + candidate + for candidate in ("CLARABEL", "ECOS", "SCS") + if candidate in installed + ), + None, + ) + solve_kwargs: dict[str, Any] = {"warm_start": False, "verbose": False} + if selected_solver is not None: + solve_kwargs["solver"] = selected_solver + try: + problem.solve(**solve_kwargs) + except Exception as exc: + raise OptimizationError(f"CVXPY solve failed: {exc}") from exc + if problem.status not in {"optimal", "optimal_inaccurate"}: + raise OptimizationError(f"CVXPY returned {problem.status}") + values = [float(item) for item in weight.value] + weights = { + item.symbol: max(0.0, values[index]) + for index, item in enumerate(ordered) + } + _post_check(ordered, constraints, weights) + return _solution( + ordered, + weights, + solver=f"cvxpy:{selected_solver or 'default'}", + status=str(problem.status).upper(), + diagnostics={ + "objective": float(problem.value), + "candidate_count": count, + }, + ) diff --git a/src/quant60/pit.py b/src/quant60/pit.py new file mode 100644 index 0000000..c8bc205 --- /dev/null +++ b/src/quant60/pit.py @@ -0,0 +1,268 @@ +"""Point-in-time records, joins, quality checks, and deterministic versions.""" + +from __future__ import annotations + +import hashlib +import math +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Iterable + +from .ledger import canonical_json, jsonable + + +class PITIntegrityError(ValueError): + """Point-in-time data is ambiguous, malformed, or future contaminated.""" + + +def _aware_datetime(value: datetime | str, name: str) -> datetime: + if isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise PITIntegrityError(f"{name} must be an ISO date-time") from exc + elif isinstance(value, datetime): + parsed = value + else: + raise PITIntegrityError(f"{name} must be a datetime or ISO string") + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise PITIntegrityError(f"{name} must include a timezone") + return parsed.astimezone(timezone.utc) + + +@dataclass(frozen=True, slots=True) +class PITRecord: + """One field value with separate economic and information timestamps.""" + + entity_id: str + field: str + effective_time: datetime + available_time: datetime + value: Any + source: str + source_version: str + revision: int = 0 + + def __post_init__(self) -> None: + entity = str(self.entity_id).strip() + field_name = str(self.field).strip() + source = str(self.source).strip() + source_version = str(self.source_version).strip() + if not entity or not field_name or not source or not source_version: + raise PITIntegrityError( + "entity_id, field, source and source_version are required" + ) + if type(self.revision) is not int or self.revision < 0: + raise PITIntegrityError("revision must be a non-negative integer") + if isinstance(self.value, float) and not math.isfinite(self.value): + raise PITIntegrityError("PIT numeric values must be finite") + try: + jsonable(self.value) + except TypeError as exc: + raise PITIntegrityError("PIT value must be JSON serializable") from exc + object.__setattr__(self, "entity_id", entity) + object.__setattr__(self, "field", field_name) + object.__setattr__(self, "source", source) + object.__setattr__(self, "source_version", source_version) + object.__setattr__( + self, + "effective_time", + _aware_datetime(self.effective_time, "effective_time"), + ) + object.__setattr__( + self, + "available_time", + _aware_datetime(self.available_time, "available_time"), + ) + + @property + def identity(self) -> tuple[str, str, datetime, datetime, int]: + return ( + self.entity_id, + self.field, + self.effective_time, + self.available_time, + self.revision, + ) + + def as_dict(self) -> dict[str, Any]: + return { + "entity_id": self.entity_id, + "field": self.field, + "effective_time": self.effective_time.isoformat(), + "available_time": self.available_time.isoformat(), + "value": jsonable(self.value), + "source": self.source, + "source_version": self.source_version, + "revision": self.revision, + } + + +@dataclass(frozen=True, slots=True) +class PITValue: + entity_id: str + field: str + value: Any + effective_time: datetime + available_time: datetime + source: str + source_version: str + revision: int + + @classmethod + def from_record(cls, record: PITRecord) -> "PITValue": + return cls( + entity_id=record.entity_id, + field=record.field, + value=record.value, + effective_time=record.effective_time, + available_time=record.available_time, + source=record.source, + source_version=record.source_version, + revision=record.revision, + ) + + +class PointInTimeTable: + """Immutable in-memory reference table used by adapters and tests.""" + + def __init__(self, records: Iterable[PITRecord]) -> None: + ordered = sorted( + tuple(records), + key=lambda record: ( + record.entity_id, + record.field, + record.effective_time, + record.available_time, + record.revision, + record.source, + record.source_version, + ), + ) + seen: set[tuple[str, str, datetime, datetime, int]] = set() + by_key: dict[tuple[str, str], list[PITRecord]] = {} + for record in ordered: + if not isinstance(record, PITRecord): + raise PITIntegrityError("all entries must be PITRecord instances") + if record.identity in seen: + raise PITIntegrityError( + "duplicate PIT identity for " + f"{record.entity_id}.{record.field}" + ) + seen.add(record.identity) + by_key.setdefault((record.entity_id, record.field), []).append(record) + self._records = ordered + self._by_key = by_key + self._data_version = hashlib.sha256( + canonical_json([record.as_dict() for record in ordered]).encode("utf-8") + ).hexdigest() + + @property + def records(self) -> tuple[PITRecord, ...]: + return self._records + + @property + def data_version(self) -> str: + return self._data_version + + def query( + self, + entity_id: str, + field: str, + *, + as_of: datetime | str, + effective_at: datetime | str | None = None, + ) -> PITValue | None: + information_clock = _aware_datetime(as_of, "as_of") + economic_clock = ( + information_clock + if effective_at is None + else _aware_datetime(effective_at, "effective_at") + ) + candidates = [ + record + for record in self._by_key.get( + (str(entity_id).strip(), str(field).strip()), + (), + ) + if record.available_time <= information_clock + and record.effective_time <= economic_clock + ] + if not candidates: + return None + selected = max( + candidates, + key=lambda record: ( + record.effective_time, + record.available_time, + record.revision, + record.source_version, + ), + ) + return PITValue.from_record(selected) + + def snapshot( + self, + *, + as_of: datetime | str, + effective_at: datetime | str | None = None, + entities: Iterable[str] | None = None, + fields: Iterable[str] | None = None, + ) -> dict[str, dict[str, PITValue]]: + requested_entities = ( + {str(entity).strip() for entity in entities} + if entities is not None + else {entity for entity, unused in self._by_key} + ) + requested_fields = ( + {str(field).strip() for field in fields} + if fields is not None + else {field for unused, field in self._by_key} + ) + result: dict[str, dict[str, PITValue]] = {} + for entity in sorted(requested_entities): + for field in sorted(requested_fields): + value = self.query( + entity, + field, + as_of=as_of, + effective_at=effective_at, + ) + if value is not None: + result.setdefault(entity, {})[field] = value + return result + + def quality_report(self) -> dict[str, Any]: + sources = sorted( + { + f"{record.source}@{record.source_version}" + for record in self._records + } + ) + return { + "ok": True, + "record_count": len(self._records), + "entity_count": len({record.entity_id for record in self._records}), + "field_count": len({record.field for record in self._records}), + "sources": sources, + "data_version": self.data_version, + } + + +def assert_information_available( + values: Iterable[PITRecord | PITValue], + *, + as_of: datetime | str, +) -> None: + """Fail if any supplied value was unavailable at the decision timestamp.""" + + decision_time = _aware_datetime(as_of, "as_of") + future = [ + f"{value.entity_id}.{value.field}@{value.available_time.isoformat()}" + for value in values + if value.available_time > decision_time + ] + if future: + raise PITIntegrityError( + "future-available values reached the decision: " + ", ".join(future) + ) diff --git a/src/quant60/portable_core.py b/src/quant60/portable_core.py new file mode 100644 index 0000000..b1373ee --- /dev/null +++ b/src/quant60/portable_core.py @@ -0,0 +1,374 @@ +"""Small strategy core that also runs on Python 3.6 platform runtimes. + +Keep this module deliberately boring: no dataclasses, no third-party imports, +no modern annotation syntax. JoinQuant/QMT/Qlib adapters can vendor this +single file when importing the complete ``quant60`` package is not possible. +""" + +from __future__ import division + +import math +import re + + +_CANONICAL_EXCHANGES = { + "SH": "XSHG", + "SSE": "XSHG", + "XSHG": "XSHG", + "SZ": "XSHE", + "SZE": "XSHE", + "SZSE": "XSHE", + "XSHE": "XSHE", + "BJ": "XBSE", + "BSE": "XBSE", + "XBSE": "XBSE", +} + +_SHORT_EXCHANGES = { + "XSHG": "SH", + "XSHE": "SZ", + "XBSE": "BJ", +} + + +def _sum_numeric(values): + """Sum numerics without trusting a hosted runtime's global ``sum`` name.""" + + total = 0.0 + for value in values: + total += value + return total + + +def _infer_exchange(code): + if code.startswith(("4", "8")): + return "XBSE" + if code.startswith(("5", "6", "9")): + return "XSHG" + return "XSHE" + + +def _split_symbol(symbol): + if not isinstance(symbol, str): + raise TypeError("symbol must be a string") + value = symbol.strip().upper().replace(" ", "") + if not value: + raise ValueError("symbol must not be empty") + + match = re.match(r"^(\d{6})\.(XSHG|XSHE|XBSE|SH|SZ|BJ)$", value) + if match: + return match.group(1), _CANONICAL_EXCHANGES[match.group(2)] + + match = re.match(r"^(XSHG|XSHE|XBSE|SH|SZ|BJ)[\.:]?(\d{6})$", value) + if match: + return match.group(2), _CANONICAL_EXCHANGES[match.group(1)] + + match = re.match(r"^(\d{6})[\.:](SSE|SZE|SZSE|BSE)$", value) + if match: + return match.group(1), _CANONICAL_EXCHANGES[match.group(2)] + + if re.match(r"^\d{6}$", value): + return value, _infer_exchange(value) + raise ValueError("unsupported A-share symbol: %s" % symbol) + + +def normalize_symbol(symbol, target="canonical"): + """Convert common A-share codes to canonical/JQ, QMT, or Qlib format.""" + + code, exchange = _split_symbol(symbol) + target_name = str(target).strip().lower() + if target_name in ("canonical", "joinquant", "jq"): + return "%s.%s" % (code, exchange) + if target_name in ("qmt", "xtquant"): + return "%s.%s" % (code, _SHORT_EXCHANGES[exchange]) + if target_name == "qlib": + return "%s%s" % (_SHORT_EXCHANGES[exchange], code) + raise ValueError("unsupported target platform: %s" % target) + + +def convert_symbol(symbol, target="canonical"): + """Alias kept as the stable adapter-facing name.""" + + return normalize_symbol(symbol, target) + + +def momentum_score(prices, lookback=20, skip=0): + """Return trailing simple momentum, or ``None`` if history is insufficient. + + ``skip=0`` uses the latest completed decision bar. Set ``skip=1`` only + when intentionally implementing a 20-1 convention. A lookback of N needs + N+skip+1 observations. + """ + + lookback = int(lookback) + skip = int(skip) + if lookback <= 0 or skip < 0: + raise ValueError("lookback must be positive and skip non-negative") + values = list(prices) + if len(values) < lookback + skip + 1: + return None + end_index = len(values) - 1 - skip + start_index = end_index - lookback + start = float(values[start_index]) + end = float(values[end_index]) + if ( + start <= 0.0 + or end <= 0.0 + or not math.isfinite(start) + or not math.isfinite(end) + ): + return None + return end / start - 1.0 + + +def rank_momentum(price_history, lookback=20, skip=0, top_n=None): + """Rank ``{symbol: closes}`` by momentum with deterministic tie-breaking.""" + + ranked = [] + for symbol in sorted(price_history): + score = momentum_score(price_history[symbol], lookback, skip) + if score is not None and math.isfinite(score): + ranked.append((normalize_symbol(symbol), score)) + ranked.sort(key=lambda item: (-item[1], item[0])) + if top_n is not None: + ranked = ranked[: max(0, int(top_n))] + return ranked + + +def capped_target_weights( + scores, max_weight=0.20, gross_target=1.0, min_score=0.0 +): + """Long-only score-proportional weights with iterative cap redistribution.""" + + max_weight = float(max_weight) + gross_target = float(gross_target) + min_score = float(min_score) + if not math.isfinite(max_weight) or not 0.0 < max_weight <= 1.0: + raise ValueError("max_weight must be finite and lie in (0, 1]") + if not math.isfinite(gross_target) or not 0.0 <= gross_target <= 1.0: + raise ValueError("gross_target must be finite and lie in [0, 1]") + if not math.isfinite(min_score): + raise ValueError("min_score must be finite") + + if hasattr(scores, "items"): + items = list(scores.items()) + else: + items = list(scores) + normalized = {} + for symbol, raw_score in items: + name = normalize_symbol(symbol) + score = float(raw_score) + if not math.isfinite(score): + raise ValueError("score must be finite for %s" % name) + normalized[name] = score + + weights = dict((symbol, 0.0) for symbol in normalized) + active = dict( + (symbol, score) + for symbol, score in normalized.items() + if score > min_score and score > 0.0 + ) + remaining = min(gross_target, len(active) * max_weight) + + while active and remaining > 1e-15: + score_sum = _sum_numeric(active.values()) + if score_sum <= 0.0: + break + capped = [] + for symbol in sorted(active): + proposed = remaining * active[symbol] / score_sum + if proposed >= max_weight - 1e-15: + weights[symbol] = max_weight + capped.append(symbol) + if not capped: + for symbol in sorted(active): + weights[symbol] = remaining * active[symbol] / score_sum + remaining = 0.0 + break + for symbol in capped: + active.pop(symbol) + remaining -= max_weight + remaining = max(0.0, remaining) + return weights + + +def round_board_lot(quantity, lot_size=100): + """Round an absolute long position down to a board-lot quantity.""" + + lot_size = int(lot_size) + if lot_size <= 0: + raise ValueError("lot_size must be positive") + if not math.isfinite(float(quantity)): + raise ValueError("quantity must be finite") + quantity = max(0, int(quantity)) + return quantity // lot_size * lot_size + + +def _is_star_market(symbol): + code, exchange = _split_symbol(symbol) + return exchange == "XSHG" and code.startswith(("688", "689")) + + +def target_quantities( + weights, prices, equity, lot_size=100, cash_buffer=0.02 +): + """Convert target weights to affordable board-lot long quantities.""" + + equity = float(equity) + cash_buffer = float(cash_buffer) + if ( + not math.isfinite(equity) + or equity < 0.0 + or not math.isfinite(cash_buffer) + or not 0.0 <= cash_buffer < 1.0 + ): + raise ValueError("invalid equity or cash_buffer") + positive_weights = [] + for raw_symbol, raw_weight in weights.items(): + weight = float(raw_weight) + if not math.isfinite(weight): + raise ValueError( + "weight must be finite for %s" % normalize_symbol(raw_symbol) + ) + positive_weights.append(max(0.0, weight)) + requested_gross = _sum_numeric(positive_weights) + gross_ceiling = 1.0 - cash_buffer + scale = ( + min(1.0, gross_ceiling / requested_gross) + if requested_gross > 0.0 + else 1.0 + ) + result = {} + for raw_symbol in sorted(weights): + symbol = normalize_symbol(raw_symbol) + weight = float(weights[raw_symbol]) + if not math.isfinite(weight): + raise ValueError("weight must be finite for %s" % symbol) + weight = max(0.0, weight) + price = prices.get(raw_symbol) + if price is None: + price = prices.get(symbol) + if price is None: + raise KeyError("missing price for %s" % symbol) + price = float(price) + if price <= 0.0 or not math.isfinite(price): + raise ValueError("invalid price for %s" % symbol) + # Weights are absolute equity weights. cash_buffer is a ceiling, not + # a second multiplicative haircut: gross=.95 and buffer=.02 stays .95. + budget = equity * weight * scale + quantity = round_board_lot(budget / price, lot_size) + # STAR Market auction orders must contain at least 200 shares. Keep + # the portable planner conservative and on the configured 100-share + # grid even though quantities above 200 may increment by one share. + if _is_star_market(symbol) and 0 < quantity < 200: + quantity = 0 + result[symbol] = quantity + return result + + +def order_deltas( + targets, current, sellable=None, lot_size=100 +): + """Return signed board-lot orders; negative sells respect sellable quantity.""" + + canonical_targets = {} + canonical_current = {} + canonical_sellable = {} + for symbol, quantity in targets.items(): + canonical_targets[normalize_symbol(symbol)] = max(0, int(quantity)) + for symbol, quantity in current.items(): + canonical_current[normalize_symbol(symbol)] = max(0, int(quantity)) + if sellable is not None: + for symbol, quantity in sellable.items(): + canonical_sellable[normalize_symbol(symbol)] = max(0, int(quantity)) + + result = {} + symbols = sorted(set(canonical_targets) | set(canonical_current)) + for symbol in symbols: + target = round_board_lot(canonical_targets.get(symbol, 0), lot_size) + if _is_star_market(symbol) and 0 < target < 200: + target = 0 + held = canonical_current.get(symbol, 0) + delta = target - held + if delta > 0: + delta = round_board_lot(delta, lot_size) + if _is_star_market(symbol) and 0 < delta < 200: + delta = 0 + elif delta < 0: + available = held + if sellable is not None: + available = min(held, canonical_sellable.get(symbol, 0)) + requested = min(-delta, available) + # A complete liquidation may submit the entire odd-lot balance. + # This also covers the STAR exception for a remaining balance + # below 200 shares. Partial orders stay on the configured grid. + full_liquidation = ( + target == 0 + and requested == held + and available >= held + ) + if full_liquidation: + delta = -held + else: + delta = -round_board_lot(requested, lot_size) + if _is_star_market(symbol) and 0 < -delta < 200: + delta = 0 + if delta: + result[symbol] = delta + return result + + +def build_rebalance_plan( + price_history, + current, + sellable, + equity, + lookback=20, + skip=0, + top_n=10, + max_weight=0.20, + gross_target=1.0, + cash_buffer=0.02, + lot_size=100, +): + """Build the complete portable signal -> weight -> quantity -> order plan.""" + + ranked = rank_momentum(price_history, lookback, skip, top_n) + scores = dict(ranked) + weights = capped_target_weights(scores, max_weight, gross_target, 0.0) + for raw_symbol in price_history: + symbol = normalize_symbol(raw_symbol) + if symbol not in scores: + scores[symbol] = momentum_score( + price_history[raw_symbol], lookback, skip + ) + if symbol not in weights: + weights[symbol] = 0.0 + prices = {} + for raw_symbol, history in price_history.items(): + if not history: + continue + prices[normalize_symbol(raw_symbol)] = history[-1] + targets = target_quantities( + weights, prices, equity, lot_size, cash_buffer + ) + orders = order_deltas(targets, current, sellable, lot_size) + return { + "scores": scores, + "weights": weights, + "targets": targets, + "orders": orders, + } + + +__all__ = [ + "build_rebalance_plan", + "capped_target_weights", + "convert_symbol", + "momentum_score", + "normalize_symbol", + "order_deltas", + "rank_momentum", + "round_board_lot", + "target_quantities", +] diff --git a/src/quant60/qmt_shadow.py b/src/quant60/qmt_shadow.py new file mode 100644 index 0000000..2ab37a1 --- /dev/null +++ b/src/quant60/qmt_shadow.py @@ -0,0 +1,3595 @@ +"""Read-only QMT/XtTrader preflight and deterministic target-diff evidence. + +This module deliberately accepts only an already-connected query boundary. +It never connects, submits, cancels, or exposes a live-order switch. A plan +that contains any blocker has every proposed delta forced to zero. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import math +import os +import tempfile +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Callable, Mapping, Protocol, Sequence + +from .backtest import _engine_source_manifest +from .contracts import validate_records +from .ledger import canonical_json, jsonable +from .portable_core import normalize_symbol, order_deltas +from .snapshot_pipeline import ( + first_executable_window, + verify_snapshot_decision, +) + + +_OPEN_ORDER_STATES = { + "NEW", + "ACCEPTED", + "PARTIALLY_FILLED", + "CANCEL_PENDING", + "UNKNOWN", +} +_TERMINAL_ORDER_STATES = {"FILLED", "REJECTED", "CANCELLED"} +_KNOWN_ORDER_STATES = _OPEN_ORDER_STATES | _TERMINAL_ORDER_STATES +DECISION_EQUITY_ABSOLUTE_TOLERANCE = 1.0 +DECISION_EQUITY_RELATIVE_TOLERANCE = 0.0001 +DECISION_EQUITY_TOLERANCE_RULE = ( + "max(absolute_tolerance," + "relative_tolerance*abs(decision_equity))" +) +MAX_SHADOW_QUERY_DURATION_SECONDS = 10.0 +MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE = 1.0 +SHADOW_SEMANTIC_INPUT_VERSION = "1.0" +AUTHENTICATED_EVIDENCE_POLICY_ID = "QMT_SHADOW_EVIDENCE_V1" +_EVIDENCE_HMAC_DOMAIN = b"quant-os-qmt-shadow-evidence-v1\0" +_SHADOW_LIMITATIONS = ( + "NO_REAL_TIME_QUOTE_OR_LIMIT_PRICE", + "NO_BUYING_POWER_CHECK_WITHOUT_EXECUTION_PRICE", + "NO_TWAP_POV_CHILD_PLAN", + "NO_ORDER_SUBMISSION_OR_CANCELLATION", + "BROKER_SOURCE_TIME_IS_LOCAL_QUERY_COMPLETION", +) +_POSITION_DIFF_BASIS = ( + "CURRENT_BROKER_POSITIONS_AND_SELLABLE_QUANTITY" +) + + +class QmtShadowPlanError(ValueError): + """The input or broker query boundary cannot produce trustworthy facts.""" + + +class ReadOnlyXtTraderQueries(Protocol): + """The only broker methods visible to the shadow planner.""" + + state: str + allow_live_orders: bool + + def query_asset(self) -> Mapping[str, Any]: ... + + def query_positions(self) -> Sequence[Mapping[str, Any]]: ... + + def query_orders( + self, cancelable_only: bool = False + ) -> Sequence[Mapping[str, Any]]: ... + + def query_trades(self) -> Sequence[Mapping[str, Any]]: ... + + +def _sha256_payload(payload: Any) -> str: + return hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _published_json_bytes(payload: Any) -> bytes: + return ( + json.dumps( + jsonable(payload), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + + +def _account_hash_key_bytes(value: bytes | str | None) -> bytes: + if isinstance(value, str): + key = value.encode("utf-8") + elif isinstance(value, bytes): + key = value + else: + raise QmtShadowPlanError( + "account_hash_key is required for keyed shadow evidence" + ) + if len(key) < 32: + raise QmtShadowPlanError( + "account_hash_key must contain at least 32 bytes" + ) + return key + + +def _authenticated_evidence_envelope( + *, + plan: Mapping[str, Any], + broker_observation: Mapping[str, Any], + broker_snapshot: Mapping[str, Any] | None, + decision_manifest_sha256: str, + planner_source_sha256: str, + engine_source_sha256: str, + published_artifact_sha256: Mapping[str, str], +) -> dict[str, Any]: + expected_artifact_names = { + "shadow_plan.json", + "broker_observation.json", + "broker_snapshot.json", + } + if set(published_artifact_sha256) != expected_artifact_names: + raise QmtShadowPlanError( + "authenticated evidence published artifact set is invalid" + ) + return { + "schema_version": "1.0", + "policy_id": AUTHENTICATED_EVIDENCE_POLICY_ID, + "plan_sha256": _sha256_payload(plan), + "broker_observation_sha256": _sha256_payload( + broker_observation + ), + "broker_snapshot_sha256": ( + _sha256_payload(broker_snapshot) + if broker_snapshot is not None + else None + ), + "decision_manifest_sha256": decision_manifest_sha256, + "planner_source_sha256": planner_source_sha256, + "engine_source_sha256": engine_source_sha256, + "published_artifact_sha256": { + name: str(published_artifact_sha256[name]) + for name in sorted(expected_artifact_names) + }, + "canonical_policy": { + "semantic_input_version": SHADOW_SEMANTIC_INPUT_VERSION, + "max_query_duration_seconds": ( + MAX_SHADOW_QUERY_DURATION_SECONDS + ), + "max_broker_asset_absolute_tolerance": ( + MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE + ), + "decision_equity_absolute_tolerance": ( + DECISION_EQUITY_ABSOLUTE_TOLERANCE + ), + "decision_equity_relative_tolerance": ( + DECISION_EQUITY_RELATIVE_TOLERANCE + ), + "decision_equity_tolerance_rule": ( + DECISION_EQUITY_TOLERANCE_RULE + ), + }, + } + + +def _authenticated_evidence_hmac( + envelope: Mapping[str, Any], + key: bytes, +) -> str: + return hmac.new( + key, + _EVIDENCE_HMAC_DOMAIN + + canonical_json(envelope).encode("utf-8"), + hashlib.sha256, + ).hexdigest() + + +def _strict_json(path: Path) -> Any: + def reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant {value}") + + return json.loads( + path.read_text(encoding="utf-8"), + parse_constant=reject_constant, + ) + + +def _atomic_json(path: Path, payload: Any) -> None: + encoded = _published_json_bytes(payload) + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=str(path.parent), + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def _aware(value: datetime, name: str) -> datetime: + if not isinstance(value, datetime): + raise QmtShadowPlanError(f"{name} must be a datetime") + if value.tzinfo is None or value.utcoffset() is None: + raise QmtShadowPlanError(f"{name} must include a timezone") + return value + + +def _parse_aware(value: Any, name: str) -> datetime: + if not isinstance(value, str): + raise QmtShadowPlanError(f"{name} must be an ISO-8601 string") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise QmtShadowPlanError(f"{name} must be ISO-8601") from exc + return _aware(parsed, name) + + +def _same_instant(left: datetime, right: datetime) -> bool: + return left.astimezone(timezone.utc) == right.astimezone(timezone.utc) + + +def _finite_non_negative(value: Any, name: str) -> float: + if isinstance(value, bool): + raise QmtShadowPlanError(f"{name} must be finite and non-negative") + try: + result = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise QmtShadowPlanError( + f"{name} must be finite and non-negative" + ) from exc + if not math.isfinite(result) or result < 0: + raise QmtShadowPlanError(f"{name} must be finite and non-negative") + return result + + +def _equity_consistency( + *, + decision_equity: float, + current_total_asset: float | None, +) -> dict[str, Any]: + allowed_difference = max( + DECISION_EQUITY_ABSOLUTE_TOLERANCE, + DECISION_EQUITY_RELATIVE_TOLERANCE * abs(decision_equity), + ) + if current_total_asset is None: + return { + "evaluated": False, + "decision_equity": decision_equity, + "current_total_asset": None, + "absolute_difference": None, + "absolute_tolerance": DECISION_EQUITY_ABSOLUTE_TOLERANCE, + "relative_tolerance": DECISION_EQUITY_RELATIVE_TOLERANCE, + "relative_reference": "decision_equity", + "allowed_difference": allowed_difference, + "threshold_rule": DECISION_EQUITY_TOLERANCE_RULE, + "within_tolerance": False, + } + absolute_difference = abs(current_total_asset - decision_equity) + return { + "evaluated": True, + "decision_equity": decision_equity, + "current_total_asset": current_total_asset, + "absolute_difference": absolute_difference, + "absolute_tolerance": DECISION_EQUITY_ABSOLUTE_TOLERANCE, + "relative_tolerance": DECISION_EQUITY_RELATIVE_TOLERANCE, + "relative_reference": "decision_equity", + "allowed_difference": allowed_difference, + "threshold_rule": DECISION_EQUITY_TOLERANCE_RULE, + "within_tolerance": absolute_difference <= allowed_difference, + } + + +def _positive_int(value: Any, name: str) -> int: + if isinstance(value, bool): + raise QmtShadowPlanError(f"{name} must be a positive integer") + try: + result = int(value) + except (TypeError, ValueError, OverflowError) as exc: + raise QmtShadowPlanError(f"{name} must be a positive integer") from exc + if result != value or result <= 0: + raise QmtShadowPlanError(f"{name} must be a positive integer") + return result + + +def _non_negative_int(value: Any, name: str) -> int: + if isinstance(value, bool): + raise QmtShadowPlanError(f"{name} must be a non-negative integer") + try: + result = int(value) + except (TypeError, ValueError, OverflowError) as exc: + raise QmtShadowPlanError( + f"{name} must be a non-negative integer" + ) from exc + if result != value or result < 0: + raise QmtShadowPlanError(f"{name} must be a non-negative integer") + return result + + +def _exact_int(value: Any, name: str) -> int: + if type(value) is not int: + raise QmtShadowPlanError(f"{name} must be an integer") + return value + + +def _blocker( + blockers: list[dict[str, Any]], + code: str, + detail: str, + *, + symbol: str | None = None, +) -> None: + item: dict[str, Any] = {"code": code, "detail": detail} + if symbol is not None: + item["symbol"] = symbol + blockers.append(item) + + +def _source_manifest() -> tuple[str, dict[str, str]]: + project = Path(__file__).resolve().parents[2] + files = { + "src/quant60/qmt_shadow.py": Path(__file__).resolve(), + "adapters/xttrader_live.py": project / "adapters" / "xttrader_live.py", + "tools/qmt_shadow_plan.py": project / "tools" / "qmt_shadow_plan.py", + } + sources = { + name: _sha256_bytes(path.read_bytes()) + for name, path in sorted(files.items()) + } + return _sha256_payload(sources), sources + + +def _load_verified_decision( + manifest_file: str | Path, +) -> tuple[Path, dict[str, Any], list[dict[str, Any]], dict[str, Any]]: + path = Path(manifest_file).expanduser().resolve() + verification = verify_snapshot_decision(path) + manifest = _strict_json(path) + targets = _strict_json(path.parent / "targets.json") + decision = _strict_json(path.parent / "decision.json") + if not isinstance(targets, list) or not isinstance(decision, dict): + raise QmtShadowPlanError("decision artifact has invalid JSON roots") + if not decision.get("risk", {}).get("approved"): + raise QmtShadowPlanError("decision risk approval is absent") + if decision.get("risk", {}).get("violations"): + raise QmtShadowPlanError("decision contains risk violations") + + target_map: dict[str, int] = {} + weight_map: dict[str, float] = {} + identity: tuple[str, str, str] | None = None + for index, row in enumerate(targets): + symbol = normalize_symbol(str(row["symbol"])) + if symbol in target_map: + raise QmtShadowPlanError(f"duplicate target symbol {symbol}") + row_identity = ( + str(row["run_id"]), + str(row["decision_id"]), + str(row["as_of"]), + ) + if identity is None: + identity = row_identity + elif row_identity != identity: + raise QmtShadowPlanError( + f"target record #{index} has inconsistent identity" + ) + target_map[symbol] = _non_negative_int( + row["target_quantity"], + f"target quantity {symbol}", + ) + weight = _finite_non_negative( + row["target_weight"], + f"target weight {symbol}", + ) + if weight > 1: + raise QmtShadowPlanError(f"target weight exceeds one for {symbol}") + weight_map[symbol] = weight + if identity is None: + raise QmtShadowPlanError("decision has no target records") + expected_identity = ( + str(decision.get("run_id")), + str(decision.get("decision_id")), + str(decision.get("as_of")), + ) + if identity != expected_identity: + raise QmtShadowPlanError("targets and decision identity disagree") + decision_targets = { + normalize_symbol(str(symbol)): _non_negative_int( + quantity, f"decision target quantity {symbol}" + ) + for symbol, quantity in decision.get("targets", {}).items() + } + decision_weights = { + normalize_symbol(str(symbol)): _finite_non_negative( + weight, f"decision target weight {symbol}" + ) + for symbol, weight in decision.get("weights", {}).items() + } + if target_map != decision_targets: + raise QmtShadowPlanError("targets.json and decision.targets disagree") + if weight_map != decision_weights: + raise QmtShadowPlanError("targets.json and decision.weights disagree") + if verification["run_id"] != identity[0]: + raise QmtShadowPlanError("verified manifest and targets run_id disagree") + return path, manifest, targets, decision + + +def _query( + adapter: ReadOnlyXtTraderQueries, + method: str, + *args: Any, +) -> Any: + try: + return getattr(adapter, method)(*args) + except Exception as exc: + raise QmtShadowPlanError( + f"read-only broker query {method} failed: " + f"{type(exc).__name__}: {exc}" + ) from exc + + +def _as_records(value: Any, name: str) -> list[Mapping[str, Any]]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise QmtShadowPlanError(f"{name} query must return a sequence") + output = list(value) + if any(not isinstance(item, Mapping) for item in output): + raise QmtShadowPlanError(f"{name} query returned a non-object record") + return output + + +def _normalise_orders( + raw_orders: Sequence[Mapping[str, Any]], + blockers: list[dict[str, Any]], + expected_account_id: str, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], bool]: + observed: list[dict[str, Any]] = [] + open_orders: list[dict[str, Any]] = [] + valid = True + seen: set[str] = set() + for index, raw in enumerate(raw_orders): + broker_id_raw = raw.get("broker_order_id") + broker_id = ( + "" + if ( + broker_id_raw is None + or broker_id_raw == "" + or broker_id_raw == 0 + or broker_id_raw == -1 + ) + else str(broker_id_raw) + ) + key = broker_id or f"missing-{index}" + try: + account_id = str(raw.get("account_id") or "").strip() + if not account_id: + raise QmtShadowPlanError("account_id is missing") + if not expected_account_id or account_id != expected_account_id: + raise QmtShadowPlanError( + "order belongs to a different broker account" + ) + if not broker_id: + raise QmtShadowPlanError("broker_order_id is missing") + if broker_id in seen: + raise QmtShadowPlanError( + f"duplicate broker_order_id {broker_id}" + ) + seen.add(broker_id) + symbol = normalize_symbol(str(raw.get("symbol", ""))) + side = str(raw.get("side", "")).upper() + if side not in {"BUY", "SELL"}: + raise QmtShadowPlanError(f"unsupported side {side!r}") + quantity = _positive_int( + raw.get("quantity"), f"order {broker_id} quantity" + ) + filled = _non_negative_int( + raw.get("filled_quantity", 0), + f"order {broker_id} filled quantity", + ) + if filled > quantity: + raise QmtShadowPlanError( + f"order {broker_id} fill exceeds quantity" + ) + state = str(raw.get("state", "")).upper() + if state not in _KNOWN_ORDER_STATES: + raise QmtShadowPlanError( + f"order {broker_id} has unsupported state {state!r}" + ) + if state == "FILLED" and filled != quantity: + raise QmtShadowPlanError( + f"filled order {broker_id} is not fully filled" + ) + if state == "PARTIALLY_FILLED" and not 0 < filled < quantity: + raise QmtShadowPlanError( + f"partially filled order {broker_id} has invalid fill" + ) + if state in {"NEW", "ACCEPTED", "REJECTED"} and filled: + raise QmtShadowPlanError( + f"order {broker_id} state {state} conflicts with fill={filled}" + ) + limit_price = _finite_non_negative( + raw.get("limit_price", 0), + f"order {broker_id} limit price", + ) + normalised = { + "broker_order_id": broker_id, + "client_order_id": ( + None + if raw.get("client_order_id") in {None, ""} + else str(raw["client_order_id"]) + ), + "symbol": symbol, + "side": side, + "quantity": quantity, + "filled_quantity": filled, + "limit_price": limit_price, + "state": state, + } + observed.append(normalised) + if state in _OPEN_ORDER_STATES: + if state == "UNKNOWN": + _blocker( + blockers, + "UNKNOWN_ORDER_STATE", + f"broker order {broker_id} state is UNKNOWN", + symbol=symbol, + ) + else: + _blocker( + blockers, + "UNFINISHED_ORDER", + f"broker order {broker_id} is {state}", + symbol=symbol, + ) + if limit_price <= 0: + valid = False + _blocker( + blockers, + "INVALID_OPEN_ORDER", + f"broker order {broker_id} has no positive limit price", + symbol=symbol, + ) + else: + open_orders.append( + { + "order_id": ( + normalised["client_order_id"] + or f"QMT-{broker_id}" + ), + "broker_order_id": broker_id, + "symbol": symbol, + "side": side, + "quantity": quantity, + "filled_quantity": filled, + "status": state, + "limit_price": limit_price, + "order_type": "LIMIT", + "time_in_force": "DAY", + } + ) + except (KeyError, QmtShadowPlanError, TypeError, ValueError) as exc: + valid = False + _blocker( + blockers, + "INVALID_ORDER_FACT", + f"order record {key}: {exc}", + ) + return ( + sorted(observed, key=lambda item: item["broker_order_id"]), + sorted(open_orders, key=lambda item: item["broker_order_id"]), + valid, + ) + + +def _normalise_positions( + raw_positions: Sequence[Mapping[str, Any]], + blockers: list[dict[str, Any]], + expected_account_id: str, +) -> tuple[list[dict[str, Any]], bool, int]: + output: list[dict[str, Any]] = [] + seen: set[str] = set() + valid = True + dropped_zero_position_count = 0 + for index, raw in enumerate(raw_positions): + try: + account_id = str(raw.get("account_id") or "").strip() + if not account_id: + raise QmtShadowPlanError("account_id is missing") + if not expected_account_id or account_id != expected_account_id: + raise QmtShadowPlanError( + "position belongs to a different broker account" + ) + symbol = normalize_symbol(str(raw.get("symbol", ""))) + if symbol in seen: + raise QmtShadowPlanError(f"duplicate position {symbol}") + seen.add(symbol) + quantity = _non_negative_int( + raw.get("volume"), f"position {symbol} quantity" + ) + sellable = _non_negative_int( + raw.get("sellable"), f"position {symbol} sellable" + ) + if sellable > quantity: + raise QmtShadowPlanError( + f"position {symbol} sellable exceeds quantity" + ) + market_value = _finite_non_negative( + raw.get("market_value"), + f"position {symbol} market value", + ) + average_cost = _finite_non_negative( + raw.get("avg_price", 0), + f"position {symbol} average cost", + ) + if quantity > 0 and market_value <= 0: + raise QmtShadowPlanError( + f"position {symbol} has quantity but no market value" + ) + if quantity: + output.append( + { + "symbol": symbol, + "quantity": quantity, + "sellable_quantity": sellable, + "average_cost": average_cost, + "market_value": market_value, + } + ) + else: + dropped_zero_position_count += 1 + except (QmtShadowPlanError, TypeError, ValueError) as exc: + valid = False + _blocker( + blockers, + "INVALID_POSITION_FACT", + f"position record #{index}: {exc}", + ) + return ( + sorted(output, key=lambda item: item["symbol"]), + valid, + dropped_zero_position_count, + ) + + +def _normalise_trades( + raw_trades: Sequence[Mapping[str, Any]], + orders: Sequence[Mapping[str, Any]], + blockers: list[dict[str, Any]], + expected_account_id: str, +) -> tuple[list[dict[str, Any]], bool]: + output: list[dict[str, Any]] = [] + valid = True + seen_trade_ids: set[str] = set() + order_ids = {item["broker_order_id"] for item in orders} + for index, raw in enumerate(raw_trades): + try: + account_id = str(raw.get("account_id") or "").strip() + if not account_id: + raise QmtShadowPlanError("account_id is missing") + if not expected_account_id or account_id != expected_account_id: + raise QmtShadowPlanError( + "trade belongs to a different broker account" + ) + broker_order_id_raw = raw.get("broker_order_id") + broker_order_id = ( + "" + if ( + broker_order_id_raw is None + or broker_order_id_raw == "" + or broker_order_id_raw == 0 + or broker_order_id_raw == -1 + ) + else str(broker_order_id_raw) + ) + if not broker_order_id: + raise QmtShadowPlanError("broker_order_id is missing") + trade_id = str(raw.get("trade_id") or "").strip() + if not trade_id: + raise QmtShadowPlanError("trade_id is missing") + if trade_id in seen_trade_ids: + raise QmtShadowPlanError(f"duplicate trade_id {trade_id}") + seen_trade_ids.add(trade_id) + symbol = normalize_symbol(str(raw.get("symbol", ""))) + side = str(raw.get("side", "")).upper() + if side not in {"BUY", "SELL"}: + raise QmtShadowPlanError(f"unsupported side {side!r}") + quantity = _positive_int( + raw.get("quantity"), f"trade #{index} quantity" + ) + price = _finite_non_negative( + raw.get("price"), f"trade #{index} price" + ) + if price <= 0: + raise QmtShadowPlanError(f"trade #{index} price must be positive") + amount = _finite_non_negative( + raw.get("amount"), f"trade #{index} amount" + ) + if broker_order_id not in order_ids: + valid = False + _blocker( + blockers, + "ORPHAN_TRADE", + ( + f"trade {trade_id} references absent order " + f"{broker_order_id}" + ), + symbol=symbol, + ) + output.append( + { + "trade_id": trade_id, + "broker_order_id": broker_order_id, + "symbol": symbol, + "side": side, + "quantity": quantity, + "price": price, + "amount": amount, + } + ) + except (QmtShadowPlanError, TypeError, ValueError) as exc: + valid = False + _blocker( + blockers, + "INVALID_TRADE_FACT", + f"trade record #{index}: {exc}", + ) + return sorted( + output, + key=lambda item: ( + item["broker_order_id"], + item["trade_id"], + item["symbol"], + item["quantity"], + item["price"], + ), + ), valid + + +def _validate_fill_totals( + orders: Sequence[Mapping[str, Any]], + trades: Sequence[Mapping[str, Any]], + blockers: list[dict[str, Any]], +) -> bool: + valid = True + order_by_id = { + str(item["broker_order_id"]): item for item in orders + } + trade_totals: dict[str, int] = {} + for trade in trades: + order_id = str(trade["broker_order_id"]) + order = order_by_id.get(order_id) + if order is None: + valid = False + continue + if ( + trade["symbol"] != order["symbol"] + or trade["side"] != order["side"] + ): + valid = False + _blocker( + blockers, + "TRADE_ORDER_IDENTITY_MISMATCH", + ( + f"trade {trade['trade_id']} symbol/side " + f"{trade['symbol']}/{trade['side']} disagrees with " + f"order {order_id} {order['symbol']}/{order['side']}" + ), + symbol=str(trade["symbol"]), + ) + trade_totals[order_id] = ( + trade_totals.get(order_id, 0) + int(trade["quantity"]) + ) + for order in orders: + order_id = str(order["broker_order_id"]) + expected = int(order["filled_quantity"]) + observed = trade_totals.get(order_id, 0) + if expected != observed: + valid = False + _blocker( + blockers, + "FILL_TRADE_MISMATCH", + ( + f"order {order_id} reports filled={expected}, " + f"queried trades sum to {observed}" + ), + symbol=str(order["symbol"]), + ) + return valid + + +def build_qmt_shadow_plan( + *, + decision_manifest: str | Path, + adapter: ReadOnlyXtTraderQueries, + clock: Callable[[], datetime] | None = None, + max_query_duration_seconds: float = MAX_SHADOW_QUERY_DURATION_SECONDS, + market_value_tolerance: float = MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE, + account_hash_key: bytes | str | None = None, +) -> dict[str, Any]: + """Query broker truth and build an inert, fail-closed target diff.""" + + if getattr(adapter, "allow_live_orders", None) is not False: + raise QmtShadowPlanError( + "shadow planning requires allow_live_orders exactly False" + ) + if getattr(adapter, "state", None) != "READY": + raise QmtShadowPlanError("XtTrader adapter must be in READY state") + query_limit = _finite_non_negative( + max_query_duration_seconds, "max_query_duration_seconds" + ) + if query_limit > MAX_SHADOW_QUERY_DURATION_SECONDS: + raise QmtShadowPlanError( + "max_query_duration_seconds cannot exceed the fixed " + f"{MAX_SHADOW_QUERY_DURATION_SECONDS:g}s safety ceiling" + ) + value_tolerance = _finite_non_negative( + market_value_tolerance, "market_value_tolerance" + ) + if value_tolerance > MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE: + raise QmtShadowPlanError( + "market_value_tolerance cannot exceed the fixed " + f"{MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE:g} CNY safety ceiling" + ) + key_bytes = _account_hash_key_bytes(account_hash_key) + decision_path, decision_manifest_body, targets, decision = ( + _load_verified_decision(decision_manifest) + ) + decision_equity = _finite_non_negative( + decision.get("equity"), + "decision equity", + ) + if decision_equity <= 0: + raise QmtShadowPlanError("decision equity must be positive") + decision_as_of = _parse_aware(decision["as_of"], "decision.as_of") + decision_signal_as_of = _parse_aware( + decision.get("signal_as_of"), + "decision.signal_as_of", + ) + if not _same_instant(decision_as_of, decision_signal_as_of): + raise QmtShadowPlanError( + "decision signal_as_of does not match decision.as_of" + ) + try: + next_trading_session = date.fromisoformat( + str(decision.get("next_trading_session")) + ) + except ValueError as exc: + raise QmtShadowPlanError( + "decision.next_trading_session must be an ISO date" + ) from exc + expected_window = first_executable_window(next_trading_session) + decision_window = decision.get("first_executable_window") + if not isinstance(decision_window, Mapping) or dict( + decision_window + ) != expected_window: + raise QmtShadowPlanError( + "decision first_executable_window is not the canonical " + "[09:00,09:30) Asia/Shanghai window" + ) + window_start = _parse_aware( + expected_window["start"], + "decision first_executable_window.start", + ) + window_end = _parse_aware( + expected_window["end"], + "decision first_executable_window.end", + ) + + now = clock or (lambda: datetime.now(timezone.utc)) + query_started = _aware(now(), "query_started") + orders_before_raw = _as_records( + _query(adapter, "query_orders", False), "orders" + ) + asset_raw = _query(adapter, "query_asset") + positions_raw = _as_records( + _query(adapter, "query_positions"), "positions" + ) + trades_raw = _as_records(_query(adapter, "query_trades"), "trades") + orders_after_raw = _as_records( + _query(adapter, "query_orders", False), "orders" + ) + query_completed = _aware(now(), "query_completed") + + if not isinstance(asset_raw, Mapping): + raise QmtShadowPlanError("asset query must return an object") + blockers: list[dict[str, Any]] = [] + adapter_ready_after_query = ( + getattr(adapter, "state", None) == "READY" + ) + if not adapter_ready_after_query: + _blocker( + blockers, + "ADAPTER_NOT_READY_AFTER_QUERY", + f"adapter state is {getattr(adapter, 'state', None)!r}", + ) + adapter_errors = getattr(adapter, "errors", []) + if adapter_errors: + _blocker( + blockers, + "ADAPTER_ERRORS_PRESENT", + f"adapter reports {len(adapter_errors)} callback/query error(s)", + ) + live_audit = getattr(adapter, "live_authorization_audit", []) + if live_audit: + _blocker( + blockers, + "LIVE_AUTHORIZATION_HISTORY_PRESENT", + "fresh read-only adapter unexpectedly contains live authorization history", + ) + live_disabled_after_query = ( + getattr(adapter, "allow_live_orders", None) is False + ) + if not live_disabled_after_query: + _blocker( + blockers, + "LIVE_MODE_CHANGED_DURING_QUERY", + "adapter allow_live_orders was not False after broker queries", + ) + + elapsed = (query_completed - query_started).total_seconds() + query_window_valid = 0 <= elapsed <= query_limit + if elapsed < 0: + _blocker( + blockers, + "CLOCK_REGRESSION", + "query completion precedes query start", + ) + elif elapsed > query_limit: + _blocker( + blockers, + "QUERY_WINDOW_TOO_LONG", + f"query window {elapsed:.6f}s exceeds {query_limit:.6f}s", + ) + window_start_utc = window_start.astimezone(timezone.utc) + window_end_utc = window_end.astimezone(timezone.utc) + query_started_utc = query_started.astimezone(timezone.utc) + query_completed_utc = query_completed.astimezone(timezone.utc) + execution_window_valid = ( + window_start_utc <= query_started_utc < window_end_utc + and window_start_utc <= query_completed_utc < window_end_utc + ) + if not execution_window_valid: + shanghai = timezone(timedelta(hours=8)) + local_started = query_started.astimezone(shanghai) + local_completed = query_completed.astimezone(shanghai) + if ( + local_started.date() != next_trading_session + or local_completed.date() != next_trading_session + ): + code = "QUERY_WRONG_TRADING_SESSION" + detail = ( + "broker query must occur on the unique next trading " + f"session {next_trading_session.isoformat()}" + ) + elif query_completed_utc >= window_end_utc: + code = "QUERY_MISSED_EXECUTABLE_WINDOW" + detail = ( + "broker query completed at or after the exclusive 09:30 " + "Asia/Shanghai pre-open boundary" + ) + else: + code = "QUERY_BEFORE_EXECUTABLE_WINDOW" + detail = ( + "broker query began before 09:00 Asia/Shanghai on the " + "unique next trading session" + ) + _blocker(blockers, code, detail) + + asset_valid = True + try: + account_id = str(asset_raw.get("account_id") or "").strip() + if not account_id: + raise QmtShadowPlanError("asset account_id is missing") + cash = _finite_non_negative(asset_raw.get("cash"), "asset cash") + market_value = _finite_non_negative( + asset_raw.get("market_value"), "asset market value" + ) + total_asset = _finite_non_negative( + asset_raw.get("total_asset"), "asset total asset" + ) + if total_asset <= 0: + raise QmtShadowPlanError("asset total_asset must be positive") + except (QmtShadowPlanError, TypeError, ValueError) as exc: + asset_valid = False + account_id = "" + cash = market_value = total_asset = 0.0 + _blocker(blockers, "INVALID_ASSET_FACT", str(exc)) + equity_consistency = _equity_consistency( + decision_equity=decision_equity, + current_total_asset=total_asset if asset_valid else None, + ) + if ( + equity_consistency["evaluated"] + and not equity_consistency["within_tolerance"] + ): + _blocker( + blockers, + "DECISION_EQUITY_DRIFT", + ( + "decision equity/current total_asset drift " + f"{equity_consistency['absolute_difference']:.6f} exceeds " + "allowed " + f"{equity_consistency['allowed_difference']:.6f}" + ), + ) + + before, unused_before_open, before_valid = _normalise_orders( + orders_before_raw, + blockers, + account_id, + ) + del unused_before_open + after, open_orders, after_valid = _normalise_orders( + orders_after_raw, + blockers, + account_id, + ) + orders_stable = before == after + if not orders_stable: + _blocker( + blockers, + "ORDERS_CHANGED_DURING_QUERY", + "broker order facts changed inside the read-only query window", + ) + ( + positions, + positions_valid, + dropped_zero_position_count, + ) = _normalise_positions( + positions_raw, + blockers, + account_id, + ) + trades, trades_valid = _normalise_trades( + trades_raw, + after, + blockers, + account_id, + ) + fill_facts_valid = _validate_fill_totals(after, trades, blockers) + + asset_identity_valid = asset_valid + if asset_valid: + position_market_value = sum( + float(item["market_value"]) for item in positions + ) + if abs(position_market_value - market_value) > value_tolerance: + asset_identity_valid = False + _blocker( + blockers, + "POSITION_MARKET_VALUE_MISMATCH", + ( + f"positions sum={position_market_value:.6f}, " + f"asset market_value={market_value:.6f}, " + f"tolerance={value_tolerance:.6f}" + ), + ) + if abs(cash + market_value - total_asset) > value_tolerance: + asset_identity_valid = False + _blocker( + blockers, + "ASSET_IDENTITY_MISMATCH", + ( + f"cash+market_value={cash + market_value:.6f}, " + f"total_asset={total_asset:.6f}, " + f"tolerance={value_tolerance:.6f}" + ), + ) + + account_hash = ( + hmac.new( + key_bytes, + ("qmt-account-v1\0" + account_id).encode("utf-8"), + hashlib.sha256, + ).hexdigest() + if account_id + else None + ) + decision_broker = decision.get("broker_snapshot") + bound_observation_as_of: datetime | None = None + bound_source_time: datetime | None = None + if decision_broker is None: + _blocker( + blockers, + "DECISION_NOT_ACCOUNT_BOUND", + ( + "decision was built without a broker snapshot; use this " + "blocked run's broker_snapshot.json to regenerate the " + "decision before shadow readiness" + ), + ) + elif not isinstance(decision_broker, Mapping): + _blocker( + blockers, + "INVALID_DECISION_BROKER_IDENTITY", + "decision broker_snapshot identity is not an object", + ) + else: + expected_account_hash = str( + decision_broker.get("account_hash") or "" + ) + if ( + account_hash is None + or not expected_account_hash + or not hmac.compare_digest( + account_hash, + expected_account_hash, + ) + ): + _blocker( + blockers, + "DECISION_ACCOUNT_MISMATCH", + "decision target and current QMT observation use different account identities", + ) + if decision_broker.get("broker") != "QMT_XTTRADER": + _blocker( + blockers, + "DECISION_BROKER_MISMATCH", + ( + "decision broker identity is not the QMT_XTTRADER " + "read-only boundary" + ), + ) + if ( + decision_broker.get("next_trading_session") + != next_trading_session.isoformat() + or decision_broker.get("first_executable_window") + != expected_window + ): + _blocker( + blockers, + "DECISION_EXECUTION_WINDOW_MISMATCH", + "decision broker identity is bound to a different trading " + "session or pre-open window", + ) + bound_signal_as_of = _parse_aware( + decision_broker.get("signal_as_of"), + "decision broker signal_as_of", + ) + bound_observation_as_of = _parse_aware( + decision_broker.get("observation_as_of"), + "decision broker observation_as_of", + ) + bound_legacy_as_of = _parse_aware( + decision_broker.get("as_of"), + "decision broker as_of", + ) + bound_source_time = _parse_aware( + decision_broker.get("source_time"), + "decision broker source_time", + ) + if not _same_instant( + bound_signal_as_of, + decision_signal_as_of, + ): + _blocker( + blockers, + "DECISION_SIGNAL_BINDING_MISMATCH", + "decision broker identity is bound to a different signal clock", + ) + if not _same_instant( + bound_observation_as_of, + bound_legacy_as_of, + ): + _blocker( + blockers, + "DECISION_OBSERVATION_CLOCK_MISMATCH", + "decision broker observation_as_of does not equal as_of", + ) + signal_utc = decision_signal_as_of.astimezone(timezone.utc) + bound_observation_utc = bound_observation_as_of.astimezone( + timezone.utc + ) + bound_source_utc = bound_source_time.astimezone(timezone.utc) + if bound_observation_utc < signal_utc: + _blocker( + blockers, + "DECISION_OBSERVATION_PRECEDES_SIGNAL", + "bound broker observation predates the completed signal close", + ) + if bound_source_utc < signal_utc: + _blocker( + blockers, + "DECISION_SOURCE_PRECEDES_SIGNAL", + "bound broker source_time predates the completed signal close", + ) + query_completed_utc = query_completed.astimezone(timezone.utc) + if ( + bound_observation_utc - query_completed_utc + ).total_seconds() > 5: + _blocker( + blockers, + "DECISION_OBSERVATION_FROM_FUTURE", + "bound broker observation is after the current query window", + ) + if (bound_source_utc - query_completed_utc).total_seconds() > 5: + _blocker( + blockers, + "DECISION_SOURCE_FROM_FUTURE", + "bound broker source_time is after the current query window", + ) + + target_quantities = { + normalize_symbol(row["symbol"]): int(row["target_quantity"]) + for row in targets + } + target_weights = { + normalize_symbol(row["symbol"]): float(row["target_weight"]) + for row in targets + } + current_quantities = { + item["symbol"]: int(item["quantity"]) for item in positions + } + sellable_quantities = { + item["symbol"]: int(item["sellable_quantity"]) for item in positions + } + lot_sizes = { + normalize_symbol(row["symbol"]): int(row.get("lot_size", 100)) + for row in targets + } + if len(set(lot_sizes.values())) > 1: + _blocker( + blockers, + "MIXED_LOT_SIZES", + "a single deterministic shadow-delta pass requires one lot size", + ) + lot_size = next(iter(lot_sizes.values()), 100) + feasible_deltas = ( + order_deltas( + target_quantities, + current_quantities, + sellable_quantities, + lot_size=lot_size, + ) + if positions_valid and target_quantities + else {} + ) + rows: list[dict[str, Any]] = [] + position_by_symbol = {item["symbol"]: item for item in positions} + for symbol in sorted(set(target_quantities) | set(current_quantities)): + position = position_by_symbol.get(symbol, {}) + current_quantity = current_quantities.get(symbol, 0) + target_quantity = target_quantities.get(symbol, 0) + desired_delta = target_quantity - current_quantity + feasible_delta = feasible_deltas.get(symbol, 0) + if desired_delta != feasible_delta: + _blocker( + blockers, + "UNEXECUTABLE_POSITION_RESIDUAL", + ( + f"desired delta {desired_delta} is clipped to " + f"{feasible_delta} by board-lot/sellable rules" + ), + symbol=symbol, + ) + current_market_value = float(position.get("market_value", 0.0)) + current_weight = ( + current_market_value / total_asset + if asset_valid and total_asset > 0 + else None + ) + rows.append( + { + "symbol": symbol, + "target_weight": target_weights.get(symbol, 0.0), + "current_weight": current_weight, + "target_quantity": target_quantity, + "current_quantity": current_quantity, + "sellable_quantity": sellable_quantities.get(symbol, 0), + "current_market_value": current_market_value, + "desired_delta": desired_delta, + "board_lot_feasible_delta": feasible_delta, + "proposed_delta": 0, + "proposed_side": None, + "proposed_quantity": 0, + "lot_size": lot_sizes.get(symbol, lot_size), + } + ) + + blockers = sorted( + { + ( + item["code"], + item.get("symbol"), + item["detail"], + ): item + for item in blockers + }.values(), + key=lambda item: ( + item["code"], + item.get("symbol") or "", + item["detail"], + ), + ) + ready = not blockers + if ready: + for row in rows: + delta = int(row["board_lot_feasible_delta"]) + row["proposed_delta"] = delta + row["proposed_side"] = ( + "BUY" if delta > 0 else "SELL" if delta < 0 else None + ) + row["proposed_quantity"] = abs(delta) + + observation_facts = { + "semantic_inputs": { + "schema_version": SHADOW_SEMANTIC_INPUT_VERSION, + "adapter_state_after_query": getattr( + adapter, "state", None + ), + "adapter_allow_live_orders_after_query": getattr( + adapter, "allow_live_orders", None + ), + "adapter_error_count": len(adapter_errors), + "live_authorization_audit_count": len(live_audit), + "query_started_at": query_started.isoformat(), + "query_completed_at": query_completed.isoformat(), + "max_query_duration_seconds": query_limit, + "market_value_tolerance": value_tolerance, + "raw_position_count": len(positions_raw), + "dropped_zero_position_count": ( + dropped_zero_position_count + ), + "raw_order_before_count": len(orders_before_raw), + "raw_order_after_count": len(orders_after_raw), + "raw_trade_count": len(trades_raw), + "fact_validation_blockers": [ + dict(item) + for item in blockers + if item["code"] + in { + "INVALID_ASSET_FACT", + "INVALID_ORDER_FACT", + "INVALID_POSITION_FACT", + "INVALID_TRADE_FACT", + } + ], + }, + "asset": { + "account_hash": account_hash, + "cash": cash, + "market_value": market_value, + "total_asset": total_asset, + }, + "positions": positions, + "orders_before": before, + "orders_after": after, + "trades": trades, + } + snapshot_valid = all( + ( + adapter_ready_after_query, + live_disabled_after_query, + not adapter_errors, + not live_audit, + query_window_valid, + execution_window_valid, + orders_stable, + asset_valid, + asset_identity_valid, + positions_valid, + before_valid, + after_valid, + trades_valid, + fill_facts_valid, + account_hash is not None, + ) + ) + broker_snapshot: dict[str, Any] | None = None + if snapshot_valid: + snapshot_body = { + "schema_version": "1.0", + "snapshot_id": "", + "signal_as_of": decision_signal_as_of.isoformat(), + "next_trading_session": next_trading_session.isoformat(), + "first_executable_window": expected_window, + "observation_as_of": query_completed.isoformat(), + "as_of": query_completed.isoformat(), + "source_time": query_completed.isoformat(), + "broker": "QMT_XTTRADER", + "account_hash": account_hash, + "cash": cash, + "total_asset": total_asset, + "market_value": market_value, + "positions": positions, + "open_orders": open_orders, + "metadata": { + "read_only": True, + "source_time_semantics": ( + "local_query_completion_not_broker_clock" + ), + "query_started_at": query_started.isoformat(), + "query_duration_ms": max(0, round(elapsed * 1000)), + "terminal_order_count": sum( + item["state"] in _TERMINAL_ORDER_STATES for item in after + ), + "trade_count": len(trades), + "observed_facts_sha256": _sha256_payload(observation_facts), + }, + } + snapshot_body["snapshot_id"] = ( + "QMT-SHADOW-" + _sha256_payload(snapshot_body)[:16] + ) + validate_records([snapshot_body], "broker_snapshot.schema.json") + broker_snapshot = snapshot_body + + planner_source_sha256, planner_sources = _source_manifest() + engine_source_sha256, unused_engine_sources = _engine_source_manifest() + del unused_engine_sources + decision_manifest_sha256 = _sha256_bytes(decision_path.read_bytes()) + plan: dict[str, Any] = { + "schema_version": "1.0", + "plan_id": "", + "mode": "QMT_XTTRADER_SHADOW_READ_ONLY", + "status": "SHADOW_READY" if ready else "BLOCKED", + "decision": { + "run_id": decision["run_id"], + "decision_id": decision["decision_id"], + "as_of": decision["as_of"], + "signal_as_of": decision_signal_as_of.isoformat(), + "next_trading_session": next_trading_session.isoformat(), + "first_executable_window": expected_window, + "bound_broker_observation_as_of": ( + bound_observation_as_of.isoformat() + if bound_observation_as_of is not None + else None + ), + "bound_broker_source_time": ( + bound_source_time.isoformat() + if bound_source_time is not None + else None + ), + "model_version": decision["model_version"], + "data_version": decision["data_version"], + "equity": decision_equity, + "decision_manifest_sha256": decision_manifest_sha256, + }, + "observation": { + "query_started_at": query_started.isoformat(), + "query_completed_at": query_completed.isoformat(), + "query_duration_ms": max(0, round(elapsed * 1000)), + "orders_queried_twice": True, + "adapter_allow_live_orders_after_query": ( + getattr(adapter, "allow_live_orders", None) + ), + "asset_count": 1, + "position_count": len(positions_raw), + "order_count": len(orders_after_raw), + "trade_count": len(trades_raw), + "observed_facts_sha256": _sha256_payload(observation_facts), + "broker_snapshot_valid": broker_snapshot is not None, + }, + "portfolio": { + "cash": cash if asset_valid else None, + "market_value": market_value if asset_valid else None, + "total_asset": total_asset if asset_valid else None, + "current_gross_weight": ( + market_value / total_asset + if asset_valid and total_asset > 0 + else None + ), + "target_gross_weight": sum(target_weights.values()), + "equity_consistency": equity_consistency, + }, + "target_diffs": rows, + "blockers": blockers, + "limitations": list(_SHADOW_LIMITATIONS), + "safety": { + "read_only": True, + "allow_live_orders": False, + "mutation_methods_invoked": False, + "ready_for_manual_review": ready, + "ready_for_live_submission": False, + "automatic_submission_supported": False, + "position_diff_basis": _POSITION_DIFF_BASIS, + }, + "broker_snapshot_sha256": ( + _sha256_payload(broker_snapshot) + if broker_snapshot is not None + else None + ), + } + plan["plan_id"] = "QSP-" + _sha256_payload(plan)[:20] + published_artifact_sha256 = { + "shadow_plan.json": _sha256_bytes( + _published_json_bytes(plan) + ), + "broker_snapshot.json": _sha256_bytes( + _published_json_bytes(broker_snapshot) + ), + "broker_observation.json": _sha256_bytes( + _published_json_bytes(observation_facts) + ), + } + manifest = { + "schema_version": "1.0", + "plan_id": plan["plan_id"], + "mode": plan["mode"], + "decision_run_id": decision["run_id"], + "decision_manifest_sha256": decision_manifest_sha256, + "decision_manifest_run_id": decision_manifest_body["run_id"], + "planner_source_sha256": planner_source_sha256, + "planner_sources": planner_sources, + "engine_source_sha256": engine_source_sha256, + "deterministic_canonical_json": True, + "broker_mutation_allowed": False, + "authenticated_evidence_policy_id": ( + AUTHENTICATED_EVIDENCE_POLICY_ID + ), + "authenticated_evidence_hmac_sha256": ( + _authenticated_evidence_hmac( + _authenticated_evidence_envelope( + plan=plan, + broker_observation=observation_facts, + broker_snapshot=broker_snapshot, + decision_manifest_sha256=decision_manifest_sha256, + planner_source_sha256=planner_source_sha256, + engine_source_sha256=engine_source_sha256, + published_artifact_sha256=( + published_artifact_sha256 + ), + ), + key_bytes, + ) + ), + } + return { + "plan": plan, + "broker_snapshot": broker_snapshot, + "broker_observation": observation_facts, + "manifest": manifest, + } + + +def write_qmt_shadow_plan( + result: Mapping[str, Any], + output_dir: str | Path, +) -> dict[str, str]: + root = Path(output_dir).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + marker = root / ".quant60-incomplete" + _atomic_json(marker, {"plan_id": result["plan"]["plan_id"]}) + paths = { + "plan": root / "shadow_plan.json", + "broker_snapshot": root / "broker_snapshot.json", + "broker_observation": root / "broker_observation.json", + } + _atomic_json(paths["plan"], result["plan"]) + _atomic_json(paths["broker_snapshot"], result["broker_snapshot"]) + _atomic_json( + paths["broker_observation"], + result["broker_observation"], + ) + manifest = dict(result["manifest"]) + manifest["artifact_sha256"] = { + path.name: _sha256_bytes(path.read_bytes()) for path in paths.values() + } + manifest_path = root / "manifest.json" + _atomic_json(manifest_path, manifest) + marker.unlink() + paths["manifest"] = manifest_path + return {name: str(path) for name, path in paths.items()} + + +def _verify_shadow_clock_contract( + plan: Mapping[str, Any], +) -> dict[str, Any]: + decision = plan.get("decision") + observation = plan.get("observation") + if not isinstance(decision, Mapping) or not isinstance( + observation, + Mapping, + ): + raise QmtShadowPlanError( + "shadow plan decision/observation clocks are missing" + ) + decision_as_of = _parse_aware( + decision.get("as_of"), + "plan decision.as_of", + ) + signal_as_of = _parse_aware( + decision.get("signal_as_of"), + "plan decision.signal_as_of", + ) + if not _same_instant(decision_as_of, signal_as_of): + raise QmtShadowPlanError( + "shadow plan decision signal clock mismatch" + ) + try: + next_trading_session = date.fromisoformat( + str(decision.get("next_trading_session")) + ) + except ValueError as exc: + raise QmtShadowPlanError( + "shadow plan next_trading_session is invalid" + ) from exc + expected_window = first_executable_window(next_trading_session) + if decision.get("first_executable_window") != expected_window: + raise QmtShadowPlanError( + "shadow plan first executable window is invalid" + ) + window_start = _parse_aware( + expected_window["start"], + "shadow plan first_executable_window.start", + ) + window_end = _parse_aware( + expected_window["end"], + "shadow plan first_executable_window.end", + ) + query_started = _parse_aware( + observation.get("query_started_at"), + "plan observation.query_started_at", + ) + query_completed = _parse_aware( + observation.get("query_completed_at"), + "plan observation.query_completed_at", + ) + if query_completed.astimezone(timezone.utc) < query_started.astimezone( + timezone.utc + ): + raise QmtShadowPlanError("shadow plan query clock regressed") + window_start_utc = window_start.astimezone(timezone.utc) + window_end_utc = window_end.astimezone(timezone.utc) + query_started_utc = query_started.astimezone(timezone.utc) + query_completed_utc = query_completed.astimezone(timezone.utc) + query_inside_window = ( + window_start_utc <= query_started_utc < window_end_utc + and window_start_utc <= query_completed_utc < window_end_utc + ) + window_blocker_codes = { + "QUERY_WRONG_TRADING_SESSION", + "QUERY_BEFORE_EXECUTABLE_WINDOW", + "QUERY_MISSED_EXECUTABLE_WINDOW", + } + blocker_codes = { + item.get("code") + for item in plan.get("blockers", []) + if isinstance(item, Mapping) + } + if query_inside_window and blocker_codes & window_blocker_codes: + raise QmtShadowPlanError( + "shadow plan has a spurious execution-window blocker" + ) + if not query_inside_window and not ( + blocker_codes & window_blocker_codes + ): + raise QmtShadowPlanError( + "shadow plan outside execution window lacks a blocker" + ) + if plan.get("status") == "SHADOW_READY" and not query_inside_window: + raise QmtShadowPlanError( + "ready shadow plan query is outside execution window" + ) + + bound_observation_raw = decision.get( + "bound_broker_observation_as_of" + ) + bound_source_raw = decision.get("bound_broker_source_time") + if (bound_observation_raw is None) != (bound_source_raw is None): + raise QmtShadowPlanError( + "shadow plan broker binding clock set is incomplete" + ) + bound_observation: datetime | None = None + bound_source: datetime | None = None + if bound_observation_raw is not None: + bound_observation = _parse_aware( + bound_observation_raw, + "plan decision.bound_broker_observation_as_of", + ) + bound_source = _parse_aware( + bound_source_raw, + "plan decision.bound_broker_source_time", + ) + signal_utc = signal_as_of.astimezone(timezone.utc) + bound_observation_utc = bound_observation.astimezone(timezone.utc) + bound_source_utc = bound_source.astimezone(timezone.utc) + if not window_start_utc <= bound_observation_utc < window_end_utc: + raise QmtShadowPlanError( + "bound broker observation is outside execution window" + ) + if not window_start_utc <= bound_source_utc < window_end_utc: + raise QmtShadowPlanError( + "bound broker source_time is outside execution window" + ) + if bound_observation_utc < signal_utc: + raise QmtShadowPlanError( + "bound broker observation predates signal_as_of" + ) + if bound_source_utc < signal_utc: + raise QmtShadowPlanError( + "bound broker source_time predates signal_as_of" + ) + source_lag = ( + bound_observation_utc - bound_source_utc + ).total_seconds() + if source_lag < -5 or source_lag > 60: + raise QmtShadowPlanError( + "bound broker source/observation freshness is invalid" + ) + if (bound_observation_utc - query_completed_utc).total_seconds() > 5: + raise QmtShadowPlanError( + "bound broker observation is after current query" + ) + if (bound_source_utc - query_completed_utc).total_seconds() > 5: + raise QmtShadowPlanError( + "bound broker source_time is after current query" + ) + if plan.get("status") == "SHADOW_READY" and bound_observation is None: + raise QmtShadowPlanError( + "ready shadow plan is missing account-binding clocks" + ) + return { + "signal_as_of": signal_as_of, + "next_trading_session": next_trading_session, + "first_executable_window": expected_window, + "window_start": window_start, + "window_end": window_end, + "query_completed": query_completed, + "bound_observation": bound_observation, + "bound_source": bound_source, + } + + +def _verify_equity_consistency_contract( + plan: Mapping[str, Any], + blockers: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + decision = plan.get("decision") + portfolio = plan.get("portfolio") + if not isinstance(decision, Mapping) or not isinstance( + portfolio, + Mapping, + ): + raise QmtShadowPlanError( + "shadow plan decision/portfolio equity facts are missing" + ) + contract = portfolio.get("equity_consistency") + if not isinstance(contract, Mapping): + raise QmtShadowPlanError( + "shadow plan equity consistency contract is missing" + ) + decision_equity = _finite_non_negative( + decision.get("equity"), + "plan decision equity", + ) + if decision_equity <= 0: + raise QmtShadowPlanError("plan decision equity must be positive") + contract_decision_equity = _finite_non_negative( + contract.get("decision_equity"), + "equity contract decision_equity", + ) + if contract_decision_equity != decision_equity: + raise QmtShadowPlanError( + "equity contract decision equity mismatch" + ) + absolute_tolerance = _finite_non_negative( + contract.get("absolute_tolerance"), + "equity contract absolute_tolerance", + ) + relative_tolerance = _finite_non_negative( + contract.get("relative_tolerance"), + "equity contract relative_tolerance", + ) + if ( + absolute_tolerance != DECISION_EQUITY_ABSOLUTE_TOLERANCE + or relative_tolerance != DECISION_EQUITY_RELATIVE_TOLERANCE + or contract.get("relative_reference") != "decision_equity" + or contract.get("threshold_rule") + != DECISION_EQUITY_TOLERANCE_RULE + ): + raise QmtShadowPlanError( + "equity tolerance constants/rule were changed" + ) + expected_allowed = max( + absolute_tolerance, + relative_tolerance * abs(decision_equity), + ) + allowed = _finite_non_negative( + contract.get("allowed_difference"), + "equity contract allowed_difference", + ) + if allowed != expected_allowed: + raise QmtShadowPlanError( + "equity contract allowed difference is inconsistent" + ) + if type(contract.get("evaluated")) is not bool: + raise QmtShadowPlanError( + "equity contract evaluated must be a bool" + ) + if type(contract.get("within_tolerance")) is not bool: + raise QmtShadowPlanError( + "equity contract within_tolerance must be a bool" + ) + blocker_codes = [str(item.get("code")) for item in blockers] + drift_blocker_count = blocker_codes.count("DECISION_EQUITY_DRIFT") + if contract["evaluated"]: + current_total_asset = _finite_non_negative( + contract.get("current_total_asset"), + "equity contract current_total_asset", + ) + if current_total_asset <= 0: + raise QmtShadowPlanError( + "equity contract current_total_asset must be positive" + ) + portfolio_total_asset = _finite_non_negative( + portfolio.get("total_asset"), + "plan portfolio total_asset", + ) + if portfolio_total_asset != current_total_asset: + raise QmtShadowPlanError( + "equity contract/portfolio total_asset mismatch" + ) + difference = _finite_non_negative( + contract.get("absolute_difference"), + "equity contract absolute_difference", + ) + expected_difference = abs( + current_total_asset - decision_equity + ) + if difference != expected_difference: + raise QmtShadowPlanError( + "equity contract absolute difference is inconsistent" + ) + expected_within = expected_difference <= expected_allowed + if contract["within_tolerance"] is not expected_within: + raise QmtShadowPlanError( + "equity contract tolerance result is inconsistent" + ) + if expected_within and drift_blocker_count: + raise QmtShadowPlanError( + "equity drift blocker is spurious" + ) + if not expected_within and drift_blocker_count != 1: + raise QmtShadowPlanError( + "equity drift lacks exactly one DECISION_EQUITY_DRIFT blocker" + ) + else: + if ( + contract.get("current_total_asset") is not None + or contract.get("absolute_difference") is not None + or contract.get("within_tolerance") is not False + or portfolio.get("total_asset") is not None + ): + raise QmtShadowPlanError( + "unevaluated equity contract contains current equity facts" + ) + if "INVALID_ASSET_FACT" not in blocker_codes: + raise QmtShadowPlanError( + "unevaluated equity contract lacks INVALID_ASSET_FACT" + ) + if drift_blocker_count: + raise QmtShadowPlanError( + "unevaluated equity contract has a drift blocker" + ) + if ( + plan.get("status") == "SHADOW_READY" + and not contract["within_tolerance"] + ): + raise QmtShadowPlanError( + "ready shadow plan has inconsistent decision/current equity" + ) + return dict(contract) + + +def _canonical_shadow_semantics( + *, + targets: Sequence[Mapping[str, Any]], + decision: Mapping[str, Any], + broker_observation: Mapping[str, Any], +) -> dict[str, Any]: + """Replay every readiness-relevant derivation from immutable inputs. + + The published plan is deliberately not an input. This function consumes + only the separately verified decision artifact and the captured read-only + broker observation, then recreates the semantic projection that the plan + is required to match exactly. + """ + + expected_observation_keys = { + "semantic_inputs", + "asset", + "positions", + "orders_before", + "orders_after", + "trades", + } + if set(broker_observation) != expected_observation_keys: + raise QmtShadowPlanError( + "broker observation semantic input set is invalid" + ) + semantic = broker_observation.get("semantic_inputs") + asset = broker_observation.get("asset") + positions_raw = broker_observation.get("positions") + orders_before_raw = broker_observation.get("orders_before") + orders_after_raw = broker_observation.get("orders_after") + trades_raw = broker_observation.get("trades") + if not isinstance(semantic, Mapping) or not isinstance(asset, Mapping): + raise QmtShadowPlanError( + "broker observation semantic/asset facts are invalid" + ) + expected_semantic_keys = { + "schema_version", + "adapter_state_after_query", + "adapter_allow_live_orders_after_query", + "adapter_error_count", + "live_authorization_audit_count", + "query_started_at", + "query_completed_at", + "max_query_duration_seconds", + "market_value_tolerance", + "raw_position_count", + "dropped_zero_position_count", + "raw_order_before_count", + "raw_order_after_count", + "raw_trade_count", + "fact_validation_blockers", + } + if ( + set(semantic) != expected_semantic_keys + or semantic.get("schema_version") + != SHADOW_SEMANTIC_INPUT_VERSION + ): + raise QmtShadowPlanError( + "broker observation semantic contract is invalid" + ) + for field in ( + "adapter_error_count", + "live_authorization_audit_count", + "raw_position_count", + "dropped_zero_position_count", + "raw_order_before_count", + "raw_order_after_count", + "raw_trade_count", + ): + _non_negative_int(semantic.get(field), f"semantic input {field}") + query_limit = _finite_non_negative( + semantic.get("max_query_duration_seconds"), + "semantic input max_query_duration_seconds", + ) + if query_limit > MAX_SHADOW_QUERY_DURATION_SECONDS: + raise QmtShadowPlanError( + "semantic query duration policy exceeds the canonical ceiling" + ) + value_tolerance = _finite_non_negative( + semantic.get("market_value_tolerance"), + "semantic input market_value_tolerance", + ) + if value_tolerance > MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE: + raise QmtShadowPlanError( + "semantic asset tolerance exceeds the canonical ceiling" + ) + query_started = _parse_aware( + semantic.get("query_started_at"), + "semantic input query_started_at", + ) + query_completed = _parse_aware( + semantic.get("query_completed_at"), + "semantic input query_completed_at", + ) + fact_validation = semantic.get("fact_validation_blockers") + if not isinstance(fact_validation, list) or any( + not isinstance(item, Mapping) for item in fact_validation + ): + raise QmtShadowPlanError( + "semantic fact validation blockers must be an array of objects" + ) + allowed_validation_codes = { + "INVALID_ASSET_FACT", + "INVALID_ORDER_FACT", + "INVALID_POSITION_FACT", + "INVALID_TRADE_FACT", + } + for item in fact_validation: + if ( + set(item) - {"code", "detail", "symbol"} + or item.get("code") not in allowed_validation_codes + or not isinstance(item.get("detail"), str) + ): + raise QmtShadowPlanError( + "semantic fact validation blocker is invalid" + ) + + expected_asset_keys = { + "account_hash", + "cash", + "market_value", + "total_asset", + } + if set(asset) != expected_asset_keys: + raise QmtShadowPlanError("broker observation asset shape is invalid") + account_hash_raw = asset.get("account_hash") + if account_hash_raw is not None and ( + not isinstance(account_hash_raw, str) + or len(account_hash_raw) != 64 + or any(character not in "0123456789abcdef" for character in account_hash_raw) + ): + raise QmtShadowPlanError( + "broker observation account_hash is invalid" + ) + account_hash = account_hash_raw + cash = _finite_non_negative( + asset.get("cash"), "broker observation asset cash" + ) + market_value = _finite_non_negative( + asset.get("market_value"), + "broker observation asset market_value", + ) + total_asset = _finite_non_negative( + asset.get("total_asset"), + "broker observation asset total_asset", + ) + + if not isinstance(positions_raw, list): + raise QmtShadowPlanError("broker observation positions must be an array") + positions: list[dict[str, Any]] = [] + seen_positions: set[str] = set() + for index, raw in enumerate(positions_raw): + if not isinstance(raw, Mapping) or set(raw) != { + "symbol", + "quantity", + "sellable_quantity", + "average_cost", + "market_value", + }: + raise QmtShadowPlanError( + f"broker observation position #{index} shape is invalid" + ) + symbol = normalize_symbol(str(raw.get("symbol", ""))) + if symbol in seen_positions: + raise QmtShadowPlanError( + f"duplicate broker observation position {symbol}" + ) + seen_positions.add(symbol) + quantity = _non_negative_int( + raw.get("quantity"), + f"broker observation {symbol} quantity", + ) + sellable = _non_negative_int( + raw.get("sellable_quantity"), + f"broker observation {symbol} sellable_quantity", + ) + if quantity <= 0 or sellable > quantity: + raise QmtShadowPlanError( + f"broker observation position {symbol} quantity is invalid" + ) + position_market_value = _finite_non_negative( + raw.get("market_value"), + f"broker observation {symbol} market_value", + ) + average_cost = _finite_non_negative( + raw.get("average_cost"), + f"broker observation {symbol} average_cost", + ) + if position_market_value <= 0: + raise QmtShadowPlanError( + f"broker observation position {symbol} has no market value" + ) + positions.append( + { + "symbol": symbol, + "quantity": quantity, + "sellable_quantity": sellable, + "average_cost": average_cost, + "market_value": position_market_value, + } + ) + if positions != sorted(positions, key=lambda item: item["symbol"]): + raise QmtShadowPlanError( + "broker observation positions are not canonical" + ) + + def canonical_orders(value: Any, name: str) -> list[dict[str, Any]]: + if not isinstance(value, list): + raise QmtShadowPlanError( + f"broker observation {name} must be an array" + ) + output: list[dict[str, Any]] = [] + seen: set[str] = set() + for index, raw in enumerate(value): + if not isinstance(raw, Mapping) or set(raw) != { + "broker_order_id", + "client_order_id", + "symbol", + "side", + "quantity", + "filled_quantity", + "limit_price", + "state", + }: + raise QmtShadowPlanError( + f"broker observation {name} #{index} shape is invalid" + ) + order_id = str(raw.get("broker_order_id") or "") + if not order_id or order_id in seen: + raise QmtShadowPlanError( + f"broker observation {name} order identity is invalid" + ) + seen.add(order_id) + symbol = normalize_symbol(str(raw.get("symbol", ""))) + side = str(raw.get("side", "")) + state = str(raw.get("state", "")) + if side not in {"BUY", "SELL"} or state not in _KNOWN_ORDER_STATES: + raise QmtShadowPlanError( + f"broker observation {name} order state is invalid" + ) + quantity = _positive_int( + raw.get("quantity"), + f"broker observation order {order_id} quantity", + ) + filled = _non_negative_int( + raw.get("filled_quantity"), + f"broker observation order {order_id} filled_quantity", + ) + if filled > quantity: + raise QmtShadowPlanError( + f"broker observation order {order_id} fill is invalid" + ) + if state == "FILLED" and filled != quantity: + raise QmtShadowPlanError( + f"broker observation filled order {order_id} is incomplete" + ) + if state == "PARTIALLY_FILLED" and not 0 < filled < quantity: + raise QmtShadowPlanError( + f"broker observation partially filled order " + f"{order_id} has invalid fill" + ) + if state in {"NEW", "ACCEPTED", "REJECTED"} and filled: + raise QmtShadowPlanError( + f"broker observation order {order_id} state/fill conflict" + ) + client_order_id = raw.get("client_order_id") + if client_order_id is not None and not isinstance( + client_order_id, str + ): + raise QmtShadowPlanError( + f"broker observation order {order_id} client id is invalid" + ) + if client_order_id == "": + raise QmtShadowPlanError( + f"broker observation order {order_id} client id is empty" + ) + output.append( + { + "broker_order_id": order_id, + "client_order_id": client_order_id, + "symbol": symbol, + "side": side, + "quantity": quantity, + "filled_quantity": filled, + "limit_price": _finite_non_negative( + raw.get("limit_price"), + f"broker observation order {order_id} limit_price", + ), + "state": state, + } + ) + canonical = sorted(output, key=lambda item: item["broker_order_id"]) + if output != canonical: + raise QmtShadowPlanError( + f"broker observation {name} is not canonical" + ) + return output + + orders_before = canonical_orders(orders_before_raw, "orders_before") + orders_after = canonical_orders(orders_after_raw, "orders_after") + + if not isinstance(trades_raw, list): + raise QmtShadowPlanError("broker observation trades must be an array") + trades: list[dict[str, Any]] = [] + seen_trades: set[str] = set() + for index, raw in enumerate(trades_raw): + if not isinstance(raw, Mapping) or set(raw) != { + "trade_id", + "broker_order_id", + "symbol", + "side", + "quantity", + "price", + "amount", + }: + raise QmtShadowPlanError( + f"broker observation trade #{index} shape is invalid" + ) + trade_id = str(raw.get("trade_id") or "") + order_id = str(raw.get("broker_order_id") or "") + if not trade_id or trade_id in seen_trades or not order_id: + raise QmtShadowPlanError( + f"broker observation trade #{index} identity is invalid" + ) + seen_trades.add(trade_id) + symbol = normalize_symbol(str(raw.get("symbol", ""))) + side = str(raw.get("side", "")) + if side not in {"BUY", "SELL"}: + raise QmtShadowPlanError( + f"broker observation trade {trade_id} side is invalid" + ) + price = _finite_non_negative( + raw.get("price"), + f"broker observation trade {trade_id} price", + ) + if price <= 0: + raise QmtShadowPlanError( + f"broker observation trade {trade_id} price is invalid" + ) + trades.append( + { + "trade_id": trade_id, + "broker_order_id": order_id, + "symbol": symbol, + "side": side, + "quantity": _positive_int( + raw.get("quantity"), + f"broker observation trade {trade_id} quantity", + ), + "price": price, + "amount": _finite_non_negative( + raw.get("amount"), + f"broker observation trade {trade_id} amount", + ), + } + ) + canonical_trades = sorted( + trades, + key=lambda item: ( + item["broker_order_id"], + item["trade_id"], + item["symbol"], + item["quantity"], + item["price"], + ), + ) + if trades != canonical_trades: + raise QmtShadowPlanError( + "broker observation trades are not canonical" + ) + + blockers: list[dict[str, Any]] = [ + dict(item) for item in fact_validation + ] + adapter_state = semantic.get("adapter_state_after_query") + adapter_ready = adapter_state == "READY" + if not adapter_ready: + _blocker( + blockers, + "ADAPTER_NOT_READY_AFTER_QUERY", + f"adapter state is {adapter_state!r}", + ) + adapter_error_count = _non_negative_int( + semantic.get("adapter_error_count"), + "semantic input adapter_error_count", + ) + if adapter_error_count: + _blocker( + blockers, + "ADAPTER_ERRORS_PRESENT", + f"adapter reports {adapter_error_count} callback/query error(s)", + ) + live_audit_count = _non_negative_int( + semantic.get("live_authorization_audit_count"), + "semantic input live_authorization_audit_count", + ) + if live_audit_count: + _blocker( + blockers, + "LIVE_AUTHORIZATION_HISTORY_PRESENT", + "fresh read-only adapter unexpectedly contains live authorization history", + ) + live_disabled = ( + semantic.get("adapter_allow_live_orders_after_query") is False + ) + if not live_disabled: + _blocker( + blockers, + "LIVE_MODE_CHANGED_DURING_QUERY", + "adapter allow_live_orders was not False after broker queries", + ) + + elapsed = (query_completed - query_started).total_seconds() + query_window_valid = 0 <= elapsed <= query_limit + if elapsed < 0: + _blocker( + blockers, + "CLOCK_REGRESSION", + "query completion precedes query start", + ) + elif elapsed > query_limit: + _blocker( + blockers, + "QUERY_WINDOW_TOO_LONG", + f"query window {elapsed:.6f}s exceeds {query_limit:.6f}s", + ) + + decision_equity = _finite_non_negative( + decision.get("equity"), "input decision equity" + ) + if decision_equity <= 0: + raise QmtShadowPlanError("input decision equity must be positive") + decision_signal_as_of = _parse_aware( + decision.get("signal_as_of"), + "input decision.signal_as_of", + ) + decision_as_of = _parse_aware( + decision.get("as_of"), + "input decision.as_of", + ) + if not _same_instant(decision_as_of, decision_signal_as_of): + raise QmtShadowPlanError( + "input decision signal_as_of does not match as_of" + ) + try: + next_trading_session = date.fromisoformat( + str(decision.get("next_trading_session")) + ) + except ValueError as exc: + raise QmtShadowPlanError( + "input decision next_trading_session is invalid" + ) from exc + expected_window = first_executable_window(next_trading_session) + if decision.get("first_executable_window") != expected_window: + raise QmtShadowPlanError( + "input decision executable window is not canonical" + ) + window_start = _parse_aware( + expected_window["start"], "input decision executable window start" + ) + window_end = _parse_aware( + expected_window["end"], "input decision executable window end" + ) + query_started_utc = query_started.astimezone(timezone.utc) + query_completed_utc = query_completed.astimezone(timezone.utc) + window_start_utc = window_start.astimezone(timezone.utc) + window_end_utc = window_end.astimezone(timezone.utc) + execution_window_valid = ( + window_start_utc <= query_started_utc < window_end_utc + and window_start_utc <= query_completed_utc < window_end_utc + ) + if not execution_window_valid: + shanghai = timezone(timedelta(hours=8)) + local_started = query_started.astimezone(shanghai) + local_completed = query_completed.astimezone(shanghai) + if ( + local_started.date() != next_trading_session + or local_completed.date() != next_trading_session + ): + code = "QUERY_WRONG_TRADING_SESSION" + detail = ( + "broker query must occur on the unique next trading " + f"session {next_trading_session.isoformat()}" + ) + elif query_completed_utc >= window_end_utc: + code = "QUERY_MISSED_EXECUTABLE_WINDOW" + detail = ( + "broker query completed at or after the exclusive 09:30 " + "Asia/Shanghai pre-open boundary" + ) + else: + code = "QUERY_BEFORE_EXECUTABLE_WINDOW" + detail = ( + "broker query began before 09:00 Asia/Shanghai on the " + "unique next trading session" + ) + _blocker(blockers, code, detail) + + validation_codes = { + str(item.get("code")) for item in fact_validation + } + count_contracts = ( + ( + "raw_position_count", + len(positions) + + _non_negative_int( + semantic.get("dropped_zero_position_count"), + "semantic input dropped_zero_position_count", + ), + "INVALID_POSITION_FACT", + ), + ( + "raw_order_before_count", + len(orders_before), + "INVALID_ORDER_FACT", + ), + ( + "raw_order_after_count", + len(orders_after), + "INVALID_ORDER_FACT", + ), + ( + "raw_trade_count", + len(trades), + "INVALID_TRADE_FACT", + ), + ) + for field, normalised_count, invalid_code in count_contracts: + raw_count = _non_negative_int( + semantic.get(field), f"semantic input {field}" + ) + if raw_count < normalised_count or ( + invalid_code not in validation_codes + and raw_count != normalised_count + ): + raise QmtShadowPlanError( + f"semantic input {field} is inconsistent with broker facts" + ) + asset_valid = "INVALID_ASSET_FACT" not in validation_codes + if asset_valid and total_asset <= 0: + raise QmtShadowPlanError( + "valid broker observation total_asset must be positive" + ) + if not asset_valid and ( + account_hash is not None + or cash != 0 + or market_value != 0 + or total_asset != 0 + ): + raise QmtShadowPlanError( + "invalid asset observation was not fail-closed" + ) + equity_consistency = _equity_consistency( + decision_equity=decision_equity, + current_total_asset=total_asset if asset_valid else None, + ) + if ( + equity_consistency["evaluated"] + and not equity_consistency["within_tolerance"] + ): + _blocker( + blockers, + "DECISION_EQUITY_DRIFT", + ( + "decision equity/current total_asset drift " + f"{equity_consistency['absolute_difference']:.6f} exceeds " + "allowed " + f"{equity_consistency['allowed_difference']:.6f}" + ), + ) + + open_orders: list[dict[str, Any]] = [] + invalid_open_order = False + for order_set, collect_open in ( + (orders_before, False), + (orders_after, True), + ): + for order in order_set: + state = order["state"] + if state not in _OPEN_ORDER_STATES: + continue + if state == "UNKNOWN": + _blocker( + blockers, + "UNKNOWN_ORDER_STATE", + ( + f"broker order {order['broker_order_id']} " + "state is UNKNOWN" + ), + symbol=order["symbol"], + ) + else: + _blocker( + blockers, + "UNFINISHED_ORDER", + ( + f"broker order {order['broker_order_id']} " + f"is {state}" + ), + symbol=order["symbol"], + ) + if order["limit_price"] <= 0: + invalid_open_order = True + _blocker( + blockers, + "INVALID_OPEN_ORDER", + ( + f"broker order {order['broker_order_id']} " + "has no positive limit price" + ), + symbol=order["symbol"], + ) + elif collect_open: + open_orders.append( + { + "order_id": ( + order["client_order_id"] + or f"QMT-{order['broker_order_id']}" + ), + "broker_order_id": order["broker_order_id"], + "symbol": order["symbol"], + "side": order["side"], + "quantity": order["quantity"], + "filled_quantity": order["filled_quantity"], + "status": order["state"], + "limit_price": order["limit_price"], + "order_type": "LIMIT", + "time_in_force": "DAY", + } + ) + open_orders = sorted( + open_orders, key=lambda item: item["broker_order_id"] + ) + orders_stable = orders_before == orders_after + if not orders_stable: + _blocker( + blockers, + "ORDERS_CHANGED_DURING_QUERY", + "broker order facts changed inside the read-only query window", + ) + + positions_valid = "INVALID_POSITION_FACT" not in validation_codes + asset_identity_valid = asset_valid + if asset_valid: + positions_market_value = sum( + float(item["market_value"]) for item in positions + ) + if abs(positions_market_value - market_value) > value_tolerance: + asset_identity_valid = False + _blocker( + blockers, + "POSITION_MARKET_VALUE_MISMATCH", + ( + f"positions sum={positions_market_value:.6f}, " + f"asset market_value={market_value:.6f}, " + f"tolerance={value_tolerance:.6f}" + ), + ) + if abs(cash + market_value - total_asset) > value_tolerance: + asset_identity_valid = False + _blocker( + blockers, + "ASSET_IDENTITY_MISMATCH", + ( + f"cash+market_value={cash + market_value:.6f}, " + f"total_asset={total_asset:.6f}, " + f"tolerance={value_tolerance:.6f}" + ), + ) + + order_by_id = { + str(item["broker_order_id"]): item for item in orders_after + } + trade_totals: dict[str, int] = {} + trades_valid = "INVALID_TRADE_FACT" not in validation_codes + fill_facts_valid = True + for trade in trades: + order_id = str(trade["broker_order_id"]) + order = order_by_id.get(order_id) + if order is None: + trades_valid = False + _blocker( + blockers, + "ORPHAN_TRADE", + ( + f"trade {trade['trade_id']} references absent order " + f"{order_id}" + ), + symbol=trade["symbol"], + ) + fill_facts_valid = False + continue + if ( + trade["symbol"] != order["symbol"] + or trade["side"] != order["side"] + ): + fill_facts_valid = False + _blocker( + blockers, + "TRADE_ORDER_IDENTITY_MISMATCH", + ( + f"trade {trade['trade_id']} symbol/side " + f"{trade['symbol']}/{trade['side']} disagrees with " + f"order {order_id} {order['symbol']}/{order['side']}" + ), + symbol=trade["symbol"], + ) + trade_totals[order_id] = ( + trade_totals.get(order_id, 0) + int(trade["quantity"]) + ) + for order in orders_after: + order_id = str(order["broker_order_id"]) + expected_fill = int(order["filled_quantity"]) + observed_fill = trade_totals.get(order_id, 0) + if expected_fill != observed_fill: + fill_facts_valid = False + _blocker( + blockers, + "FILL_TRADE_MISMATCH", + ( + f"order {order_id} reports filled={expected_fill}, " + f"queried trades sum to {observed_fill}" + ), + symbol=order["symbol"], + ) + + decision_broker = decision.get("broker_snapshot") + bound_observation_as_of: datetime | None = None + bound_source_time: datetime | None = None + if decision_broker is None: + _blocker( + blockers, + "DECISION_NOT_ACCOUNT_BOUND", + ( + "decision was built without a broker snapshot; use this " + "blocked run's broker_snapshot.json to regenerate the " + "decision before shadow readiness" + ), + ) + elif not isinstance(decision_broker, Mapping): + _blocker( + blockers, + "INVALID_DECISION_BROKER_IDENTITY", + "decision broker_snapshot identity is not an object", + ) + else: + expected_account_hash = str( + decision_broker.get("account_hash") or "" + ) + if ( + account_hash is None + or not expected_account_hash + or not hmac.compare_digest(account_hash, expected_account_hash) + ): + _blocker( + blockers, + "DECISION_ACCOUNT_MISMATCH", + "decision target and current QMT observation use different account identities", + ) + if decision_broker.get("broker") != "QMT_XTTRADER": + _blocker( + blockers, + "DECISION_BROKER_MISMATCH", + ( + "decision broker identity is not the QMT_XTTRADER " + "read-only boundary" + ), + ) + if ( + decision_broker.get("next_trading_session") + != next_trading_session.isoformat() + or decision_broker.get("first_executable_window") + != expected_window + ): + _blocker( + blockers, + "DECISION_EXECUTION_WINDOW_MISMATCH", + "decision broker identity is bound to a different trading " + "session or pre-open window", + ) + bound_signal_as_of = _parse_aware( + decision_broker.get("signal_as_of"), + "input decision broker signal_as_of", + ) + bound_observation_as_of = _parse_aware( + decision_broker.get("observation_as_of"), + "input decision broker observation_as_of", + ) + bound_legacy_as_of = _parse_aware( + decision_broker.get("as_of"), + "input decision broker as_of", + ) + bound_source_time = _parse_aware( + decision_broker.get("source_time"), + "input decision broker source_time", + ) + if not _same_instant( + bound_signal_as_of, decision_signal_as_of + ): + _blocker( + blockers, + "DECISION_SIGNAL_BINDING_MISMATCH", + "decision broker identity is bound to a different signal clock", + ) + if not _same_instant( + bound_observation_as_of, bound_legacy_as_of + ): + _blocker( + blockers, + "DECISION_OBSERVATION_CLOCK_MISMATCH", + "decision broker observation_as_of does not equal as_of", + ) + signal_utc = decision_signal_as_of.astimezone(timezone.utc) + bound_observation_utc = bound_observation_as_of.astimezone( + timezone.utc + ) + bound_source_utc = bound_source_time.astimezone(timezone.utc) + if bound_observation_utc < signal_utc: + _blocker( + blockers, + "DECISION_OBSERVATION_PRECEDES_SIGNAL", + "bound broker observation predates the completed signal close", + ) + if bound_source_utc < signal_utc: + _blocker( + blockers, + "DECISION_SOURCE_PRECEDES_SIGNAL", + "bound broker source_time predates the completed signal close", + ) + if ( + bound_observation_utc - query_completed_utc + ).total_seconds() > 5: + _blocker( + blockers, + "DECISION_OBSERVATION_FROM_FUTURE", + "bound broker observation is after the current query window", + ) + if ( + bound_source_utc - query_completed_utc + ).total_seconds() > 5: + _blocker( + blockers, + "DECISION_SOURCE_FROM_FUTURE", + "bound broker source_time is after the current query window", + ) + + target_quantities: dict[str, int] = {} + target_weights: dict[str, float] = {} + lot_sizes: dict[str, int] = {} + for index, target in enumerate(targets): + if not isinstance(target, Mapping): + raise QmtShadowPlanError( + f"input target #{index} must be an object" + ) + symbol = normalize_symbol(str(target.get("symbol", ""))) + if symbol in target_quantities: + raise QmtShadowPlanError(f"duplicate input target {symbol}") + target_quantities[symbol] = _non_negative_int( + target.get("target_quantity"), + f"input target {symbol} target_quantity", + ) + weight = _finite_non_negative( + target.get("target_weight"), + f"input target {symbol} target_weight", + ) + if weight > 1: + raise QmtShadowPlanError( + f"input target {symbol} weight exceeds one" + ) + target_weights[symbol] = weight + lot_sizes[symbol] = _positive_int( + target.get("lot_size", 100), + f"input target {symbol} lot_size", + ) + if not target_quantities: + raise QmtShadowPlanError("input decision has no targets") + target_gross_weight = sum(target_weights.values()) + if target_gross_weight > 1: + raise QmtShadowPlanError( + "input decision target weights exceed the cash budget" + ) + if len(set(lot_sizes.values())) > 1: + _blocker( + blockers, + "MIXED_LOT_SIZES", + "a single deterministic shadow-delta pass requires one lot size", + ) + lot_size = next(iter(lot_sizes.values())) + current_quantities = { + item["symbol"]: int(item["quantity"]) for item in positions + } + sellable_quantities = { + item["symbol"]: int(item["sellable_quantity"]) + for item in positions + } + feasible_deltas = ( + order_deltas( + target_quantities, + current_quantities, + sellable_quantities, + lot_size=lot_size, + ) + if positions_valid + else {} + ) + position_by_symbol = { + item["symbol"]: item for item in positions + } + rows: list[dict[str, Any]] = [] + for symbol in sorted(set(target_quantities) | set(current_quantities)): + position = position_by_symbol.get(symbol, {}) + current_quantity = current_quantities.get(symbol, 0) + target_quantity = target_quantities.get(symbol, 0) + desired_delta = target_quantity - current_quantity + feasible_delta = feasible_deltas.get(symbol, 0) + if desired_delta != feasible_delta: + _blocker( + blockers, + "UNEXECUTABLE_POSITION_RESIDUAL", + ( + f"desired delta {desired_delta} is clipped to " + f"{feasible_delta} by board-lot/sellable rules" + ), + symbol=symbol, + ) + current_market_value = float(position.get("market_value", 0.0)) + rows.append( + { + "symbol": symbol, + "target_weight": target_weights.get(symbol, 0.0), + "current_weight": ( + current_market_value / total_asset + if asset_valid and total_asset > 0 + else None + ), + "target_quantity": target_quantity, + "current_quantity": current_quantity, + "sellable_quantity": sellable_quantities.get(symbol, 0), + "current_market_value": current_market_value, + "desired_delta": desired_delta, + "board_lot_feasible_delta": feasible_delta, + "proposed_delta": 0, + "proposed_side": None, + "proposed_quantity": 0, + "lot_size": lot_sizes.get(symbol, lot_size), + } + ) + + blockers = sorted( + { + ( + item["code"], + item.get("symbol"), + item["detail"], + ): item + for item in blockers + }.values(), + key=lambda item: ( + item["code"], + item.get("symbol") or "", + item["detail"], + ), + ) + ready = not blockers + if ready: + for row in rows: + delta = int(row["board_lot_feasible_delta"]) + row["proposed_delta"] = delta + row["proposed_side"] = ( + "BUY" if delta > 0 else "SELL" if delta < 0 else None + ) + row["proposed_quantity"] = abs(delta) + + order_fact_valid = ( + "INVALID_ORDER_FACT" not in validation_codes + and not invalid_open_order + ) + snapshot_valid = all( + ( + adapter_ready, + live_disabled, + adapter_error_count == 0, + live_audit_count == 0, + query_window_valid, + execution_window_valid, + orders_stable, + asset_valid, + asset_identity_valid, + positions_valid, + order_fact_valid, + trades_valid, + fill_facts_valid, + account_hash is not None, + ) + ) + observation_projection = { + "query_started_at": query_started.isoformat(), + "query_completed_at": query_completed.isoformat(), + "query_duration_ms": max(0, round(elapsed * 1000)), + "orders_queried_twice": True, + "adapter_allow_live_orders_after_query": semantic.get( + "adapter_allow_live_orders_after_query" + ), + "asset_count": 1, + "position_count": semantic["raw_position_count"], + "order_count": semantic["raw_order_after_count"], + "trade_count": semantic["raw_trade_count"], + "observed_facts_sha256": _sha256_payload(broker_observation), + "broker_snapshot_valid": snapshot_valid, + } + portfolio = { + "cash": cash if asset_valid else None, + "market_value": market_value if asset_valid else None, + "total_asset": total_asset if asset_valid else None, + "current_gross_weight": ( + market_value / total_asset + if asset_valid and total_asset > 0 + else None + ), + "target_gross_weight": target_gross_weight, + "equity_consistency": equity_consistency, + } + broker_snapshot: dict[str, Any] | None = None + if snapshot_valid: + broker_snapshot = { + "schema_version": "1.0", + "snapshot_id": "", + "signal_as_of": decision_signal_as_of.isoformat(), + "next_trading_session": next_trading_session.isoformat(), + "first_executable_window": expected_window, + "observation_as_of": query_completed.isoformat(), + "as_of": query_completed.isoformat(), + "source_time": query_completed.isoformat(), + "broker": "QMT_XTTRADER", + "account_hash": account_hash, + "cash": cash, + "total_asset": total_asset, + "market_value": market_value, + "positions": positions, + "open_orders": open_orders, + "metadata": { + "read_only": True, + "source_time_semantics": ( + "local_query_completion_not_broker_clock" + ), + "query_started_at": query_started.isoformat(), + "query_duration_ms": max(0, round(elapsed * 1000)), + "terminal_order_count": sum( + item["state"] in _TERMINAL_ORDER_STATES + for item in orders_after + ), + "trade_count": len(trades), + "observed_facts_sha256": _sha256_payload( + broker_observation + ), + }, + } + broker_snapshot["snapshot_id"] = ( + "QMT-SHADOW-" + _sha256_payload(broker_snapshot)[:16] + ) + return { + "blockers": blockers, + "status": "SHADOW_READY" if ready else "BLOCKED", + "ready": ready, + "target_diffs": rows, + "observation": observation_projection, + "portfolio": portfolio, + "broker_snapshot": broker_snapshot, + "broker_snapshot_sha256": ( + _sha256_payload(broker_snapshot) + if broker_snapshot is not None + else None + ), + "bound_observation_as_of": ( + bound_observation_as_of.isoformat() + if bound_observation_as_of is not None + else None + ), + "bound_source_time": ( + bound_source_time.isoformat() + if bound_source_time is not None + else None + ), + } + + +def verify_qmt_shadow_plan( + manifest_file: str | Path, + *, + decision_manifest: str | Path | None = None, + account_hash_key: bytes | str | None = None, +) -> dict[str, Any]: + path = Path(manifest_file).expanduser().resolve() + root = path.parent + if (root / ".quant60-incomplete").exists(): + raise QmtShadowPlanError("shadow plan publication is incomplete") + manifest_bytes = path.read_bytes() + manifest = _strict_json(path) + if manifest_bytes != _published_json_bytes(manifest): + raise QmtShadowPlanError( + "shadow plan manifest is not in deterministic published encoding" + ) + if not isinstance(manifest, Mapping) or set(manifest) != { + "schema_version", + "plan_id", + "mode", + "decision_run_id", + "decision_manifest_sha256", + "decision_manifest_run_id", + "planner_source_sha256", + "planner_sources", + "engine_source_sha256", + "deterministic_canonical_json", + "broker_mutation_allowed", + "authenticated_evidence_policy_id", + "authenticated_evidence_hmac_sha256", + "artifact_sha256", + }: + raise QmtShadowPlanError( + "shadow plan manifest field set is not canonical" + ) + if ( + manifest.get("schema_version") != "1.0" + or manifest.get("mode") != "QMT_XTTRADER_SHADOW_READ_ONLY" + or manifest.get("deterministic_canonical_json") is not True + ): + raise QmtShadowPlanError( + "shadow plan manifest policy is invalid" + ) + expected_names = { + "shadow_plan.json", + "broker_snapshot.json", + "broker_observation.json", + } + if set(manifest.get("artifact_sha256", {})) != expected_names: + raise QmtShadowPlanError("shadow plan artifact set is invalid") + if {entry.name for entry in root.iterdir()} != expected_names | {path.name}: + raise QmtShadowPlanError("shadow plan directory is not an exact set") + for name, expected in manifest["artifact_sha256"].items(): + if _sha256_bytes((root / name).read_bytes()) != expected: + raise QmtShadowPlanError(f"shadow plan artifact hash mismatch: {name}") + source_sha256, sources = _source_manifest() + if sources != manifest.get("planner_sources"): + raise QmtShadowPlanError("current shadow planner sources have changed") + if source_sha256 != manifest.get("planner_source_sha256"): + raise QmtShadowPlanError("shadow planner source hash mismatch") + current_engine_sha256, unused_sources = _engine_source_manifest() + del unused_sources + if current_engine_sha256 != manifest.get("engine_source_sha256"): + raise QmtShadowPlanError("current Quant OS source has changed") + + plan = _strict_json(root / "shadow_plan.json") + broker_snapshot = _strict_json(root / "broker_snapshot.json") + broker_observation = _strict_json(root / "broker_observation.json") + for name, payload in ( + ("shadow_plan.json", plan), + ("broker_snapshot.json", broker_snapshot), + ("broker_observation.json", broker_observation), + ): + if (root / name).read_bytes() != _published_json_bytes(payload): + raise QmtShadowPlanError( + f"{name} is not in deterministic published encoding" + ) + key_bytes = _account_hash_key_bytes(account_hash_key) + plan_body = dict(plan) + plan_id = plan_body.pop("plan_id", None) + expected_plan_id = "QSP-" + _sha256_payload( + dict(plan_body, plan_id="") + )[:20] + if plan_id != expected_plan_id or plan_id != manifest.get("plan_id"): + raise QmtShadowPlanError("shadow plan identity mismatch") + if plan.get("mode") != "QMT_XTTRADER_SHADOW_READ_ONLY": + raise QmtShadowPlanError("shadow plan mode is not read-only") + if set(plan) != { + "schema_version", + "plan_id", + "mode", + "status", + "decision", + "observation", + "portfolio", + "target_diffs", + "blockers", + "limitations", + "safety", + "broker_snapshot_sha256", + }: + raise QmtShadowPlanError("shadow plan field set is not canonical") + if plan.get("schema_version") != "1.0": + raise QmtShadowPlanError("shadow plan schema version is invalid") + safety = plan.get("safety", {}) + if ( + safety.get("read_only") is not True + or safety.get("allow_live_orders") is not False + or safety.get("mutation_methods_invoked") is not False + or safety.get("ready_for_live_submission") is not False + or safety.get("automatic_submission_supported") is not False + or safety.get("position_diff_basis") + != _POSITION_DIFF_BASIS + or manifest.get("broker_mutation_allowed") is not False + ): + raise QmtShadowPlanError("shadow plan safety invariant failed") + blockers = plan.get("blockers", []) + if ( + not isinstance(blockers, list) + or any(not isinstance(item, Mapping) for item in blockers) + ): + raise QmtShadowPlanError( + "shadow plan blockers must be an array of objects" + ) + blocked = bool(blockers) + if plan.get("status") not in {"SHADOW_READY", "BLOCKED"}: + raise QmtShadowPlanError("shadow plan has an invalid status") + if (plan.get("status") == "BLOCKED") != blocked: + raise QmtShadowPlanError("shadow plan blocker/status mismatch") + if safety.get("ready_for_manual_review") is not (not blocked): + raise QmtShadowPlanError("shadow plan review-readiness mismatch") + limitations = plan.get("limitations") + if limitations != list(_SHADOW_LIMITATIONS): + raise QmtShadowPlanError( + "shadow plan limitations are not canonical" + ) + _verify_equity_consistency_contract(plan, blockers) + clock_contract = _verify_shadow_clock_contract(plan) + if not isinstance(broker_observation, Mapping): + raise QmtShadowPlanError("broker observation must be an object") + observation_hash = _sha256_payload(broker_observation) + if observation_hash != plan.get("observation", {}).get( + "observed_facts_sha256" + ): + raise QmtShadowPlanError("broker observation hash mismatch") + if decision_manifest is None: + raise QmtShadowPlanError( + "original decision manifest is required for semantic replay" + ) + ( + input_decision_path, + input_decision_manifest, + input_targets, + input_decision, + ) = _load_verified_decision(decision_manifest) + input_decision_manifest_sha256 = _sha256_bytes( + input_decision_path.read_bytes() + ) + if input_decision_manifest_sha256 != manifest.get( + "decision_manifest_sha256" + ): + raise QmtShadowPlanError("input decision manifest hash mismatch") + saved_evidence_hmac = manifest.get( + "authenticated_evidence_hmac_sha256" + ) + published_artifact_sha256 = { + name: _sha256_bytes((root / name).read_bytes()) + for name in ( + "shadow_plan.json", + "broker_observation.json", + "broker_snapshot.json", + ) + } + expected_evidence_hmac = _authenticated_evidence_hmac( + _authenticated_evidence_envelope( + plan=plan, + broker_observation=broker_observation, + broker_snapshot=broker_snapshot, + decision_manifest_sha256=input_decision_manifest_sha256, + planner_source_sha256=source_sha256, + engine_source_sha256=current_engine_sha256, + published_artifact_sha256=published_artifact_sha256, + ), + key_bytes, + ) + if ( + manifest.get("authenticated_evidence_policy_id") + != AUTHENTICATED_EVIDENCE_POLICY_ID + or not isinstance(saved_evidence_hmac, str) + or len(saved_evidence_hmac) != 64 + or not hmac.compare_digest( + saved_evidence_hmac, + expected_evidence_hmac, + ) + ): + raise QmtShadowPlanError( + "authenticated shadow evidence HMAC verification failed" + ) + replay = _canonical_shadow_semantics( + targets=input_targets, + decision=input_decision, + broker_observation=broker_observation, + ) + expected_decision = { + "run_id": input_decision["run_id"], + "decision_id": input_decision["decision_id"], + "as_of": input_decision["as_of"], + "signal_as_of": _parse_aware( + input_decision.get("signal_as_of"), + "input decision.signal_as_of", + ).isoformat(), + "next_trading_session": input_decision["next_trading_session"], + "first_executable_window": input_decision[ + "first_executable_window" + ], + "bound_broker_observation_as_of": replay[ + "bound_observation_as_of" + ], + "bound_broker_source_time": replay["bound_source_time"], + "model_version": input_decision["model_version"], + "data_version": input_decision["data_version"], + "equity": _finite_non_negative( + input_decision.get("equity"), + "input decision equity", + ), + "decision_manifest_sha256": input_decision_manifest_sha256, + } + if plan.get("decision") != expected_decision: + raise QmtShadowPlanError( + "shadow plan decision projection does not exactly match input" + ) + if ( + manifest.get("decision_run_id") != input_decision["run_id"] + or manifest.get("decision_manifest_run_id") + != input_decision_manifest.get("run_id") + ): + raise QmtShadowPlanError( + "shadow plan manifest decision identity mismatch" + ) + if blockers != replay["blockers"]: + raise QmtShadowPlanError( + "shadow plan blockers do not match semantic replay" + ) + if plan.get("status") != replay["status"]: + raise QmtShadowPlanError( + "shadow plan status does not match semantic replay" + ) + if plan.get("observation") != replay["observation"]: + raise QmtShadowPlanError( + "shadow plan observation projection does not match semantic replay" + ) + if plan.get("portfolio") != replay["portfolio"]: + raise QmtShadowPlanError( + "shadow plan cash/portfolio projection does not match semantic replay" + ) + if plan.get("target_diffs") != replay["target_diffs"]: + raise QmtShadowPlanError( + "shadow plan target/order projection does not match semantic replay" + ) + if ( + safety.get("ready_for_manual_review") + is not replay["ready"] + ): + raise QmtShadowPlanError( + "shadow plan readiness does not match semantic replay" + ) + expected_safety = { + "read_only": True, + "allow_live_orders": False, + "mutation_methods_invoked": False, + "ready_for_manual_review": replay["ready"], + "ready_for_live_submission": False, + "automatic_submission_supported": False, + "position_diff_basis": _POSITION_DIFF_BASIS, + } + if safety != expected_safety: + raise QmtShadowPlanError( + "shadow plan safety projection does not match semantic replay" + ) + if broker_snapshot != replay["broker_snapshot"]: + raise QmtShadowPlanError( + "broker snapshot does not match semantic replay" + ) + if plan.get("broker_snapshot_sha256") != replay[ + "broker_snapshot_sha256" + ]: + raise QmtShadowPlanError( + "broker snapshot hash does not match semantic replay" + ) + if replay["ready"] and replay["broker_snapshot"] is None: + raise QmtShadowPlanError( + "ready shadow plan lacks a valid replayed broker snapshot" + ) + rows = plan.get("target_diffs", []) + if not isinstance(rows, list): + raise QmtShadowPlanError("shadow plan target_diffs must be an array") + symbols: list[str] = [] + row_by_symbol: dict[str, Mapping[str, Any]] = {} + for index, row in enumerate(rows): + if not isinstance(row, Mapping): + raise QmtShadowPlanError( + f"shadow plan target diff #{index} must be an object" + ) + symbol = normalize_symbol(str(row.get("symbol", ""))) + symbols.append(symbol) + row_by_symbol[symbol] = row + target = _non_negative_int( + row.get("target_quantity"), + f"target diff {symbol} target_quantity", + ) + current = _non_negative_int( + row.get("current_quantity"), + f"target diff {symbol} current_quantity", + ) + sellable = _non_negative_int( + row.get("sellable_quantity"), + f"target diff {symbol} sellable_quantity", + ) + if sellable > current: + raise QmtShadowPlanError( + f"target diff {symbol} sellable exceeds current" + ) + desired = int(row.get("desired_delta")) + feasible = int(row.get("board_lot_feasible_delta")) + proposed = int(row.get("proposed_delta")) + if desired != target - current: + raise QmtShadowPlanError( + f"target diff {symbol} desired delta is inconsistent" + ) + if blocked and proposed != 0: + raise QmtShadowPlanError( + "blocked shadow plan contains proposed trades" + ) + if not blocked and proposed != feasible: + raise QmtShadowPlanError( + f"ready shadow plan {symbol} proposed delta is inconsistent" + ) + expected_side = ( + "BUY" if proposed > 0 else "SELL" if proposed < 0 else None + ) + if row.get("proposed_side") != expected_side: + raise QmtShadowPlanError( + f"target diff {symbol} proposed side is inconsistent" + ) + if _non_negative_int( + row.get("proposed_quantity"), + f"target diff {symbol} proposed_quantity", + ) != abs(proposed): + raise QmtShadowPlanError( + f"target diff {symbol} proposed quantity is inconsistent" + ) + if symbols != sorted(set(symbols)): + raise QmtShadowPlanError( + "shadow plan target diffs must be unique and symbol-sorted" + ) + observed_positions = broker_observation.get("positions") + observed_asset = broker_observation.get("asset") + if not isinstance(observed_positions, list) or not isinstance( + observed_asset, + Mapping, + ): + raise QmtShadowPlanError( + "broker observation asset/positions facts are invalid" + ) + observed_position_map: dict[str, Mapping[str, Any]] = {} + for index, position in enumerate(observed_positions): + if not isinstance(position, Mapping): + raise QmtShadowPlanError( + f"broker observation position #{index} is invalid" + ) + symbol = normalize_symbol(str(position.get("symbol", ""))) + if symbol in observed_position_map: + raise QmtShadowPlanError( + f"duplicate broker observation position {symbol}" + ) + observed_position_map[symbol] = position + for symbol, row in row_by_symbol.items(): + position = observed_position_map.get(symbol) + expected_quantity = ( + _non_negative_int( + position.get("quantity"), + f"broker observation {symbol} quantity", + ) + if position is not None + else 0 + ) + expected_sellable = ( + _non_negative_int( + position.get("sellable_quantity"), + f"broker observation {symbol} sellable_quantity", + ) + if position is not None + else 0 + ) + expected_market_value = ( + _finite_non_negative( + position.get("market_value"), + f"broker observation {symbol} market_value", + ) + if position is not None + else 0.0 + ) + if ( + row.get("current_quantity") != expected_quantity + or row.get("sellable_quantity") != expected_sellable + or row.get("current_market_value") + != expected_market_value + ): + raise QmtShadowPlanError( + f"target diff {symbol} is not based on current broker position" + ) + if not set(observed_position_map).issubset(row_by_symbol): + raise QmtShadowPlanError( + "broker observation position is absent from target diffs" + ) + equity_contract = plan.get("portfolio", {}).get( + "equity_consistency", + {}, + ) + if equity_contract.get("evaluated"): + observed_total_asset = _finite_non_negative( + observed_asset.get("total_asset"), + "broker observation total_asset", + ) + if ( + observed_total_asset + != equity_contract.get("current_total_asset") + ): + raise QmtShadowPlanError( + "equity contract is not based on current broker asset" + ) + if ( + plan.get("decision", {}).get("run_id") + != manifest.get("decision_run_id") + or plan.get("decision", {}).get("decision_manifest_sha256") + != manifest.get("decision_manifest_sha256") + ): + raise QmtShadowPlanError("shadow plan/manifest decision identity mismatch") + if broker_snapshot is not None: + validate_records([broker_snapshot], "broker_snapshot.schema.json") + if _sha256_payload(broker_snapshot) != plan.get( + "broker_snapshot_sha256" + ): + raise QmtShadowPlanError("broker snapshot identity mismatch") + if ( + broker_snapshot.get("metadata", {}).get( + "observed_facts_sha256" + ) + != plan.get("observation", {}).get("observed_facts_sha256") + or broker_snapshot.get("as_of") + != plan.get("observation", {}).get("query_completed_at") + ): + raise QmtShadowPlanError( + "broker snapshot/plan observation identity mismatch" + ) + broker_signal_as_of = _parse_aware( + broker_snapshot.get("signal_as_of"), + "broker snapshot signal_as_of", + ) + broker_observation_as_of = _parse_aware( + broker_snapshot.get("observation_as_of"), + "broker snapshot observation_as_of", + ) + broker_as_of = _parse_aware( + broker_snapshot.get("as_of"), + "broker snapshot as_of", + ) + broker_source_time = _parse_aware( + broker_snapshot.get("source_time"), + "broker snapshot source_time", + ) + if not _same_instant( + broker_signal_as_of, + clock_contract["signal_as_of"], + ): + raise QmtShadowPlanError( + "broker snapshot signal binding mismatch" + ) + if ( + broker_snapshot.get("next_trading_session") + != clock_contract["next_trading_session"].isoformat() + or broker_snapshot.get("first_executable_window") + != clock_contract["first_executable_window"] + ): + raise QmtShadowPlanError( + "broker snapshot execution-window binding mismatch" + ) + if ( + not _same_instant(broker_observation_as_of, broker_as_of) + or not _same_instant( + broker_observation_as_of, + clock_contract["query_completed"], + ) + or not _same_instant(broker_source_time, broker_as_of) + ): + raise QmtShadowPlanError( + "broker snapshot observation/source clock mismatch" + ) + if broker_observation_as_of.astimezone( + timezone.utc + ) < broker_signal_as_of.astimezone(timezone.utc): + raise QmtShadowPlanError( + "broker snapshot observation predates signal binding" + ) + portfolio = plan.get("portfolio", {}) + for field in ("cash", "market_value", "total_asset"): + if portfolio.get(field) != broker_snapshot.get(field): + raise QmtShadowPlanError( + f"broker snapshot/plan portfolio {field} mismatch" + ) + elif plan.get("broker_snapshot_sha256") is not None: + raise QmtShadowPlanError("missing broker snapshot has a hash") + if plan.get("observation", {}).get( + "broker_snapshot_valid" + ) is not (broker_snapshot is not None): + raise QmtShadowPlanError("broker snapshot validity flag mismatch") + + if decision_manifest is not None: + decision_path = Path(decision_manifest).expanduser().resolve() + verify_snapshot_decision(decision_path) + if _sha256_bytes(decision_path.read_bytes()) != manifest.get( + "decision_manifest_sha256" + ): + raise QmtShadowPlanError("input decision manifest hash mismatch") + input_decision = _strict_json( + decision_path.parent / "decision.json" + ) + if ( + input_decision.get("run_id") + != plan.get("decision", {}).get("run_id") + or input_decision.get("decision_id") + != plan.get("decision", {}).get("decision_id") + ): + raise QmtShadowPlanError( + "input decision identity does not match shadow plan" + ) + input_decision_equity = _finite_non_negative( + input_decision.get("equity"), + "input decision equity", + ) + if ( + input_decision_equity <= 0 + or input_decision_equity + != plan.get("decision", {}).get("equity") + ): + raise QmtShadowPlanError( + "input decision equity does not match shadow plan" + ) + input_signal_as_of = _parse_aware( + input_decision.get("signal_as_of"), + "input decision.signal_as_of", + ) + if not _same_instant( + input_signal_as_of, + clock_contract["signal_as_of"], + ): + raise QmtShadowPlanError( + "input decision signal clock does not match shadow plan" + ) + if ( + input_decision.get("next_trading_session") + != clock_contract["next_trading_session"].isoformat() + or input_decision.get("first_executable_window") + != clock_contract["first_executable_window"] + ): + raise QmtShadowPlanError( + "input decision execution window does not match shadow plan" + ) + input_broker = input_decision.get("broker_snapshot") + if input_broker is None: + if clock_contract["bound_observation"] is not None: + raise QmtShadowPlanError( + "unbound input decision has bound shadow clocks" + ) + elif not isinstance(input_broker, Mapping): + raise QmtShadowPlanError( + "input decision broker identity is invalid" + ) + else: + input_observation = _parse_aware( + input_broker.get("observation_as_of"), + "input decision broker observation_as_of", + ) + input_source = _parse_aware( + input_broker.get("source_time"), + "input decision broker source_time", + ) + if ( + clock_contract["bound_observation"] is None + or not _same_instant( + input_observation, + clock_contract["bound_observation"], + ) + or clock_contract["bound_source"] is None + or not _same_instant( + input_source, + clock_contract["bound_source"], + ) + ): + raise QmtShadowPlanError( + "input decision broker clocks do not match shadow plan" + ) + return { + "ok": True, + "plan_id": plan_id, + "status": plan["status"], + "blocker_count": len(blockers), + "target_diff_count": len(plan.get("target_diffs", [])), + "broker_snapshot_valid": broker_snapshot is not None, + "read_only": True, + } + + +__all__ = [ + "DECISION_EQUITY_ABSOLUTE_TOLERANCE", + "DECISION_EQUITY_RELATIVE_TOLERANCE", + "MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE", + "MAX_SHADOW_QUERY_DURATION_SECONDS", + "QmtShadowPlanError", + "ReadOnlyXtTraderQueries", + "build_qmt_shadow_plan", + "verify_qmt_shadow_plan", + "write_qmt_shadow_plan", +] diff --git a/src/quant60/reconcile.py b/src/quant60/reconcile.py new file mode 100644 index 0000000..d8660c9 --- /dev/null +++ b/src/quant60/reconcile.py @@ -0,0 +1,846 @@ +"""Broker-snapshot reconciliation with explicit fail-closed semantics. + +The original cash/position-only keyword API remains supported. Passing +``local_snapshot``/``broker_snapshot`` (or ``require_complete=True``) enables +the complete contract: cash, positions, sellable quantities, open orders and +fresh, timezone-aware snapshot timestamps must all be present. + +This module validates normalized snapshots. It is not evidence that a real +broker's proprietary fields or callback lifecycle have been certified. +""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation +from typing import Any, Iterable, Mapping + +from .domain import decimal +from .portable_core import normalize_symbol + + +_OPEN_STATUSES = { + "NEW", + "ACCEPTED", + "PARTIALLY_FILLED", + "CANCEL_PENDING", + "UNKNOWN", +} +_TERMINAL_STATUSES = {"FILLED", "REJECTED", "CANCELLED"} +_MISSING = object() + + +@dataclass(frozen=True, slots=True) +class ReconciliationDifference: + kind: str + key: str + expected: str + observed: str + + +@dataclass(frozen=True, slots=True) +class ReconciliationReport: + balanced: bool + differences: tuple[ReconciliationDifference, ...] + complete: bool = False + + @property + def safe_to_open(self) -> bool: + """New opening orders are permitted only for an exact safe report.""" + + return self.complete and self.balanced + + +def _render(value: Any) -> str: + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, (dict, list, tuple)): + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return str(value) + + +def _difference( + differences: list[ReconciliationDifference], + kind: str, + key: str, + expected: Any, + observed: Any, +) -> None: + differences.append( + ReconciliationDifference( + kind=kind, + key=key, + expected=_render(expected), + observed=_render(observed), + ) + ) + + +def _mapping_value( + snapshot: Mapping[str, Any] | None, + *names: str, +) -> Any: + if snapshot is None: + return _MISSING + for name in names: + if name in snapshot: + return snapshot[name] + return _MISSING + + +def _quantity(value: Any, label: str) -> int: + if isinstance(value, bool): + raise ValueError(f"{label} must be an integer") + result = int(value) + if result != value and not ( + isinstance(value, str) and str(result) == value.strip() + ): + raise ValueError(f"{label} must be an integer") + if result < 0: + raise ValueError(f"{label} must be non-negative") + return result + + +def _position_maps( + raw: Any, + *, + label: str, +) -> tuple[dict[str, int], dict[str, int], bool]: + """Return total positions, embedded sellable and sellable-presence.""" + + if raw is _MISSING or raw is None: + raise ValueError(f"{label} positions are missing") + positions: dict[str, int] = {} + sellable: dict[str, int] = {} + sellable_present = True + + if isinstance(raw, Mapping): + if all(not isinstance(value, Mapping) for value in raw.values()): + for symbol, value in raw.items(): + canonical = normalize_symbol(str(symbol)) + quantity = _quantity(value, f"{label} position {canonical}") + if quantity: + positions[canonical] = quantity + # With no holdings, an empty sellable map is unambiguous. A + # non-empty scalar position map still needs an explicit sellable + # snapshot in complete mode. + return positions, sellable, not raw + records: Iterable[Any] = ( + dict(value, symbol=value.get("symbol", key)) + if isinstance(value, Mapping) + else {"symbol": key, "quantity": value} + for key, value in raw.items() + ) + elif isinstance(raw, Iterable) and not isinstance(raw, (str, bytes)): + records = raw + else: + raise ValueError(f"{label} positions must be a mapping or iterable") + + seen: set[str] = set() + for index, item in enumerate(records): + if not isinstance(item, Mapping): + raise ValueError(f"{label} position #{index} must be an object") + symbol = item.get("symbol", item.get("stock_code")) + if not symbol: + raise ValueError(f"{label} position #{index} has no symbol") + canonical = normalize_symbol(str(symbol)) + if canonical in seen: + raise ValueError(f"duplicate {label} position {canonical}") + seen.add(canonical) + total_raw = item.get( + "quantity", + item.get("volume", item.get("total_amount", _MISSING)), + ) + if total_raw is _MISSING: + raise ValueError(f"{label} position {canonical} has no quantity") + total = _quantity(total_raw, f"{label} position {canonical}") + if total: + positions[canonical] = total + available_raw = item.get( + "sellable_quantity", + item.get( + "sellable", + item.get("can_use_volume", item.get("closeable_amount", _MISSING)), + ), + ) + if available_raw is _MISSING: + sellable_present = False + continue + available = _quantity( + available_raw, + f"{label} sellable {canonical}", + ) + if available > total: + raise ValueError( + f"{label} sellable {canonical} exceeds total position" + ) + if available: + sellable[canonical] = available + return positions, sellable, sellable_present + + +def _quantity_map(raw: Any, *, label: str) -> dict[str, int]: + if raw is _MISSING or raw is None: + raise ValueError(f"{label} is missing") + if not isinstance(raw, Mapping): + raise ValueError(f"{label} must be a mapping") + output: dict[str, int] = {} + for symbol, value in raw.items(): + canonical = normalize_symbol(str(symbol)) + quantity = _quantity(value, f"{label} {canonical}") + if quantity: + output[canonical] = quantity + return output + + +def _normalise_open_orders( + raw: Any, + *, + label: str, + differences: list[ReconciliationDifference], + require_intent_fields: bool = False, +) -> dict[str, dict[str, Any]]: + if raw is _MISSING or raw is None: + raise ValueError(f"{label} open_orders are missing") + def record_from(value: Any, default_id: Any = None) -> Any: + if isinstance(value, Mapping): + result = dict(value) + if default_id is not None: + result.setdefault("order_id", default_id) + return result + if hasattr(value, "order_id"): + side = getattr(value, "side", None) + status = getattr(value, "status", None) + return { + "order_id": getattr(value, "order_id", default_id), + "symbol": getattr(value, "symbol", None), + "side": getattr(side, "value", side), + "quantity": getattr(value, "quantity", None), + "filled_quantity": getattr(value, "filled_quantity", 0), + "status": getattr(status, "value", status), + "broker_order_id": getattr(value, "broker_order_id", None), + "limit_price": getattr(value, "limit_price", None), + "order_type": getattr(value, "order_type", None), + "time_in_force": getattr(value, "time_in_force", None), + } + return {"order_id": default_id, "status": value} + + if isinstance(raw, Mapping): + records: Iterable[Any] = ( + record_from(value, key) for key, value in raw.items() + ) + elif isinstance(raw, Iterable) and not isinstance(raw, (str, bytes)): + records = (record_from(value) for value in raw) + else: + raise ValueError(f"{label} open_orders must be a mapping or iterable") + + output: dict[str, dict[str, Any]] = {} + for index, item in enumerate(records): + if not isinstance(item, Mapping): + _difference( + differences, + "INVALID_OPEN_ORDER", + f"{label}[{index}]", + "object", + type(item).__name__, + ) + continue + identity = item.get( + "client_order_id", + item.get("order_id", item.get("broker_order_id")), + ) + if identity in {None, ""}: + _difference( + differences, + "INVALID_OPEN_ORDER", + f"{label}[{index}]", + "non-empty order identity", + "missing", + ) + continue + order_id = str(identity) + try: + symbol = normalize_symbol(str(item["symbol"])) + side = str(item["side"]).upper() + if side not in {"BUY", "SELL"}: + raise ValueError("side must be BUY or SELL") + quantity = _quantity( + item.get("order_quantity", item.get("quantity")), + f"{label} order {order_id} quantity", + ) + filled = _quantity( + item.get( + "cumulative_filled_quantity", + item.get("filled_quantity", 0), + ), + f"{label} order {order_id} filled quantity", + ) + if quantity <= 0: + raise ValueError("quantity must be positive") + if filled > quantity: + raise ValueError("filled quantity exceeds order quantity") + status = str(item.get("status", item.get("state", ""))).upper() + if status not in _OPEN_STATUSES | _TERMINAL_STATUSES: + raise ValueError(f"unsupported status {status!r}") + broker_order_id = item.get("broker_order_id") + limit_price_raw = item.get("limit_price") + order_type_raw = item.get("order_type") + time_in_force_raw = item.get("time_in_force") + if require_intent_fields: + if broker_order_id in {None, ""}: + raise ValueError("broker_order_id is required") + if limit_price_raw is None: + raise ValueError("limit_price is required") + if order_type_raw is None: + raise ValueError("order_type is required") + if time_in_force_raw is None: + raise ValueError("time_in_force is required") + limit_price = None + if limit_price_raw is not None: + limit_price = decimal(limit_price_raw) + if not limit_price.is_finite() or limit_price <= 0: + raise ValueError( + "limit_price must be finite and positive" + ) + order_type = ( + None + if order_type_raw is None + else str(order_type_raw).upper() + ) + if order_type is not None and order_type != "LIMIT": + raise ValueError("order_type must be LIMIT") + time_in_force = ( + None + if time_in_force_raw is None + else str(time_in_force_raw).upper() + ) + if time_in_force is not None and time_in_force != "DAY": + raise ValueError("time_in_force must be DAY") + except ( + InvalidOperation, + KeyError, + OverflowError, + TypeError, + ValueError, + ) as exc: + _difference( + differences, + "INVALID_OPEN_ORDER", + order_id, + "valid normalized open order", + f"{type(exc).__name__}: {exc}", + ) + continue + + normalized = { + "symbol": symbol, + "side": side, + "quantity": quantity, + "filled_quantity": filled, + "status": status, + "broker_order_id": ( + None if broker_order_id is None else str(broker_order_id) + ), + "limit_price": ( + None if limit_price is None else format(limit_price, "f") + ), + "order_type": order_type, + "time_in_force": time_in_force, + } + prior = output.get(order_id) + if prior is not None: + _difference( + differences, + "DUPLICATE_OPEN_ORDER", + order_id, + "unique order identity", + ( + "identical duplicate" + if prior == normalized + else normalized + ), + ) + continue + output[order_id] = normalized + if status == "UNKNOWN": + _difference( + differences, + "UNKNOWN_ORDER", + order_id, + "known broker state", + "UNKNOWN", + ) + elif status in _TERMINAL_STATUSES: + _difference( + differences, + "TERMINAL_OPEN_ORDER", + order_id, + "active state", + status, + ) + return output + + +def _parse_timestamp(value: Any, label: str) -> datetime: + if isinstance(value, datetime): + result = value + elif isinstance(value, str): + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + result = datetime.fromisoformat(text) + except ValueError as exc: + raise ValueError(f"{label} must be ISO-8601") from exc + else: + raise ValueError(f"{label} must be a datetime or ISO-8601 string") + if result.tzinfo is None or result.utcoffset() is None: + raise ValueError(f"{label} must include a timezone offset") + return result.astimezone(timezone.utc) + + +def _finite_non_negative_seconds(value: Any, label: str) -> float: + """Return a usable tolerance or reject unsafe numeric edge cases.""" + + if isinstance(value, bool): + raise ValueError(f"{label} must be a finite non-negative number") + try: + result = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError( + f"{label} must be a finite non-negative number" + ) from exc + if not math.isfinite(result) or result < 0: + raise ValueError(f"{label} must be a finite non-negative number") + return result + + +def _check_freshness( + *, + differences: list[ReconciliationDifference], + key: str, + value: Any, + now: datetime, + max_age_seconds: float, + max_clock_skew_seconds: float, +) -> datetime | None: + try: + timestamp = _parse_timestamp(value, key) + except ValueError as exc: + _difference( + differences, + "INVALID_TIMESTAMP", + key, + "timezone-aware ISO-8601", + str(exc), + ) + return None + age = (now - timestamp).total_seconds() + if age < -max_clock_skew_seconds: + _difference( + differences, + "FUTURE_TIMESTAMP", + key, + f"<= {max_clock_skew_seconds:g}s in future", + f"{-age:g}s in future", + ) + elif age > max_age_seconds: + _difference( + differences, + ( + "STALE_SOURCE" + if key == "broker.source_time" + else "STALE_SNAPSHOT" + ), + key, + f"age <= {max_age_seconds:g}s", + f"age={age:g}s", + ) + return timestamp + + +def reconcile_snapshot( + *, + local_cash: Decimal | float | str | None = None, + local_positions: Mapping[str, int] | Iterable[Mapping[str, Any]] | None = None, + broker_cash: Decimal | float | str | None = None, + broker_positions: Mapping[str, int] | Iterable[Mapping[str, Any]] | None = None, + cash_tolerance: Decimal | float | str = "0.01", + local_sellable: Mapping[str, int] | None = None, + broker_sellable: Mapping[str, int] | None = None, + local_open_orders: Mapping[str, Any] | Iterable[Mapping[str, Any]] | None = None, + broker_open_orders: Mapping[str, Any] | Iterable[Mapping[str, Any]] | None = None, + local_as_of: datetime | str | None = None, + broker_as_of: datetime | str | None = None, + broker_source_time: datetime | str | None = None, + as_of: datetime | str | None = None, + source_time: datetime | str | None = None, + now: datetime | str | None = None, + max_snapshot_age_seconds: float = 60.0, + max_source_age_seconds: float | None = None, + max_age_seconds: float | None = None, + max_clock_skew_seconds: float = 5.0, + local_snapshot: Mapping[str, Any] | None = None, + broker_snapshot: Mapping[str, Any] | None = None, + require_complete: bool | None = None, +) -> ReconciliationReport: + """Compare local intent/accounting with a broker snapshot. + + Legacy callers may continue to provide only the original four + cash/position arguments. Complete mode is selected automatically when a + snapshot object is provided, or explicitly with ``require_complete=True``. + Complete mode fails closed on any missing sellable/open-order/timestamp + field and on an ``UNKNOWN`` order even when both sides report it. + + ``as_of`` and ``source_time`` are aliases for the broker timestamps. They + exist for direct use with the JSON broker-snapshot contract. + """ + + differences: list[ReconciliationDifference] = [] + complete = ( + bool(require_complete) + if require_complete is not None + else local_snapshot is not None or broker_snapshot is not None + ) + snapshot_age_limit = _finite_non_negative_seconds( + max_snapshot_age_seconds, + "max_snapshot_age_seconds", + ) + source_age_limit = ( + None + if max_source_age_seconds is None + else _finite_non_negative_seconds( + max_source_age_seconds, + "max_source_age_seconds", + ) + ) + clock_skew_limit = _finite_non_negative_seconds( + max_clock_skew_seconds, + "max_clock_skew_seconds", + ) + if max_age_seconds is not None: + alias_age_limit = _finite_non_negative_seconds( + max_age_seconds, + "max_age_seconds", + ) + snapshot_age_limit = alias_age_limit + if source_age_limit is None: + source_age_limit = alias_age_limit + if source_age_limit is None: + source_age_limit = snapshot_age_limit + + def pick(explicit: Any, snapshot: Mapping[str, Any] | None, *names: str) -> Any: + if explicit is not None: + return explicit + return _mapping_value(snapshot, *names) + + local_cash_raw = pick(local_cash, local_snapshot, "cash") + broker_cash_raw = pick(broker_cash, broker_snapshot, "cash") + local_positions_raw = pick(local_positions, local_snapshot, "positions") + broker_positions_raw = pick(broker_positions, broker_snapshot, "positions") + + required = { + "local.cash": local_cash_raw, + "broker.cash": broker_cash_raw, + "local.positions": local_positions_raw, + "broker.positions": broker_positions_raw, + } + missing_required = [ + key for key, value in required.items() if value is _MISSING or value is None + ] + if missing_required and not complete: + raise ValueError("missing reconciliation inputs: " + ", ".join(missing_required)) + for key in missing_required: + _difference(differences, "MISSING_FIELD", key, "present", "missing") + + tolerance = decimal(cash_tolerance) + if not tolerance.is_finite() or tolerance < 0: + raise ValueError("cash_tolerance must be a finite non-negative number") + if not missing_required: + local_cash_value = decimal(local_cash_raw) + broker_cash_value = decimal(broker_cash_raw) + if not local_cash_value.is_finite() or not broker_cash_value.is_finite(): + raise ValueError("cash values must be finite") + if abs(local_cash_value - broker_cash_value) > tolerance: + _difference( + differences, + "CASH", + "CNY", + local_cash_value, + broker_cash_value, + ) + + local_position_map: dict[str, int] = {} + broker_position_map: dict[str, int] = {} + embedded_local_sellable: dict[str, int] = {} + embedded_broker_sellable: dict[str, int] = {} + local_has_sellable = False + broker_has_sellable = False + if local_positions_raw is not _MISSING and local_positions_raw is not None: + ( + local_position_map, + embedded_local_sellable, + local_has_sellable, + ) = _position_maps(local_positions_raw, label="local") + if broker_positions_raw is not _MISSING and broker_positions_raw is not None: + ( + broker_position_map, + embedded_broker_sellable, + broker_has_sellable, + ) = _position_maps(broker_positions_raw, label="broker") + for symbol in sorted(set(local_position_map) | set(broker_position_map)): + expected = local_position_map.get(symbol, 0) + observed = broker_position_map.get(symbol, 0) + if expected != observed: + _difference(differences, "POSITION", symbol, expected, observed) + + local_sellable_raw = pick( + local_sellable, + local_snapshot, + "sellable", + "sellable_positions", + ) + broker_sellable_raw = pick( + broker_sellable, + broker_snapshot, + "sellable", + "sellable_positions", + ) + if local_sellable_raw is _MISSING and local_has_sellable: + local_sellable_map = embedded_local_sellable + elif local_sellable_raw is not _MISSING and local_sellable_raw is not None: + local_sellable_map = _quantity_map( + local_sellable_raw, + label="local sellable", + ) + else: + local_sellable_map = None + if broker_sellable_raw is _MISSING and broker_has_sellable: + broker_sellable_map = embedded_broker_sellable + elif broker_sellable_raw is not _MISSING and broker_sellable_raw is not None: + broker_sellable_map = _quantity_map( + broker_sellable_raw, + label="broker sellable", + ) + else: + broker_sellable_map = None + + if complete and local_sellable_map is None: + _difference( + differences, + "MISSING_FIELD", + "local.sellable", + "present", + "missing", + ) + if complete and broker_sellable_map is None: + _difference( + differences, + "MISSING_FIELD", + "broker.sellable", + "present", + "missing", + ) + if local_sellable_map is not None: + for symbol, quantity in local_sellable_map.items(): + if quantity > local_position_map.get(symbol, 0): + _difference( + differences, + "INVALID_SELLABLE", + f"local:{symbol}", + f"<= {local_position_map.get(symbol, 0)}", + quantity, + ) + if broker_sellable_map is not None: + for symbol, quantity in broker_sellable_map.items(): + if quantity > broker_position_map.get(symbol, 0): + _difference( + differences, + "INVALID_SELLABLE", + f"broker:{symbol}", + f"<= {broker_position_map.get(symbol, 0)}", + quantity, + ) + if local_sellable_map is not None and broker_sellable_map is not None: + for symbol in sorted(set(local_sellable_map) | set(broker_sellable_map)): + expected = local_sellable_map.get(symbol, 0) + observed = broker_sellable_map.get(symbol, 0) + if expected != observed: + _difference(differences, "SELLABLE", symbol, expected, observed) + + local_orders_raw = pick( + local_open_orders, + local_snapshot, + "open_orders", + ) + broker_orders_raw = pick( + broker_open_orders, + broker_snapshot, + "open_orders", + ) + local_orders: dict[str, dict[str, Any]] | None = None + broker_orders: dict[str, dict[str, Any]] | None = None + if local_orders_raw is not _MISSING and local_orders_raw is not None: + local_orders = _normalise_open_orders( + local_orders_raw, + label="local", + differences=differences, + require_intent_fields=complete, + ) + if broker_orders_raw is not _MISSING and broker_orders_raw is not None: + broker_orders = _normalise_open_orders( + broker_orders_raw, + label="broker", + differences=differences, + require_intent_fields=complete, + ) + if complete and local_orders is None: + _difference( + differences, + "MISSING_FIELD", + "local.open_orders", + "present", + "missing", + ) + if complete and broker_orders is None: + _difference( + differences, + "MISSING_FIELD", + "broker.open_orders", + "present", + "missing", + ) + if local_orders is not None and broker_orders is not None: + for order_id in sorted(set(local_orders) | set(broker_orders)): + expected = local_orders.get(order_id) + observed = broker_orders.get(order_id) + if expected != observed: + _difference( + differences, + "OPEN_ORDER", + order_id, + expected if expected is not None else "missing", + observed if observed is not None else "missing", + ) + + local_as_of_raw = pick(local_as_of, local_snapshot, "as_of") + broker_as_of_explicit = broker_as_of if broker_as_of is not None else as_of + broker_as_of_raw = pick( + broker_as_of_explicit, + broker_snapshot, + "as_of", + ) + source_explicit = ( + broker_source_time if broker_source_time is not None else source_time + ) + source_time_raw = pick( + source_explicit, + broker_snapshot, + "source_time", + ) + if ( + broker_source_time is not None + and source_time is not None + and _parse_timestamp(broker_source_time, "broker_source_time") + != _parse_timestamp(source_time, "source_time") + ): + raise ValueError("broker_source_time and source_time disagree") + if ( + broker_as_of is not None + and as_of is not None + and _parse_timestamp(broker_as_of, "broker_as_of") + != _parse_timestamp(as_of, "as_of") + ): + raise ValueError("broker_as_of and as_of disagree") + + freshness_requested = complete or any( + value is not _MISSING and value is not None + for value in (local_as_of_raw, broker_as_of_raw, source_time_raw) + ) + if freshness_requested: + now_value = ( + datetime.now(timezone.utc) + if now is None + else _parse_timestamp(now, "now") + ) + timestamp_values = { + "local.as_of": local_as_of_raw, + "broker.as_of": broker_as_of_raw, + "broker.source_time": source_time_raw, + } + parsed: dict[str, datetime] = {} + for key, value in timestamp_values.items(): + if value is _MISSING or value is None: + if complete: + _difference( + differences, + "MISSING_FIELD", + key, + "present", + "missing", + ) + continue + limit = ( + source_age_limit + if key == "broker.source_time" + else snapshot_age_limit + ) + timestamp = _check_freshness( + differences=differences, + key=key, + value=value, + now=now_value, + max_age_seconds=limit, + max_clock_skew_seconds=clock_skew_limit, + ) + if timestamp is not None: + parsed[key] = timestamp + + local_time = parsed.get("local.as_of") + broker_time = parsed.get("broker.as_of") + broker_source = parsed.get("broker.source_time") + if ( + local_time is not None + and broker_time is not None + and abs((local_time - broker_time).total_seconds()) + > clock_skew_limit + ): + _difference( + differences, + "AS_OF_SKEW", + "local_vs_broker", + f"<= {clock_skew_limit:g}s", + f"{abs((local_time - broker_time).total_seconds()):g}s", + ) + if ( + broker_time is not None + and broker_source is not None + and broker_source > broker_time + ): + lead = (broker_source - broker_time).total_seconds() + if lead > clock_skew_limit: + _difference( + differences, + "SOURCE_AFTER_AS_OF", + "broker.source_time", + f"<= broker.as_of + {clock_skew_limit:g}s", + f"lead={lead:g}s", + ) + + differences = sorted( + set(differences), + key=lambda item: (item.kind, item.key, item.expected, item.observed) + ) + return ReconciliationReport( + not differences, + tuple(differences), + complete=complete, + ) diff --git a/src/quant60/replay.py b/src/quant60/replay.py new file mode 100644 index 0000000..728f89a --- /dev/null +++ b/src/quant60/replay.py @@ -0,0 +1,837 @@ +"""Strict projector for normalized ``ORDER_STATUS`` and ``FILL`` events. + +The projector rebuilds order cumulative fills, cash delta and position delta. +It accepts at-least-once *identical* callbacks idempotently, but rejects +conflicting duplicates, unknown-order fills, overfills, sequence/time +regressions and illegal order-state regressions. + +The canonical JSON wire shape is the ledger envelope documented by +``schemas/order_event.schema.json``: ``sequence``, ``timestamp``, ``event_id``, +``event_type`` and ``payload``, plus an optional *paired* ``previous_hash`` / +``hash``. ``SimBroker`` emits that shape through ``LedgerRecord.as_dict()``. +Legacy quantity aliases remain accepted at the Python boundary, but are not +part of the canonical schema. + +It operates on local normalized ledger events. Passing its tests is not +evidence that a real broker callback mapping has been certified. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field +from datetime import datetime, timezone +from decimal import Decimal, ROUND_HALF_UP +from typing import Any, Iterable, Mapping + +from .domain import ( + ORDER_TRANSITIONS, + PRICE_QUANT, + OrderStatus, + Side, + decimal, + money, +) +from .ledger import HashChainLedger, LedgerRecord, canonical_json +from .portable_core import normalize_symbol + + +class LedgerReplayError(ValueError): + """Base class for events that cannot be projected safely.""" + + +class OutOfOrderError(LedgerReplayError): + """A sequence, event-time or state transition moved backwards.""" + + +class ConflictingDuplicateError(LedgerReplayError): + """One event/fill identity was reused for different content.""" + + +class UnknownOrderError(LedgerReplayError): + """A fill referred to an order that has not been registered.""" + + +class OverfillError(LedgerReplayError): + """Cumulative fills exceeded the submitted order quantity.""" + + +# Descriptive aliases kept for callers that prefer the longer names. +ReplayError = LedgerReplayError +ReplayOrderingError = OutOfOrderError +ReplayConflictError = ConflictingDuplicateError + + +_OPEN_STATUSES = { + OrderStatus.NEW, + OrderStatus.ACCEPTED, + OrderStatus.PARTIALLY_FILLED, + OrderStatus.CANCEL_PENDING, + OrderStatus.UNKNOWN, +} +_ENVELOPE_KEYS = { + "sequence", + "timestamp", + "event_id", + "event_type", + "payload", + "previous_hash", + "hash", +} +_ORDER_STATUS_PAYLOAD_KEYS = { + "order_id", + "broker_order_id", + "symbol", + "side", + "quantity", + "order_quantity", + "filled_quantity", + "cumulative_filled_quantity", + "average_fill_price", + "status", + "reason", + "submitted_at", + "metadata", +} +_FILL_PAYLOAD_KEYS = { + "fill_id", + "order_id", + "trading_date", + "symbol", + "side", + "quantity", + "last_fill_quantity", + "price", + "commission", + "stamp_duty", + "transfer_fee", + "cash_delta", +} +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_MISSING = object() + + +def _is_present(value: Any) -> bool: + return value is not None and value != "" + + +def _timestamp(value: Any, label: str) -> datetime: + if isinstance(value, datetime): + parsed = value + elif isinstance(value, str): + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError as exc: + raise LedgerReplayError(f"{label} must be ISO-8601") from exc + else: + raise LedgerReplayError( + f"{label} must be a datetime or ISO-8601 string" + ) + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise LedgerReplayError(f"{label} must include a timezone offset") + return parsed.astimezone(timezone.utc) + + +def _required(payload: Mapping[str, Any], key: str, event_id: str) -> Any: + if key not in payload or not _is_present(payload[key]): + raise LedgerReplayError(f"{event_id}: missing {key}") + return payload[key] + + +def _positive_int(value: Any, label: str) -> int: + if isinstance(value, bool): + raise LedgerReplayError(f"{label} must be an integer") + try: + result = int(value) + except (TypeError, ValueError) as exc: + raise LedgerReplayError(f"{label} must be an integer") from exc + if isinstance(value, str): + exact = value.strip() == str(result) + else: + exact = value == result + if not exact or result <= 0: + raise LedgerReplayError(f"{label} must be a positive integer") + return result + + +def _nonnegative_int(value: Any, label: str) -> int: + if isinstance(value, bool): + raise LedgerReplayError(f"{label} must be an integer") + try: + result = int(value) + except (TypeError, ValueError) as exc: + raise LedgerReplayError(f"{label} must be an integer") from exc + if isinstance(value, str): + exact = value.strip() == str(result) + else: + exact = value == result + if not exact or result < 0: + raise LedgerReplayError(f"{label} must be a non-negative integer") + return result + + +def _aliased_int( + payload: Mapping[str, Any], + keys: tuple[str, ...], + event_id: str, + *, + positive: bool, +) -> int: + """Read canonical/legacy integer names and reject conflicting copies.""" + + present = [ + (key, payload[key]) + for key in keys + if key in payload and _is_present(payload[key]) + ] + if not present: + raise LedgerReplayError( + f"{event_id}: missing one of {', '.join(keys)}" + ) + parser = _positive_int if positive else _nonnegative_int + parsed = [ + (key, parser(value, f"{event_id}.{key}")) + for key, value in present + ] + if any(value != parsed[0][1] for _, value in parsed[1:]): + names = ", ".join(key for key, _ in parsed) + raise ConflictingDuplicateError( + f"{event_id}: conflicting alias values for {names}" + ) + return parsed[0][1] + + +def _reject_unknown_keys( + payload: Mapping[str, Any], + allowed: set[str], + event_id: str, +) -> None: + unknown = sorted(str(key) for key in set(payload) - allowed) + if unknown: + raise LedgerReplayError( + f"{event_id}: unsupported payload fields {unknown}" + ) + + +def _finite_decimal(value: Any, label: str) -> Decimal: + try: + result = decimal(value) + except Exception as exc: + raise LedgerReplayError(f"{label} must be numeric") from exc + if not result.is_finite(): + raise LedgerReplayError(f"{label} must be finite") + return result + + +def _record_value(record: Any, name: str, default: Any = None) -> Any: + if isinstance(record, Mapping): + return record.get(name, default) + return getattr(record, name, default) + + +@dataclass(frozen=True, slots=True) +class ReplayedOrder: + order_id: str + symbol: str + side: str + quantity: int + status: str + filled_quantity: int + average_fill_price: Decimal + cash_delta: Decimal + position_delta: int + fill_ids: tuple[str, ...] + last_sequence: int + last_timestamp: str + + @property + def remaining_quantity(self) -> int: + return self.quantity - self.filled_quantity + + @property + def is_open(self) -> bool: + return OrderStatus(self.status) in _OPEN_STATUSES + + def accounting_dict(self) -> dict[str, Any]: + return { + "order_id": self.order_id, + "symbol": self.symbol, + "side": self.side, + "quantity": self.quantity, + "status": self.status, + "filled_quantity": self.filled_quantity, + "average_fill_price": format(self.average_fill_price, "f"), + "cash_delta": format(self.cash_delta, "f"), + "position_delta": self.position_delta, + "fill_ids": list(self.fill_ids), + } + + +@dataclass(frozen=True, slots=True) +class LedgerProjection: + orders: dict[str, ReplayedOrder] + cash_delta: Decimal + position_deltas: dict[str, int] + fill_ids: tuple[str, ...] + source_last_sequence: int + relevant_event_count: int + + @property + def has_unknown_orders(self) -> bool: + return any( + order.status == OrderStatus.UNKNOWN.value + for order in self.orders.values() + ) + + @property + def safe_to_reconcile(self) -> bool: + return not self.has_unknown_orders + + @property + def open_orders(self) -> dict[str, ReplayedOrder]: + return { + order_id: order + for order_id, order in self.orders.items() + if order.is_open + } + + def accounting_dict(self) -> dict[str, Any]: + """Stable accounting state, excluding delivery-order metadata.""" + + return { + "schema_version": 1, + "cash_delta": format(self.cash_delta, "f"), + "position_deltas": dict(sorted(self.position_deltas.items())), + "orders": [ + self.orders[order_id].accounting_dict() + for order_id in sorted(self.orders) + ], + "fill_ids": list(self.fill_ids), + "has_unknown_orders": self.has_unknown_orders, + } + + @property + def digest(self) -> str: + return hashlib.sha256( + canonical_json(self.accounting_dict()).encode("utf-8") + ).hexdigest() + + +@dataclass(slots=True) +class _MutableOrder: + order_id: str + symbol: str + side: Side + quantity: int + status: OrderStatus + filled_quantity: int = 0 + average_fill_price: Decimal = Decimal("0") + cash_delta: Decimal = Decimal("0") + position_delta: int = 0 + fill_ids: list[str] = field(default_factory=list) + last_sequence: int = 0 + last_timestamp: datetime | None = None + last_status_signature: str | None = None + + +class LedgerProjector: + """Stateful strict projector; one instance processes one ordered stream.""" + + def __init__(self) -> None: + self._orders: dict[str, _MutableOrder] = {} + self._event_signatures: dict[str, tuple[int, str]] = {} + self._fill_signatures: dict[str, str] = {} + self._fill_ids: list[str] = [] + self._cash_delta = Decimal("0") + self._position_deltas: dict[str, int] = {} + self._last_sequence = 0 + self._relevant_event_count = 0 + self._failure: LedgerReplayError | None = None + + def _check_order_time( + self, + order: _MutableOrder, + event_time: datetime, + event_id: str, + ) -> None: + if order.last_timestamp is not None and event_time < order.last_timestamp: + raise OutOfOrderError( + f"{event_id}: timestamp precedes prior event for {order.order_id}" + ) + + @staticmethod + def _validate_status_semantics( + order: _MutableOrder, + event_id: str, + ) -> None: + if ( + order.status == OrderStatus.FILLED + and order.filled_quantity != order.quantity + ): + raise LedgerReplayError( + f"{event_id}: FILLED requires cumulative quantity == order quantity" + ) + if order.status == OrderStatus.PARTIALLY_FILLED and not ( + 0 < order.filled_quantity < order.quantity + ): + raise LedgerReplayError( + f"{event_id}: PARTIALLY_FILLED has invalid cumulative quantity" + ) + + def _status_event( + self, + *, + payload: Mapping[str, Any], + event_id: str, + sequence: int, + event_time: datetime, + ) -> None: + _reject_unknown_keys( + payload, + _ORDER_STATUS_PAYLOAD_KEYS, + event_id, + ) + order_id = str(_required(payload, "order_id", event_id)) + symbol = normalize_symbol(str(_required(payload, "symbol", event_id))) + try: + side = Side(str(_required(payload, "side", event_id)).upper()) + except ValueError as exc: + raise LedgerReplayError(f"{event_id}: unsupported order side") from exc + quantity = _aliased_int( + payload, + ("quantity", "order_quantity"), + event_id, + positive=True, + ) + filled = _aliased_int( + payload, + ("filled_quantity", "cumulative_filled_quantity"), + event_id, + positive=False, + ) + if filled > quantity: + raise OverfillError( + f"{event_id}: reported cumulative fill exceeds order quantity" + ) + try: + status = OrderStatus( + str(_required(payload, "status", event_id)).upper() + ) + except ValueError as exc: + raise LedgerReplayError(f"{event_id}: unsupported order status") from exc + + order = self._orders.get(order_id) + if order is None: + if filled: + raise UnknownOrderError( + f"{event_id}: initial order status refers to unprojected fills" + ) + order = _MutableOrder( + order_id=order_id, + symbol=symbol, + side=side, + quantity=quantity, + status=status, + last_sequence=sequence, + last_timestamp=event_time, + ) + self._orders[order_id] = order + else: + if ( + order.symbol != symbol + or order.side != side + or order.quantity != quantity + ): + raise ConflictingDuplicateError( + f"{event_id}: immutable order fields changed for {order_id}" + ) + if filled != order.filled_quantity: + relation = "ahead of" if filled > order.filled_quantity else "behind" + raise OutOfOrderError( + f"{event_id}: status cumulative fill is {relation} projected fills" + ) + + reported_average = payload.get("average_fill_price") + if reported_average is not None: + observed_average = _finite_decimal( + reported_average, + f"{event_id}.average_fill_price", + ).quantize(PRICE_QUANT, rounding=ROUND_HALF_UP) + if observed_average != order.average_fill_price: + raise ConflictingDuplicateError( + f"{event_id}: average fill price disagrees with FILL events" + ) + signature = canonical_json( + { + "order_id": order_id, + "symbol": symbol, + "side": side.value, + "quantity": quantity, + "status": status.value, + "filled_quantity": filled, + "average_fill_price": format(order.average_fill_price, "f"), + } + ) + if order.last_status_signature == signature: + return + if order.last_sequence: + self._check_order_time(order, event_time, event_id) + if status != order.status: + if status not in ORDER_TRANSITIONS[order.status]: + raise OutOfOrderError( + f"{event_id}: illegal state regression " + f"{order.status.value}->{status.value}" + ) + order.status = status + order.last_sequence = sequence + order.last_timestamp = event_time + self._validate_status_semantics(order, event_id) + if order.last_status_signature != signature: + self._relevant_event_count += 1 + order.last_status_signature = signature + + def _fill_event( + self, + *, + payload: Mapping[str, Any], + event_id: str, + sequence: int, + event_time: datetime, + ) -> None: + _reject_unknown_keys( + payload, + _FILL_PAYLOAD_KEYS, + event_id, + ) + fill_id = str(_required(payload, "fill_id", event_id)) + order_id = str(_required(payload, "order_id", event_id)) + order = self._orders.get(order_id) + if order is None: + raise UnknownOrderError( + f"{event_id}: fill {fill_id} references unknown order {order_id}" + ) + symbol = normalize_symbol(str(_required(payload, "symbol", event_id))) + try: + side = Side(str(_required(payload, "side", event_id)).upper()) + except ValueError as exc: + raise LedgerReplayError(f"{event_id}: unsupported fill side") from exc + if symbol != order.symbol or side != order.side: + raise ConflictingDuplicateError( + f"{event_id}: fill identity disagrees with order {order_id}" + ) + quantity = _aliased_int( + payload, + ("quantity", "last_fill_quantity"), + event_id, + positive=True, + ) + price = _finite_decimal( + _required(payload, "price", event_id), + f"{event_id}.price", + ) + if price <= 0: + raise LedgerReplayError(f"{event_id}: fill price must be positive") + fees: dict[str, Decimal] = {} + for name in ("commission", "stamp_duty", "transfer_fee"): + value = _finite_decimal(payload.get(name, 0), f"{event_id}.{name}") + if value < 0: + raise LedgerReplayError(f"{event_id}: {name} must be non-negative") + fees[name] = money(value) + total_fees = money(sum(fees.values(), Decimal("0"))) + notional = money(price * quantity) + cash_delta = ( + money(-notional - total_fees) + if side == Side.BUY + else money(notional - total_fees) + ) + if "cash_delta" in payload: + reported_cash = money( + _finite_decimal(payload["cash_delta"], f"{event_id}.cash_delta") + ) + if reported_cash != cash_delta: + raise ConflictingDuplicateError( + f"{event_id}: cash_delta disagrees with price, quantity and fees" + ) + fill_signature = canonical_json( + { + "fill_id": fill_id, + "order_id": order_id, + "symbol": symbol, + "side": side.value, + "quantity": quantity, + "price": format(price, "f"), + "commission": format(fees["commission"], "f"), + "stamp_duty": format(fees["stamp_duty"], "f"), + "transfer_fee": format(fees["transfer_fee"], "f"), + "cash_delta": format(cash_delta, "f"), + } + ) + prior_fill = self._fill_signatures.get(fill_id) + if prior_fill is not None: + if prior_fill != fill_signature: + raise ConflictingDuplicateError( + f"{event_id}: fill_id {fill_id} has conflicting content" + ) + return + self._check_order_time(order, event_time, event_id) + if order.status not in { + OrderStatus.ACCEPTED, + OrderStatus.PARTIALLY_FILLED, + OrderStatus.CANCEL_PENDING, + OrderStatus.UNKNOWN, + }: + raise OutOfOrderError( + f"{event_id}: fill cannot follow order state {order.status.value}" + ) + if order.filled_quantity + quantity > order.quantity: + raise OverfillError( + f"{event_id}: fill would exceed order quantity {order.quantity}" + ) + + previous_quantity = order.filled_quantity + weighted = order.average_fill_price * previous_quantity + price * quantity + order.filled_quantity += quantity + order.average_fill_price = ( + weighted / order.filled_quantity + ).quantize(PRICE_QUANT, rounding=ROUND_HALF_UP) + order.cash_delta = money(order.cash_delta + cash_delta) + signed_quantity = quantity if side == Side.BUY else -quantity + order.position_delta += signed_quantity + order.fill_ids.append(fill_id) + order.last_sequence = sequence + order.last_timestamp = event_time + inferred = ( + OrderStatus.FILLED + if order.filled_quantity == order.quantity + else OrderStatus.PARTIALLY_FILLED + ) + if inferred != order.status: + if inferred not in ORDER_TRANSITIONS[order.status]: + raise OutOfOrderError( + f"{event_id}: fill implies illegal state " + f"{order.status.value}->{inferred.value}" + ) + order.status = inferred + + self._fill_signatures[fill_id] = fill_signature + self._fill_ids.append(fill_id) + self._cash_delta = money(self._cash_delta + cash_delta) + self._position_deltas[symbol] = ( + self._position_deltas.get(symbol, 0) + signed_quantity + ) + if not self._position_deltas[symbol]: + del self._position_deltas[symbol] + self._relevant_event_count += 1 + + def process(self, record: LedgerRecord | Mapping[str, Any]) -> None: + """Process one event, permanently poisoning the instance on failure.""" + + if self._failure is not None: + raise LedgerReplayError( + "projector is fail-closed after a prior replay error" + ) from self._failure + try: + self._process(record) + except LedgerReplayError as exc: + self._failure = exc + raise + except Exception as exc: + wrapped = LedgerReplayError( + f"replay rejected malformed event: {type(exc).__name__}: {exc}" + ) + self._failure = wrapped + raise wrapped from exc + + def _process(self, record: LedgerRecord | Mapping[str, Any]) -> None: + if isinstance(record, Mapping): + unknown_envelope = sorted( + str(key) for key in set(record) - _ENVELOPE_KEYS + ) + if unknown_envelope: + raise LedgerReplayError( + f"record has unsupported envelope fields {unknown_envelope}" + ) + event_id_value = _record_value(record, "event_id") + event_type_value = _record_value(record, "event_type") + timestamp_value = _record_value(record, "timestamp") + sequence_value = _record_value(record, "sequence") + payload = _record_value(record, "payload") + previous_hash = _record_value(record, "previous_hash", _MISSING) + record_hash = _record_value(record, "hash", _MISSING) + if (previous_hash is _MISSING) != (record_hash is _MISSING): + raise LedgerReplayError( + "record previous_hash and hash must be supplied together" + ) + for name, value in ( + ("previous_hash", previous_hash), + ("hash", record_hash), + ): + if value is not _MISSING and ( + not isinstance(value, str) + or _SHA256_PATTERN.fullmatch(value) is None + ): + raise LedgerReplayError( + f"record {name} must be a lowercase SHA-256 hex string" + ) + if not isinstance(event_id_value, str) or not event_id_value: + raise LedgerReplayError("record event_id must be a non-empty string") + if not isinstance(event_type_value, str) or not event_type_value: + raise LedgerReplayError(f"{event_id_value}: record has no event_type") + if ( + event_type_value.upper() in {"ORDER_STATUS", "FILL"} + and event_type_value not in {"ORDER_STATUS", "FILL"} + ): + raise LedgerReplayError( + f"{event_id_value}: event_type must use canonical uppercase" + ) + if ( + isinstance(sequence_value, bool) + or not isinstance(sequence_value, int) + or sequence_value <= 0 + ): + raise LedgerReplayError( + f"{event_id_value}.sequence must be a positive integer" + ) + sequence = sequence_value + if not isinstance(timestamp_value, str) or not timestamp_value: + raise LedgerReplayError( + f"{event_id_value}: timestamp must be a non-empty string" + ) + if not isinstance(payload, Mapping): + raise LedgerReplayError(f"{event_id_value}: payload must be an object") + event_id = str(event_id_value) + event_type = event_type_value + event_time = _timestamp(timestamp_value, f"{event_id}.timestamp") + event_signature = canonical_json( + { + "event_type": event_type, + "timestamp": timestamp_value, + "payload": dict(payload), + } + ) + prior_event = self._event_signatures.get(event_id) + expected_sequence = self._last_sequence + 1 + if sequence < self._last_sequence: + raise OutOfOrderError( + f"{event_id}: sequence {sequence} follows {self._last_sequence}" + ) + if sequence > expected_sequence: + raise OutOfOrderError( + f"{event_id}: sequence gap; expected {expected_sequence}, " + f"observed {sequence}" + ) + if sequence == self._last_sequence: + if prior_event is None or prior_event[0] != sequence: + raise OutOfOrderError( + f"{event_id}: sequence {sequence} is already occupied" + ) + if prior_event[1] != event_signature: + raise ConflictingDuplicateError( + f"event_id {event_id} has conflicting content" + ) + return + if prior_event is not None: + raise ConflictingDuplicateError( + f"event_id {event_id} was first observed at sequence " + f"{prior_event[0]}, not {sequence}" + ) + self._event_signatures[event_id] = (sequence, event_signature) + self._last_sequence = sequence + if event_type not in {"ORDER_STATUS", "FILL"}: + return + if event_type == "ORDER_STATUS": + self._status_event( + payload=payload, + event_id=event_id, + sequence=sequence, + event_time=event_time, + ) + else: + self._fill_event( + payload=payload, + event_id=event_id, + sequence=sequence, + event_time=event_time, + ) + + def projection(self) -> LedgerProjection: + if self._failure is not None: + raise LedgerReplayError( + "cannot materialize a projection after a replay error" + ) from self._failure + orders = { + order_id: ReplayedOrder( + order_id=order.order_id, + symbol=order.symbol, + side=order.side.value, + quantity=order.quantity, + status=order.status.value, + filled_quantity=order.filled_quantity, + average_fill_price=order.average_fill_price, + cash_delta=money(order.cash_delta), + position_delta=order.position_delta, + fill_ids=tuple(order.fill_ids), + last_sequence=order.last_sequence, + last_timestamp=( + order.last_timestamp.isoformat() + if order.last_timestamp is not None + else "" + ), + ) + for order_id, order in sorted(self._orders.items()) + } + return LedgerProjection( + orders=orders, + cash_delta=money(self._cash_delta), + position_deltas=dict(sorted(self._position_deltas.items())), + fill_ids=tuple(self._fill_ids), + source_last_sequence=self._last_sequence, + relevant_event_count=self._relevant_event_count, + ) + + +def project_ledger( + records: HashChainLedger + | Iterable[LedgerRecord | Mapping[str, Any]], +) -> LedgerProjection: + """Project a verified ledger or an explicitly ordered event iterable.""" + + if isinstance(records, HashChainLedger): + records.verify() + source: Iterable[LedgerRecord | Mapping[str, Any]] = records.records + else: + source = records + projector = LedgerProjector() + for record in source: + projector.process(record) + return projector.projection() + + +def replay_ledger( + records: HashChainLedger + | Iterable[LedgerRecord | Mapping[str, Any]], +) -> LedgerProjection: + """Compatibility-friendly verb alias for :func:`project_ledger`.""" + + return project_ledger(records) + + +__all__ = [ + "ConflictingDuplicateError", + "LedgerProjection", + "LedgerProjector", + "LedgerReplayError", + "OutOfOrderError", + "OverfillError", + "ReplayConflictError", + "ReplayError", + "ReplayOrderingError", + "ReplayedOrder", + "UnknownOrderError", + "project_ledger", + "replay_ledger", +] diff --git a/src/quant60/research.py b/src/quant60/research.py new file mode 100644 index 0000000..db35f47 --- /dev/null +++ b/src/quant60/research.py @@ -0,0 +1,274 @@ +"""Small research utilities with explicit time-ordering semantics.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Iterator, Sequence + + +class ResearchDependencyUnavailable(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class WalkForwardFold: + train: tuple[int, ...] + test: tuple[int, ...] + purged: tuple[int, ...] + embargoed: tuple[int, ...] + + +def purged_walk_forward( + sample_count: int, + *, + train_size: int, + test_size: int, + purge: int, + embargo: int = 0, + step: int | None = None, +) -> Iterator[WalkForwardFold]: + """Yield expanding chronological folds without train/test label overlap.""" + + values = (sample_count, train_size, test_size, purge, embargo) + if min(values) < 0 or train_size == 0 or test_size == 0: + raise ValueError("invalid walk-forward sizes") + stride = test_size if step is None else int(step) + if stride <= 0: + raise ValueError("step must be positive") + test_start = train_size + purge + while test_start + test_size <= sample_count: + train_end = test_start - purge + train = tuple(range(0, train_end)) + purged = tuple(range(train_end, test_start)) + test = tuple(range(test_start, test_start + test_size)) + embargo_end = min(sample_count, test_start + test_size + embargo) + embargoed = tuple(range(test_start + test_size, embargo_end)) + yield WalkForwardFold(train, test, purged, embargoed) + test_start += stride + embargo + + +class NumpyRidge: + """Minimal deterministic Ridge challenger; NumPy is imported lazily.""" + + def __init__(self, alpha: float = 1.0, fit_intercept: bool = True): + if alpha < 0: + raise ValueError("alpha must be non-negative") + self.alpha = float(alpha) + self.fit_intercept = bool(fit_intercept) + self.coef_ = None + self.intercept_ = 0.0 + + @staticmethod + def _numpy(): + try: + import numpy as np + except ImportError as exc: + raise ResearchDependencyUnavailable( + "NumpyRidge requires NumPy; install the research environment" + ) from exc + return np + + def fit(self, x: Sequence[Sequence[float]], y: Sequence[float]) -> "NumpyRidge": + np = self._numpy() + matrix = np.asarray(x, dtype=float) + target = np.asarray(y, dtype=float) + if matrix.ndim != 2 or target.ndim != 1: + raise ValueError("x must be 2-D and y must be 1-D") + if matrix.shape[0] != target.shape[0] or matrix.shape[0] == 0: + raise ValueError("x/y row count mismatch or empty data") + if not np.isfinite(matrix).all() or not np.isfinite(target).all(): + raise ValueError("x/y must be finite") + if self.fit_intercept: + x_mean = matrix.mean(axis=0) + y_mean = float(target.mean()) + centered_x = matrix - x_mean + centered_y = target - y_mean + else: + x_mean = np.zeros(matrix.shape[1], dtype=float) + y_mean = 0.0 + centered_x, centered_y = matrix, target + gram = centered_x.T @ centered_x + penalty = np.eye(gram.shape[0], dtype=float) * self.alpha + self.coef_ = np.linalg.solve(gram + penalty, centered_x.T @ centered_y) + self.intercept_ = y_mean - float(x_mean @ self.coef_) + return self + + def predict(self, x: Sequence[Sequence[float]]): + if self.coef_ is None: + raise ValueError("model is not fitted") + np = self._numpy() + matrix = np.asarray(x, dtype=float) + if matrix.ndim != 2 or matrix.shape[1] != len(self.coef_): + raise ValueError("prediction feature shape mismatch") + return matrix @ self.coef_ + self.intercept_ + + +def _solve_linear_system( + matrix: list[list[float]], + target: list[float], +) -> list[float]: + """Solve a dense system with deterministic partial-pivot elimination.""" + + size = len(matrix) + if size == 0 or len(target) != size or any( + len(row) != size for row in matrix + ): + raise ValueError("linear system must be non-empty and square") + augmented = [ + [float(value) for value in row] + [float(target[index])] + for index, row in enumerate(matrix) + ] + for column in range(size): + pivot = max( + range(column, size), + key=lambda row: (abs(augmented[row][column]), -row), + ) + if abs(augmented[pivot][column]) <= 1e-14: + raise ValueError("ridge normal equation is singular") + if pivot != column: + augmented[column], augmented[pivot] = ( + augmented[pivot], + augmented[column], + ) + divisor = augmented[column][column] + augmented[column] = [value / divisor for value in augmented[column]] + for row in range(size): + if row == column: + continue + factor = augmented[row][column] + if factor == 0: + continue + augmented[row] = [ + value - factor * pivot_value + for value, pivot_value in zip( + augmented[row], + augmented[column], + ) + ] + return [augmented[row][-1] for row in range(size)] + + +class DeterministicRidge: + """Dependency-free Ridge challenger for CI and Colab smoke experiments.""" + + def __init__(self, alpha: float = 1.0, fit_intercept: bool = True): + if not math.isfinite(float(alpha)) or alpha < 0: + raise ValueError("alpha must be finite and non-negative") + if type(fit_intercept) is not bool: + raise ValueError("fit_intercept must be a bool") + self.alpha = float(alpha) + self.fit_intercept = fit_intercept + self.coef_: tuple[float, ...] | None = None + self.intercept_: float = 0.0 + + @staticmethod + def _validate( + x: Sequence[Sequence[float]], + y: Sequence[float] | None = None, + ) -> tuple[list[list[float]], list[float] | None]: + matrix = [[float(value) for value in row] for row in x] + if not matrix or not matrix[0]: + raise ValueError("x must be a non-empty 2-D matrix") + width = len(matrix[0]) + if any(len(row) != width for row in matrix): + raise ValueError("x rows have inconsistent widths") + if any(not math.isfinite(value) for row in matrix for value in row): + raise ValueError("x must be finite") + if y is None: + return matrix, None + target = [float(value) for value in y] + if len(target) != len(matrix): + raise ValueError("x/y row count mismatch") + if any(not math.isfinite(value) for value in target): + raise ValueError("y must be finite") + return matrix, target + + def fit( + self, + x: Sequence[Sequence[float]], + y: Sequence[float], + ) -> "DeterministicRidge": + matrix, target = self._validate(x, y) + assert target is not None + design = ( + [[1.0] + row for row in matrix] + if self.fit_intercept + else matrix + ) + width = len(design[0]) + gram = [[0.0 for unused in range(width)] for unused in range(width)] + rhs = [0.0 for unused in range(width)] + for row, observed in zip(design, target): + for left in range(width): + rhs[left] += row[left] * observed + for right in range(width): + gram[left][right] += row[left] * row[right] + for index in range(width): + if not (self.fit_intercept and index == 0): + gram[index][index] += self.alpha + solution = _solve_linear_system(gram, rhs) + if self.fit_intercept: + self.intercept_ = solution[0] + self.coef_ = tuple(solution[1:]) + else: + self.intercept_ = 0.0 + self.coef_ = tuple(solution) + return self + + def predict( + self, + x: Sequence[Sequence[float]], + ) -> list[float]: + if self.coef_ is None: + raise ValueError("model is not fitted") + matrix, unused = self._validate(x) + del unused + if any(len(row) != len(self.coef_) for row in matrix): + raise ValueError("prediction feature shape mismatch") + return [ + self.intercept_ + + sum(coefficient * value for coefficient, value in zip(self.coef_, row)) + for row in matrix + ] + + +def pearson_correlation(left: Sequence[float], right: Sequence[float]) -> float: + if len(left) != len(right) or len(left) < 2: + raise ValueError("correlation needs equal vectors with at least two rows") + x = [float(value) for value in left] + y = [float(value) for value in right] + if any(not math.isfinite(value) for value in x + y): + raise ValueError("correlation inputs must be finite") + x_mean = sum(x) / len(x) + y_mean = sum(y) / len(y) + numerator = sum( + (x_value - x_mean) * (y_value - y_mean) + for x_value, y_value in zip(x, y) + ) + x_norm = math.sqrt(sum((value - x_mean) ** 2 for value in x)) + y_norm = math.sqrt(sum((value - y_mean) ** 2 for value in y)) + if x_norm <= 1e-15 or y_norm <= 1e-15: + return 0.0 + return numerator / (x_norm * y_norm) + + +def _average_ranks(values: Sequence[float]) -> list[float]: + indexed = sorted(enumerate(values), key=lambda item: (item[1], item[0])) + ranks = [0.0] * len(values) + start = 0 + while start < len(indexed): + end = start + 1 + while end < len(indexed) and indexed[end][1] == indexed[start][1]: + end += 1 + average = (start + 1 + end) / 2.0 + for position in range(start, end): + ranks[indexed[position][0]] = average + start = end + return ranks + + +def spearman_correlation(left: Sequence[float], right: Sequence[float]) -> float: + if len(left) != len(right) or len(left) < 2: + raise ValueError("correlation needs equal vectors with at least two rows") + return pearson_correlation(_average_ranks(left), _average_ranks(right)) diff --git a/src/quant60/risk.py b/src/quant60/risk.py new file mode 100644 index 0000000..6d20fad --- /dev/null +++ b/src/quant60/risk.py @@ -0,0 +1,192 @@ +"""Deterministic pre-trade checks for the portable reference strategy.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Mapping + +from .portable_core import normalize_symbol + + +@dataclass(frozen=True, slots=True) +class RiskViolation: + code: str + message: str + symbol: str | None = None + + +@dataclass(frozen=True, slots=True) +class RiskDecision: + approved: bool + violations: tuple[RiskViolation, ...] + gross_weight: float + one_way_turnover: float + + def require_approved(self) -> None: + if not self.approved: + details = "; ".join(item.message for item in self.violations) + raise ValueError(f"pre-trade risk rejected target: {details}") + + +def weights_from_positions( + quantities: Mapping[str, int], + prices: Mapping[str, float], + equity: float, +) -> dict[str, float]: + if not math.isfinite(float(equity)) or equity <= 0: + raise ValueError("equity must be finite and positive") + output: dict[str, float] = {} + for raw_symbol, quantity in quantities.items(): + symbol = normalize_symbol(raw_symbol) + price = prices.get(raw_symbol, prices.get(symbol)) + if ( + price is None + or not math.isfinite(float(price)) + or float(price) <= 0 + ): + raise ValueError(f"missing finite positive price for {symbol}") + output[symbol] = ( + max(0, int(quantity)) * float(price) / float(equity) + ) + return output + + +def check_target_weights( + targets: Mapping[str, float], + current: Mapping[str, float] | None = None, + *, + max_single_weight: float, + max_gross_weight: float, + max_one_way_turnover: float = 1.0, + tolerance: float = 1e-10, +) -> RiskDecision: + """Check concentration, gross exposure and one-way turnover. + + The function is intentionally independent of any optimizer. It is run + after target generation so an optimizer cannot silently return a + numerically invalid solution. + """ + + if not math.isfinite(float(max_single_weight)) or not ( + 0 < max_single_weight <= 1 + ): + raise ValueError("max_single_weight must lie in (0, 1]") + if not math.isfinite(float(max_gross_weight)) or not ( + 0 <= max_gross_weight <= 1 + ): + raise ValueError("max_gross_weight must lie in [0, 1]") + if not math.isfinite(float(max_one_way_turnover)) or not ( + 0 <= max_one_way_turnover <= 1 + ): + raise ValueError("max_one_way_turnover must lie in [0, 1]") + if not math.isfinite(float(tolerance)) or tolerance < 0: + raise ValueError("tolerance must be finite and non-negative") + + normalized = { + normalize_symbol(symbol): float(weight) + for symbol, weight in targets.items() + } + prior = { + normalize_symbol(symbol): float(weight) + for symbol, weight in (current or {}).items() + } + violations: list[RiskViolation] = [] + for symbol, weight in sorted(normalized.items()): + if not math.isfinite(weight): + violations.append( + RiskViolation( + "NON_FINITE_WEIGHT", + f"{symbol} target weight is not finite", + symbol, + ) + ) + continue + if weight < -tolerance: + violations.append( + RiskViolation("NEGATIVE_WEIGHT", f"{symbol} weight is negative", symbol) + ) + if weight > max_single_weight + tolerance: + violations.append( + RiskViolation( + "SINGLE_NAME_CAP", + f"{symbol} weight {weight:.12f} exceeds " + f"{max_single_weight:.12f}", + symbol, + ) + ) + for symbol, weight in sorted(prior.items()): + if not math.isfinite(weight): + violations.append( + RiskViolation( + "NON_FINITE_CURRENT_WEIGHT", + f"{symbol} current weight is not finite", + symbol, + ) + ) + has_non_finite = any( + not math.isfinite(weight) + for weight in list(normalized.values()) + list(prior.values()) + ) + gross = ( + float("inf") + if has_non_finite + else sum(max(0.0, weight) for weight in normalized.values()) + ) + if gross > max_gross_weight + tolerance: + violations.append( + RiskViolation( + "GROSS_CAP", + f"gross weight {gross:.12f} exceeds {max_gross_weight:.12f}", + ) + ) + symbols = set(normalized) | set(prior) + turnover = ( + float("inf") + if has_non_finite + else 0.5 + * sum( + abs(normalized.get(symbol, 0.0) - prior.get(symbol, 0.0)) + for symbol in symbols + ) + ) + if turnover > max_one_way_turnover + tolerance: + violations.append( + RiskViolation( + "TURNOVER_CAP", + f"one-way turnover {turnover:.12f} exceeds " + f"{max_one_way_turnover:.12f}", + ) + ) + return RiskDecision( + approved=not violations, + violations=tuple(violations), + gross_weight=gross, + one_way_turnover=turnover, + ) + + +def diagonal_variance( + returns_by_symbol: Mapping[str, list[float]], + *, + variance_floor: float = 1e-8, +) -> dict[str, float]: + """Transparent risk fallback used when a full factor model is unavailable.""" + + if not math.isfinite(float(variance_floor)) or variance_floor <= 0: + raise ValueError("variance_floor must be finite and positive") + output: dict[str, float] = {} + for raw_symbol, values in returns_by_symbol.items(): + symbol = normalize_symbol(raw_symbol) + observations = [float(value) for value in values] + if any(not math.isfinite(value) for value in observations): + raise ValueError(f"returns must be finite for {symbol}") + if len(observations) < 2: + output[symbol] = variance_floor + continue + mean = sum(observations) / len(observations) + variance = sum((value - mean) ** 2 for value in observations) / ( + len(observations) - 1 + ) + output[symbol] = max(variance_floor, variance) + return output diff --git a/src/quant60/snapshot_backtest.py b/src/quant60/snapshot_backtest.py new file mode 100644 index 0000000..755d713 --- /dev/null +++ b/src/quant60/snapshot_backtest.py @@ -0,0 +1,341 @@ +"""Historical local backtest driven by a verified provider data snapshot.""" + +from __future__ import annotations + +from datetime import date +from decimal import Decimal +import hashlib +from pathlib import Path +from typing import Any + +from .backtest import ( + BacktestResult, + _contract_manifest, + _engine_source_manifest, + _metrics, +) +from .broker import SimBroker +from .config import BacktestConfig +from .data_snapshot import load_verified_data_snapshot +from .domain import Bar, Portfolio, Side +from .ledger import HashChainLedger, canonical_json +from .snapshot_pipeline import ( + _build_snapshot_decision_from_verified_snapshot, + first_executable_window, +) +from .synthetic import group_bars_by_date + + +def _sha256(payload: Any) -> str: + return hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + + +def _decimal_or_none(value: Any) -> Decimal | None: + return Decimal(str(value)) if value is not None else None + + +def _domain_bars(rows: list[dict[str, Any]]) -> list[Bar]: + bars = [] + for row in rows: + bars.append( + Bar( + trading_date=date.fromisoformat(row["trading_date"]), + symbol=row["symbol"], + open=Decimal(str(row["open"])), + high=Decimal(str(row["high"])), + low=Decimal(str(row["low"])), + close=Decimal(str(row["close"])), + volume=int(row["volume"]), + previous_close=_decimal_or_none(row.get("previous_close")), + limit_up=_decimal_or_none(row.get("limit_up")), + limit_down=_decimal_or_none(row.get("limit_down")), + suspended=bool(row["paused"]), + ) + ) + return bars + + +def run_snapshot_backtest( + *, + snapshot_manifest: str | Path, + config: BacktestConfig, +) -> BacktestResult: + """Run portable-momentum with daily PIT membership and provider rules.""" + + config.validate() + snapshot = load_verified_data_snapshot(snapshot_manifest) + data_manifest = snapshot["manifest"] + if data_manifest.get("query", {}).get("adjustment") != "none": + raise ValueError( + "snapshot backtest requires raw OHLC plus adjustment factors" + ) + if config.universe.mode != "pit_index": + raise ValueError( + "provider snapshot backtest requires universe.mode=pit_index" + ) + all_bars = _domain_bars(snapshot["bars"]) + sessions = group_bars_by_date(all_bars) + if len(sessions) <= config.strategy.lookback + config.strategy.skip + 1: + raise ValueError("provider snapshot is too short for the strategy") + + config_body = config.as_dict() + config_body["symbols"] = [""] + config_body["universe_query"] = data_manifest.get("query", {}) + config_hash = _sha256(config_body) + engine_source_sha256, engine_sources = _engine_source_manifest() + contracts_sha256, contract_schemas = _contract_manifest() + portable_path = Path(__file__).with_name("portable_core.py") + portable_core_sha256 = hashlib.sha256( + portable_path.read_bytes() + ).hexdigest() + strategy_sha256 = _sha256( + { + "version": "portable-momentum-v1", + "parameters": config_body["strategy"], + "portable_core_sha256": portable_core_sha256, + } + ) + rules_sha256 = _sha256( + { + "version": "provider-cn-a-v1", + "provider": data_manifest["provider"], + "provider_version": data_manifest["provider_version"], + "fees": config_body["fees"], + "execution": config_body["execution"], + "membership": "exact_effective_date", + "price_adjustment": "PIT_FRONT_FROM_RAW_AND_FACTOR", + "price_limits": "provider_daily_fields", + "clock": "weekly_first_close_to_next_open", + } + ) + run_id = "q60p-" + _sha256( + { + "config_hash": config_hash, + "data_version": data_manifest["data_version"], + "engine_source_sha256": engine_source_sha256, + "strategy_sha256": strategy_sha256, + "rules_sha256": rules_sha256, + "contracts_sha256": contracts_sha256, + } + )[:16] + + portfolio = Portfolio(Decimal(str(config.initial_cash))) + ledger = HashChainLedger() + broker = SimBroker( + portfolio, + config.execution, + config.fees, + ledger, + ) + signals: list[dict[str, Any]] = [] + targets: list[dict[str, Any]] = [] + equity_curve: list[dict[str, Any]] = [] + decision_count = 0 + session_items = list(sessions.items()) + + for session_index, (trading_date, session_bars) in enumerate(session_items): + portfolio.start_session() + broker.execute_session(session_bars) + if config.execution.cancel_open_orders_at_end: + broker.cancel_open_orders( + trading_date, + "DAILY_WINDOW_END", + phase="post_open", + ) + portfolio.mark(session_bars.values()) + snapshot_row = { + "date": trading_date.isoformat(), + "cash": float(portfolio.cash), + "market_value": float(portfolio.market_value), + "equity": float(portfolio.equity), + "positions": portfolio.quantities(), + } + equity_curve.append(snapshot_row) + ledger.append( + "PORTFOLIO_SNAPSHOT", + snapshot_row, + timestamp=f"{trading_date.isoformat()}T15:00:00+08:00", + event_id=f"portfolio:{trading_date.isoformat()}", + ) + + first_session_of_week = ( + session_index == 0 + or session_items[session_index - 1][0].isocalendar()[:2] + != trading_date.isocalendar()[:2] + ) + warmed_up = ( + session_index + >= config.strategy.lookback + config.strategy.skip + ) + if ( + not first_session_of_week + or not warmed_up + or session_index >= len(session_items) - 1 + ): + continue + positions = [] + for symbol, quantity in sorted(portfolio.quantities().items()): + position = portfolio.position(symbol) + positions.append( + { + "symbol": symbol, + "quantity": quantity, + "sellable_quantity": position.sellable_quantity, + "average_cost": ( + float(position.average_cost) + if quantity > 0 + else None + ), + "market_value": None, + } + ) + as_of_timestamp = ( + f"{trading_date.isoformat()}T15:00:00+08:00" + ) + next_trading_session = session_items[session_index + 1][0] + executable_window = first_executable_window( + next_trading_session + ) + observation_timestamp = executable_window["start"] + broker_snapshot = { + "schema_version": "1.0", + "snapshot_id": ( + f"local-backtest-{run_id}-{trading_date.strftime('%Y%m%d')}" + ), + "signal_as_of": as_of_timestamp, + "next_trading_session": next_trading_session.isoformat(), + "first_executable_window": executable_window, + "observation_as_of": observation_timestamp, + "as_of": observation_timestamp, + "source_time": observation_timestamp, + "broker": "quant-os-local-sim", + "account_hash": _sha256({"run_id": run_id})[:16], + "cash": float(portfolio.cash), + "total_asset": float(portfolio.equity), + "market_value": float(portfolio.market_value), + "positions": positions, + "open_orders": [], + "metadata": { + "evidence_class": "provider_snapshot_local_simulation" + }, + } + decision_result = _build_snapshot_decision_from_verified_snapshot( + snapshot=snapshot, + config=config, + as_of=trading_date, + broker_snapshot=broker_snapshot, + ) + decision = decision_result["decision"] + decision_count += 1 + for record in decision_result["signals"]: + normalized = dict(record) + normalized["run_id"] = run_id + signals.append(normalized) + for record in decision_result["targets"]: + normalized = dict(record) + normalized["run_id"] = run_id + normalized["config_hash"] = config_hash + targets.append(normalized) + ledger.append( + "DECISION", + { + "run_id": run_id, + "decision_id": decision["decision_id"], + "as_of": decision["as_of"], + "provider_decision_run_id": decision["run_id"], + "weights": decision["weights"], + "targets": decision["targets"], + "orders": decision["orders"], + "risk_approved": decision["risk"]["approved"], + "data_version": decision["data_version"], + }, + timestamp=decision["as_of"], + event_id=f"decision:{decision['decision_id']}", + ) + for symbol, delta in sorted( + decision["orders"].items(), + key=lambda item: (item[1] > 0, item[0]), + ): + if int(delta) == 0: + continue + broker.submit( + symbol, + Side.BUY if int(delta) > 0 else Side.SELL, + abs(int(delta)), + trading_date, + metadata={ + "run_id": run_id, + "decision_id": decision["decision_id"], + "data_version": data_manifest["data_version"], + }, + ) + + if decision_count == 0: + raise ValueError("provider snapshot produced no weekly decision") + last_date = session_items[-1][0] + broker.cancel_open_orders(last_date, "BACKTEST_END") + ledger.verify() + report = _metrics( + equity_curve, + broker, + float(config.initial_cash), + ) + report.update( + { + "run_id": run_id, + "config_hash": config_hash, + "data_version": data_manifest["data_version"], + "engine_source_sha256": engine_source_sha256, + "rules_sha256": rules_sha256, + "strategy_sha256": strategy_sha256, + "contracts_sha256": contracts_sha256, + "final_cash": float(portfolio.cash), + "final_positions": portfolio.quantities(), + "ledger_records": len(ledger.records), + "ledger_head_hash": ledger.head_hash, + "decision_count": decision_count, + "provider": data_manifest["provider"], + "provider_version": data_manifest["provider_version"], + "evidence_class": "provider-snapshot-local-backtest", + "real_platform_backtest": False, + "investment_value_claim": False, + } + ) + manifest = { + "schema_version": "1.0", + "run_id": run_id, + "engine": "local", + "mode": "provider_snapshot", + "evidence_class": "provider-snapshot-local-backtest", + "config_hash": config_hash, + "data_version": data_manifest["data_version"], + "input_data_manifest_sha256": hashlib.sha256( + Path(snapshot["manifest_file"]).read_bytes() + ).hexdigest(), + "engine_source_sha256": engine_source_sha256, + "engine_sources": engine_sources, + "portable_core_sha256": portable_core_sha256, + "rules_sha256": rules_sha256, + "strategy_sha256": strategy_sha256, + "contracts_sha256": contracts_sha256, + "contract_schemas": contract_schemas, + "strategy_version": "portable-momentum-v1", + "rules_version": "provider-cn-a-v1", + "timezone": "Asia/Shanghai", + "signal_clock": "T_CLOSE_COMPLETE", + "first_executable_clock": "T_PLUS_1_OPEN", + "price_adjustment": "PIT_FRONT_FROM_RAW_AND_FACTOR", + "fill_model": "prior_day_provider_volume_capped_open_slippage_v2", + "deterministic": True, + } + return BacktestResult( + run_id=run_id, + config_hash=config_hash, + data_version=data_manifest["data_version"], + signals=signals, + targets=targets, + equity_curve=equity_curve, + report=report, + manifest=manifest, + ledger=ledger, + ) diff --git a/src/quant60/snapshot_pipeline.py b/src/quant60/snapshot_pipeline.py new file mode 100644 index 0000000..2090eda --- /dev/null +++ b/src/quant60/snapshot_pipeline.py @@ -0,0 +1,1050 @@ +"""Verified provider snapshot to portable signal and target decision.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import tempfile +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Mapping + +from .backtest import _contract_manifest, _engine_source_manifest +from .config import BacktestConfig +from .contracts import validate_records +from .data_snapshot import load_verified_data_snapshot +from .ledger import canonical_json, jsonable +from .portable_core import ( + capped_target_weights, + momentum_score, + normalize_symbol, + order_deltas, + rank_momentum, + target_quantities, +) +from .risk import check_target_weights, weights_from_positions +from .universe import UniverseState + + +class SnapshotDecisionError(ValueError): + pass + + +SHANGHAI_TIMEZONE = timezone(timedelta(hours=8)) +BROKER_SOURCE_CLOCK_SKEW_SECONDS = 5.0 +BROKER_SOURCE_MAX_LAG_SECONDS = 60.0 +FIRST_EXECUTABLE_WINDOW_START = (9, 0, 0) +FIRST_EXECUTABLE_WINDOW_END = (9, 30, 0) + + +def _sha256_payload(payload: Any) -> str: + return hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _atomic_json(path: Path, payload: Any) -> None: + encoded = ( + json.dumps( + jsonable(payload), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=str(path.parent), + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def _finite_positive(value: Any, name: str) -> float: + numeric = float(value) + if not math.isfinite(numeric) or numeric <= 0: + raise SnapshotDecisionError(f"{name} must be finite and positive") + return numeric + + +def _parse_aware_timestamp(value: Any, name: str) -> datetime: + if not isinstance(value, str): + raise SnapshotDecisionError(f"{name} must be an ISO-8601 string") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise SnapshotDecisionError( + f"{name} must be a valid ISO-8601 timestamp" + ) from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise SnapshotDecisionError(f"{name} must include a timezone") + return parsed + + +def _signal_close_timestamp(as_of: date) -> datetime: + return datetime( + as_of.year, + as_of.month, + as_of.day, + 15, + 0, + tzinfo=SHANGHAI_TIMEZONE, + ) + + +def first_executable_window(next_trading_session: date) -> dict[str, Any]: + """Return the canonical broker-observation window for a known session.""" + + if not isinstance(next_trading_session, date): + raise SnapshotDecisionError("next_trading_session must be a date") + start = datetime( + next_trading_session.year, + next_trading_session.month, + next_trading_session.day, + *FIRST_EXECUTABLE_WINDOW_START, + tzinfo=SHANGHAI_TIMEZONE, + ) + end = datetime( + next_trading_session.year, + next_trading_session.month, + next_trading_session.day, + *FIRST_EXECUTABLE_WINDOW_END, + tzinfo=SHANGHAI_TIMEZONE, + ) + return { + "session": next_trading_session.isoformat(), + "timezone": "Asia/Shanghai", + "start": start.isoformat(), + "end": end.isoformat(), + "boundary": "[start,end)", + "purpose": "PRE_OPEN_BROKER_OBSERVATION", + } + + +def _validated_first_executable_window( + value: Any, + *, + next_trading_session: date, + name: str, +) -> tuple[datetime, datetime]: + if not isinstance(value, Mapping): + raise SnapshotDecisionError(f"{name} must be an object") + expected = first_executable_window(next_trading_session) + if dict(value) != expected: + raise SnapshotDecisionError( + f"{name} does not match the canonical [09:00,09:30) " + "Asia/Shanghai pre-open window" + ) + return ( + _parse_aware_timestamp(expected["start"], f"{name}.start"), + _parse_aware_timestamp(expected["end"], f"{name}.end"), + ) + + +def _frozen_trading_calendar( + data_manifest: Mapping[str, Any], +) -> list[date]: + query = data_manifest.get("query") + if not isinstance(query, Mapping): + raise SnapshotDecisionError("snapshot query metadata is missing") + raw = query.get("trading_sessions") + if not isinstance(raw, list): + raise SnapshotDecisionError( + "snapshot query.trading_sessions calendar is missing" + ) + try: + sessions = [date.fromisoformat(str(item)) for item in raw] + except (TypeError, ValueError) as exc: + raise SnapshotDecisionError( + "snapshot trading calendar contains an invalid date" + ) from exc + if len(sessions) < 2 or sessions != sorted(set(sessions)): + raise SnapshotDecisionError( + "snapshot trading calendar must be unique and increasing" + ) + declared_next = data_manifest.get("next_trading_session") + if sessions[-1].isoformat() != declared_next: + raise SnapshotDecisionError( + "snapshot next_trading_session/calendar identity mismatch" + ) + return sessions + + +def _next_session_for_decision( + data_manifest: Mapping[str, Any], + as_of: date, +) -> date: + sessions = _frozen_trading_calendar(data_manifest) + try: + index = sessions.index(as_of) + except ValueError as exc: + raise SnapshotDecisionError( + f"decision date {as_of.isoformat()} is absent from the " + "frozen trading calendar" + ) from exc + if index >= len(sessions) - 1: + raise SnapshotDecisionError( + "decision date has no next session in the frozen calendar" + ) + return sessions[index + 1] + + +def _same_instant(left: datetime, right: datetime) -> bool: + return left.astimezone(timezone.utc) == right.astimezone(timezone.utc) + + +def _validate_broker_binding_times( + *, + signal_as_of: datetime, + next_trading_session: date, + first_window: Mapping[str, Any], + declared_signal_as_of: Any, + declared_next_trading_session: Any, + declared_first_window: Any, + observation_as_of: Any, + snapshot_as_of: Any, + source_time: Any, +) -> tuple[datetime, datetime]: + if str(declared_next_trading_session) != next_trading_session.isoformat(): + raise SnapshotDecisionError( + "broker snapshot next_trading_session does not match decision" + ) + if dict(declared_first_window or {}) != dict(first_window): + raise SnapshotDecisionError( + "broker snapshot first_executable_window does not match decision" + ) + window_start, window_end = _validated_first_executable_window( + first_window, + next_trading_session=next_trading_session, + name="first_executable_window", + ) + declared_signal = _parse_aware_timestamp( + declared_signal_as_of, + "broker snapshot signal_as_of", + ) + if not _same_instant(declared_signal, signal_as_of): + raise SnapshotDecisionError( + "broker snapshot signal_as_of must equal decision signal_as_of" + ) + observation = _parse_aware_timestamp( + observation_as_of, + "broker snapshot observation_as_of", + ) + legacy_as_of = _parse_aware_timestamp( + snapshot_as_of, + "broker snapshot as_of", + ) + if not _same_instant(observation, legacy_as_of): + raise SnapshotDecisionError( + "broker snapshot observation_as_of must equal as_of" + ) + source = _parse_aware_timestamp( + source_time, + "broker snapshot source_time", + ) + observation_utc = observation.astimezone(timezone.utc) + source_utc = source.astimezone(timezone.utc) + window_start_utc = window_start.astimezone(timezone.utc) + window_end_utc = window_end.astimezone(timezone.utc) + if not window_start_utc <= observation_utc < window_end_utc: + raise SnapshotDecisionError( + "broker snapshot observation_as_of is outside the unique next " + "session [09:00,09:30) pre-open window" + ) + if not window_start_utc <= source_utc < window_end_utc: + raise SnapshotDecisionError( + "broker snapshot source_time is outside the unique next session " + "[09:00,09:30) pre-open window" + ) + source_lag = (observation_utc - source_utc).total_seconds() + if source_lag < -BROKER_SOURCE_CLOCK_SKEW_SECONDS: + raise SnapshotDecisionError( + "broker snapshot source_time is more than 5s after observation_as_of" + ) + if source_lag > BROKER_SOURCE_MAX_LAG_SECONDS: + raise SnapshotDecisionError( + "broker snapshot source_time is more than 60s stale" + ) + return observation, source + + +def _broker_state( + broker_snapshot: Mapping[str, Any] | None, + *, + signal_as_of: datetime, + next_trading_session: date, + first_window: Mapping[str, Any], + equity: float | None, +) -> tuple[float, dict[str, int], dict[str, int], dict[str, Any] | None]: + if broker_snapshot is None: + if equity is None: + raise SnapshotDecisionError( + "equity is required when broker_snapshot is absent" + ) + return _finite_positive(equity, "equity"), {}, {}, None + if equity is not None: + raise SnapshotDecisionError( + "equity and broker_snapshot are mutually exclusive" + ) + snapshot = dict(broker_snapshot) + try: + validate_records([snapshot], "broker_snapshot.schema.json") + except ValueError as exc: + raise SnapshotDecisionError( + f"broker snapshot contract is invalid: {exc}" + ) from exc + if snapshot["open_orders"]: + raise SnapshotDecisionError( + "broker snapshot has open orders; reconcile before a new decision" + ) + signal_text = signal_as_of.isoformat() + for field in ("signal_as_of", "observation_as_of"): + if field not in snapshot: + raise SnapshotDecisionError( + f"broker snapshot {field} is required for an " + "account-bound decision" + ) + observation, source = _validate_broker_binding_times( + signal_as_of=signal_as_of, + next_trading_session=next_trading_session, + first_window=first_window, + declared_signal_as_of=snapshot["signal_as_of"], + declared_next_trading_session=snapshot.get( + "next_trading_session" + ), + declared_first_window=snapshot.get("first_executable_window"), + observation_as_of=snapshot["observation_as_of"], + snapshot_as_of=snapshot["as_of"], + source_time=snapshot["source_time"], + ) + current: dict[str, int] = {} + sellable: dict[str, int] = {} + for position in snapshot["positions"]: + symbol = normalize_symbol(position["symbol"]) + if symbol in current: + raise SnapshotDecisionError( + f"broker snapshot has duplicate position {symbol}" + ) + quantity = int(position["quantity"]) + available = int(position["sellable_quantity"]) + if available > quantity: + raise SnapshotDecisionError( + f"sellable quantity exceeds holdings for {symbol}" + ) + current[symbol] = quantity + sellable[symbol] = available + identity = { + "snapshot_id": snapshot["snapshot_id"], + "signal_as_of": signal_text, + "next_trading_session": next_trading_session.isoformat(), + "first_executable_window": dict(first_window), + "observation_as_of": observation.isoformat(), + "as_of": snapshot["as_of"], + "source_time": source.isoformat(), + "broker": snapshot["broker"], + "account_hash": snapshot["account_hash"], + "snapshot_sha256": _sha256_payload(snapshot), + } + return ( + _finite_positive(snapshot["total_asset"], "broker total_asset"), + current, + sellable, + identity, + ) + + +def _dated_rows( + rows: list[dict[str, Any]], + *, + field: str, + as_of: date, +) -> list[tuple[date, dict[str, Any]]]: + output: list[tuple[date, dict[str, Any]]] = [] + for row in rows: + row_date = date.fromisoformat(str(row[field])) + if row_date <= as_of: + output.append((row_date, row)) + return output + + +def build_snapshot_decision( + *, + snapshot_manifest: str | Path, + config: BacktestConfig, + as_of: date, + equity: float | None = None, + broker_snapshot: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build a PIT decision from an immutable JQData-compatible snapshot. + + Raw prices remain the execution/notional price. Momentum uses the + provider adjustment factor normalized to the decision date, so a later + corporate action never changes an earlier decision. + """ + + snapshot = load_verified_data_snapshot(snapshot_manifest) + return _build_snapshot_decision_from_verified_snapshot( + snapshot=snapshot, + config=config, + as_of=as_of, + equity=equity, + broker_snapshot=broker_snapshot, + ) + + +def _build_snapshot_decision_from_verified_snapshot( + *, + snapshot: Mapping[str, Any], + config: BacktestConfig, + as_of: date, + equity: float | None = None, + broker_snapshot: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build a decision from a snapshot already verified by this process. + + This private entry point lets a historical backtest validate and parse a + large provider snapshot once, then reuse that immutable in-memory object + for every decision date. External callers must use + :func:`build_snapshot_decision`, which performs the verification itself. + """ + + config.validate() + if snapshot.get("verification", {}).get("ok") is not True: + raise SnapshotDecisionError("provider snapshot is not verified") + data_manifest = snapshot["manifest"] + query = data_manifest.get("query", {}) + if query.get("adjustment") != "none": + raise SnapshotDecisionError( + "snapshot decision requires raw OHLC plus adjustment factor" + ) + if config.universe.mode != "pit_index": + raise SnapshotDecisionError( + "provider snapshot decisions require universe.mode=pit_index" + ) + query_index = query.get("index_symbol") + if not query_index or normalize_symbol(query_index) != normalize_symbol( + config.universe.index_symbol + ): + raise SnapshotDecisionError( + "snapshot index does not match configured PIT index" + ) + signal_as_of = _signal_close_timestamp(as_of) + as_of_timestamp = signal_as_of.isoformat() + next_trading_session = _next_session_for_decision( + data_manifest, + as_of, + ) + if next_trading_session <= as_of: + raise SnapshotDecisionError( + "next_trading_session must strictly follow decision date" + ) + executable_window = first_executable_window(next_trading_session) + account_equity, current, sellable, broker_identity = _broker_state( + broker_snapshot, + signal_as_of=signal_as_of, + next_trading_session=next_trading_session, + first_window=executable_window, + equity=equity, + ) + memberships = { + normalize_symbol(row["member_symbol"]) + for row_date, row in _dated_rows( + snapshot["memberships"], + field="effective_date", + as_of=as_of, + ) + if row_date == as_of + } + if not memberships: + raise SnapshotDecisionError( + f"no exact PIT index membership on {as_of.isoformat()}" + ) + + histories: dict[str, list[tuple[date, dict[str, Any]]]] = {} + for row_date, row in _dated_rows( + snapshot["bars"], + field="trading_date", + as_of=as_of, + ): + symbol = normalize_symbol(row["symbol"]) + histories.setdefault(symbol, []).append((row_date, row)) + for symbol in histories: + histories[symbol].sort(key=lambda item: item[0]) + + required = ( + config.strategy.lookback + config.strategy.skip + 1 + ) + universe: dict[str, dict[str, Any]] = {} + portable_history: dict[str, list[float]] = {} + raw_prices: dict[str, float] = {} + fixed_symbols: set[str] = set() + exit_symbols: set[str] = set() + + for symbol in sorted(memberships | set(current)): + history = histories.get(symbol, []) + latest = history[-1] if history else None + exact_bar = latest is not None and latest[0] == as_of + latest_row = latest[1] if latest is not None else None + paused = bool(latest_row.get("paused")) if exact_bar else True + is_st = ( + bool(latest_row.get("is_st")) + if exact_bar and config.universe.exclude_st + else False + ) + has_factor_history = ( + len(history) >= required + and all( + item[1].get("adjustment_factor") is not None + and _finite_positive( + item[1]["adjustment_factor"], + f"{symbol} adjustment_factor", + ) + > 0 + for item in history[-required:] + ) + ) + held = current.get(symbol, 0) > 0 + is_member = symbol in memberships + reasons: list[str] = [] + if ( + is_member + and exact_bar + and not paused + and not is_st + and has_factor_history + ): + state = UniverseState.OPENABLE + base_factor = _finite_positive( + latest_row["adjustment_factor"], + f"{symbol} decision adjustment_factor", + ) + portable_history[symbol] = [ + _finite_positive(row["close"], f"{symbol} close") + * _finite_positive( + row["adjustment_factor"], + f"{symbol} adjustment_factor", + ) + / base_factor + for _, row in history + ] + raw_prices[symbol] = _finite_positive( + latest_row["close"], + f"{symbol} raw close", + ) + elif held and (not exact_bar or paused): + state = UniverseState.FROZEN + reasons.append( + "NO_CURRENT_BAR" if not exact_bar else "PAUSED" + ) + fixed_symbols.add(symbol) + elif held and is_st: + state = UniverseState.SELL_ONLY + reasons.append("ST_EXCLUDED") + exit_symbols.add(symbol) + elif held and is_member: + state = UniverseState.HOLD_ONLY + reasons.append("INSUFFICIENT_ADJUSTED_HISTORY") + fixed_symbols.add(symbol) + elif held: + state = UniverseState.SELL_ONLY + reasons.append("NOT_IN_PIT_INDEX") + exit_symbols.add(symbol) + else: + state = UniverseState.EXCLUDED + if not exact_bar: + reasons.append("NO_CURRENT_BAR") + elif paused: + reasons.append("PAUSED") + elif is_st: + reasons.append("ST_EXCLUDED") + else: + reasons.append("INSUFFICIENT_ADJUSTED_HISTORY") + if latest_row is not None: + raw_prices[symbol] = _finite_positive( + latest_row["close"], + f"{symbol} latest raw close", + ) + universe[symbol] = { + "state": state.value, + "reasons": reasons, + "member": is_member, + "held_quantity": current.get(symbol, 0), + "is_st": is_st, + "last_bar_date": latest[0].isoformat() if latest else None, + } + + if len(portable_history) < config.strategy.top_n: + raise SnapshotDecisionError( + "openable universe is smaller than top_n after PIT/data gates: " + f"{len(portable_history)} < {config.strategy.top_n}" + ) + for symbol in current: + if symbol not in raw_prices: + raise SnapshotDecisionError( + f"held security {symbol} has no price at or before as_of" + ) + + current_weights = weights_from_positions( + current, + raw_prices, + account_equity, + ) + fixed_weight = sum( + current_weights.get(symbol, 0.0) for symbol in fixed_symbols + ) + gross_ceiling = min( + config.strategy.gross_target, + 1.0 - config.strategy.cash_buffer, + ) + remaining_gross = max(0.0, gross_ceiling - fixed_weight) + ranked = rank_momentum( + portable_history, + lookback=config.strategy.lookback, + skip=config.strategy.skip, + top_n=config.strategy.top_n, + ) + scores = { + symbol: momentum_score( + history, + config.strategy.lookback, + config.strategy.skip, + ) + for symbol, history in sorted(portable_history.items()) + } + active_weights = capped_target_weights( + dict(ranked), + max_weight=config.strategy.max_weight, + gross_target=remaining_gross, + min_score=0.0, + ) + for symbol in portable_history: + active_weights.setdefault(symbol, 0.0) + active_targets = target_quantities( + active_weights, + raw_prices, + account_equity, + lot_size=config.execution.lot_size, + cash_buffer=config.strategy.cash_buffer, + ) + weights = dict(active_weights) + targets = dict(active_targets) + for symbol in fixed_symbols: + weights[symbol] = current_weights[symbol] + targets[symbol] = current[symbol] + for symbol in exit_symbols: + weights[symbol] = 0.0 + targets[symbol] = 0 + orders = order_deltas( + targets, + current, + sellable, + lot_size=config.execution.lot_size, + ) + for symbol in fixed_symbols: + orders.pop(symbol, None) + + risk = check_target_weights( + weights, + current_weights, + max_single_weight=config.strategy.max_weight, + max_gross_weight=gross_ceiling, + max_one_way_turnover=1.0, + ) + risk.require_approved() + + config_body = { + "universe": config.as_dict()["universe"], + "strategy": config.as_dict()["strategy"], + "execution": config.as_dict()["execution"], + "dynamic_universe": query.get("index_symbol"), + "clock": "T_CLOSE_COMPLETE_TO_T_PLUS_1_OPEN", + } + config_hash = _sha256_payload(config_body) + engine_source_sha256, engine_sources = _engine_source_manifest() + contracts_sha256, contract_schemas = _contract_manifest() + input_manifest_sha256 = _sha256_bytes( + Path(snapshot["manifest_file"]).read_bytes() + ) + run_id = "q60d-" + _sha256_payload( + { + "as_of": as_of.isoformat(), + "next_trading_session": next_trading_session.isoformat(), + "first_executable_window": executable_window, + "data_version": data_manifest["data_version"], + "config_hash": config_hash, + "equity": account_equity, + "current": current, + "sellable": sellable, + "broker": broker_identity, + "engine_source_sha256": engine_source_sha256, + } + )[:16] + decision_id = f"D-{as_of.strftime('%Y%m%d')}-{run_id[-8:]}" + signal_records = [] + for symbol, score in sorted(scores.items()): + if score is None or not math.isfinite(float(score)): + raise SnapshotDecisionError(f"non-finite score for {symbol}") + signal_records.append( + { + "schema_version": "1.0", + "run_id": run_id, + "signal_id": f"S-{as_of.strftime('%Y%m%d')}-{symbol}", + "as_of": as_of_timestamp, + "symbol": symbol, + "horizon_trading_days": config.strategy.rebalance_every, + "score": float(score), + "expected_excess_return": None, + "confidence": None, + "model_version": "portable-momentum-v1", + "data_version": data_manifest["data_version"], + "feature_version": ( + "pit_front_adjusted_close_momentum_" + f"{config.strategy.lookback}_skip_{config.strategy.skip}" + ), + "metadata": { + "selected": weights.get(symbol, 0.0) > 0, + "raw_price": raw_prices[symbol], + "universe_state": universe[symbol]["state"], + }, + } + ) + target_records = [] + for symbol, quantity in sorted(targets.items()): + target_records.append( + { + "schema_version": "1.0", + "run_id": run_id, + "decision_id": decision_id, + "as_of": as_of_timestamp, + "symbol": symbol, + "target_weight": float(weights.get(symbol, 0.0)), + "target_quantity": int(quantity), + "lot_size": config.execution.lot_size, + "signal_id": ( + f"S-{as_of.strftime('%Y%m%d')}-{symbol}" + if symbol in scores + else None + ), + "constraint_state": "FEASIBLE", + "config_hash": config_hash, + "metadata": { + "universe_state": universe[symbol]["state"], + "reasons": universe[symbol]["reasons"], + }, + } + ) + validate_records(signal_records, "signal.schema.json") + validate_records(target_records, "target.schema.json") + decision = { + "schema_version": "1.0", + "run_id": run_id, + "decision_id": decision_id, + "as_of": as_of_timestamp, + "signal_as_of": as_of_timestamp, + "next_trading_session": next_trading_session.isoformat(), + "first_executable_window": executable_window, + "evidence_class": "provider_snapshot_decision", + "provider": data_manifest["provider"], + "provider_version": data_manifest["provider_version"], + "data_version": data_manifest["data_version"], + "input_manifest_sha256": input_manifest_sha256, + "config_hash": config_hash, + "model_version": "portable-momentum-v1", + "equity": account_equity, + "weights": dict(sorted(weights.items())), + "targets": dict(sorted(targets.items())), + "orders": dict(sorted(orders.items())), + "universe": universe, + "risk": { + "approved": risk.approved, + "gross_weight": risk.gross_weight, + "one_way_turnover": risk.one_way_turnover, + "violations": [], + }, + "broker_snapshot": broker_identity, + "investment_value_claim": False, + "real_platform_backtest": False, + } + manifest = { + "schema_version": "1.0", + "run_id": run_id, + "evidence_class": "provider_snapshot_decision", + "data_version": data_manifest["data_version"], + "input_manifest_sha256": input_manifest_sha256, + "config_hash": config_hash, + "engine_source_sha256": engine_source_sha256, + "engine_sources": engine_sources, + "contracts_sha256": contracts_sha256, + "contract_schemas": contract_schemas, + "model_version": "portable-momentum-v1", + "signal_as_of": as_of_timestamp, + "signal_clock": "T_CLOSE_COMPLETE", + "first_executable_clock": "T_PLUS_1_OPEN", + "next_trading_session": next_trading_session.isoformat(), + "first_executable_window": executable_window, + "price_adjustment": "PIT_FRONT_FROM_RAW_AND_FACTOR", + "deterministic": True, + } + return { + "signals": signal_records, + "targets": target_records, + "decision": decision, + "input_data_manifest": data_manifest, + "manifest": manifest, + } + + +def write_snapshot_decision( + result: Mapping[str, Any], + output_dir: str | Path, +) -> dict[str, str]: + root = Path(output_dir).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + marker = root / ".quant60-incomplete" + _atomic_json(marker, {"run_id": result["manifest"]["run_id"]}) + paths = { + "signals": root / "signals.json", + "targets": root / "targets.json", + "decision": root / "decision.json", + "input_data_manifest": root / "input_data_manifest.json", + } + for name, path in paths.items(): + _atomic_json(path, result[name]) + manifest = dict(result["manifest"]) + manifest["artifact_sha256"] = { + path.name: _sha256_bytes(path.read_bytes()) + for path in paths.values() + } + manifest_path = root / "manifest.json" + _atomic_json(manifest_path, manifest) + marker.unlink() + paths["manifest"] = manifest_path + return {name: str(path) for name, path in paths.items()} + + +def _validate_saved_decision_clocks( + *, + manifest: Mapping[str, Any], + decision: Mapping[str, Any], + signals: list[Mapping[str, Any]], + targets: list[Mapping[str, Any]], +) -> datetime: + signal_as_of = _parse_aware_timestamp( + decision.get("signal_as_of"), + "decision.signal_as_of", + ) + decision_as_of = _parse_aware_timestamp( + decision.get("as_of"), + "decision.as_of", + ) + manifest_signal_as_of = _parse_aware_timestamp( + manifest.get("signal_as_of"), + "manifest.signal_as_of", + ) + if not _same_instant(signal_as_of, decision_as_of): + raise SnapshotDecisionError( + "decision signal_as_of/as_of identity mismatch" + ) + if not _same_instant(signal_as_of, manifest_signal_as_of): + raise SnapshotDecisionError( + "decision/manifest signal_as_of identity mismatch" + ) + if ( + manifest.get("signal_clock") != "T_CLOSE_COMPLETE" + or manifest.get("first_executable_clock") != "T_PLUS_1_OPEN" + ): + raise SnapshotDecisionError( + "snapshot decision clock labels are invalid" + ) + shanghai_signal = signal_as_of.astimezone(SHANGHAI_TIMEZONE) + if ( + shanghai_signal.hour, + shanghai_signal.minute, + shanghai_signal.second, + shanghai_signal.microsecond, + ) != (15, 0, 0, 0): + raise SnapshotDecisionError( + "decision signal_as_of must be the 15:00 Asia/Shanghai " + "completed-close clock" + ) + try: + next_trading_session = date.fromisoformat( + str(decision.get("next_trading_session")) + ) + except ValueError as exc: + raise SnapshotDecisionError( + "decision.next_trading_session must be an ISO date" + ) from exc + if next_trading_session <= shanghai_signal.date(): + raise SnapshotDecisionError( + "decision.next_trading_session must follow signal date" + ) + if ( + manifest.get("next_trading_session") + != next_trading_session.isoformat() + ): + raise SnapshotDecisionError( + "decision/manifest next_trading_session mismatch" + ) + decision_window = decision.get("first_executable_window") + manifest_window = manifest.get("first_executable_window") + _validated_first_executable_window( + decision_window, + next_trading_session=next_trading_session, + name="decision.first_executable_window", + ) + if dict(manifest_window or {}) != dict(decision_window or {}): + raise SnapshotDecisionError( + "decision/manifest first_executable_window mismatch" + ) + for collection_name, records in ( + ("signal", signals), + ("target", targets), + ): + for index, record in enumerate(records): + record_as_of = _parse_aware_timestamp( + record.get("as_of"), + f"{collection_name} #{index} as_of", + ) + if not _same_instant(signal_as_of, record_as_of): + raise SnapshotDecisionError( + f"{collection_name} #{index} as_of does not match " + "decision signal_as_of" + ) + broker_identity = decision.get("broker_snapshot") + if broker_identity is not None: + if not isinstance(broker_identity, Mapping): + raise SnapshotDecisionError( + "decision broker_snapshot identity must be an object" + ) + _validate_broker_binding_times( + signal_as_of=signal_as_of, + next_trading_session=next_trading_session, + first_window=decision_window, + declared_signal_as_of=broker_identity.get("signal_as_of"), + declared_next_trading_session=broker_identity.get( + "next_trading_session" + ), + declared_first_window=broker_identity.get( + "first_executable_window" + ), + observation_as_of=broker_identity.get("observation_as_of"), + snapshot_as_of=broker_identity.get("as_of"), + source_time=broker_identity.get("source_time"), + ) + return signal_as_of + + +def verify_snapshot_decision(manifest_file: str | Path) -> dict[str, Any]: + path = Path(manifest_file).expanduser().resolve() + root = path.parent + if (root / ".quant60-incomplete").exists(): + raise SnapshotDecisionError("snapshot decision publication is incomplete") + manifest = json.loads(path.read_text(encoding="utf-8")) + expected_names = { + "signals.json", + "targets.json", + "decision.json", + "input_data_manifest.json", + } + if set(manifest.get("artifact_sha256", {})) != expected_names: + raise SnapshotDecisionError("snapshot decision artifact set is invalid") + if {entry.name for entry in root.iterdir()} != expected_names | {path.name}: + raise SnapshotDecisionError( + "snapshot decision directory is not an exact set" + ) + for name, expected_hash in manifest["artifact_sha256"].items(): + if _sha256_bytes((root / name).read_bytes()) != expected_hash: + raise SnapshotDecisionError( + f"snapshot decision artifact hash mismatch: {name}" + ) + signals = json.loads((root / "signals.json").read_text(encoding="utf-8")) + targets = json.loads((root / "targets.json").read_text(encoding="utf-8")) + decision = json.loads((root / "decision.json").read_text(encoding="utf-8")) + input_manifest = json.loads( + (root / "input_data_manifest.json").read_text(encoding="utf-8") + ) + validate_records(signals, "signal.schema.json") + validate_records(targets, "target.schema.json") + signal_as_of = _validate_saved_decision_clocks( + manifest=manifest, + decision=decision, + signals=signals, + targets=targets, + ) + input_manifest_file_sha256 = _sha256_bytes( + (root / "input_data_manifest.json").read_bytes() + ) + if ( + input_manifest_file_sha256 + != manifest["artifact_sha256"]["input_data_manifest.json"] + ): + raise SnapshotDecisionError("input data manifest artifact mismatch") + if ( + input_manifest_file_sha256 + != manifest.get("input_manifest_sha256") + or input_manifest_file_sha256 + != decision.get("input_manifest_sha256") + ): + raise SnapshotDecisionError( + "decision input calendar manifest hash binding mismatch" + ) + if decision["run_id"] != manifest["run_id"]: + raise SnapshotDecisionError("decision/manifest run_id mismatch") + if input_manifest["data_version"] != manifest["data_version"]: + raise SnapshotDecisionError("input data_version mismatch") + signal_date = signal_as_of.astimezone(SHANGHAI_TIMEZONE).date() + expected_next_session = _next_session_for_decision( + input_manifest, + signal_date, + ) + if ( + expected_next_session.isoformat() + != manifest.get("next_trading_session") + ): + raise SnapshotDecisionError( + "decision next_trading_session is not the adjacent session in " + "the verified input calendar" + ) + current_source, unused_sources = _engine_source_manifest() + del unused_sources + current_contracts, unused_contracts = _contract_manifest() + del unused_contracts + if current_source != manifest["engine_source_sha256"]: + raise SnapshotDecisionError( + "current Quant OS source does not match decision manifest" + ) + if current_contracts != manifest["contracts_sha256"]: + raise SnapshotDecisionError( + "current Quant OS schemas do not match decision manifest" + ) + return { + "ok": True, + "run_id": manifest["run_id"], + "data_version": manifest["data_version"], + "signal_count": len(signals), + "target_count": len(targets), + "order_count": len(decision["orders"]), + "signal_as_of": signal_as_of.isoformat(), + "next_trading_session": expected_next_session.isoformat(), + "first_executable_window": manifest["first_executable_window"], + "evidence_class": manifest["evidence_class"], + } diff --git a/src/quant60/synthetic.py b/src/quant60/synthetic.py new file mode 100644 index 0000000..88d51b9 --- /dev/null +++ b/src/quant60/synthetic.py @@ -0,0 +1,194 @@ +"""Deterministic synthetic A-share daily data for offline smoke tests.""" + +from __future__ import annotations + +import math +from datetime import date, timedelta +from decimal import Decimal, ROUND_HALF_UP +from typing import Iterable + +from .domain import Bar +from .portable_core import normalize_symbol + + +CENT = Decimal("0.01") + + +class _StableRng: + """Tiny fixed LCG whose output does not depend on Python's random module.""" + + def __init__(self, seed: int) -> None: + self.state = int(seed) & 0x7FFFFFFF + + def uniform(self) -> float: + self.state = (1103515245 * self.state + 12345) & 0x7FFFFFFF + return self.state / 2147483648.0 + + def centered(self) -> float: + return sum(self.uniform() for _ in range(6)) - 3.0 + + +def trading_dates(start: date, count: int) -> list[date]: + if count <= 0: + return [] + result: list[date] = [] + current = start + while len(result) < count: + if current.weekday() < 5: + result.append(current) + current += timedelta(days=1) + return result + + +def _limit_ratio(symbol: str) -> Decimal: + code = symbol[:6] + if symbol.endswith(".XBSE"): + return Decimal("0.30") + if code.startswith(("300", "301", "688", "689")): + return Decimal("0.20") + return Decimal("0.10") + + +def _tick(value: Decimal | float) -> Decimal: + return Decimal(str(value)).quantize(CENT, rounding=ROUND_HALF_UP) + + +def generate_synthetic_bars( + symbols: Iterable[str], + start_date: date | str = date(2024, 1, 2), + trading_days_count: int = 90, + seed: int = 60, + base_volume: int = 100_000, + include_market_events: bool = True, +) -> list[Bar]: + """Generate stable OHLCV with suspensions and locked limit events. + + Each symbol gets one suspension, one locked limit-up and one locked + limit-down session when the requested horizon is long enough. The event + placement is deterministic and staggered across symbols. + """ + + canonical_symbols = sorted({normalize_symbol(symbol) for symbol in symbols}) + if not canonical_symbols: + raise ValueError("at least one symbol is required") + if isinstance(start_date, str): + start_date = date.fromisoformat(start_date) + if trading_days_count <= 1 or base_volume < 100: + raise ValueError("invalid synthetic horizon/base_volume") + + dates = trading_dates(start_date, trading_days_count) + rng = _StableRng(seed) + result: list[Bar] = [] + previous: dict[str, Decimal] = { + symbol: _tick(8 + index * 7.25) + for index, symbol in enumerate(canonical_symbols) + } + + for day_index, trading_date in enumerate(dates): + for symbol_index, symbol in enumerate(canonical_symbols): + previous_close = previous[symbol] + ratio = _limit_ratio(symbol) + limit_up = _tick(previous_close * (Decimal("1") + ratio)) + limit_down = _tick(previous_close * (Decimal("1") - ratio)) + suspended_day = 11 + symbol_index * 3 + limit_up_day = 24 + symbol_index * 2 + limit_down_day = 39 + symbol_index * 2 + + if include_market_events and day_index == suspended_day: + bar = Bar( + trading_date=trading_date, + symbol=symbol, + open=previous_close, + high=previous_close, + low=previous_close, + close=previous_close, + volume=0, + previous_close=previous_close, + limit_up=limit_up, + limit_down=limit_down, + suspended=True, + ) + elif include_market_events and day_index == limit_up_day: + bar = Bar( + trading_date=trading_date, + symbol=symbol, + open=limit_up, + high=limit_up, + low=limit_up, + close=limit_up, + volume=max(100, base_volume // 20), + previous_close=previous_close, + limit_up=limit_up, + limit_down=limit_down, + ) + elif include_market_events and day_index == limit_down_day: + bar = Bar( + trading_date=trading_date, + symbol=symbol, + open=limit_down, + high=limit_down, + low=limit_down, + close=limit_down, + volume=max(100, base_volume // 20), + previous_close=previous_close, + limit_up=limit_up, + limit_down=limit_down, + ) + else: + drift = (symbol_index - (len(canonical_symbols) - 1) / 2) * 0.0007 + cycle = math.sin((day_index + symbol_index * 5) / 8.0) * 0.003 + noise = rng.centered() * 0.004 + close_return = max( + -float(ratio) * 0.85, + min(float(ratio) * 0.85, drift + cycle + noise), + ) + gap = rng.centered() * 0.0015 + open_price = _tick(previous_close * Decimal(str(1 + gap))) + close_price = _tick( + previous_close * Decimal(str(1 + close_return)) + ) + open_price = min(limit_up, max(limit_down, open_price)) + close_price = min(limit_up, max(limit_down, close_price)) + intraday = Decimal( + str(abs(rng.centered()) * 0.0025 + 0.001) + ) + high = _tick( + max(open_price, close_price) + * (Decimal("1") + intraday) + ) + low = _tick( + min(open_price, close_price) + * (Decimal("1") - intraday) + ) + high = min(limit_up, max(high, open_price, close_price)) + low = max(limit_down, min(low, open_price, close_price)) + volume_factor = 0.7 + rng.uniform() * 0.8 + volume = int(base_volume * volume_factor) // 100 * 100 + bar = Bar( + trading_date=trading_date, + symbol=symbol, + open=open_price, + high=high, + low=low, + close=close_price, + volume=volume, + previous_close=previous_close, + limit_up=limit_up, + limit_down=limit_down, + ) + result.append(bar) + previous[symbol] = bar.close + result.sort(key=lambda bar: (bar.trading_date, bar.symbol)) + return result + + +def group_bars_by_date(bars: Iterable[Bar]) -> dict[date, dict[str, Bar]]: + grouped: dict[date, dict[str, Bar]] = {} + for bar in bars: + session = grouped.setdefault(bar.trading_date, {}) + if bar.symbol in session: + raise ValueError( + f"duplicate bar for {bar.symbol} on {bar.trading_date}" + ) + session[bar.symbol] = bar + return dict(sorted(grouped.items())) diff --git a/src/quant60/universe.py b/src/quant60/universe.py new file mode 100644 index 0000000..5e61fae --- /dev/null +++ b/src/quant60/universe.py @@ -0,0 +1,181 @@ +"""Point-in-time A-share universe state machine with explicit reason codes.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from enum import Enum + +from .portable_core import normalize_symbol + + +class UniverseState(str, Enum): + OPENABLE = "openable" + HOLD_ONLY = "hold_only" + SELL_ONLY = "sell_only" + FROZEN = "frozen" + EXCLUDED = "excluded" + + +@dataclass(frozen=True, slots=True) +class SecurityEligibility: + symbol: str + index_member: bool + listing_sessions: int + median_turnover: float + minimum_turnover: float + is_st: bool = False + delisting: bool = False + suspended: bool = False + valid_quote: bool = True + locked_limit_up: bool = False + locked_limit_down: bool = False + held_quantity: int = 0 + sellable_quantity: int = 0 + + def __post_init__(self) -> None: + object.__setattr__(self, "symbol", normalize_symbol(self.symbol)) + for name in ( + "index_member", + "is_st", + "delisting", + "suspended", + "valid_quote", + "locked_limit_up", + "locked_limit_down", + ): + if type(getattr(self, name)) is not bool: + raise ValueError(f"{name} must be a bool") + if type(self.listing_sessions) is not int or self.listing_sessions < 0: + raise ValueError("listing_sessions must be a non-negative integer") + if type(self.held_quantity) is not int or self.held_quantity < 0: + raise ValueError("held_quantity must be a non-negative integer") + if ( + type(self.sellable_quantity) is not int + or self.sellable_quantity < 0 + or self.sellable_quantity > self.held_quantity + ): + raise ValueError("sellable_quantity must lie within held quantity") + for name in ("median_turnover", "minimum_turnover"): + value = float(getattr(self, name)) + if not math.isfinite(value) or value < 0: + raise ValueError(f"{name} must be finite and non-negative") + + +@dataclass(frozen=True, slots=True) +class UniverseDecision: + symbol: str + state: UniverseState + reasons: tuple[str, ...] + can_open: bool + can_hold: bool + can_sell: bool + + +def classify_security( + eligibility: SecurityEligibility, + *, + seasoning_sessions: int = 120, +) -> UniverseDecision: + if type(seasoning_sessions) is not int or seasoning_sessions < 0: + raise ValueError("seasoning_sessions must be a non-negative integer") + item = eligibility + held = item.held_quantity > 0 + reasons: list[str] = [] + + if not item.index_member: + reasons.append("NOT_PIT_INDEX_MEMBER") + if item.listing_sessions < seasoning_sessions: + reasons.append("IPO_SEASONING") + if item.is_st: + reasons.append("RISK_WARNING") + if item.delisting: + reasons.append("DELISTING") + if item.median_turnover < item.minimum_turnover: + reasons.append("LIQUIDITY_BELOW_FLOOR") + if item.suspended: + reasons.append("SUSPENDED") + if not item.valid_quote: + reasons.append("NO_VALID_QUOTE") + if item.locked_limit_up: + reasons.append("LOCKED_LIMIT_UP") + if item.locked_limit_down: + reasons.append("LOCKED_LIMIT_DOWN") + if held and item.sellable_quantity <= 0: + reasons.append("NO_SELLABLE_QUANTITY") + + completely_frozen = ( + item.suspended + or not item.valid_quote + or (held and item.locked_limit_down) + or (held and item.sellable_quantity <= 0) + ) + structural_exit = ( + not item.index_member + or item.is_st + or item.delisting + ) + disallow_new = ( + structural_exit + or item.listing_sessions < seasoning_sessions + or item.median_turnover < item.minimum_turnover + or item.suspended + or not item.valid_quote + or item.locked_limit_up + or item.locked_limit_down + ) + + if held and completely_frozen: + state = UniverseState.FROZEN + can_open = False + can_hold = True + can_sell = False + elif held and disallow_new: + state = ( + UniverseState.SELL_ONLY + if structural_exit + else UniverseState.HOLD_ONLY + ) + can_open = False + can_hold = True + can_sell = item.sellable_quantity > 0 + elif held: + state = UniverseState.OPENABLE + can_open = True + can_hold = True + can_sell = item.sellable_quantity > 0 + elif disallow_new: + state = UniverseState.EXCLUDED + can_open = False + can_hold = False + can_sell = False + else: + state = UniverseState.OPENABLE + can_open = True + can_hold = True + can_sell = False + + return UniverseDecision( + symbol=item.symbol, + state=state, + reasons=tuple(reasons or ["ELIGIBLE"]), + can_open=can_open, + can_hold=can_hold, + can_sell=can_sell, + ) + + +def build_universe( + entries: list[SecurityEligibility], + *, + seasoning_sessions: int = 120, +) -> dict[str, UniverseDecision]: + result: dict[str, UniverseDecision] = {} + for entry in sorted(entries, key=lambda item: item.symbol): + if entry.symbol in result: + raise ValueError(f"duplicate universe symbol {entry.symbol}") + result[entry.symbol] = classify_security( + entry, + seasoning_sessions=seasoning_sessions, + ) + return result diff --git a/tests/platform_bundle_probe_test.py b/tests/platform_bundle_probe_test.py new file mode 100644 index 0000000..f0bed93 --- /dev/null +++ b/tests/platform_bundle_probe_test.py @@ -0,0 +1,132 @@ +import hashlib +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "src")) + +from platforms import joinquant_strategy, qmt_builtin_strategy +from tools.bundle_platforms import bundle_all, effective_platform_config +from tools.capability_probe import probe_capabilities + + +class BundleTest(unittest.TestCase): + def test_source_wrappers_match_canonical_baseline_config(self): + baseline = json.loads( + (ROOT / "configs/baseline.json").read_text(encoding="utf-8") + ) + self.assertEqual( + joinquant_strategy.CONFIG, + effective_platform_config(baseline, "joinquant"), + ) + self.assertEqual( + qmt_builtin_strategy.CONFIG, + effective_platform_config(baseline, "qmt_builtin"), + ) + + def test_bundles_inline_same_core_compile_and_match_manifest(self): + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) + manifest = bundle_all(ROOT, output) + manifest_file = output / "bundle_manifest.json" + stored = json.loads(manifest_file.read_text(encoding="ascii")) + for name, filename in { + "joinquant": "joinquant_strategy.py", + "qmt_builtin": "qmt_builtin_strategy.py", + }.items(): + artifact = output / filename + payload = artifact.read_bytes() + compile(payload, str(artifact), "exec") + self.assertNotIn(b"from quant60.portable_core import", payload) + self.assertEqual( + hashlib.sha256(payload).hexdigest(), + stored["artifacts"][name]["output_sha256"], + ) + (output / "qmt_builtin_strategy.py").read_bytes().decode("ascii") + self.assertEqual( + stored["artifacts"]["joinquant"]["core"], + "src/quant60/portable_core.py", + ) + self.assertNotIn( + "/Users/", json.dumps(stored, ensure_ascii=True) + ) + self.assertEqual( + stored["portable_core_sha256"], + manifest["portable_core_sha256"], + ) + self.assertEqual(stored["baseline_config"], "configs/baseline.json") + self.assertEqual( + stored["baseline_config_sha256"], + hashlib.sha256( + (ROOT / "configs/baseline.json").read_bytes() + ).hexdigest(), + ) + + def test_committed_dist_is_byte_for_byte_fresh(self): + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) + bundle_all(ROOT, output) + for name in ( + "joinquant_strategy.py", + "qmt_builtin_strategy.py", + "bundle_manifest.json", + ): + self.assertEqual( + (ROOT / "dist" / name).read_bytes(), + (output / name).read_bytes(), + f"committed dist artifact is stale: {name}", + ) + + def test_joinquant_bundle_ignores_host_shadowed_sum_name(self): + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) + bundle_all(ROOT, output) + source = (output / "joinquant_strategy.py").read_text( + encoding="utf-8" + ) + namespace = {"sum": lambda values: values} + exec(compile(source, "joinquant_strategy.py", "exec"), namespace) + weights = namespace["capped_target_weights"]( + { + "600000.XSHG": 3.0, + "000001.XSHE": 2.0, + "300750.XSHE": 1.0, + }, + max_weight=0.4, + gross_target=0.95, + ) + self.assertAlmostEqual( + namespace["_sum_numeric"](weights.values()), + 0.95, + ) + quantities = namespace["target_quantities"]( + weights, + {symbol: 10.0 for symbol in weights}, + equity=1_000_000, + cash_buffer=0.02, + ) + self.assertTrue(all(value >= 0 for value in quantities.values())) + + +class CapabilityProbeTest(unittest.TestCase): + def test_shallow_probe_does_not_claim_external_verification(self): + report = probe_capabilities(ROOT, deep=False) + self.assertTrue(report["artifacts"]["portable_core"]) + self.assertTrue(report["artifacts"]["colab_notebook"]) + self.assertTrue(report["artifacts"]["jqdata_snapshot_adapter"]) + self.assertTrue(report["artifacts"]["qmt_shadow_planner"]) + self.assertTrue(report["artifacts"]["qmt_shadow_cli"]) + self.assertIn("no external platform", report["verification_boundary"]) + self.assertTrue(report["matrix"]["xttrader"]["safe_default"]) + self.assertIn("rejects BJ", report["matrix"]["xttrader"]["market_scope"]) + for probe in report["optional_runtimes"].values(): + self.assertIs(probe["import_ok"], None) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/platform_joinquant_qmt_parity_test.py b/tests/platform_joinquant_qmt_parity_test.py new file mode 100644 index 0000000..0bcddd7 --- /dev/null +++ b/tests/platform_joinquant_qmt_parity_test.py @@ -0,0 +1,460 @@ +import datetime +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "src")) + +from platforms import joinquant_strategy, qmt_builtin_strategy +from platforms.fake_joinquant_harness import FakeJoinQuantHarness, FakePosition +from platforms.fake_qmt_harness import FakeQmtContext, FakeQmtHarness + + +def price_histories(): + return { + "600000.XSHG": [15.0 - index * 0.02 for index in range(21)], + "000001.XSHE": [10.0 + index * 0.10 for index in range(21)], + "300750.XSHE": [100.0 + index * 1.5 for index in range(21)], + "000333.XSHE": [20.0 + index * 0.05 for index in range(21)], + } + + +class JoinQuantContractTest(unittest.TestCase): + def setUp(self): + self.harness = FakeJoinQuantHarness(price_histories()) + self.harness.initialize(joinquant_strategy) + + def test_initialize_enables_future_data_guard_and_weekly_open(self): + self.assertIs(self.harness.options["avoid_future_data"], True) + self.assertIs(self.harness.options["use_real_price"], True) + self.assertEqual( + self.harness.options["order_volume_ratio"], + 0.1, + ) + self.assertEqual(self.harness.benchmark, "000905.XSHG") + self.assertEqual(self.harness.order_cost["type"], "stock") + self.assertAlmostEqual( + self.harness.order_cost["cost"].open_commission, + 0.00021, + ) + self.assertAlmostEqual( + self.harness.order_cost["cost"].close_tax, + 0.0005, + ) + self.assertEqual(self.harness.slippage["type"], "stock") + self.assertAlmostEqual( + self.harness.slippage["slippage"].value, + 0.0004, + ) + self.assertEqual(len(self.harness.schedules), 1) + _, run_time = self.harness.schedules[0] + self.assertEqual(run_time, "open") + + def test_rebalance_submits_exact_portable_executable_deltas(self): + self.assertIsNone(self.harness.run_scheduled()) + self.harness.advance_session() + plan = self.harness.run_scheduled() + self.assertEqual(plan, self.harness.g.quant60_last_plan) + self.assertTrue(self.harness.orders) + ordered_symbols = {symbol for symbol, _ in self.harness.orders} + expected = { + symbol: int(delta) + for symbol, delta in plan["orders"].items() + if int(delta) != 0 + } + self.assertEqual(ordered_symbols, set(expected)) + self.assertEqual( + dict(self.harness.orders), + expected, + ) + + def test_missing_current_data_fails_closed_for_new_position(self): + best = "300750.XSHE" + del self.harness.current_data[best] + self.assertIsNone(self.harness.run_scheduled()) + self.harness.advance_session() + self.harness.run_scheduled() + ordered_symbols = {symbol for symbol, _ in self.harness.orders} + self.assertNotIn(best, ordered_symbols) + self.assertEqual( + self.harness.g.quant60_skipped_orders[-1]["reason"], + "current_data_unavailable", + ) + + def test_star_market_buy_uses_daily_high_limit_protection(self): + histories = price_histories() + star = "688301.XSHG" + histories[star] = [20.0 + index * 0.3 for index in range(21)] + harness = FakeJoinQuantHarness(histories) + harness.current_data[star].high_limit = 31.28 + harness.initialize(joinquant_strategy) + self.assertIsNone(harness.run_scheduled()) + harness.advance_session() + plan = harness.run_scheduled() + self.assertGreater(plan["orders"][star], 0) + styles = dict(harness.order_styles) + self.assertEqual(styles[star].kind, "limit") + self.assertAlmostEqual(styles[star].limit_price, 31.28) + + def test_star_market_sell_uses_daily_low_limit_protection(self): + histories = price_histories() + star = "688301.XSHG" + histories[star] = [20.0 + index * 0.3 for index in range(21)] + harness = FakeJoinQuantHarness( + histories, + total_value=1000.0, + positions={star: FakePosition(1000)}, + ) + harness.current_data[star].low_limit = 17.06 + harness.initialize(joinquant_strategy) + self.assertIsNone(harness.run_scheduled()) + harness.advance_session() + plan = harness.run_scheduled() + self.assertEqual(plan["orders"][star], -1000) + styles = dict(harness.order_styles) + self.assertEqual(styles[star].kind, "limit") + self.assertAlmostEqual(styles[star].limit_price, 17.06) + + def test_invalid_star_protection_price_fails_closed(self): + histories = price_histories() + star = "688301.XSHG" + histories[star] = [20.0 + index * 0.3 for index in range(21)] + harness = FakeJoinQuantHarness(histories) + harness.current_data[star].high_limit = float("nan") + harness.initialize(joinquant_strategy) + self.assertIsNone(harness.run_scheduled()) + harness.advance_session() + plan = harness.run_scheduled() + self.assertGreater(plan["orders"][star], 0) + self.assertNotIn(star, dict(harness.orders)) + self.assertEqual( + harness.g.quant60_skipped_orders[-1]["reason"], + "star_protection_price_invalid", + ) + + def test_incomplete_history_aborts_entire_rebalance_without_orders(self): + histories = price_histories() + histories["300750.XSHE"] = histories["300750.XSHE"][1:] + harness = FakeJoinQuantHarness(histories) + harness.initialize(joinquant_strategy) + self.assertIsNone(harness.run_scheduled()) + harness.advance_session() + with self.assertRaises(joinquant_strategy.PriceHistoryUnavailable): + harness.run_scheduled() + self.assertEqual(harness.orders, []) + + def test_joinquant_unmanaged_position_aborts_without_orders(self): + harness = FakeJoinQuantHarness( + price_histories(), + positions={"601318.XSHG": FakePosition(1000)}, + ) + harness.initialize(joinquant_strategy) + harness.g.quant60_config["universe_mode"] = "fixed" + self.assertIsNone(harness.run_scheduled()) + harness.advance_session() + with self.assertRaises(joinquant_strategy.UnmanagedPositionError): + harness.run_scheduled() + self.assertEqual(harness.orders, []) + + def test_executable_sell_delta_preserves_t1_cap_and_odd_lot(self): + position = FakePosition(total_amount=1050, closeable_amount=200) + harness = FakeJoinQuantHarness( + price_histories(), + total_value=1000.0, + positions={"600000.XSHG": position}, + ) + harness.initialize(joinquant_strategy) + self.assertIsNone(harness.run_scheduled()) + harness.advance_session() + plan = harness.run_scheduled() + self.assertEqual(plan["orders"]["600000.XSHG"], -200) + self.assertEqual(dict(harness.orders)["600000.XSHG"], 850) + + def test_one_session_week_executes_on_next_available_session(self): + self.assertIsNone(self.harness.run_scheduled()) + # Model a holiday week with no second session: the next available + # session is the first session of the next ISO week. + self.harness.advance_session(days=7) + plan = self.harness.run_scheduled() + self.assertIsNotNone(plan) + self.assertTrue(self.harness.orders) + self.assertEqual( + self.harness.g.quant60_pending_first_session, + self.harness.context.current_dt.date(), + ) + + +class CrossPlatformParityTest(unittest.TestCase): + def test_same_completed_close_produces_same_portable_plan(self): + jq = FakeJoinQuantHarness(price_histories()) + jq.initialize(joinquant_strategy) + jq.context.previous_date = datetime.date(2026, 7, 24) + jq.context.current_dt = datetime.datetime(2026, 7, 27, 9, 30) + jq_plan = joinquant_strategy.compute_plan(jq.context) + + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + context = FakeQmtContext( + qmt_histories, + bar_time=datetime.datetime(2026, 7, 24, 15, 0), + ) + qmt = FakeQmtHarness(context).install(qmt_builtin_strategy) + qmt_builtin_strategy.init(context) + self.assertEqual(context.commission["type"], 0) + self.assertEqual( + context.commission["values"], + [0.0, 0.0005, 0.00021, 0.00021, 0.0, 5.0], + ) + self.assertEqual( + context.slippage, + {"type": 2, "value": 0.0002}, + ) + qmt_plan = qmt_builtin_strategy.compute_plan( + context, datetime.datetime(2026, 7, 24, 15, 0) + ) + self.assertEqual(jq_plan, qmt_plan) + self.assertEqual(joinquant_strategy.CONFIG["skip"], 0) + self.assertEqual(qmt_builtin_strategy.CONFIG["skip"], 0) + self.assertEqual(context.last_market_request["end_time"], "20260724") + self.assertIs(context.last_market_request["fill_data"], False) + self.assertEqual( + jq.last_index_request, + { + "index_symbol": "000905.XSHG", + "date": jq.context.previous_date, + }, + ) + self.assertEqual( + jq.last_extras_request["info"], + "is_st", + ) + self.assertEqual( + jq.last_extras_request["end_date"], + jq.context.previous_date, + ) + self.assertEqual( + context.last_sector_request, + { + "sector_name": "中证500", + "timetag": context.get_bar_timetag(context.barpos), + }, + ) + + def test_pit_st_exclusion_is_identical_across_hosted_wrappers(self): + st_canonical = "300750.XSHE" + jq = FakeJoinQuantHarness( + price_histories(), + st_symbols={st_canonical}, + ) + jq.initialize(joinquant_strategy) + jq.context.previous_date = datetime.date(2026, 7, 24) + jq.context.current_dt = datetime.datetime(2026, 7, 27, 9, 30) + jq_plan = joinquant_strategy.compute_plan(jq.context) + + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + qmt_symbol = "300750.SZ" + context = FakeQmtContext( + qmt_histories, + bar_time=datetime.datetime(2026, 7, 24, 15, 0), + st_periods={ + qmt_symbol: { + "ST": [["20260701", "20260731"]], + } + }, + ) + FakeQmtHarness(context).install(qmt_builtin_strategy) + qmt_builtin_strategy.init(context) + qmt_plan = qmt_builtin_strategy.compute_plan( + context, + datetime.datetime(2026, 7, 24, 15, 0), + ) + self.assertEqual(jq_plan, qmt_plan) + self.assertNotIn(st_canonical, jq_plan["scores"]) + self.assertNotIn(st_canonical, qmt_plan["scores"]) + + def test_qmt_live_orders_are_off_by_default(self): + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + context = FakeQmtContext( + qmt_histories, + trade_mode="trading", + bar_time=datetime.datetime(2026, 7, 24, 15, 0), + ) + harness = FakeQmtHarness(context) + self.assertIsNone(harness.run(qmt_builtin_strategy)) + context._bar_time += datetime.timedelta(days=7) + plan = qmt_builtin_strategy.handlebar(context) + self.assertIsNotNone(plan) + self.assertEqual(harness.orders, []) + + def test_qmt_builtin_live_mutation_cannot_be_enabled_by_parameters(self): + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + context = FakeQmtContext( + qmt_histories, + trade_mode="trading", + bar_time=datetime.datetime(2026, 7, 24, 15, 0), + ) + harness = FakeQmtHarness(context).install(qmt_builtin_strategy) + qmt_builtin_strategy.init(context) + context.do_back_test = "False" + self.assertFalse(qmt_builtin_strategy._is_backtest(context)) + self.assertFalse(qmt_builtin_strategy._orders_allowed(context)) + # Even an injected ContextInfo knob cannot enable broker mutation. + context.q60_config = { + "enable_live_orders": True, + "live_confirmation": "Q60_LIVE_ACK", + } + self.assertFalse(qmt_builtin_strategy._orders_allowed(context)) + + def test_qmt_module_state_survives_contextinfo_rollback_between_bars(self): + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + first_context = FakeQmtContext( + qmt_histories, + trade_mode="backtest", + bar_time=datetime.datetime(2026, 7, 20, 15, 0), + ) + harness = FakeQmtHarness(first_context).install(qmt_builtin_strategy) + qmt_builtin_strategy.init(first_context) + self.assertFalse( + any(name.startswith("q60_") for name in vars(first_context)) + ) + self.assertIsNone(qmt_builtin_strategy.handlebar(first_context)) + + # QMT documents that ContextInfo user attributes may roll back between + # handlebar calls. Rebuild a clean platform context to model that + # boundary; the module-global clock must still recognize a new week. + rebuilt_context = FakeQmtContext( + qmt_histories, + trade_mode="backtest", + bar_time=datetime.datetime(2026, 7, 27, 15, 0), + ) + self.assertFalse( + any(name.startswith("q60_") for name in vars(rebuilt_context)) + ) + plan = qmt_builtin_strategy.handlebar(rebuilt_context) + self.assertIsNotNone(plan) + self.assertGreater(len(harness.orders), 0) + self.assertIs(qmt_builtin_strategy.g.last_plan, plan) + + def test_qmt_backtest_submits_once_per_iso_week(self): + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + context = FakeQmtContext(qmt_histories, trade_mode="backtest") + harness = FakeQmtHarness(context) + self.assertIsNone(harness.run(qmt_builtin_strategy)) + context._bar_time += datetime.timedelta(days=7) + first = qmt_builtin_strategy.handlebar(context) + first_count = len(harness.orders) + second = qmt_builtin_strategy.handlebar(context) + self.assertIsNotNone(first) + self.assertGreater(first_count, 0) + self.assertIsNone(second) + self.assertEqual(len(harness.orders), first_count) + first_ids = {item["user_order_id"] for item in harness.orders} + context._bar_time += datetime.timedelta(days=7) + third = qmt_builtin_strategy.handlebar(context) + second_ids = { + item["user_order_id"] for item in harness.orders[first_count:] + } + self.assertIsNotNone(third) + self.assertTrue(first_ids.isdisjoint(second_ids)) + self.assertTrue(all(len(value.encode("ascii")) <= 24 for value in second_ids)) + + def test_qmt_incomplete_history_aborts_without_passorder(self): + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + qmt_histories["300750.SZ"] = qmt_histories["300750.SZ"][1:] + context = FakeQmtContext(qmt_histories, trade_mode="backtest") + harness = FakeQmtHarness(context) + self.assertIsNone(harness.run(qmt_builtin_strategy)) + context._bar_time += datetime.timedelta(days=7) + with self.assertRaises(qmt_builtin_strategy.PriceHistoryUnavailable): + qmt_builtin_strategy.handlebar(context) + self.assertEqual(harness.orders, []) + + def test_qmt_unmanaged_position_aborts_without_passorder(self): + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + position = type( + "Position", + (), + { + "stock_code": "601318.SH", + "volume": 1000, + "can_use_volume": 1000, + }, + )() + context = FakeQmtContext( + qmt_histories, + positions=[position], + trade_mode="backtest", + ) + harness = FakeQmtHarness(context) + self.assertIsNone(harness.run(qmt_builtin_strategy)) + qmt_builtin_strategy.g.config["universe_mode"] = "fixed" + context._bar_time += datetime.timedelta(days=7) + with self.assertRaises(qmt_builtin_strategy.UnmanagedPositionError): + qmt_builtin_strategy.handlebar(context) + self.assertEqual(harness.orders, []) + + def test_qmt_minute_driver_is_rejected_before_orders(self): + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + context = FakeQmtContext( + qmt_histories, + trade_mode="backtest", + params={"period": "1m"}, + ) + harness = FakeQmtHarness(context) + with self.assertRaises(qmt_builtin_strategy.UnsupportedStrategyPeriod): + harness.run(qmt_builtin_strategy) + self.assertEqual(harness.orders, []) + + def test_qmt_builtin_do_back_test_flag_enables_backtest_orders(self): + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in price_histories().items() + } + context = FakeQmtContext(qmt_histories, trade_mode="backtest") + del context.trade_mode + context._param.pop("trade_mode") + context.do_back_test = True + harness = FakeQmtHarness(context) + self.assertIsNone(harness.run(qmt_builtin_strategy)) + context._bar_time += datetime.timedelta(days=7) + plan = qmt_builtin_strategy.handlebar(context) + self.assertIsNotNone(plan) + self.assertGreater(len(harness.orders), 0) + + def test_qmt_source_is_ascii_despite_gbk_header(self): + payload = (ROOT / "platforms/qmt_builtin_strategy.py").read_bytes() + payload.decode("ascii") + self.assertTrue(payload.startswith(b"# coding: gbk")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/platform_qlib_test.py b/tests/platform_qlib_test.py new file mode 100644 index 0000000..03c5df9 --- /dev/null +++ b/tests/platform_qlib_test.py @@ -0,0 +1,363 @@ +import importlib.util +import os +import struct +import sys +import tempfile +import unittest +from contextlib import redirect_stderr +from io import StringIO +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "src")) + +from platforms.qlib_runner import ( + QlibUnavailableError, + _main, + build_alpha158_lightgbm_task, + generate_portable_momentum_signal, + run_native_momentum_backtest, +) +from tools.build_qlib_tiny_fixture import build_default_fixture + + +class QlibFixtureTest(unittest.TestCase): + def test_standard_library_fixture_has_expected_bin_header(self): + with tempfile.TemporaryDirectory() as temp: + report = build_default_fixture(Path(temp), days=40) + feature = Path(temp) / "features/sh600000/close.day.bin" + payload = feature.read_bytes() + values = struct.unpack("<" + "f" * (len(payload) // 4), payload) + self.assertEqual(values[0], 0.0) + self.assertEqual(len(values), 41) + self.assertEqual(report["calendar_count"], 40) + self.assertTrue((Path(temp) / "calendars/day.txt").is_file()) + self.assertTrue((Path(temp) / "instruments/csi300.txt").is_file()) + + def test_alpha158_lightgbm_config_uses_stable_097_paths(self): + config = build_alpha158_lightgbm_task( + market="csi300", + benchmark="SH000300", + train=("2018-01-01", "2020-12-31"), + valid=("2021-01-01", "2021-12-31"), + test=("2022-01-01", "2022-12-31"), + ) + self.assertEqual( + config["task"]["model"]["module_path"], + "qlib.contrib.model.gbdt", + ) + self.assertEqual( + config["task"]["dataset"]["kwargs"]["handler"]["module_path"], + "qlib.contrib.data.handler", + ) + self.assertEqual( + config["portfolio_analysis"]["executor"]["class"], + "SimulatorExecutor", + ) + self.assertEqual( + config["portfolio_analysis"]["strategy"]["class"], + "TopkDropoutStrategy", + ) + + @unittest.skipUnless( + importlib.util.find_spec("qlib") is not None + and os.environ.get("QUANT60_RUN_QLIB_NATIVE_SMOKE") == "1", + "set QUANT60_RUN_QLIB_NATIVE_SMOKE=1 in a pyqlib 0.9.7 env", + ) + def test_optional_native_momentum_smoke(self): + with tempfile.TemporaryDirectory() as temp: + build_default_fixture(Path(temp), days=80) + result = run_native_momentum_backtest( + provider_uri=temp, + market="csi300", + benchmark="SH000300", + start_time="2024-02-01", + end_time="2024-04-19", + topk=1, + n_drop=1, + ) + self.assertGreater(len(result["signal"]), 0) + + def test_missing_provider_fails_before_optional_import(self): + with self.assertRaises(FileNotFoundError): + run_native_momentum_backtest( + provider_uri="/definitely/missing/quant60-provider", + market="csi300", + benchmark="SH000300", + start_time="2024-01-01", + end_time="2024-02-01", + ) + + @unittest.skipUnless( + importlib.util.find_spec("pandas") is not None, + "pandas is optional outside the Qlib environment", + ) + def test_weekly_signal_uses_true_first_provider_session(self): + import pandas + + dates = pandas.bdate_range("2024-01-02", periods=20) + index = pandas.MultiIndex.from_product( + [["SH600000"], dates], + names=["instrument", "datetime"], + ) + frame = pandas.DataFrame( + {"$close": [10.0 + index * 0.1 for index in range(len(dates))]}, + index=index, + ) + + class FakeD: + @staticmethod + def features(*args, **kwargs): + del args, kwargs + return frame + + signal = generate_portable_momentum_signal( + FakeD, + ["SH600000"], + start_time="2024-01-02", + end_time="2024-01-31", + feature_start_time="2024-01-02", + lookback=2, + rebalance="weekly", + ) + selected_dates = list( + signal.index.get_level_values("datetime").unique() + ) + # Warm-up finishes mid-week, so that partial week is skipped. Every + # emitted date thereafter is the actual first provider session. + self.assertNotIn(pandas.Timestamp("2024-01-04"), selected_dates) + for selected in selected_dates: + week_dates = [ + value + for value in dates + if value.to_period("W") == selected.to_period("W") + ] + self.assertEqual(selected, min(week_dates)) + + +class QlibCliTest(unittest.TestCase): + def test_legacy_momentum_cli_remains_default_and_writes_json(self): + fake_result = { + "qlib_version": "0.9.7", + "signal": [0.1, 0.2], + "portfolio_metrics": {"1day": object()}, + "clock_contract": "test-clock", + "strategy_fidelity": "signal-only", + "a_share_rule_fidelity": "approximation", + } + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) / "nested" / "summary.json" + with mock.patch( + "platforms.qlib_runner.run_native_momentum_backtest", + return_value=fake_result, + ) as runner, mock.patch("builtins.print") as printer: + status = _main( + [ + "--provider-uri", + temp, + "--start", + "2024-01-02", + "--end", + "2024-03-29", + "--feature-start", + "2023-12-01", + "--lookback", + "10", + "--skip", + "2", + "--topk", + "20", + "--n-drop", + "3", + "--result-json", + str(output), + ] + ) + self.assertEqual(status, 0) + runner.assert_called_once_with( + provider_uri=temp, + market="csi300", + benchmark="SH000300", + start_time="2024-01-02", + end_time="2024-03-29", + feature_start_time="2023-12-01", + lookback=10, + skip=2, + topk=20, + n_drop=3, + rebalance="weekly", + ) + payload = __import__("json").loads(output.read_text(encoding="utf-8")) + self.assertEqual(payload["workflow"], "momentum") + self.assertEqual(payload["signal_rows"], 2) + self.assertEqual(payload["portfolio_frequencies"], ["1day"]) + self.assertIn('"workflow": "momentum"', printer.call_args.args[0]) + + def test_alpha158_cli_dispatches_with_validated_segments(self): + fake_result = { + "qlib_version": "0.9.7", + "experiment_name": "quant-os-test", + "recorder_id": "rec-123", + } + with tempfile.TemporaryDirectory() as temp: + with mock.patch( + "platforms.qlib_runner.run_alpha158_lightgbm_workflow", + return_value=fake_result, + ) as runner, mock.patch("builtins.print"): + status = _main( + [ + "--workflow", + "alpha158", + "--provider-uri", + temp, + "--market", + "csi500", + "--benchmark", + "SH000905", + "--train-start", + "2018-01-01", + "--train-end", + "2020-12-31", + "--valid-start", + "2021-01-01", + "--valid-end", + "2021-12-31", + "--test-start", + "2022-01-01", + "--test-end", + "2022-12-31", + "--experiment-name", + "quant-os-test", + ] + ) + self.assertEqual(status, 0) + runner.assert_called_once_with( + provider_uri=temp, + market="csi500", + benchmark="SH000905", + train=("2018-01-01", "2020-12-31"), + valid=("2021-01-01", "2021-12-31"), + test=("2022-01-01", "2022-12-31"), + experiment_name="quant-os-test", + topk=50, + n_drop=5, + ) + + def test_alpha158_cli_rejects_missing_or_overlapping_segments(self): + base = [ + "--workflow", + "alpha158", + "--provider-uri", + "/provider", + "--train-start", + "2018-01-01", + "--train-end", + "2020-12-31", + "--valid-start", + "2020-12-31", + "--valid-end", + "2021-12-31", + "--test-start", + "2022-01-01", + "--test-end", + "2022-12-31", + ] + with mock.patch( + "platforms.qlib_runner.run_alpha158_lightgbm_workflow" + ) as runner, redirect_stderr(StringIO()): + with self.assertRaises(SystemExit) as overlap: + _main(base) + with self.assertRaises(SystemExit) as missing: + _main(base[:-2]) + self.assertEqual(overlap.exception.code, 2) + self.assertEqual(missing.exception.code, 2) + runner.assert_not_called() + + def test_momentum_cli_rejects_invalid_dates_and_position_counts(self): + with mock.patch( + "platforms.qlib_runner.run_native_momentum_backtest" + ) as runner, redirect_stderr(StringIO()): + with self.assertRaises(SystemExit) as dates: + _main( + [ + "--provider-uri", + "/provider", + "--start", + "2024-02-01", + "--end", + "2024-01-01", + ] + ) + with self.assertRaises(SystemExit) as counts: + _main( + [ + "--provider-uri", + "/provider", + "--start", + "2024-01-01", + "--end", + "2024-02-01", + "--topk", + "2", + "--n-drop", + "3", + ] + ) + self.assertEqual(dates.exception.code, 2) + self.assertEqual(counts.exception.code, 2) + runner.assert_not_called() + + def test_cli_rejects_date_arguments_from_the_other_workflow(self): + with mock.patch( + "platforms.qlib_runner.run_native_momentum_backtest" + ) as momentum_runner, redirect_stderr(StringIO()): + with self.assertRaises(SystemExit) as momentum: + _main( + [ + "--provider-uri", + "/provider", + "--start", + "2024-01-01", + "--end", + "2024-02-01", + "--train-start", + "2020-01-01", + ] + ) + with mock.patch( + "platforms.qlib_runner.run_alpha158_lightgbm_workflow" + ) as alpha_runner, redirect_stderr(StringIO()): + with self.assertRaises(SystemExit) as alpha: + _main( + [ + "--workflow", + "alpha158", + "--provider-uri", + "/provider", + "--start", + "2022-01-01", + "--train-start", + "2018-01-01", + "--train-end", + "2020-12-31", + "--valid-start", + "2021-01-01", + "--valid-end", + "2021-12-31", + "--test-start", + "2022-01-01", + "--test-end", + "2022-12-31", + ] + ) + self.assertEqual(momentum.exception.code, 2) + self.assertEqual(alpha.exception.code, 2) + momentum_runner.assert_not_called() + alpha_runner.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/platform_qmt_runner_test.py b/tests/platform_qmt_runner_test.py new file mode 100644 index 0000000..9c3ca5d --- /dev/null +++ b/tests/platform_qmt_runner_test.py @@ -0,0 +1,155 @@ +import sys +import json +import hashlib +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "src")) + +from platforms import qmt_research_runner + + +class FakeFrame: + def __len__(self): + return 1 + + +class QmtResearchRunnerTest(unittest.TestCase): + def test_build_params_normalizes_official_timestamps(self): + params = qmt_research_runner.build_qmt_parameters( + stock_code="600000.sh", + start_time="20260102", + end_time="2026-01-30", + ) + self.assertEqual(params["stock_code"], "600000.SH") + self.assertEqual(params["start_time"], "2026-01-02 00:00:00") + self.assertEqual(params["end_time"], "2026-01-30 23:59:59") + self.assertEqual(params["trade_mode"], "backtest") + self.assertEqual(params["quote_mode"], "history") + self.assertEqual(params["account_id"], "test") + self.assertEqual(params["benchmark"], "000905.SH") + baseline_bytes = (ROOT / "configs/baseline.json").read_bytes() + baseline = json.loads(baseline_bytes) + self.assertEqual( + params["max_vol_rate"], + baseline["execution"]["participation_rate"], + ) + self.assertEqual( + params["slippage"], + baseline["execution"]["slippage_bps"] / 10_000, + ) + expected_commission = ( + baseline["fees"]["commission_rate"] + + baseline["fees"]["transfer_fee_rate"] + ) + self.assertEqual(params["open_commission"], expected_commission) + self.assertEqual(params["close_commission"], expected_commission) + self.assertEqual( + params["q60_baseline_config_sha256"], + hashlib.sha256(baseline_bytes).hexdigest(), + ) + + def test_live_mode_overrides_are_rejected(self): + with self.assertRaises(ValueError): + qmt_research_runner.build_qmt_parameters( + stock_code="600000.SH", + start_time="20260102", + end_time="20260130", + overrides={"trade_mode": "trading"}, + ) + with self.assertRaises(ValueError): + qmt_research_runner.build_qmt_parameters( + stock_code="600000.SH", + start_time="20260102", + end_time="20260130", + overrides={"period": "1m"}, + ) + with self.assertRaises(ValueError): + qmt_research_runner.build_qmt_parameters( + stock_code="600000.SH", + start_time="20260102", + end_time="20260130", + overrides={"benchmark": "000300.SH"}, + ) + + def test_non_finite_asset_and_rate_are_rejected(self): + for kwargs in ( + {"asset": float("nan")}, + {"asset": float("inf")}, + {"overrides": {"max_vol_rate": float("nan")}}, + {"overrides": {"open_commission": float("inf")}}, + ): + with self.subTest(kwargs=kwargs): + with self.assertRaisesRegex(ValueError, "finite"): + qmt_research_runner.build_qmt_parameters( + stock_code="600000.SH", + start_time="20260102", + end_time="20260130", + **kwargs, + ) + + def test_intraday_period_is_rejected_to_prevent_daily_bar_lookahead(self): + with self.assertRaisesRegex(ValueError, "period must be 1d"): + qmt_research_runner.build_qmt_parameters( + stock_code="600000.SH", + start_time="20260102", + end_time="20260130", + period="1m", + ) + + def test_preflight_and_run_use_param_keyword(self): + params = qmt_research_runner.build_qmt_parameters( + stock_code="600000.SH", + start_time="20260102", + end_time="20260130", + ) + calls = {} + + def run_strategy_file(user_script, *, param): + calls["user_script"] = user_script + calls["param"] = param + return object() + + fake_qmttools = types.SimpleNamespace(run_strategy_file=run_strategy_file) + + def get_market_data_ex(**kwargs): + calls["data"] = kwargs + return {"600000.SH": FakeFrame()} + + fake_xtdata = types.SimpleNamespace(get_market_data_ex=get_market_data_ex) + fake_xtquant = types.SimpleNamespace(__version__="fake") + + def import_module(name): + return { + "xtquant": fake_xtquant, + "xtquant.qmttools": fake_qmttools, + "xtquant.xtdata": fake_xtdata, + }[name] + + with tempfile.TemporaryDirectory() as temp: + strategy = Path(temp) / "strategy.py" + strategy.write_text("def init(C): pass\n", encoding="ascii") + with mock.patch.object( + qmt_research_runner.importlib.util, + "find_spec", + return_value=object(), + ), mock.patch.object( + qmt_research_runner.importlib, + "import_module", + side_effect=import_module, + ): + result = qmt_research_runner.run_qmt_backtest(strategy, params) + self.assertIsNotNone(result) + self.assertEqual(calls["param"], params) + self.assertEqual(calls["data"]["start_time"], "20260102000000") + self.assertEqual(calls["data"]["end_time"], "20260130235959") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/platform_xttrader_test.py b/tests/platform_xttrader_test.py new file mode 100644 index 0000000..2a137c4 --- /dev/null +++ b/tests/platform_xttrader_test.py @@ -0,0 +1,859 @@ +import datetime as dt +import sys +import threading +import unittest +from pathlib import Path +from types import SimpleNamespace + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "src")) + +from adapters.xttrader_live import ( + IdempotencyConflict, + LiveOrderBlocked, + NotConnectedError, + ReconnectFailed, + XtTraderLiveAdapter, + map_order_status, +) + + +class Constants: + STOCK_BUY = 23 + STOCK_SELL = 24 + FIX_PRICE = 11 + + +class FakeTrader: + def __init__(self, connect_code=0, orders=None): + self.connect_code = connect_code + self.callback = None + self.started = False + self.stopped = False + self.order_calls = [] + self.cancel_calls = [] + self.orders = list(orders or []) + + def register_callback(self, callback): + self.callback = callback + + def start(self): + self.started = True + + def connect(self): + return self.connect_code + + def subscribe(self, account): + self.account = account + return 0 + + def stop(self): + self.stopped = True + + def query_stock_asset(self, account): + del account + return SimpleNamespace( + account_id="test", cash=10_000, market_value=20_000, total_asset=30_000 + ) + + def query_stock_orders(self, account, cancelable_only=False): + del account, cancelable_only + return list(self.orders) + + def query_stock_positions(self, account): + del account + return [ + SimpleNamespace( + account_id="test", + stock_code="600000.SH", + volume=1000, + can_use_volume=700, + market_value=10_000, + open_price=10, + ) + ] + + def query_stock_trades(self, account): + del account + return [] + + def order_stock_async(self, *args): + self.order_calls.append(args) + return 101 + + def cancel_order_stock(self, account, order_id): + self.cancel_calls.append((account, order_id)) + return 0 + + +def passing_live_decision(**overrides): + decision = { + "authorization_id": "auth-test-001", + "snapshot_id": "snapshot-test-001", + "as_of": dt.datetime.now(dt.timezone.utc).isoformat(), + "reconciliation_safe": True, + "account_allowed": True, + "within_notional_limit": True, + "within_order_rate_limit": True, + "operator_approved": True, + "kill_switch_ready": True, + "compliance_approved": True, + } + decision.update(overrides) + return decision + + +def adapter_for(trader, allow=False, live_guard=None): + if allow is True and live_guard is None: + live_guard = lambda intent: passing_live_decision() + return XtTraderLiveAdapter( + trader_factory=lambda: trader, + account=SimpleNamespace(account_id="test"), + constants=Constants, + allow_live_orders=allow, + live_guard=live_guard, + ) + + +class XtTraderLiveAdapterTest(unittest.TestCase): + def test_real_status_mapping_uses_domain_spelling(self): + self.assertEqual(map_order_status(48), "NEW") + self.assertEqual(map_order_status(50), "ACCEPTED") + self.assertEqual(map_order_status(51), "CANCEL_PENDING") + self.assertEqual(map_order_status(54), "CANCELLED") + self.assertEqual(map_order_status(55), "PARTIALLY_FILLED") + self.assertEqual(map_order_status(56), "FILLED") + self.assertEqual(map_order_status(57), "REJECTED") + self.assertEqual(map_order_status(999), "UNKNOWN") + + def test_default_is_dry_run_and_idempotent_without_broker_call(self): + trader = FakeTrader() + adapter = adapter_for(trader) + first = adapter.submit_order( + client_order_id="decision-001", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + ) + second = adapter.submit_order( + client_order_id="decision-001", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + ) + self.assertEqual(first, second) + self.assertEqual(first["state"], "DRY_RUN") + self.assertEqual(trader.order_calls, []) + + def test_allow_live_orders_requires_a_real_bool(self): + trader = FakeTrader() + for value in ("False", "true", 1, None): + with self.subTest(value=value): + with self.assertRaisesRegex(TypeError, "must be a bool"): + adapter_for(trader, allow=value) + self.assertEqual(trader.order_calls, []) + + def test_live_mode_requires_callable_guard_but_dry_run_does_not(self): + trader = FakeTrader() + dry_run = XtTraderLiveAdapter( + trader_factory=lambda: trader, + account=SimpleNamespace(account_id="test"), + constants=Constants, + ) + self.assertFalse(dry_run.allow_live_orders) + with self.assertRaisesRegex(TypeError, "requires a callable live_guard"): + XtTraderLiveAdapter( + trader_factory=lambda: trader, + account=SimpleNamespace(account_id="test"), + constants=Constants, + allow_live_orders=True, + ) + with self.assertRaisesRegex(TypeError, "live_guard must be callable"): + XtTraderLiveAdapter( + trader_factory=lambda: trader, + account=SimpleNamespace(account_id="test"), + constants=Constants, + allow_live_orders=True, + live_guard="allow", + ) + + def test_idempotency_key_conflict_is_rejected(self): + adapter = adapter_for(FakeTrader()) + adapter.submit_order( + client_order_id="decision-001", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + ) + with self.assertRaises(IdempotencyConflict): + adapter.submit_order( + client_order_id="decision-001", + symbol="600000.SH", + side="BUY", + quantity=200, + limit_price=10.5, + ) + + def test_non_finite_live_price_is_rejected_before_broker_call(self): + trader = FakeTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "finite and positive"): + adapter.submit_order( + client_order_id="bad-price", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=value, + confirm_live=True, + ) + self.assertEqual(trader.order_calls, []) + + def test_limit_price_rejects_bool_non_real_and_non_positive(self): + trader = FakeTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + for value in (True, False, "10.5", None, 10 + 0j, 0, -1): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "limit_price"): + adapter.submit_order( + client_order_id="strict-price", + symbol="600000.SH", + side="SELL", + quantity=100, + limit_price=value, + confirm_live=True, + ) + self.assertEqual(trader.order_calls, []) + + def test_submit_confirm_live_requires_a_real_bool(self): + trader = FakeTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + for value in ("False", "true", 1, None): + with self.subTest(value=value): + with self.assertRaisesRegex(TypeError, "must be a bool"): + adapter.submit_order( + client_order_id="strict-confirm", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=value, + ) + self.assertEqual(trader.order_calls, []) + + def test_quantity_rejects_bool_fraction_float_and_non_positive(self): + trader = FakeTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + for value in (True, False, 100.0, 100.5, "100", None, 0, -100): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "positive integer"): + adapter.submit_order( + client_order_id="strict-quantity", + symbol="600000.SH", + side="BUY", + quantity=value, + limit_price=10.5, + confirm_live=True, + ) + self.assertEqual(trader.order_calls, []) + + def test_real_order_requires_double_opt_in_and_remark_round_trip(self): + trader = FakeTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + with self.assertRaises(LiveOrderBlocked): + adapter.submit_order( + client_order_id="12345678901234567890", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + ) + result = adapter.submit_order( + client_order_id="12345678901234567890", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + self.assertEqual(result["state"], "NEW") + self.assertEqual(len(trader.order_calls), 1) + remark = trader.order_calls[0][-1] + self.assertEqual(remark, "q60:12345678901234567890") + self.assertLessEqual(len(remark), 24) + self.assertEqual( + result["live_policy"]["authorization_id"], "auth-test-001" + ) + + def test_live_guard_receives_normalized_submit_and_cancel_intents(self): + intents = [] + + def guard(intent): + intents.append(intent) + return passing_live_decision( + authorization_id=f"auth-{len(intents)}" + ) + + trader = FakeTrader() + adapter = adapter_for(trader, allow=True, live_guard=guard) + adapter.connect() + result = adapter.submit_order( + client_order_id="guard-intent", + symbol="600000.SH", + side="buy", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + adapter.cancel_order(88, confirm_live=True) + self.assertEqual( + intents, + [ + { + "action": "SUBMIT_ORDER", + "account_id": "test", + "client_order_id": "guard-intent", + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "limit_price": 10.5, + "notional": 1050.0, + "strategy_name": "quant60", + }, + { + "action": "CANCEL_ORDER", + "account_id": "test", + "broker_order_id": 88, + "strategy_name": "quant60", + }, + ], + ) + self.assertEqual(result["live_policy"]["authorization_id"], "auth-1") + self.assertEqual(len(trader.order_calls), 1) + self.assertEqual(len(trader.cancel_calls), 1) + audit = adapter.live_authorization_audit + self.assertEqual(len(audit), 2) + self.assertEqual(audit[0]["decision"]["account_id"], "test") + self.assertEqual(audit[1]["intent"]["action"], "CANCEL_ORDER") + self.assertEqual(audit[1]["outcome"], "BROKER_ACCEPTED") + + def test_live_guard_is_bound_to_a_non_empty_account_id(self): + trader = FakeTrader() + calls = [] + adapter = XtTraderLiveAdapter( + trader_factory=lambda: trader, + account=SimpleNamespace(account_id=" "), + constants=Constants, + allow_live_orders=True, + live_guard=lambda intent: ( + calls.append(intent) or passing_live_decision() + ), + ) + with self.assertRaisesRegex(NotConnectedError, "account_id"): + adapter.connect() + self.assertEqual(calls, []) + self.assertEqual(trader.order_calls, []) + + def test_live_guard_missing_false_non_bool_or_exception_blocks_submit(self): + def raising_guard(intent): + del intent + raise RuntimeError("policy backend unavailable") + + cases = [ + ( + "missing", + lambda intent: { + key: value + for key, value in passing_live_decision().items() + if key != "compliance_approved" + }, + ), + ( + "false", + lambda intent: passing_live_decision( + reconciliation_safe=False + ), + ), + ( + "non-bool", + lambda intent: passing_live_decision(account_allowed=1), + ), + ( + "empty-id", + lambda intent: passing_live_decision( + authorization_id=" " + ), + ), + ("exception", raising_guard), + ] + for name, guard in cases: + with self.subTest(name=name): + trader = FakeTrader() + adapter = adapter_for( + trader, allow=True, live_guard=guard + ) + adapter.connect() + with self.assertRaises(LiveOrderBlocked): + adapter.submit_order( + client_order_id=f"blocked-{name}", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + self.assertEqual(trader.order_calls, []) + + def test_live_guard_rejects_stale_naive_and_future_decisions(self): + now = dt.datetime.now(dt.timezone.utc) + cases = { + "stale": (now - dt.timedelta(seconds=61)).isoformat(), + "naive": now.replace(tzinfo=None).isoformat(), + "future": (now + dt.timedelta(seconds=6)).isoformat(), + } + for name, as_of in cases.items(): + with self.subTest(name=name): + trader = FakeTrader() + adapter = adapter_for( + trader, + allow=True, + live_guard=lambda intent, value=as_of: ( + passing_live_decision(as_of=value) + ), + ) + adapter.connect() + with self.assertRaises(LiveOrderBlocked): + adapter.submit_order( + client_order_id=f"clock-{name}", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + self.assertEqual(trader.order_calls, []) + + def test_live_guard_failure_blocks_cancel_before_broker(self): + trader = FakeTrader() + adapter = adapter_for( + trader, + allow=True, + live_guard=lambda intent: passing_live_decision( + kill_switch_ready=False + ), + ) + adapter.connect() + with self.assertRaisesRegex( + LiveOrderBlocked, "kill_switch_ready" + ): + adapter.cancel_order(88, confirm_live=True) + self.assertEqual(trader.cancel_calls, []) + + def test_concurrent_same_client_id_submits_once(self): + class BlockingTrader(FakeTrader): + def __init__(self): + super().__init__() + self.submit_entered = threading.Event() + self.release_submit = threading.Event() + + def order_stock_async(self, *args): + self.order_calls.append(args) + self.submit_entered.set() + if not self.release_submit.wait(2): + raise AssertionError("test did not release broker submit") + return 101 + + trader = BlockingTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + results = [] + failures = [] + + def submit(): + try: + results.append( + adapter.submit_order( + client_order_id="decision-001", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + ) + except BaseException as exc: + failures.append(exc) + + first = threading.Thread(target=submit) + first.start() + self.assertTrue(trader.submit_entered.wait(1)) + # The broker boundary is inside the idempotency critical section. + acquired = adapter._lock.acquire(blocking=False) + if acquired: + adapter._lock.release() + self.assertFalse(acquired) + + second = threading.Thread(target=submit) + second.start() + trader.release_submit.set() + first.join(2) + second.join(2) + self.assertFalse(first.is_alive()) + self.assertFalse(second.is_alive()) + self.assertEqual(failures, []) + self.assertEqual(len(trader.order_calls), 1) + self.assertEqual(len(results), 2) + self.assertEqual(results[0], results[1]) + + def test_synchronous_order_error_matches_pre_submit_reservation(self): + class SynchronousErrorTrader(FakeTrader): + def order_stock_async(self, *args): + self.order_calls.append(args) + self.callback.on_order_error( + SimpleNamespace( + order_id=0, + order_remark=args[-1], + error_id=42, + error_msg="synchronous rejection", + ) + ) + return 101 + + trader = SynchronousErrorTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + result = adapter.submit_order( + client_order_id="decision-001", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + self.assertEqual(result["state"], "REJECTED") + self.assertEqual(result["error_id"], 42) + self.assertEqual(len(trader.order_calls), 1) + + def test_synchronous_order_error_by_seq_is_drained_from_orphans(self): + class SynchronousErrorTrader(FakeTrader): + def order_stock_async(self, *args): + self.order_calls.append(args) + self.callback.on_order_error( + SimpleNamespace( + order_id=None, + order_remark="", + seq=101, + error_id=43, + error_msg="fast seq rejection", + ) + ) + return 101 + + trader = SynchronousErrorTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + result = adapter.submit_order( + client_order_id="decision-002", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + self.assertEqual(result["state"], "REJECTED") + self.assertEqual(result["error_id"], 43) + self.assertEqual(adapter._orphan_order_errors, []) + + def test_malformed_async_seq_never_leaves_submitting_reservation(self): + class MalformedSeqTrader(FakeTrader): + def __init__(self, seq): + super().__init__() + self.seq = seq + + def order_stock_async(self, *args): + self.order_calls.append(args) + return self.seq + + for index, seq in enumerate(("bad", 1.5, True, object())): + with self.subTest(seq=repr(seq)): + trader = MalformedSeqTrader(seq) + adapter = adapter_for(trader, allow=True) + adapter.connect() + client_id = f"bad-seq-{index}" + with self.assertRaisesRegex( + Exception, "malformed async request sequence" + ): + adapter.submit_order( + client_order_id=client_id, + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + record = adapter.journal.get(client_id) + self.assertEqual(record["state"], "UNKNOWN") + self.assertNotEqual(record["state"], "SUBMITTING") + self.assertEqual(len(trader.order_calls), 1) + + def test_query_normalizes_position_and_disconnect_fails_fast(self): + trader = FakeTrader() + adapter = adapter_for(trader) + adapter.connect() + position = adapter.query_positions()[0] + self.assertEqual(position["sellable"], 700) + self.assertEqual(position["symbol"], "600000.XSHG") + trader.callback.on_disconnected() + with self.assertRaises(NotConnectedError): + adapter.query_orders() + + def test_connect_recovers_idempotency_from_broker_remark(self): + broker_order = SimpleNamespace( + account_id="test", + order_id=88, + order_sysid="sys-88", + order_remark="q60:decision-001", + stock_code="600000.SH", + order_type=23, + order_volume=100, + traded_volume=0, + price=10.5, + traded_price=0, + order_status=50, + status_msg="", + ) + trader = FakeTrader(orders=[broker_order]) + adapter = adapter_for(trader, allow=True) + adapter.connect() + recovered = adapter.submit_order( + client_order_id="decision-001", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + self.assertEqual(recovered["broker_order_id"], 88) + self.assertEqual(recovered["state"], "ACCEPTED") + self.assertEqual(trader.order_calls, []) + + def test_broker_payload_conflict_for_same_remark_fails_closed(self): + orders = [ + SimpleNamespace( + account_id="test", + order_id=88 + index, + order_sysid="sys", + order_remark="q60:decision-001", + stock_code="600000.SH", + order_type=23, + order_volume=quantity, + traded_volume=0, + price=10.5, + traded_price=0, + order_status=50, + status_msg="", + ) + for index, quantity in enumerate((100, 200)) + ] + adapter = adapter_for(FakeTrader(orders=orders), allow=True) + with self.assertRaises(IdempotencyConflict): + adapter.connect() + self.assertNotEqual(adapter.state, "READY") + + def test_order_error_updates_matching_journal_to_rejected(self): + trader = FakeTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + adapter.submit_order( + client_order_id="decision-001", + symbol="600000.SH", + side="BUY", + quantity=100, + limit_price=10.5, + confirm_live=True, + ) + trader.callback.on_order_error( + SimpleNamespace( + order_id=999, + order_remark="q60:decision-001", + error_id=42, + error_msg="rejected", + ) + ) + record = adapter.journal.get("decision-001") + self.assertEqual(record["state"], "REJECTED") + self.assertEqual(record["error_id"], 42) + + def test_cancel_none_is_not_reported_as_success(self): + trader = FakeTrader() + trader.cancel_order_stock = lambda account, order_id: None + adapter = adapter_for(trader, allow=True) + adapter.connect() + with self.assertRaisesRegex(Exception, "cancel failed"): + adapter.cancel_order(88, confirm_live=True) + + def test_cancel_confirm_live_requires_a_real_bool(self): + trader = FakeTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + for value in ("False", "true", 1, None): + with self.subTest(value=value): + with self.assertRaisesRegex(TypeError, "must be a bool"): + adapter.cancel_order(88, confirm_live=value) + self.assertEqual(trader.cancel_calls, []) + + def test_cancel_order_id_requires_exact_positive_integer(self): + trader = FakeTrader() + adapter = adapter_for(trader, allow=True) + adapter.connect() + for value in (True, False, 88.0, 88.9, "88", None, 0, -1): + with self.subTest(value=value): + with self.assertRaisesRegex( + ValueError, "positive integer" + ): + adapter.cancel_order(value, confirm_live=True) + self.assertEqual(trader.cancel_calls, []) + + def test_stale_order_callback_cannot_regress_filled_terminal_state(self): + filled = SimpleNamespace( + account_id="test", + order_id=88, + order_sysid="sys-88", + order_remark="q60:decision-001", + stock_code="600000.SH", + order_type=23, + order_volume=100, + traded_volume=100, + price=10.5, + traded_price=10.5, + order_status=56, + status_msg="filled", + ) + stale = SimpleNamespace( + account_id="test", + order_id=88, + order_sysid="sys-88", + order_remark="q60:decision-001", + stock_code="600000.SH", + order_type=23, + order_volume=100, + traded_volume=0, + price=10.5, + traded_price=0, + order_status=50, + status_msg="stale accepted", + ) + trader = FakeTrader(orders=[filled]) + adapter = adapter_for(trader) + adapter.connect() + # An exact duplicate is harmless and idempotent. + trader.callback.on_stock_order(filled) + with self.assertRaisesRegex( + IdempotencyConflict, "cumulative fill regressed" + ): + trader.callback.on_stock_order(stale) + record = adapter.journal.get("decision-001") + self.assertEqual(adapter.state, "DEGRADED") + self.assertEqual(record["state"], "FILLED") + self.assertEqual(record["filled_quantity"], 100) + self.assertEqual(adapter._orders[88]["state"], "FILLED") + + def test_configured_account_anchors_every_broker_query(self): + class WrongAccountTrader(FakeTrader): + def query_stock_asset(self, account): + del account + return SimpleNamespace( + account_id="wrong", + cash=10_000, + market_value=20_000, + total_asset=30_000, + ) + + trader = WrongAccountTrader() + adapter = adapter_for(trader) + with self.assertRaisesRegex( + NotConnectedError, + "configured account", + ): + adapter.connect() + self.assertEqual(adapter.state, "FAILED") + self.assertTrue(trader.stopped) + + def test_trade_query_preserves_account_side_and_stable_trade_id(self): + class TradeTrader(FakeTrader): + def query_stock_trades(self, account): + del account + return [ + SimpleNamespace( + account_id="test", + traded_id="trade-1", + order_id=88, + stock_code="600000.SH", + order_type=23, + traded_volume=100, + traded_price=10.5, + traded_amount=1050, + ) + ] + + adapter = adapter_for(TradeTrader()) + adapter.connect() + self.assertEqual( + adapter.query_trades(), + [ + { + "account_id": "test", + "trade_id": "trade-1", + "broker_order_id": 88, + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "price": 10.5, + "amount": 1050.0, + } + ], + ) + + def test_reconnect_uses_fresh_instance_and_exhausts_bounded_attempts(self): + created = [] + + def factory(): + trader = FakeTrader(connect_code=9) + created.append(trader) + return trader + + adapter = XtTraderLiveAdapter( + trader_factory=factory, + account=SimpleNamespace(account_id="test"), + constants=Constants, + ) + with self.assertRaises(ReconnectFailed): + adapter.reconnect(attempts=2) + self.assertEqual(len(created), 2) + self.assertEqual(adapter.state, "FAILED") + + def test_bj_is_explicitly_outside_v1_live_adapter(self): + adapter = adapter_for(FakeTrader()) + with self.assertRaises(ValueError): + adapter.submit_order( + client_order_id="bj", + symbol="430001.BJ", + side="BUY", + quantity=100, + limit_price=10, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_backtest_cli.py b/tests/test_backtest_cli.py new file mode 100644 index 0000000..9db160f --- /dev/null +++ b/tests/test_backtest_cli.py @@ -0,0 +1,212 @@ +import hashlib +import json +import tempfile +import unittest +from datetime import datetime +from pathlib import Path + +from quant60.backtest import BacktestEngine, verify_artifact_manifest +from quant60.cli import main +from quant60.config import config_from_dict, load_config +from quant60.ledger import HashChainLedger +from quant60.synthetic import generate_synthetic_bars + + +PROJECT = Path(__file__).resolve().parents[1] + + +class BacktestCliTests(unittest.TestCase): + def test_empty_config_uses_valid_defaults(self): + config = config_from_dict({}) + config.validate() + + def test_non_finite_config_values_are_rejected(self): + cases = ( + {"initial_cash": "NaN"}, + {"fees": {"commission_rate": float("inf")}}, + {"execution": {"participation_rate": float("nan")}}, + {"execution": {"slippage_bps": float("inf")}}, + {"strategy": {"max_weight": float("nan")}}, + {"strategy": {"gross_target": float("inf")}}, + {"strategy": {"cash_buffer": float("nan")}}, + {"initial_cash": True}, + {"execution": {"lot_size": 100.0}}, + {"execution": {"cancel_open_orders_at_end": 1}}, + {"strategy": {"lookback": True}}, + {"strategy": {"rebalance_every": 5.0}}, + {"synthetic": {"trading_days": 90.0}}, + {"symbols": "600000.XSHG"}, + ) + for raw in cases: + with self.subTest(raw=raw): + with self.assertRaises(ValueError): + config_from_dict(raw) + + def test_backtest_is_byte_deterministic(self): + config = load_config(PROJECT / "configs" / "baseline.json") + bars = generate_synthetic_bars( + config.symbols, + config.synthetic.start_date, + config.synthetic.trading_days, + config.synthetic.seed, + config.synthetic.base_volume, + ) + first = BacktestEngine(config).run(bars) + second = BacktestEngine(config).run(bars) + self.assertEqual(first.run_id, second.run_id) + self.assertEqual(first.report, second.report) + with tempfile.TemporaryDirectory() as left, tempfile.TemporaryDirectory() as right: + left_paths = first.write_artifacts(left) + right_paths = second.write_artifacts(right) + for key in left_paths: + left_hash = hashlib.sha256(left_paths[key].read_bytes()).hexdigest() + right_hash = hashlib.sha256(right_paths[key].read_bytes()).hexdigest() + self.assertEqual(left_hash, right_hash, key) + self.assertTrue(HashChainLedger.read_jsonl(left_paths["events"]).verify()) + report = json.loads(left_paths["report"].read_text(encoding="utf-8")) + self.assertFalse(report["investment_value_claim"]) + self.assertGreater(report["fill_count"], 0) + records = [ + json.loads(line) + for line in left_paths["events"].read_text( + encoding="utf-8" + ).splitlines() + ] + timestamps = [ + datetime.fromisoformat(record["timestamp"]) for record in records + ] + self.assertEqual(timestamps, sorted(timestamps)) + + def test_decisions_use_first_trading_session_close_of_each_week(self): + config = load_config(PROJECT / "configs" / "baseline.json") + bars = generate_synthetic_bars( + config.symbols, + config.synthetic.start_date, + config.synthetic.trading_days, + config.synthetic.seed, + config.synthetic.base_volume, + ) + result = BacktestEngine(config).run(bars) + sessions = sorted({item.trading_date for item in bars}) + first_by_week = {} + for session in sessions: + first_by_week.setdefault(session.isocalendar()[:2], session) + decision_dates = { + datetime.fromisoformat(item["as_of"]).date() + for item in result.targets + } + self.assertTrue(decision_dates) + for decision_date in decision_dates: + self.assertEqual( + decision_date, + first_by_week[decision_date.isocalendar()[:2]], + ) + + def test_incomplete_market_session_fails_closed(self): + config = load_config(PROJECT / "configs" / "baseline.json") + bars = generate_synthetic_bars( + config.symbols, + config.synthetic.start_date, + config.synthetic.trading_days, + config.synthetic.seed, + config.synthetic.base_volume, + ) + removed = bars.pop(len(config.symbols) * 30) + with self.assertRaisesRegex( + ValueError, + "incomplete market-data session.*" + removed.symbol.replace(".", r"\."), + ): + BacktestEngine(config).run(bars) + + def test_cli_smoke_and_ledger_verify(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "run" + self.assertEqual( + main( + [ + "smoke", + "--config", + str(PROJECT / "configs" / "baseline.json"), + "--output", + str(output), + ] + ), + 0, + ) + self.assertEqual(main(["verify-ledger", str(output / "events.jsonl")]), 0) + self.assertEqual( + main(["verify-manifest", str(output / "manifest.json")]), + 0, + ) + self.assertTrue( + verify_artifact_manifest(output / "manifest.json")["ok"] + ) + + def test_manifest_verifier_rejects_tampered_or_incomplete_runs(self): + config = load_config(PROJECT / "configs" / "baseline.json") + bars = generate_synthetic_bars( + config.symbols, + config.synthetic.start_date, + config.synthetic.trading_days, + config.synthetic.seed, + config.synthetic.base_volume, + ) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "run" + BacktestEngine(config).run(bars).write_artifacts(output) + report_path = output / "report.json" + report_path.write_bytes(report_path.read_bytes() + b" ") + with self.assertRaisesRegex(ValueError, "report artifact hash mismatch"): + verify_artifact_manifest(output / "manifest.json") + + (output / ".quant60-incomplete").write_text( + "interrupted\n", + encoding="ascii", + ) + with self.assertRaisesRegex(ValueError, "publication is incomplete"): + verify_artifact_manifest(output / "manifest.json") + + second_output = Path(directory) / "second-run" + BacktestEngine(config).run(bars).write_artifacts(second_output) + manifest_path = second_output / "manifest.json" + unexpected = second_output / "untracked-result.txt" + unexpected.write_text("not part of the run\n", encoding="utf-8") + with self.assertRaisesRegex( + ValueError, + "artifact directory is not an exact completed set", + ): + verify_artifact_manifest(manifest_path) + unexpected.unlink() + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["engine_sources"]["portable_core.py"] = "0" * 64 + manifest_path.write_text( + json.dumps(manifest, sort_keys=True), + encoding="utf-8", + ) + with self.assertRaisesRegex( + ValueError, + "engine_sources aggregate hash mismatch", + ): + verify_artifact_manifest(manifest_path) + + def test_cli_backtest_alias(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "run" + self.assertEqual( + main( + [ + "backtest", + "--config", + str(PROJECT / "configs" / "baseline.json"), + "--output", + str(output), + ] + ), + 0, + ) + self.assertTrue((output / "report.json").is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cost_optimizer.py b/tests/test_cost_optimizer.py new file mode 100644 index 0000000..586ae86 --- /dev/null +++ b/tests/test_cost_optimizer.py @@ -0,0 +1,155 @@ +import unittest +from datetime import date + +from quant60.config import FeeConfig +from quant60.costs import ( + CostModelParameters, + DatedFeeRule, + VersionedFeeSchedule, + forecast_trade_cost, +) +from quant60.domain import Side +from quant60.optimizer import ( + OptimizationError, + PortfolioCandidate, + PortfolioConstraints, + optimize_cvxpy, + optimize_deterministic, +) +from quant60.universe import UniverseState + + +class CostOptimizerTests(unittest.TestCase): + def test_fee_schedule_switches_by_effective_date(self): + first = DatedFeeRule( + date(2024, 1, 1), + date(2024, 6, 30), + FeeConfig(stamp_duty_rate=0.001), + "old", + ) + second = DatedFeeRule( + date(2024, 7, 1), + None, + FeeConfig(stamp_duty_rate=0.0005), + "new", + ) + schedule = VersionedFeeSchedule([second, first]) + self.assertEqual(schedule.at(date(2024, 6, 30)).version, "old") + self.assertEqual(schedule.at(date(2024, 7, 1)).version, "new") + + def test_exact_minimum_fee_and_capacity_are_visible(self): + rule = DatedFeeRule( + date(2024, 1, 1), + None, + FeeConfig( + commission_rate=0.0002, + minimum_commission=5, + stamp_duty_rate=0.0005, + transfer_fee_rate=0.00001, + ), + "account-v1", + ) + forecast = forecast_trade_cost( + symbol="600000.SH", + side=Side.BUY, + quantity=100, + price=10, + adv_quantity=500, + daily_volatility=0.02, + fee_rule=rule, + parameters=CostModelParameters(max_participation=0.10), + ) + self.assertGreaterEqual(forecast.explicit_fees, 5) + self.assertTrue(forecast.capacity_breached) + self.assertLess(forecast.low, forecast.base) + self.assertLess(forecast.base, forecast.high) + + def _candidates(self): + return [ + PortfolioCandidate( + "600000.SH", + score=0.04, + variance=0.04, + current_weight=0.10, + cost_bps=10, + max_adv_weight=0.30, + industry="bank", + ), + PortfolioCandidate( + "000001.SZ", + score=0.03, + variance=0.03, + current_weight=0.10, + cost_bps=12, + max_adv_weight=0.30, + industry="bank", + ), + PortfolioCandidate( + "300750.SZ", + score=0.02, + variance=0.05, + current_weight=0.05, + cost_bps=15, + max_adv_weight=0.30, + industry="industry", + state=UniverseState.FROZEN, + ), + PortfolioCandidate( + "000333.SZ", + score=-0.01, + variance=0.04, + current_weight=0.10, + cost_bps=10, + max_adv_weight=0.30, + industry="consumer", + state=UniverseState.SELL_ONLY, + ), + ] + + def test_deterministic_optimizer_honors_frozen_caps_and_turnover(self): + constraints = PortfolioConstraints( + gross_target=0.70, + single_name_cap=0.30, + industry_cap=0.45, + one_way_turnover_cap=0.10, + ) + result = optimize_deterministic(self._candidates(), constraints) + self.assertAlmostEqual(result.weights["300750.XSHE"], 0.05) + self.assertLessEqual(result.weights["000333.XSHE"], 0.10) + self.assertLessEqual(result.one_way_turnover, 0.10 + 1e-9) + self.assertLessEqual(result.gross_weight, 0.70 + 1e-9) + self.assertEqual(result.solver, "deterministic_reference") + + def test_held_excluded_candidate_fails_closed(self): + candidate = PortfolioCandidate( + "600000.SH", + score=0, + variance=0.04, + current_weight=0.1, + cost_bps=1, + max_adv_weight=0.2, + industry="bank", + state=UniverseState.EXCLUDED, + ) + with self.assertRaisesRegex(OptimizationError, "cannot disappear"): + optimize_deterministic([candidate]) + + def test_cvxpy_path_when_dependency_is_available(self): + constraints = PortfolioConstraints( + gross_target=0.70, + single_name_cap=0.30, + industry_cap=0.45, + one_way_turnover_cap=0.20, + ) + try: + result = optimize_cvxpy(self._candidates(), constraints) + except OptimizationError as exc: + if "CVXPY is unavailable" in str(exc): + self.skipTest(str(exc)) + raise + self.assertIn(result.status, {"OPTIMAL", "OPTIMAL_INACCURATE"}) + self.assertLessEqual(result.one_way_turnover, 0.20 + 1e-6) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_data_jqdata.py b/tests/test_data_jqdata.py new file mode 100644 index 0000000..3382b21 --- /dev/null +++ b/tests/test_data_jqdata.py @@ -0,0 +1,418 @@ +import json +import tempfile +import unittest +from datetime import date, datetime, timezone +from pathlib import Path + +from adapters.jqdata_local import ( + JQDataUnavailableError, + authenticate, + fetch_index_daily_snapshot, +) +from quant60.data_snapshot import ( + CanonicalBarRecord, + DataQualityError, + verify_data_snapshot, +) + + +class FakeFrame: + def __init__(self, rows): + self.rows = rows + + def iterrows(self): + return iter(self.rows) + + +class FakeJQData: + __version__ = "fake-1" + + def __init__(self): + self.authenticated = False + self.last_get_bars_kwargs = None + self.last_get_extras_kwargs = None + self.trade_day_queries = [] + self.full_calendar_queries = 0 + + def auth(self, username, password): + self.authenticated = username == "user" and password == "password" + + def is_auth(self): + return self.authenticated + + def get_trade_days(self, *, start_date, end_date): + self.trade_day_queries.append( + { + "start_date": start_date, + "end_date": end_date, + } + ) + return [start_date, end_date] + + def get_all_trade_days(self): + self.full_calendar_queries += 1 + return [ + date(2024, 1, 2), + date(2024, 1, 3), + date(2024, 1, 4), + ] + + def get_index_stocks(self, index_symbol, *, date): + return ["600000.XSHG", "000001.XSHE"] + + def get_extras( + self, + info, + security_list, + *, + start_date, + end_date, + df, + ): + self.last_get_extras_kwargs = { + "info": info, + "security_list": list(security_list), + "start_date": start_date, + "end_date": end_date, + "df": df, + } + return FakeFrame( + [ + ( + start_date, + {symbol: False for symbol in security_list}, + ), + ( + end_date, + {symbol: False for symbol in security_list}, + ), + ] + ) + + def get_bars(self, symbol, **kwargs): + self.last_get_bars_kwargs = dict(kwargs) + offset = 0 if symbol.startswith("600000") else 10 + return FakeFrame( + [ + ( + 0, + { + "date": kwargs["start_dt"], + "open": 10 + offset, + "high": 11 + offset, + "low": 9 + offset, + "close": 10.5 + offset, + "volume": 1000, + "money": 10_500 + offset * 1000, + "factor": 1.0, + "paused": False, + "high_limit": 11.5 + offset, + "low_limit": 8.5 + offset, + "pre_close": 10 + offset, + }, + ), + ( + 1, + { + "date": kwargs["end_dt"], + "open": 10.5 + offset, + "high": 11.5 + offset, + "low": 10 + offset, + "close": 11 + offset, + "volume": 1200, + "money": 13_200 + offset * 1200, + "factor": 1.0, + "paused": False, + "high_limit": 12 + offset, + "low_limit": 9 + offset, + "pre_close": 10.5 + offset, + }, + ), + ] + ) + + +class DataJQDataTests(unittest.TestCase): + def test_authentication_never_accepts_failed_login(self): + sdk = FakeJQData() + with self.assertRaisesRegex(JQDataUnavailableError, "not accepted"): + authenticate( + sdk, + username="user", + password="wrong", + interactive=False, + ) + authenticate( + sdk, + username="user", + password="password", + interactive=False, + ) + self.assertTrue(sdk.authenticated) + + def test_fake_jqdata_snapshot_is_canonical_and_verifiable(self): + sdk = FakeJQData() + authenticate( + sdk, + username="user", + password="password", + interactive=False, + ) + with tempfile.TemporaryDirectory() as directory: + paths = fetch_index_daily_snapshot( + sdk, + index_symbol="000905.XSHG", + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 3), + output_dir=directory, + retrieved_at=datetime(2024, 1, 4, tzinfo=timezone.utc), + ) + verified = verify_data_snapshot(paths["manifest"]) + self.assertTrue(verified["ok"]) + self.assertEqual(verified["bar_count"], 4) + self.assertEqual(verified["membership_count"], 4) + self.assertEqual( + sdk.last_get_bars_kwargs["end_dt"].date(), + date(2024, 1, 3), + ) + self.assertEqual( + sdk.last_get_bars_kwargs["end_dt"].time(), + datetime.strptime("23:59:59", "%H:%M:%S").time(), + ) + self.assertIs( + sdk.last_get_bars_kwargs["include_now"], + True, + ) + self.assertEqual( + sdk.last_get_extras_kwargs["info"], + "is_st", + ) + manifest = json.loads( + Path(paths["manifest"]).read_text(encoding="utf-8") + ) + self.assertEqual( + manifest["next_trading_session"], + "2024-01-04", + ) + self.assertEqual( + manifest["query"]["trading_sessions"], + ["2024-01-02", "2024-01-03", "2024-01-04"], + ) + self.assertEqual(sdk.full_calendar_queries, 1) + content = Path(paths["manifest"]).read_text(encoding="utf-8") + self.assertNotIn("password", content) + self.assertNotIn("user", content) + bars = ( + Path(directory) / "bars.jsonl" + ).read_text(encoding="utf-8") + self.assertIn('"adjustment_factor":1.0', bars) + self.assertIn('"limit_up":', bars) + + def test_next_session_comes_from_provider_calendar_across_holiday(self): + class NationalDayCalendarJQData(FakeJQData): + def get_all_trade_days(self): + self.full_calendar_queries += 1 + return [ + date(2024, 9, 27), + date(2024, 9, 30), + date(2024, 10, 8), + date(2024, 10, 9), + ] + + sdk = NationalDayCalendarJQData() + with tempfile.TemporaryDirectory() as directory: + paths = fetch_index_daily_snapshot( + sdk, + index_symbol="000905.XSHG", + start_date=date(2024, 9, 27), + end_date=date(2024, 9, 30), + output_dir=directory, + retrieved_at=datetime(2024, 10, 9, tzinfo=timezone.utc), + ) + manifest = json.loads( + Path(paths["manifest"]).read_text(encoding="utf-8") + ) + self.assertEqual( + manifest["next_trading_session"], + "2024-10-08", + ) + self.assertEqual( + manifest["query"]["trading_sessions"][-1], + "2024-10-08", + ) + + def test_invalid_next_session_calendar_response_fails_closed(self): + class BadCalendarJQData(FakeJQData): + def get_all_trade_days(self): + return [date(2024, 1, 2), date(2024, 1, 3)] + + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex( + JQDataUnavailableError, + "unique next trading session", + ): + fetch_index_daily_snapshot( + BadCalendarJQData(), + index_symbol="000905.XSHG", + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 3), + output_dir=directory, + retrieved_at=datetime( + 2024, + 1, + 4, + tzinfo=timezone.utc, + ), + ) + + def test_inconsistent_range_and_full_calendars_fail_closed(self): + class MissingMiddleSessionJQData(FakeJQData): + def get_all_trade_days(self): + return [ + date(2024, 1, 2), + date(2024, 1, 3), + date(2024, 1, 4), + ] + + def get_trade_days(self, *, start_date, end_date): + return [start_date] + + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex( + JQDataUnavailableError, + "inconsistent sessions", + ): + fetch_index_daily_snapshot( + MissingMiddleSessionJQData(), + index_symbol="000905.XSHG", + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 3), + output_dir=directory, + retrieved_at=datetime( + 2024, + 1, + 4, + tzinfo=timezone.utc, + ), + ) + + def test_same_shanghai_day_is_rejected_before_query(self): + sdk = FakeJQData() + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex( + JQDataUnavailableError, + "24:00 finalization", + ): + fetch_index_daily_snapshot( + sdk, + index_symbol="000905.XSHG", + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 3), + output_dir=directory, + retrieved_at=datetime( + 2024, + 1, + 3, + 6, + tzinfo=timezone.utc, + ), + ) + self.assertIsNone(sdk.last_get_bars_kwargs) + self.assertIsNone(sdk.last_get_extras_kwargs) + + def test_missing_pit_st_status_fails_closed(self): + class MissingStatusJQData(FakeJQData): + def get_extras(self, *args, **kwargs): + frame = super().get_extras(*args, **kwargs) + frame.rows[-1][1].pop("600000.XSHG") + return frame + + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex( + JQDataUnavailableError, + "omitted required field 600000.XSHG", + ): + fetch_index_daily_snapshot( + MissingStatusJQData(), + index_symbol="000905.XSHG", + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 3), + output_dir=directory, + retrieved_at=datetime( + 2024, + 1, + 4, + tzinfo=timezone.utc, + ), + ) + + def test_missing_required_provider_rule_field_fails_closed(self): + class MissingLimitJQData(FakeJQData): + def get_bars(self, symbol, **kwargs): + frame = super().get_bars(symbol, **kwargs) + del frame.rows[0][1]["high_limit"] + return frame + + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex( + JQDataUnavailableError, + "omitted required field high_limit", + ): + fetch_index_daily_snapshot( + MissingLimitJQData(), + index_symbol="000905.XSHG", + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 3), + output_dir=directory, + retrieved_at=datetime( + 2024, + 1, + 4, + tzinfo=timezone.utc, + ), + ) + + def test_membership_without_same_session_bar_fails_quality_gate(self): + class MissingEndBarJQData(FakeJQData): + def get_bars(self, symbol, **kwargs): + frame = super().get_bars(symbol, **kwargs) + frame.rows.pop() + return frame + + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex( + DataQualityError, + "membership has no same-session canonical bar", + ): + fetch_index_daily_snapshot( + MissingEndBarJQData(), + index_symbol="000905.XSHG", + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 3), + output_dir=directory, + retrieved_at=datetime( + 2024, + 1, + 4, + tzinfo=timezone.utc, + ), + ) + + def test_bad_ohlc_fails_quality_gate(self): + with self.assertRaisesRegex(DataQualityError, "open must lie"): + CanonicalBarRecord( + trading_date=date(2024, 1, 2), + symbol="600000.SH", + open=12, + high=11, + low=9, + close=10, + volume=100, + money=1000, + paused=False, + source="fixture", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_domain_broker.py b/tests/test_domain_broker.py new file mode 100644 index 0000000..6cee06a --- /dev/null +++ b/tests/test_domain_broker.py @@ -0,0 +1,247 @@ +import unittest +from datetime import date +from decimal import Decimal + +from quant60.broker import SimBroker +from quant60.config import ExecutionConfig, FeeConfig +from quant60.domain import ( + Bar, + Fill, + Order, + OrderStatus, + Portfolio, + Position, + Side, +) +from quant60.ledger import HashChainLedger + + +def bar( + day, + *, + symbol="600000.XSHG", + price="10", + volume=1000, + suspended=False, + limit_up=None, + limit_down=None, +): + value = Decimal(price) + return Bar( + trading_date=day, + symbol=symbol, + open=value, + high=value, + low=value, + close=value, + volume=volume, + suspended=suspended, + limit_up=Decimal(limit_up) if limit_up else None, + limit_down=Decimal(limit_down) if limit_down else None, + ) + + +class DomainBrokerTests(unittest.TestCase): + def setUp(self): + self.portfolio = Portfolio(Decimal("100000")) + self.ledger = HashChainLedger() + self.broker = SimBroker( + self.portfolio, + ExecutionConfig( + lot_size=100, + participation_rate=0.1, + slippage_bps=0, + ), + FeeConfig( + commission_rate=0, + minimum_commission=0, + stamp_duty_rate=0, + transfer_fee_rate=0, + ), + self.ledger, + ) + + def test_duplicate_state_callback_is_idempotent(self): + order = Order("O1", "600000.SH", Side.BUY, 100, date(2024, 1, 2)) + order.transition(OrderStatus.ACCEPTED) + order.transition(OrderStatus.ACCEPTED) + self.assertEqual(order.status, OrderStatus.ACCEPTED) + order.transition(OrderStatus.UNKNOWN) + order.transition(OrderStatus.UNKNOWN) + order.transition(OrderStatus.ACCEPTED) + self.assertEqual(order.status, OrderStatus.ACCEPTED) + + def test_late_fill_during_cancel_pending_is_accepted(self): + order = Order("O1", "600000.SH", Side.BUY, 200, date(2024, 1, 2)) + order.transition(OrderStatus.ACCEPTED) + order.transition(OrderStatus.CANCEL_PENDING) + order.apply_fill(100, Decimal("10")) + self.assertEqual(order.status, OrderStatus.PARTIALLY_FILLED) + self.assertEqual(order.filled_quantity, 100) + + order.transition(OrderStatus.CANCEL_PENDING) + order.apply_fill(100, Decimal("10.1")) + self.assertEqual(order.status, OrderStatus.FILLED) + self.assertEqual(order.filled_quantity, 200) + + def test_partial_fill_and_t_plus_one(self): + submit_day = date(2024, 1, 2) + fill_day = date(2024, 1, 3) + self.broker.execute_session( + {"600000.XSHG": bar(submit_day, volume=2000)} + ) + self.broker.submit("600000.SH", Side.BUY, 500, submit_day) + fills = self.broker.execute_session( + {"600000.XSHG": bar(fill_day, volume=100_000)} + ) + self.assertEqual(sum(item.quantity for item in fills), 200) + position = self.portfolio.position("600000.XSHG") + self.assertEqual(position.quantity, 200) + self.assertEqual(position.sellable_quantity, 0) + with self.assertRaises(ValueError): + position.sell(100) + self.portfolio.start_session() + self.assertEqual(position.sellable_quantity, 200) + + def test_suspension_and_locked_limit_block(self): + submit_day = date(2024, 1, 2) + next_day = date(2024, 1, 3) + self.broker.execute_session( + {"600000.XSHG": bar(submit_day, volume=1000)} + ) + self.broker.submit("600000.SH", Side.BUY, 100, submit_day) + self.assertEqual( + self.broker.execute_session( + {"600000.XSHG": bar(next_day, suspended=True, volume=0)} + ), + [], + ) + reasons = [ + record.payload["reason"] + for record in self.ledger.records + if record.event_type == "ORDER_BLOCKED" + ] + self.assertIn("SUSPENDED", reasons) + + other = SimBroker( + Portfolio(Decimal("100000")), + self.broker.execution, + self.broker.fees, + HashChainLedger(), + ) + other.execute_session( + {"600000.XSHG": bar(submit_day, volume=1000)} + ) + other.submit("600000.SH", Side.BUY, 100, submit_day) + self.assertEqual( + other.execute_session( + { + "600000.XSHG": bar( + next_day, + price="11", + limit_up="11", + limit_down="9", + ) + } + ), + [], + ) + + def test_open_fill_capacity_uses_only_prior_session_volume(self): + submit_day = date(2024, 1, 2) + fill_day = date(2024, 1, 3) + + def executed_quantity(current_day_volume): + broker = SimBroker( + Portfolio(Decimal("100000")), + self.broker.execution, + self.broker.fees, + HashChainLedger(), + ) + broker.execute_session( + {"600000.XSHG": bar(submit_day, volume=2000)} + ) + broker.submit("600000.SH", Side.BUY, 500, submit_day) + fills = broker.execute_session( + { + "600000.XSHG": bar( + fill_day, + volume=current_day_volume, + ) + } + ) + return sum(item.quantity for item in fills) + + self.assertEqual(executed_quantity(0), 200) + self.assertEqual(executed_quantity(10_000_000), 200) + + def test_fill_cash_and_position_conservation(self): + fill = Fill( + "F1", + "O1", + date(2024, 1, 3), + "600000.SH", + Side.BUY, + 100, + Decimal("10"), + Decimal("5"), + Decimal("0"), + Decimal("0.01"), + ) + self.portfolio.apply_fill(fill) + self.assertEqual(self.portfolio.position("600000.SH").quantity, 100) + self.assertEqual(self.portfolio.cash, Decimal("98994.990000")) + + def test_complete_odd_lot_sell_is_accepted_and_filled(self): + submit_day = date(2024, 1, 2) + fill_day = date(2024, 1, 3) + symbol = "600000.XSHG" + self.portfolio.positions[symbol] = Position( + symbol, + quantity=1050, + sellable_quantity=1050, + today_bought=0, + average_cost=Decimal("9"), + ) + self.broker.execute_session( + {symbol: bar(submit_day, volume=200_000)} + ) + order = self.broker.submit( + symbol, + Side.SELL, + 1050, + submit_day, + ) + self.assertEqual(order.status, OrderStatus.ACCEPTED) + fills = self.broker.execute_session( + {symbol: bar(fill_day, volume=200_000)} + ) + self.assertEqual([item.quantity for item in fills], [1050]) + self.assertEqual(order.status, OrderStatus.FILLED) + self.assertEqual(self.portfolio.position(symbol).quantity, 0) + + def test_partial_odd_lot_and_star_subminimum_are_rejected(self): + submit_day = date(2024, 1, 2) + symbol = "600000.XSHG" + self.portfolio.positions[symbol] = Position( + symbol, + quantity=1050, + sellable_quantity=1050, + today_bought=0, + average_cost=Decimal("9"), + ) + odd = self.broker.submit(symbol, Side.SELL, 50, submit_day) + self.assertEqual(odd.status, OrderStatus.REJECTED) + self.assertEqual(odd.status_reason, "NOT_BOARD_LOT") + star = self.broker.submit( + "688301.XSHG", + Side.BUY, + 100, + submit_day, + ) + self.assertEqual(star.status, OrderStatus.REJECTED) + self.assertEqual(star.status_reason, "BELOW_MARKET_MINIMUM") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_execution_planner.py b/tests/test_execution_planner.py new file mode 100644 index 0000000..39bcb39 --- /dev/null +++ b/tests/test_execution_planner.py @@ -0,0 +1,104 @@ +import unittest +from datetime import datetime, timedelta, timezone + +from quant60.domain import Side +from quant60.execution import ( + ExecutionBucket, + ParentOrderIntent, + ResidualPolicy, + market_data_is_fresh, + plan_guarded_twap_pov, +) + + +UTC8 = timezone(timedelta(hours=8)) + + +class ExecutionPlannerTests(unittest.TestCase): + def _parent(self, quantity=1000): + return ParentOrderIntent( + decision_id="D-20260726", + symbol="600000.SH", + side=Side.BUY, + quantity=quantity, + limit_price=10.0, + window_start=datetime(2026, 7, 27, 9, 35, tzinfo=UTC8), + window_end=datetime(2026, 7, 27, 10, 0, tzinfo=UTC8), + market_data_as_of=datetime(2026, 7, 27, 9, 34, 59, tzinfo=UTC8), + max_participation=0.10, + max_child_quantity=400, + max_child_notional=3000, + residual_policy=ResidualPolicy.DEFER, + ) + + def test_plan_conserves_quantity_and_honors_all_caps(self): + parent = self._parent() + buckets = [ + ExecutionBucket( + datetime(2026, 7, 27, 9, 35 + index * 5, tzinfo=UTC8), + 2000, + ) + for index in range(5) + ] + plan = plan_guarded_twap_pov(parent, buckets) + self.assertEqual( + plan.planned_quantity + plan.residual_quantity, + parent.quantity, + ) + self.assertTrue(plan.children) + for child in plan.children: + self.assertLessEqual(child.quantity, 200) + self.assertLessEqual(child.quantity * child.limit_price, 3000) + self.assertLessEqual( + child.quantity, + child.participation_cap_quantity, + ) + self.assertEqual( + len({child.child_id for child in plan.children}), + len(plan.children), + ) + + def test_zero_volume_buckets_leave_explicit_residual(self): + parent = self._parent(quantity=500) + buckets = [ + ExecutionBucket( + datetime(2026, 7, 27, 9, 35, tzinfo=UTC8), + 0, + ) + ] + plan = plan_guarded_twap_pov(parent, buckets) + self.assertEqual(plan.planned_quantity, 0) + self.assertEqual(plan.residual_quantity, 500) + self.assertEqual(plan.residual_policy, ResidualPolicy.DEFER) + + def test_market_data_freshness_rejects_old_and_future_quotes(self): + now = datetime(2026, 7, 27, 9, 35, tzinfo=UTC8) + self.assertTrue( + market_data_is_fresh( + as_of=now - timedelta(seconds=2), + now=now, + ) + ) + self.assertFalse( + market_data_is_fresh( + as_of=now - timedelta(seconds=6), + now=now, + ) + ) + self.assertFalse( + market_data_is_fresh( + as_of=now + timedelta(seconds=2), + now=now, + ) + ) + + def test_parent_identity_is_decision_symbol_side(self): + parent = self._parent() + self.assertEqual( + parent.idempotency_key, + "D-20260726:600000.XSHG:BUY", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_experiment.py b/tests/test_experiment.py new file mode 100644 index 0000000..7ef41c2 --- /dev/null +++ b/tests/test_experiment.py @@ -0,0 +1,46 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from quant60.experiment import ( + run_research_smoke, + verify_research_manifest, + write_research_artifacts, +) + + +class ResearchExperimentTests(unittest.TestCase): + def test_research_smoke_is_deterministic_and_verifiable(self): + first = run_research_smoke(trading_days=260, seed=11) + second = run_research_smoke(trading_days=260, seed=11) + self.assertEqual(first, second) + self.assertGreater(first["report"]["fold_count"], 0) + self.assertFalse(first["report"]["investment_value_claim"]) + self.assertTrue(first["targets"]) + + with tempfile.TemporaryDirectory() as left_dir, tempfile.TemporaryDirectory() as right_dir: + left = write_research_artifacts(first, left_dir) + right = write_research_artifacts(second, right_dir) + for name in left: + self.assertEqual( + hashlib.sha256(Path(left[name]).read_bytes()).hexdigest(), + hashlib.sha256(Path(right[name]).read_bytes()).hexdigest(), + name, + ) + verified = verify_research_manifest(left["manifest"]) + self.assertTrue(verified["ok"]) + + def test_research_verifier_rejects_tamper(self): + result = run_research_smoke(trading_days=260, seed=12) + with tempfile.TemporaryDirectory() as directory: + paths = write_research_artifacts(result, directory) + report = Path(paths["report"]) + report.write_bytes(report.read_bytes() + b" ") + with self.assertRaisesRegex(ValueError, "hash mismatch"): + verify_research_manifest(paths["manifest"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_factor_risk.py b/tests/test_factor_risk.py new file mode 100644 index 0000000..2473f0e --- /dev/null +++ b/tests/test_factor_risk.py @@ -0,0 +1,91 @@ +import unittest + +from quant60.factor_risk import ( + FactorExposureSnapshot, + RiskModelError, + build_factor_risk_forecast, + estimate_factor_returns, + factor_stress_loss, + portfolio_factor_variance, +) + + +class FactorRiskTests(unittest.TestCase): + def _exposure(self): + return FactorExposureSnapshot( + as_of="2026-07-24T15:00:00+08:00", + exposures={ + "600000.SH": {"market": 1.0, "style": 1.0}, + "000001.SZ": {"market": 1.0, "style": 1.0}, + "300750.SZ": {"market": 1.0, "style": -1.0}, + }, + ) + + def _forecast(self): + return build_factor_risk_forecast( + factor_return_history=[ + {"market": 0.01, "style": 0.02}, + {"market": -0.01, "style": -0.02}, + {"market": 0.02, "style": 0.01}, + {"market": -0.02, "style": -0.01}, + ], + residual_history={ + "600000.SH": [0.01, -0.01, 0.005], + "000001.SZ": [0.01, -0.01, 0.005], + "300750.SZ": [0.01, -0.01, 0.005], + }, + shrinkage=0.2, + ) + + def test_factor_return_fit_and_residuals_are_complete(self): + fit = estimate_factor_returns( + returns={ + "600000.SH": 0.03, + "000001.SZ": 0.02, + "300750.SZ": -0.01, + }, + exposure=self._exposure(), + ) + self.assertEqual(set(fit.factor_returns), {"market", "style"}) + self.assertEqual(len(fit.residuals), 3) + + def test_correlated_pair_has_more_risk_than_style_diversified_pair(self): + exposure = self._exposure() + forecast = self._forecast() + correlated = portfolio_factor_variance( + portfolio_weights={ + "600000.SH": 0.5, + "000001.SZ": 0.5, + }, + exposure=exposure, + forecast=forecast, + ) + diversified = portfolio_factor_variance( + portfolio_weights={ + "600000.SH": 0.5, + "300750.SZ": 0.5, + }, + exposure=exposure, + forecast=forecast, + ) + self.assertGreater(correlated, diversified) + + def test_missing_risk_input_fails_closed(self): + with self.assertRaisesRegex(RiskModelError, "missing"): + portfolio_factor_variance( + portfolio_weights={"000333.SZ": 1.0}, + exposure=self._exposure(), + forecast=self._forecast(), + ) + + def test_market_down_stress_is_positive_loss(self): + loss = factor_stress_loss( + portfolio_weights={"600000.SH": 0.5, "300750.SZ": 0.5}, + exposure=self._exposure(), + factor_shocks={"market": -0.10}, + ) + self.assertAlmostEqual(loss, 0.10) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 0000000..5c4c313 --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,83 @@ +import math +import unittest +from datetime import date + +from quant60.features import ( + TrainOnlyFeaturePreprocessor, + build_executable_label, + compute_price_features, +) +from quant60.synthetic import generate_synthetic_bars + + +class FeatureTests(unittest.TestCase): + def setUp(self): + self.bars = generate_synthetic_bars( + ["600000.XSHG"], + start_date="2024-01-02", + trading_days_count=150, + seed=7, + include_market_events=False, + ) + + def test_feature_as_of_does_not_change_when_future_bars_are_appended(self): + decision = self.bars[99].trading_date + early = compute_price_features(self.bars[:100], as_of=decision) + with_future = compute_price_features(self.bars, as_of=decision) + self.assertEqual(early.values, with_future.values) + self.assertEqual(early.history_end, decision) + + def test_long_window_features_are_explicitly_missing(self): + snapshot = compute_price_features(self.bars[:30]) + self.assertIsNone(snapshot.values["momentum_60"]) + self.assertIsNotNone(snapshot.values["momentum_20"]) + + def test_preprocessor_uses_training_statistics_and_missing_indicator(self): + processor = TrainOnlyFeaturePreprocessor(winsor_fraction=0).fit( + [{"a": 1.0}, {"a": 2.0}, {"a": 3.0}] + ) + transformed = processor.transform_one({"a": None}) + self.assertEqual(transformed["a__missing"], 1.0) + self.assertAlmostEqual(transformed["a"], 0.0) + extreme = processor.transform_one({"a": 1_000_000.0}) + expected = processor.transform_one({"a": 3.0}) + self.assertEqual(extreme["a"], expected["a"]) + + def test_executable_label_uses_sessions_after_decision(self): + decision_index = 80 + label = build_executable_label( + self.bars, + decision_date=self.bars[decision_index].trading_date, + horizon_sessions=5, + ) + self.assertEqual(label.status, "OK") + self.assertEqual( + label.entry_date, + self.bars[decision_index + 1].trading_date, + ) + self.assertEqual( + label.exit_date, + self.bars[decision_index + 6].trading_date, + ) + self.assertTrue(math.isfinite(label.value)) + + def test_suspended_entry_is_censored_not_dropped(self): + bars = generate_synthetic_bars( + ["600000.XSHG"], + start_date=date(2024, 1, 2), + trading_days_count=40, + seed=7, + include_market_events=True, + ) + decision = bars[10].trading_date + label = build_executable_label( + bars, + decision_date=decision, + horizon_sessions=5, + ) + self.assertEqual(label.status, "NON_EXECUTABLE_CENSORED") + self.assertIsNone(label.value) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ledger_reconcile.py b/tests/test_ledger_reconcile.py new file mode 100644 index 0000000..0e15f4f --- /dev/null +++ b/tests/test_ledger_reconcile.py @@ -0,0 +1,987 @@ +import copy +import json +import math +import re +import tempfile +import unittest +from datetime import date, datetime +from decimal import Decimal +from pathlib import Path + +from quant60.broker import SimBroker +from quant60.config import ExecutionConfig, FeeConfig +from quant60.contracts import ( + ContractValidationError as RuntimeContractValidationError, + validate_broker_snapshot, +) +from quant60.domain import Bar, Portfolio, Side +from quant60.ledger import ( + DuplicateEventError, + HashChainLedger, + LedgerIntegrityError, +) +from quant60.reconcile import reconcile_snapshot +from quant60.replay import ( + ConflictingDuplicateError, + LedgerProjector, + LedgerReplayError, + OutOfOrderError, + OverfillError, + UnknownOrderError, + project_ledger, +) +from quant60.snapshot_pipeline import first_executable_window + + +class ContractValidationError(AssertionError): + pass + + +def validate_json_schema(instance, schema, *, root=None, path="$"): + """Validate the JSON-Schema subset used by the replay contract. + + This deliberately small validator keeps the contract test dependency-free; + it is not exposed as a general JSON-Schema implementation. + """ + + root = schema if root is None else root + if "$ref" in schema: + reference = schema["$ref"] + if not reference.startswith("#/"): + raise ContractValidationError( + f"{path}: only local schema references are supported" + ) + target = root + for component in reference[2:].split("/"): + target = target[component.replace("~1", "/").replace("~0", "~")] + validate_json_schema(instance, target, root=root, path=path) + return + + expected_type = schema.get("type") + if expected_type is not None: + expected = ( + expected_type + if isinstance(expected_type, list) + else [expected_type] + ) + + def matches_type(name): + if name == "null": + return instance is None + if name == "object": + return isinstance(instance, dict) + if name == "array": + return isinstance(instance, list) + if name == "string": + return isinstance(instance, str) + if name == "boolean": + return isinstance(instance, bool) + if name == "integer": + return isinstance(instance, int) and not isinstance(instance, bool) + if name == "number": + return ( + isinstance(instance, (int, float)) + and not isinstance(instance, bool) + and math.isfinite(float(instance)) + ) + raise ContractValidationError( + f"{path}: unsupported schema type {name}" + ) + + if not any(matches_type(name) for name in expected): + raise ContractValidationError( + f"{path}: expected type {expected}, got {type(instance).__name__}" + ) + + if "const" in schema and instance != schema["const"]: + raise ContractValidationError( + f"{path}: expected constant {schema['const']!r}" + ) + if "enum" in schema and instance not in schema["enum"]: + raise ContractValidationError(f"{path}: value is outside enum") + if "minLength" in schema and len(instance) < schema["minLength"]: + raise ContractValidationError(f"{path}: string is too short") + if "pattern" in schema and re.search(schema["pattern"], instance) is None: + raise ContractValidationError(f"{path}: string does not match pattern") + if "minimum" in schema and instance < schema["minimum"]: + raise ContractValidationError(f"{path}: value is below minimum") + if ( + "exclusiveMinimum" in schema + and instance <= schema["exclusiveMinimum"] + ): + raise ContractValidationError( + f"{path}: value is not above exclusive minimum" + ) + + if schema.get("format") == "date": + try: + date.fromisoformat(instance) + except (TypeError, ValueError) as exc: + raise ContractValidationError( + f"{path}: value is not an ISO date" + ) from exc + if schema.get("format") == "date-time": + try: + parsed = datetime.fromisoformat(instance.replace("Z", "+00:00")) + except (AttributeError, ValueError) as exc: + raise ContractValidationError( + f"{path}: value is not an ISO date-time" + ) from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ContractValidationError( + f"{path}: date-time has no timezone offset" + ) + + if isinstance(instance, dict): + required = schema.get("required", []) + missing = [key for key in required if key not in instance] + if missing: + raise ContractValidationError( + f"{path}: missing required fields {missing}" + ) + properties = schema.get("properties", {}) + if schema.get("additionalProperties") is False: + unknown = sorted(set(instance) - set(properties)) + if unknown: + raise ContractValidationError( + f"{path}: unsupported fields {unknown}" + ) + for key, subschema in properties.items(): + if key in instance: + validate_json_schema( + instance[key], + subschema, + root=root, + path=f"{path}.{key}", + ) + for trigger, dependencies in schema.get( + "dependentRequired", + {}, + ).items(): + if trigger in instance: + absent = [key for key in dependencies if key not in instance] + if absent: + raise ContractValidationError( + f"{path}: {trigger} requires {absent}" + ) + + if "oneOf" in schema: + matches = 0 + failures = [] + for candidate in schema["oneOf"]: + try: + validate_json_schema( + instance, + candidate, + root=root, + path=path, + ) + matches += 1 + except ContractValidationError as exc: + failures.append(str(exc)) + if matches != 1: + raise ContractValidationError( + f"{path}: expected exactly one schema branch, got {matches}; " + f"{failures}" + ) + + +def order_payload( + order_id, + side, + quantity, + status, + *, + filled=0, + average="0", +): + return { + "order_id": order_id, + "symbol": "600000.XSHG", + "side": side, + "quantity": quantity, + "filled_quantity": filled, + "average_fill_price": average, + "status": status, + "submitted_at": "2024-01-02", + "metadata": {}, + "reason": None, + } + + +def fill_payload( + fill_id, + order_id, + side, + quantity, + price, + *, + commission="5", + stamp_duty="0", + transfer_fee="0", + cash_delta=None, +): + payload = { + "fill_id": fill_id, + "order_id": order_id, + "trading_date": "2024-01-03", + "symbol": "600000.XSHG", + "side": side, + "quantity": quantity, + "price": price, + "commission": commission, + "stamp_duty": stamp_duty, + "transfer_fee": transfer_fee, + } + if cash_delta is not None: + payload["cash_delta"] = cash_delta + return payload + + +def golden_ledger(): + ledger = HashChainLedger() + ledger.append( + "ORDER_STATUS", + order_payload("O1", "BUY", 200, "ACCEPTED"), + timestamp="2024-01-03T09:30:00+08:00", + event_id="order:O1:accepted", + ) + ledger.append( + "FILL", + fill_payload("F1", "O1", "BUY", 100, "10", cash_delta="-1005"), + timestamp="2024-01-03T09:31:00+08:00", + event_id="fill:F1", + ) + ledger.append( + "ORDER_STATUS", + order_payload( + "O1", + "BUY", + 200, + "PARTIALLY_FILLED", + filled=100, + average="10", + ), + timestamp="2024-01-03T09:31:00+08:00", + event_id="order:O1:partial:100", + ) + ledger.append( + "FILL", + fill_payload("F2", "O1", "BUY", 100, "11", cash_delta="-1105"), + timestamp="2024-01-03T09:32:00+08:00", + event_id="fill:F2", + ) + ledger.append( + "ORDER_STATUS", + order_payload( + "O1", + "BUY", + 200, + "FILLED", + filled=200, + average="10.5", + ), + timestamp="2024-01-03T09:32:00+08:00", + event_id="order:O1:filled", + ) + ledger.append( + "ORDER_STATUS", + order_payload("O2", "SELL", 100, "ACCEPTED"), + timestamp="2024-01-03T09:33:00+08:00", + event_id="order:O2:accepted", + ) + ledger.append( + "FILL", + fill_payload( + "F3", + "O2", + "SELL", + 100, + "12", + commission="5", + stamp_duty="0.6", + transfer_fee="0.01", + cash_delta="1194.39", + ), + timestamp="2024-01-03T09:34:00+08:00", + event_id="fill:F3", + ) + ledger.append( + "ORDER_STATUS", + order_payload( + "O2", + "SELL", + 100, + "FILLED", + filled=100, + average="12", + ), + timestamp="2024-01-03T09:34:00+08:00", + event_id="order:O2:filled", + ) + return ledger + + +class LedgerReconcileTests(unittest.TestCase): + def test_broker_snapshot_schema_requires_fresh_source_and_order_intent(self): + snapshot = { + "schema_version": "1.0", + "snapshot_id": "snapshot-1", + "signal_as_of": "2024-01-02T15:00:00+08:00", + "next_trading_session": "2024-01-03", + "first_executable_window": first_executable_window( + date(2024, 1, 3) + ), + "observation_as_of": "2024-01-03T09:10:00+08:00", + "as_of": "2024-01-03T09:10:00+08:00", + "source_time": "2024-01-03T09:09:59+08:00", + "broker": "fake", + "account_hash": "account-hash", + "cash": 1000.0, + "total_asset": 2000.0, + "positions": [ + { + "symbol": "600000.XSHG", + "quantity": 100, + "sellable_quantity": 100, + } + ], + "open_orders": [ + { + "order_id": "C1", + "broker_order_id": "B1", + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "filled_quantity": 0, + "status": "ACCEPTED", + "limit_price": 10.0, + "order_type": "LIMIT", + "time_in_force": "DAY", + } + ], + } + validate_broker_snapshot(snapshot) + + no_source = copy.deepcopy(snapshot) + del no_source["source_time"] + with self.assertRaises(RuntimeContractValidationError): + validate_broker_snapshot(no_source) + no_price = copy.deepcopy(snapshot) + del no_price["open_orders"][0]["limit_price"] + with self.assertRaises(RuntimeContractValidationError): + validate_broker_snapshot(no_price) + + def test_schema_validates_actual_sim_broker_replay_envelopes(self): + ledger = HashChainLedger() + broker = SimBroker( + Portfolio(Decimal("100000")), + ExecutionConfig( + lot_size=100, + participation_rate=0.1, + slippage_bps=0, + ), + FeeConfig( + commission_rate=0, + minimum_commission=0, + stamp_duty_rate=0, + transfer_fee_rate=0, + ), + ledger, + ) + broker.submit( + "600000.SH", + Side.BUY, + 100, + date(2024, 1, 2), + ) + value = Decimal("10") + broker.execute_session( + { + "600000.XSHG": Bar( + trading_date=date(2024, 1, 2), + symbol="600000.XSHG", + open=value, + high=value, + low=value, + close=value, + volume=1000, + ) + } + ) + broker.execute_session( + { + "600000.XSHG": Bar( + trading_date=date(2024, 1, 3), + symbol="600000.XSHG", + open=value, + high=value, + low=value, + close=value, + volume=1000, + ) + } + ) + envelopes = [ + record.as_dict() + for record in ledger.records + if record.event_type in {"ORDER_STATUS", "FILL"} + ] + self.assertEqual( + [record["event_type"] for record in envelopes], + ["ORDER_STATUS", "FILL", "ORDER_STATUS"], + ) + schema_path = ( + Path(__file__).resolve().parents[1] + / "schemas" + / "order_event.schema.json" + ) + schema = json.loads(schema_path.read_text(encoding="utf-8")) + for envelope in envelopes: + with self.subTest(event_id=envelope["event_id"]): + validate_json_schema(envelope, schema) + + projection = project_ledger(envelopes) + self.assertEqual(projection.cash_delta, Decimal("-1000.000000")) + self.assertEqual( + projection.position_deltas, + {"600000.XSHG": 100}, + ) + self.assertEqual(projection.orders["O00000001"].status, "FILLED") + + def test_schema_discriminates_payloads_and_projector_enforces_envelope(self): + schema_path = ( + Path(__file__).resolve().parents[1] + / "schemas" + / "order_event.schema.json" + ) + schema = json.loads(schema_path.read_text(encoding="utf-8")) + accepted = golden_ledger().records[0].as_dict() + + flat_legacy = { + "schema_version": "1.0", + "event_id": accepted["event_id"], + "event_time": accepted["timestamp"], + **accepted["payload"], + } + with self.assertRaises(ContractValidationError): + validate_json_schema(flat_legacy, schema) + with self.assertRaises(LedgerReplayError): + project_ledger([flat_legacy]) + + crossed = copy.deepcopy(accepted) + crossed["event_type"] = "FILL" + with self.assertRaises(ContractValidationError): + validate_json_schema(crossed, schema) + + unexpected = copy.deepcopy(accepted) + unexpected["payload"]["undocumented"] = True + with self.assertRaises(ContractValidationError): + validate_json_schema(unexpected, schema) + with self.assertRaises(LedgerReplayError): + project_ledger([unexpected]) + + unpaired_hash = copy.deepcopy(accepted) + del unpaired_hash["hash"] + with self.assertRaises(ContractValidationError): + validate_json_schema(unpaired_hash, schema) + with self.assertRaises(LedgerReplayError): + project_ledger([unpaired_hash]) + + no_sequence = copy.deepcopy(accepted) + del no_sequence["sequence"] + with self.assertRaises(ContractValidationError): + validate_json_schema(no_sequence, schema) + with self.assertRaises(LedgerReplayError): + project_ledger([no_sequence]) + + alias_fixture = copy.deepcopy(accepted) + del alias_fixture["previous_hash"] + del alias_fixture["hash"] + alias_fixture["payload"]["order_quantity"] = ( + alias_fixture["payload"].pop("quantity") + ) + alias_fixture["payload"]["cumulative_filled_quantity"] = ( + alias_fixture["payload"].pop("filled_quantity") + ) + with self.assertRaises(ContractValidationError): + validate_json_schema(alias_fixture, schema) + self.assertEqual( + project_ledger([alias_fixture]).orders["O1"].quantity, + 200, + ) + conflicting_alias = copy.deepcopy(accepted) + del conflicting_alias["previous_hash"] + del conflicting_alias["hash"] + conflicting_alias["payload"]["order_quantity"] = 300 + with self.assertRaises(ConflictingDuplicateError): + project_ledger([conflicting_alias]) + + def test_hash_chain_duplicate_and_tamper_detection(self): + ledger = HashChainLedger() + first = ledger.append( + "TEST", + {"value": 1}, + timestamp="2024-01-02T15:00:00+08:00", + event_id="event-1", + ) + self.assertIs( + ledger.append( + "TEST", + {"value": 1}, + timestamp="2024-01-02T15:00:00+08:00", + event_id="event-1", + ), + first, + ) + with self.assertRaises(DuplicateEventError): + ledger.append( + "TEST", + {"value": 2}, + timestamp="2024-01-02T15:00:00+08:00", + event_id="event-1", + ) + with tempfile.TemporaryDirectory() as directory: + path = ledger.write_jsonl(Path(directory) / "events.jsonl") + self.assertTrue(HashChainLedger.read_jsonl(path).verify()) + line = json.loads(path.read_text(encoding="utf-8").splitlines()[0]) + line["payload"]["value"] = 99 + path.write_text(json.dumps(line) + "\n", encoding="utf-8") + with self.assertRaises(LedgerIntegrityError): + HashChainLedger.read_jsonl(path) + + def test_legacy_reconciliation_api_remains_compatible(self): + okay = reconcile_snapshot( + local_cash="100", + local_positions={"600000.SH": 100}, + broker_cash="100.001", + broker_positions={"SH600000": 100}, + cash_tolerance="0.01", + ) + self.assertTrue(okay.balanced) + self.assertFalse(okay.complete) + self.assertFalse(okay.safe_to_open) + bad = reconcile_snapshot( + local_cash="100", + local_positions={"600000.SH": 100}, + broker_cash="99", + broker_positions={"600000.SH": 0}, + ) + self.assertFalse(bad.safe_to_open) + self.assertEqual( + {item.kind for item in bad.differences}, + {"CASH", "POSITION"}, + ) + + def test_identical_duplicate_open_order_identity_fails_closed(self): + order = { + "order_id": "DUPLICATE", + "broker_order_id": "B-DUPLICATE", + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "filled_quantity": 0, + "status": "ACCEPTED", + "limit_price": 10.0, + "order_type": "LIMIT", + "time_in_force": "DAY", + } + local = { + "cash": "100", + "positions": [], + "open_orders": [order, copy.deepcopy(order)], + "as_of": "2024-01-03T12:00:00+08:00", + } + broker = { + **local, + "open_orders": [copy.deepcopy(order), copy.deepcopy(order)], + "source_time": "2024-01-03T12:00:00+08:00", + } + report = reconcile_snapshot( + local_snapshot=local, + broker_snapshot=broker, + now="2024-01-03T12:00:00+08:00", + ) + self.assertTrue(report.complete) + self.assertFalse(report.balanced) + self.assertFalse(report.safe_to_open) + self.assertIn( + ("DUPLICATE_OPEN_ORDER", "DUPLICATE"), + {(item.kind, item.key) for item in report.differences}, + ) + + def test_complete_snapshot_checks_sellable_orders_and_freshness(self): + local = { + "cash": "1000", + "positions": [ + { + "symbol": "600000.XSHG", + "quantity": 200, + "sellable_quantity": 100, + } + ], + "open_orders": [ + { + "order_id": "C1", + "broker_order_id": "B-C1", + "symbol": "600000.XSHG", + "side": "SELL", + "quantity": 100, + "filled_quantity": 0, + "status": "ACCEPTED", + "limit_price": 10.0, + "order_type": "LIMIT", + "time_in_force": "DAY", + } + ], + "as_of": "2024-01-03T12:00:09+08:00", + } + broker = { + "cash": "1000.001", + "positions": [ + { + "symbol": "600000.SH", + "quantity": 200, + "sellable_quantity": 100, + } + ], + "open_orders": [ + { + "order_id": "C1", + "broker_order_id": "B-C1", + "symbol": "SH600000", + "side": "SELL", + "quantity": 100, + "filled_quantity": 0, + "status": "ACCEPTED", + "limit_price": 10.0, + "order_type": "LIMIT", + "time_in_force": "DAY", + } + ], + "as_of": "2024-01-03T12:00:08+08:00", + "source_time": "2024-01-03T12:00:05+08:00", + } + report = reconcile_snapshot( + local_snapshot=local, + broker_snapshot=broker, + now="2024-01-03T12:00:10+08:00", + max_snapshot_age_seconds=15, + max_source_age_seconds=15, + ) + self.assertTrue(report.safe_to_open) + + wrong_price = copy.deepcopy(broker) + wrong_price["open_orders"][0]["limit_price"] = 10.01 + price_report = reconcile_snapshot( + local_snapshot=local, + broker_snapshot=wrong_price, + now="2024-01-03T12:00:10+08:00", + max_snapshot_age_seconds=15, + max_source_age_seconds=15, + ) + self.assertFalse(price_report.safe_to_open) + self.assertIn( + ("OPEN_ORDER", "C1"), + { + (item.kind, item.key) + for item in price_report.differences + }, + ) + + unsafe_broker = copy.deepcopy(broker) + unsafe_broker["positions"][0]["sellable_quantity"] = 50 + unsafe_broker["open_orders"][0]["status"] = "UNKNOWN" + unsafe_broker["source_time"] = "2024-01-03T11:55:00+08:00" + unsafe = reconcile_snapshot( + local_snapshot=local, + broker_snapshot=unsafe_broker, + now="2024-01-03T12:00:10+08:00", + max_snapshot_age_seconds=15, + max_source_age_seconds=15, + ) + self.assertFalse(unsafe.safe_to_open) + self.assertEqual( + {item.kind for item in unsafe.differences}, + {"OPEN_ORDER", "SELLABLE", "STALE_SOURCE", "UNKNOWN_ORDER"}, + ) + + def test_complete_snapshot_missing_or_stale_as_of_fails_closed(self): + local = { + "cash": "100", + "positions": [], + "open_orders": [], + "as_of": "2024-01-03T11:00:00+08:00", + } + broker = { + "cash": "100", + "positions": [], + "open_orders": [], + "as_of": "2024-01-03T11:00:00+08:00", + "source_time": "2024-01-03T11:00:00+08:00", + } + stale = reconcile_snapshot( + local_snapshot=local, + broker_snapshot=broker, + now="2024-01-03T12:00:00+08:00", + ) + self.assertFalse(stale.safe_to_open) + self.assertIn( + "STALE_SNAPSHOT", + {item.kind for item in stale.differences}, + ) + missing = copy.deepcopy(broker) + del missing["open_orders"] + report = reconcile_snapshot( + local_snapshot={ + **local, + "as_of": "2024-01-03T12:00:00+08:00", + }, + broker_snapshot={ + **missing, + "as_of": "2024-01-03T12:00:00+08:00", + "source_time": "2024-01-03T12:00:00+08:00", + }, + now="2024-01-03T12:00:00+08:00", + ) + self.assertFalse(report.safe_to_open) + self.assertIn( + ("MISSING_FIELD", "broker.open_orders"), + {(item.kind, item.key) for item in report.differences}, + ) + + def test_freshness_tolerances_reject_nonfinite_and_negative_values(self): + snapshot = { + "cash": "100", + "positions": [], + "open_orders": [], + "as_of": "2024-01-03T12:00:00+08:00", + "source_time": "2024-01-03T12:00:00+08:00", + } + base = { + "local_snapshot": { + key: value + for key, value in snapshot.items() + if key != "source_time" + }, + "broker_snapshot": snapshot, + "now": "2024-01-03T12:00:00+08:00", + } + tolerance_names = ( + "max_snapshot_age_seconds", + "max_source_age_seconds", + "max_clock_skew_seconds", + "max_age_seconds", + ) + invalid_values = ( + float("nan"), + float("inf"), + float("-inf"), + -0.001, + ) + for name in tolerance_names: + for value in invalid_values: + with self.subTest(tolerance=name, value=value): + with self.assertRaisesRegex( + ValueError, + rf"{name} must be a finite non-negative number", + ): + reconcile_snapshot(**base, **{name: value}) + + def test_invalid_specific_tolerance_is_not_hidden_by_age_alias(self): + legacy = { + "local_cash": "100", + "local_positions": {}, + "broker_cash": "100", + "broker_positions": {}, + "max_age_seconds": 10, + } + for name in ( + "max_snapshot_age_seconds", + "max_source_age_seconds", + ): + with self.subTest(tolerance=name): + with self.assertRaisesRegex( + ValueError, + rf"{name} must be a finite non-negative number", + ): + reconcile_snapshot(**legacy, **{name: float("nan")}) + + def test_replay_frozen_golden_digest_and_accounting(self): + projection = project_ledger(golden_ledger()) + self.assertEqual(projection.cash_delta, Decimal("-915.610000")) + self.assertEqual( + projection.position_deltas, + {"600000.XSHG": 100}, + ) + self.assertEqual(projection.orders["O1"].filled_quantity, 200) + self.assertEqual( + projection.orders["O1"].average_fill_price, + Decimal("10.5000"), + ) + self.assertEqual(projection.orders["O2"].position_delta, -100) + self.assertTrue(projection.safe_to_reconcile) + self.assertEqual( + projection.digest, + "b9b48e5f11f43262e435318f6b6a39be74d3090506a3e5952bef4af9944545e5", + ) + + def test_replay_identical_duplicates_are_idempotent(self): + ledger = golden_ledger() + baseline = project_ledger(ledger) + records = [record.as_dict() for record in ledger.records] + duplicate_fill = copy.deepcopy(records[6]) + duplicate_fill["sequence"] = 9 + duplicate_fill["event_id"] = "fill:F3:duplicate-delivery" + duplicate_status = copy.deepcopy(records[7]) + duplicate_status["sequence"] = 10 + duplicate_status["event_id"] = "order:O2:filled:duplicate-delivery" + replayed = project_ledger(records + [duplicate_fill, duplicate_status]) + self.assertEqual(replayed.digest, baseline.digest) + self.assertEqual(replayed.cash_delta, baseline.cash_delta) + self.assertEqual(replayed.fill_ids, baseline.fill_ids) + + def test_replay_duplicate_id_cannot_hide_sequence_jump(self): + first = { + "sequence": 1, + "event_id": "order:e1", + "event_type": "ORDER_STATUS", + "timestamp": "2024-01-03T09:30:00+08:00", + "payload": order_payload( + "E1", + "BUY", + 100, + "ACCEPTED", + ), + } + exact_retry = copy.deepcopy(first) + second = copy.deepcopy(first) + second["sequence"] = 2 + second["event_id"] = "order:e2" + + legitimate = LedgerProjector() + legitimate.process(first) + legitimate.process(exact_retry) + legitimate.process(second) + projection = legitimate.projection() + self.assertEqual(projection.source_last_sequence, 2) + self.assertEqual(projection.relevant_event_count, 1) + + renumbered_retry = copy.deepcopy(first) + renumbered_retry["sequence"] = 2 + wrong_sequence = LedgerProjector() + wrong_sequence.process(first) + with self.assertRaisesRegex( + ConflictingDuplicateError, + "first observed at sequence 1, not 2", + ): + wrong_sequence.process(renumbered_retry) + + changed_timestamp = copy.deepcopy(first) + changed_timestamp["timestamp"] = "2024-01-03T09:30:01+08:00" + wrong_timestamp = LedgerProjector() + wrong_timestamp.process(first) + with self.assertRaises(ConflictingDuplicateError): + wrong_timestamp.process(changed_timestamp) + + jumped_retry = copy.deepcopy(first) + jumped_retry["sequence"] = 100 + poisoned = LedgerProjector() + poisoned.process(first) + with self.assertRaisesRegex(OutOfOrderError, "sequence gap"): + poisoned.process(jumped_retry) + with self.assertRaisesRegex( + LedgerReplayError, + "fail-closed after a prior replay error", + ): + poisoned.process(second) + + late_retry = LedgerProjector() + late_retry.process(first) + late_retry.process(second) + with self.assertRaises(OutOfOrderError): + late_retry.process(exact_retry) + + def test_replay_rejects_conflicting_duplicates(self): + ledger = golden_ledger() + records = [record.as_dict() for record in ledger.records] + conflict = copy.deepcopy(records[6]) + conflict["sequence"] = 9 + conflict["event_id"] = "fill:F3:conflicting-delivery" + conflict["payload"]["price"] = "13" + with self.assertRaises(ConflictingDuplicateError): + project_ledger(records + [conflict]) + + event_conflict = copy.deepcopy(records[0]) + event_conflict["payload"]["quantity"] = 300 + with self.assertRaises(ConflictingDuplicateError): + project_ledger([records[0], event_conflict]) + + def test_replay_rejects_unknown_order_overfill_and_out_of_order(self): + unknown_fill = { + "sequence": 1, + "event_id": "fill:unknown", + "event_type": "FILL", + "timestamp": "2024-01-03T09:31:00+08:00", + "payload": fill_payload("FU", "MISSING", "BUY", 100, "10"), + } + with self.assertRaises(UnknownOrderError): + project_ledger([unknown_fill]) + + accepted = { + "sequence": 1, + "event_id": "order:small", + "event_type": "ORDER_STATUS", + "timestamp": "2024-01-03T09:30:00+08:00", + "payload": order_payload("SMALL", "BUY", 100, "ACCEPTED"), + } + overfill = { + "sequence": 2, + "event_id": "fill:too-much", + "event_type": "FILL", + "timestamp": "2024-01-03T09:31:00+08:00", + "payload": fill_payload("F-OVER", "SMALL", "BUY", 200, "10"), + } + with self.assertRaises(OverfillError): + project_ledger([accepted, overfill]) + projector = LedgerProjector() + projector.process(accepted) + with self.assertRaises(OverfillError): + projector.process(overfill) + with self.assertRaises(LedgerReplayError): + projector.projection() + with self.assertRaises(LedgerReplayError): + projector.process( + { + **accepted, + "sequence": 3, + "event_id": "order:after-failure", + } + ) + + second = copy.deepcopy(accepted) + second["sequence"] = 2 + second["event_id"] = "order:second" + first_late = copy.deepcopy(accepted) + first_late["event_id"] = "order:first-late" + with self.assertRaises(OutOfOrderError): + project_ledger([second, first_late]) + + regression = copy.deepcopy(accepted) + regression["sequence"] = 2 + regression["event_id"] = "order:regression" + regression["payload"]["status"] = "NEW" + with self.assertRaises(OutOfOrderError): + project_ledger([accepted, regression]) + + time_regression = copy.deepcopy(accepted) + time_regression["sequence"] = 2 + time_regression["event_id"] = "order:time-regression" + time_regression["timestamp"] = "2024-01-03T09:29:59+08:00" + time_regression["payload"]["status"] = "UNKNOWN" + with self.assertRaises(OutOfOrderError): + project_ledger([accepted, time_regression]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mock_parity_report.py b/tests/test_mock_parity_report.py new file mode 100644 index 0000000..01be470 --- /dev/null +++ b/tests/test_mock_parity_report.py @@ -0,0 +1,35 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from tools.export_mock_parity import verify_report, write_report + + +class MockParityReportTests(unittest.TestCase): + def test_report_is_deterministic_and_explicitly_not_platform_evidence(self): + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "first.json" + second = Path(directory) / "second.json" + report = write_report(first) + write_report(second) + self.assertEqual(first.read_bytes(), second.read_bytes()) + self.assertTrue(report["comparisons"]["whole_plan_exact"]) + self.assertEqual(report["evidence_class"], "mock_contract_only") + self.assertFalse(report["real_platform_pass"]) + self.assertEqual(report["gate_credit"], []) + self.assertTrue(verify_report(first)["ok"]) + + def test_tampering_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "report.json" + write_report(path) + report = json.loads(path.read_text(encoding="utf-8")) + report["comparisons"]["whole_plan_exact"] = False + path.write_text(json.dumps(report), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "hash mismatch"): + verify_report(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pit_universe.py b/tests/test_pit_universe.py new file mode 100644 index 0000000..8688fe4 --- /dev/null +++ b/tests/test_pit_universe.py @@ -0,0 +1,192 @@ +import unittest +from datetime import datetime, timezone + +from quant60.pit import ( + PITIntegrityError, + PITRecord, + PointInTimeTable, + assert_information_available, +) +from quant60.universe import ( + SecurityEligibility, + UniverseState, + build_universe, + classify_security, +) + + +UTC = timezone.utc + + +def record( + *, + value, + effective="2024-03-31T00:00:00+08:00", + available="2024-04-30T18:00:00+08:00", + revision=0, +): + return PITRecord( + entity_id="600000.XSHG", + field="net_profit", + effective_time=effective, + available_time=available, + value=value, + source="fixture", + source_version="v1", + revision=revision, + ) + + +class PITUniverseTests(unittest.TestCase): + def test_pit_query_never_uses_future_announcement(self): + table = PointInTimeTable([record(value=100)]) + before = table.query( + "600000.XSHG", + "net_profit", + as_of="2024-04-30T17:59:59+08:00", + ) + after = table.query( + "600000.XSHG", + "net_profit", + as_of="2024-04-30T18:00:00+08:00", + ) + self.assertIsNone(before) + self.assertEqual(after.value, 100) + + def test_revision_is_visible_only_after_its_own_available_time(self): + table = PointInTimeTable( + [ + record(value=100), + record( + value=90, + available="2024-05-05T09:00:00+08:00", + revision=1, + ), + ] + ) + first = table.query( + "600000.XSHG", + "net_profit", + as_of="2024-05-01T15:00:00+08:00", + ) + revised = table.query( + "600000.XSHG", + "net_profit", + as_of="2024-05-06T15:00:00+08:00", + ) + self.assertEqual(first.value, 100) + self.assertEqual(revised.value, 90) + + def test_future_injection_gate_fails_closed(self): + future = record( + value=100, + available="2024-05-02T09:00:00+08:00", + ) + with self.assertRaisesRegex(PITIntegrityError, "future-available"): + assert_information_available( + [future], + as_of="2024-05-01T15:00:00+08:00", + ) + + def test_naive_timestamps_and_duplicate_identity_are_rejected(self): + with self.assertRaisesRegex(PITIntegrityError, "timezone"): + record( + value=100, + effective=datetime(2024, 3, 31), + ) + item = record(value=100) + with self.assertRaisesRegex(PITIntegrityError, "duplicate"): + PointInTimeTable([item, item]) + + def test_data_version_is_order_independent(self): + left = record(value=100) + right = PITRecord( + entity_id="000001.XSHE", + field="net_profit", + effective_time=datetime(2024, 3, 31, tzinfo=UTC), + available_time=datetime(2024, 4, 30, tzinfo=UTC), + value=80, + source="fixture", + source_version="v1", + ) + self.assertEqual( + PointInTimeTable([left, right]).data_version, + PointInTimeTable([right, left]).data_version, + ) + + def test_universe_states_keep_untradable_holdings_visible(self): + frozen = classify_security( + SecurityEligibility( + symbol="600000.SH", + index_member=False, + listing_sessions=500, + median_turnover=1_000_000, + minimum_turnover=100_000, + suspended=True, + held_quantity=1000, + sellable_quantity=1000, + ) + ) + sell_only = classify_security( + SecurityEligibility( + symbol="000001.SZ", + index_member=True, + listing_sessions=500, + median_turnover=1_000_000, + minimum_turnover=100_000, + is_st=True, + held_quantity=1000, + sellable_quantity=1000, + ) + ) + self.assertEqual(frozen.state, UniverseState.FROZEN) + self.assertTrue(frozen.can_hold) + self.assertFalse(frozen.can_sell) + self.assertEqual(sell_only.state, UniverseState.SELL_ONLY) + self.assertTrue(sell_only.can_sell) + hold_only = classify_security( + SecurityEligibility( + symbol="000333.SZ", + index_member=True, + listing_sessions=500, + median_turnover=50_000, + minimum_turnover=100_000, + held_quantity=1000, + sellable_quantity=1000, + ) + ) + self.assertEqual(hold_only.state, UniverseState.HOLD_ONLY) + self.assertTrue(hold_only.can_hold) + self.assertFalse(hold_only.can_open) + + def test_openable_and_excluded_are_distinct(self): + decisions = build_universe( + [ + SecurityEligibility( + symbol="600000.SH", + index_member=True, + listing_sessions=500, + median_turnover=1_000_000, + minimum_turnover=100_000, + ), + SecurityEligibility( + symbol="300750.SZ", + index_member=True, + listing_sessions=20, + median_turnover=1_000_000, + minimum_turnover=100_000, + ), + ] + ) + self.assertEqual( + decisions["600000.XSHG"].state, + UniverseState.OPENABLE, + ) + self.assertEqual( + decisions["300750.XSHE"].state, + UniverseState.EXCLUDED, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_portable_core.py b/tests/test_portable_core.py new file mode 100644 index 0000000..bbc929c --- /dev/null +++ b/tests/test_portable_core.py @@ -0,0 +1,171 @@ +import unittest + +from quant60.portable_core import ( + build_rebalance_plan, + capped_target_weights, + momentum_score, + normalize_symbol, + order_deltas, + target_quantities, +) + + +class PortableCoreTests(unittest.TestCase): + def test_symbol_round_trip(self): + inputs = [ + ("600000.XSHG", "600000.SH", "SH600000"), + ("000001.XSHE", "000001.SZ", "SZ000001"), + ("830799.XBSE", "830799.BJ", "BJ830799"), + ] + for canonical, qmt, qlib in inputs: + self.assertEqual(normalize_symbol(qmt), canonical) + self.assertEqual(normalize_symbol(qlib), canonical) + self.assertEqual(normalize_symbol(canonical, "qmt"), qmt) + self.assertEqual(normalize_symbol(canonical, "qlib"), qlib) + + def test_momentum_uses_latest_completed_bar(self): + self.assertAlmostEqual(momentum_score([10, 11, 12], 2, 0), 0.2) + self.assertAlmostEqual(momentum_score([10, 11, 12], 1, 1), 0.1) + + def test_caps_and_cash_buffer_are_not_double_counted(self): + weights = capped_target_weights( + {"600000.XSHG": 3, "000001.XSHE": 2, "300750.XSHE": 1}, + max_weight=0.4, + gross_target=0.95, + ) + self.assertAlmostEqual(sum(weights.values()), 0.95) + self.assertLessEqual(max(weights.values()), 0.4 + 1e-12) + quantities = target_quantities( + weights, + {symbol: 10 for symbol in weights}, + equity=1_000_000, + lot_size=100, + cash_buffer=0.02, + ) + invested = sum(quantity * 10 for quantity in quantities.values()) + self.assertGreater(invested, 930_000) + self.assertLessEqual(invested, 980_000) + + def test_non_finite_or_out_of_range_portfolio_inputs_are_rejected(self): + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(kind="score", value=value): + with self.assertRaises(ValueError): + capped_target_weights({"600000.SH": value}) + with self.subTest(kind="weight", value=value): + with self.assertRaises(ValueError): + target_quantities( + {"600000.SH": value}, + {"600000.SH": 10.0}, + 100_000, + ) + with self.subTest(kind="equity", value=value): + with self.assertRaises(ValueError): + target_quantities( + {"600000.SH": 0.5}, + {"600000.SH": 10.0}, + value, + ) + for max_weight, gross_target in ((1.1, 0.9), (0.2, 1.1)): + with self.subTest(max_weight=max_weight, gross_target=gross_target): + with self.assertRaises(ValueError): + capped_target_weights( + {"600000.SH": 1.0}, + max_weight=max_weight, + gross_target=gross_target, + ) + + def test_sell_delta_respects_sellable(self): + deltas = order_deltas( + {"600000.XSHG": 0}, + {"600000.XSHG": 1000}, + {"600000.XSHG": 300}, + lot_size=100, + ) + self.assertEqual(deltas, {"600000.XSHG": -300}) + + def test_star_targets_and_partial_orders_respect_200_share_minimum(self): + self.assertEqual( + target_quantities( + {"688301.XSHG": 1.0}, + {"688301.XSHG": 300.0}, + equity=50_000, + lot_size=100, + cash_buffer=0.02, + ), + {"688301.XSHG": 0}, + ) + self.assertEqual( + target_quantities( + {"688301.XSHG": 1.0}, + {"688301.XSHG": 200.0}, + equity=50_000, + lot_size=100, + cash_buffer=0.02, + ), + {"688301.XSHG": 200}, + ) + self.assertEqual( + order_deltas( + {"688301.XSHG": 300}, + {"688301.XSHG": 200}, + {"688301.XSHG": 200}, + lot_size=100, + ), + {}, + ) + self.assertEqual( + order_deltas( + {"688301.XSHG": 1300}, + {"688301.XSHG": 1400}, + {"688301.XSHG": 1400}, + lot_size=100, + ), + {}, + ) + + def test_complete_odd_lot_liquidation_is_preserved(self): + self.assertEqual( + order_deltas( + {"600000.XSHG": 0}, + {"600000.XSHG": 1050}, + {"600000.XSHG": 1050}, + lot_size=100, + ), + {"600000.XSHG": -1050}, + ) + self.assertEqual( + order_deltas( + {"688301.XSHG": 0}, + {"688301.XSHG": 150}, + {"688301.XSHG": 150}, + lot_size=100, + ), + {"688301.XSHG": -150}, + ) + + def test_complete_plan_is_deterministic(self): + histories = { + "600000.SH": [10 + index * 0.1 for index in range(25)], + "000001.SZ": [10 + index * 0.05 for index in range(25)], + } + arguments = dict( + price_history=histories, + current={}, + sellable={}, + equity=100_000, + lookback=20, + skip=0, + top_n=2, + max_weight=0.5, + gross_target=0.95, + cash_buffer=0.02, + lot_size=100, + ) + self.assertEqual( + build_rebalance_plan(**arguments), + build_rebalance_plan(**arguments), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_project_scaffold.py b/tests/test_project_scaffold.py new file mode 100644 index 0000000..72525ad --- /dev/null +++ b/tests/test_project_scaffold.py @@ -0,0 +1,49 @@ +import json +import subprocess +import sys +import unittest +from pathlib import Path + + +PROJECT = Path(__file__).resolve().parents[1] + + +class ProjectScaffoldTests(unittest.TestCase): + def test_colab_notebook_is_valid_plain_python(self): + result = subprocess.run( + [ + sys.executable, + "tools/run_colab_notebook.py", + "notebooks/quant_os_colab.ipynb", + ], + cwd=PROJECT, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout.strip().splitlines()[-1]) + self.assertTrue(payload["ok"]) + self.assertGreaterEqual(payload["code_cells"], 5) + self.assertFalse(payload["executed"]) + + def test_secret_files_are_ignored(self): + ignore = (PROJECT / ".gitignore").read_text(encoding="utf-8") + for entry in (".env", "secrets/", "credentials/", "userdata_mini/"): + self.assertIn(entry, ignore) + + def test_claude_project_contract_exists(self): + content = (PROJECT / "CLAUDE.md").read_text(encoding="utf-8") + self.assertIn("executable source of truth for Quant OS", content) + self.assertIn("make local", content) + + def test_gitea_workflow_is_at_monorepo_root(self): + workflow = PROJECT.parent / ".gitea/workflows/quant-os-ci.yml" + self.assertTrue(workflow.is_file()) + content = workflow.read_text(encoding="utf-8") + self.assertIn("working-directory: quant-os", content) + self.assertIn("bash scripts/validate_all.sh", content) + self.assertFalse((PROJECT / ".gitea/workflows/ci.yml").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_qmt_shadow.py b/tests/test_qmt_shadow.py new file mode 100644 index 0000000..d4b6255 --- /dev/null +++ b/tests/test_qmt_shadow.py @@ -0,0 +1,2179 @@ +import hashlib +import hmac +import json +import tempfile +import unittest +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import patch + +from quant60.backtest import _contract_manifest, _engine_source_manifest +from quant60.qmt_shadow import ( + DECISION_EQUITY_ABSOLUTE_TOLERANCE, + DECISION_EQUITY_RELATIVE_TOLERANCE, + MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE, + MAX_SHADOW_QUERY_DURATION_SECONDS, + QmtShadowPlanError, + _authenticated_evidence_envelope, + _authenticated_evidence_hmac, + build_qmt_shadow_plan, + verify_qmt_shadow_plan, + write_qmt_shadow_plan, +) +from quant60.ledger import canonical_json +from quant60.snapshot_pipeline import ( + first_executable_window, + write_snapshot_decision, +) + + +UTC8 = timezone(timedelta(hours=8)) +ACCOUNT_HASH_KEY = b"k" * 32 + + +def fixture_account_hash(account_id: str = "account-fixture") -> str: + return hmac.new( + ACCOUNT_HASH_KEY, + ("qmt-account-v1\0" + account_id).encode("utf-8"), + hashlib.sha256, + ).hexdigest() + + +def decision_artifact( + root: Path, + as_of: datetime, + *, + bind_account: bool = True, + broker_account_hash: str | None = None, + broker_observed_at: datetime | None = None, + next_session: date = date(2026, 7, 27), + decision_equity: float | None = 100_000, + first_target_quantity: int = 1000, +) -> str: + run_id = "q60d-shadow-fixture" + decision_id = "D-shadow-fixture" + data_version = "fixture-data-v1" + targets = [ + { + "schema_version": "1.0", + "run_id": run_id, + "decision_id": decision_id, + "as_of": as_of.isoformat(), + "symbol": "000001.XSHE", + "target_weight": 0.10, + "target_quantity": first_target_quantity, + "lot_size": 100, + "signal_id": "S-000001", + "constraint_state": "FEASIBLE", + "config_hash": "fixture-config", + "metadata": {}, + }, + { + "schema_version": "1.0", + "run_id": run_id, + "decision_id": decision_id, + "as_of": as_of.isoformat(), + "symbol": "600000.XSHG", + "target_weight": 0.05, + "target_quantity": 500, + "lot_size": 100, + "signal_id": "S-600000", + "constraint_state": "FEASIBLE", + "config_hash": "fixture-config", + "metadata": {}, + }, + ] + engine_hash, engine_sources = _engine_source_manifest() + contract_hash, contracts = _contract_manifest() + executable_window = first_executable_window(next_session) + default_observation = datetime( + next_session.year, + next_session.month, + next_session.day, + 9, + 0, + tzinfo=UTC8, + ) + input_data_manifest = { + "data_version": data_version, + "next_trading_session": next_session.isoformat(), + "query": { + "trading_sessions": [ + as_of.date().isoformat(), + next_session.isoformat(), + ] + }, + } + input_manifest_sha256 = hashlib.sha256( + ( + json.dumps( + input_data_manifest, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + ).hexdigest() + result = { + "signals": [], + "targets": targets, + "decision": { + "schema_version": "1.0", + "run_id": run_id, + "decision_id": decision_id, + "as_of": as_of.isoformat(), + "signal_as_of": as_of.isoformat(), + "next_trading_session": next_session.isoformat(), + "first_executable_window": executable_window, + "evidence_class": "provider_snapshot_decision", + "provider": "fixture", + "provider_version": "1", + "data_version": data_version, + "input_manifest_sha256": input_manifest_sha256, + "config_hash": "fixture-config", + "model_version": "portable-momentum-v1", + "equity": decision_equity, + "weights": { + "000001.XSHE": 0.10, + "600000.XSHG": 0.05, + }, + "targets": { + "000001.XSHE": first_target_quantity, + "600000.XSHG": 500, + }, + "orders": {}, + "universe": {}, + "risk": { + "approved": True, + "gross_weight": 0.15, + "one_way_turnover": 0.10, + "violations": [], + }, + "broker_snapshot": ( + { + "snapshot_id": "QMT-SHADOW-fixture", + "signal_as_of": as_of.isoformat(), + "next_trading_session": next_session.isoformat(), + "first_executable_window": executable_window, + "observation_as_of": ( + broker_observed_at or default_observation + ).isoformat(), + "as_of": ( + broker_observed_at or default_observation + ).isoformat(), + "source_time": ( + broker_observed_at or default_observation + ).isoformat(), + "broker": "QMT_XTTRADER", + "account_hash": ( + broker_account_hash or fixture_account_hash() + ), + "snapshot_sha256": "fixture-snapshot-sha256", + } + if bind_account + else None + ), + "investment_value_claim": False, + "real_platform_backtest": False, + }, + "input_data_manifest": input_data_manifest, + "manifest": { + "schema_version": "1.0", + "run_id": run_id, + "evidence_class": "provider_snapshot_decision", + "data_version": data_version, + "input_manifest_sha256": input_manifest_sha256, + "config_hash": "fixture-config", + "engine_source_sha256": engine_hash, + "engine_sources": engine_sources, + "contracts_sha256": contract_hash, + "contract_schemas": contracts, + "model_version": "portable-momentum-v1", + "signal_as_of": as_of.isoformat(), + "signal_clock": "T_CLOSE_COMPLETE", + "first_executable_clock": "T_PLUS_1_OPEN", + "next_trading_session": next_session.isoformat(), + "first_executable_window": executable_window, + "price_adjustment": "PIT_FRONT_FROM_RAW_AND_FACTOR", + "deterministic": True, + }, + } + return write_snapshot_decision(result, root)["manifest"] + + +class FakeReadOnlyAdapter: + def __init__( + self, + *, + orders=None, + second_orders=None, + trades=None, + asset=None, + positions=None, + errors=None, + live_authorization_audit=None, + flip_live_after_query=False, + flip_state_after_query=False, + ): + self.state = "READY" + self.allow_live_orders = False + self.errors = list(errors or []) + self.live_authorization_audit = list( + live_authorization_audit or [] + ) + self.calls = [] + self.mutation_calls = [] + self._orders = list(orders or []) + self._trades = list(trades or []) + self._positions = list( + positions + if positions is not None + else [ + { + "account_id": "account-fixture", + "symbol": "600000.XSHG", + "volume": 1000, + "sellable": 1000, + "market_value": 10_000, + "avg_price": 10, + } + ] + ) + self._asset = dict( + asset + or { + "account_id": "account-fixture", + "cash": 90_000, + "market_value": 10_000, + "total_asset": 100_000, + } + ) + self._second_orders = ( + list(second_orders) if second_orders is not None else None + ) + self._order_queries = 0 + self._flip_live_after_query = bool(flip_live_after_query) + self._flip_state_after_query = bool(flip_state_after_query) + + def query_asset(self): + self.calls.append("asset") + return dict(self._asset) + + def query_positions(self): + self.calls.append("positions") + return list(self._positions) + + def query_orders(self, cancelable_only=False): + self.calls.append(("orders", cancelable_only)) + self._order_queries += 1 + if self._order_queries == 2 and self._flip_live_after_query: + self.allow_live_orders = True + if self._order_queries == 2 and self._flip_state_after_query: + self.state = "DEGRADED" + if self._order_queries == 2 and self._second_orders is not None: + return list(self._second_orders) + return list(self._orders) + + def query_trades(self): + self.calls.append("trades") + return list(self._trades) + + def submit_order(self, *args, **kwargs): + self.mutation_calls.append(("submit", args, kwargs)) + raise AssertionError("shadow planner must never submit") + + def cancel_order(self, *args, **kwargs): + self.mutation_calls.append(("cancel", args, kwargs)) + raise AssertionError("shadow planner must never cancel") + + +def fixed_clock(moment: datetime): + values = iter((moment, moment)) + return lambda: next(values) + + +_UNCHANGED = object() + + +def payload_sha256(payload) -> str: + return hashlib.sha256( + canonical_json(payload).encode("utf-8") + ).hexdigest() + + +def reseal_shadow_artifacts( + paths: dict[str, str], + plan: dict, + *, + broker_observation=_UNCHANGED, + broker_snapshot=_UNCHANGED, + evidence_hmac_key: bytes | None = None, + manifest_updates: dict | None = None, +) -> None: + if broker_observation is not _UNCHANGED: + Path(paths["broker_observation"]).write_text( + json.dumps( + broker_observation, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + if broker_snapshot is not _UNCHANGED: + Path(paths["broker_snapshot"]).write_text( + json.dumps( + broker_snapshot, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + plan["plan_id"] = "" + plan["plan_id"] = "QSP-" + payload_sha256(plan)[:20] + plan_path = Path(paths["plan"]) + plan_path.write_text( + json.dumps( + plan, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + manifest_path = Path(paths["manifest"]) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["plan_id"] = plan["plan_id"] + if manifest_updates: + manifest.update(manifest_updates) + if evidence_hmac_key is not None: + if broker_observation is _UNCHANGED: + broker_observation = json.loads( + Path(paths["broker_observation"]).read_text( + encoding="utf-8" + ) + ) + if broker_snapshot is _UNCHANGED: + broker_snapshot = json.loads( + Path(paths["broker_snapshot"]).read_text( + encoding="utf-8" + ) + ) + manifest["authenticated_evidence_hmac_sha256"] = ( + _authenticated_evidence_hmac( + _authenticated_evidence_envelope( + plan=plan, + broker_observation=broker_observation, + broker_snapshot=broker_snapshot, + decision_manifest_sha256=manifest[ + "decision_manifest_sha256" + ], + planner_source_sha256=manifest[ + "planner_source_sha256" + ], + engine_source_sha256=manifest[ + "engine_source_sha256" + ], + published_artifact_sha256={ + name: hashlib.sha256( + Path(paths[path_key]).read_bytes() + ).hexdigest() + for name, path_key in ( + ("shadow_plan.json", "plan"), + ( + "broker_snapshot.json", + "broker_snapshot", + ), + ( + "broker_observation.json", + "broker_observation", + ), + ) + }, + ), + evidence_hmac_key, + ) + ) + for artifact_name, path_key in ( + ("shadow_plan.json", "plan"), + ("broker_snapshot.json", "broker_snapshot"), + ("broker_observation.json", "broker_observation"), + ): + manifest["artifact_sha256"][artifact_name] = hashlib.sha256( + Path(paths[path_key]).read_bytes() + ).hexdigest() + manifest_path.write_text( + json.dumps( + manifest, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + + +def reseal_shadow_plan(paths: dict[str, str], plan: dict) -> None: + reseal_shadow_artifacts( + paths, + plan, + evidence_hmac_key=ACCOUNT_HASH_KEY, + ) + + +def force_ready_projection(plan: dict) -> None: + plan["blockers"] = [] + plan["status"] = "SHADOW_READY" + plan["safety"]["ready_for_manual_review"] = True + for row in plan["target_diffs"]: + delta = int(row["board_lot_feasible_delta"]) + row["proposed_delta"] = delta + row["proposed_side"] = ( + "BUY" if delta > 0 else "SELL" if delta < 0 else None + ) + row["proposed_quantity"] = abs(delta) + + +class QmtShadowPlanTests(unittest.TestCase): + def test_ready_plan_is_deterministic_read_only_and_verifiable(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, 5, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + bound_observation = observed - timedelta(seconds=5) + decision = decision_artifact( + root / "decision", + decision_time, + broker_observed_at=bound_observation, + ) + first_adapter = FakeReadOnlyAdapter() + second_adapter = FakeReadOnlyAdapter() + first = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=first_adapter, + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + second = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=second_adapter, + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertEqual(first, second) + self.assertEqual(first["plan"]["status"], "SHADOW_READY") + self.assertFalse(first["plan"]["blockers"]) + equity_contract = first["plan"]["portfolio"][ + "equity_consistency" + ] + self.assertTrue(equity_contract["within_tolerance"]) + self.assertEqual( + equity_contract["absolute_tolerance"], + DECISION_EQUITY_ABSOLUTE_TOLERANCE, + ) + self.assertEqual( + equity_contract["relative_tolerance"], + DECISION_EQUITY_RELATIVE_TOLERANCE, + ) + self.assertEqual( + first["plan"]["decision"]["signal_as_of"], + decision_time.isoformat(), + ) + self.assertEqual( + first["plan"]["decision"][ + "bound_broker_observation_as_of" + ], + bound_observation.isoformat(), + ) + rows = { + row["symbol"]: row + for row in first["plan"]["target_diffs"] + } + self.assertEqual(rows["000001.XSHE"]["proposed_delta"], 1000) + self.assertEqual(rows["600000.XSHG"]["proposed_delta"], -500) + self.assertAlmostEqual( + rows["600000.XSHG"]["current_weight"], 0.10 + ) + self.assertFalse( + first["plan"]["safety"]["ready_for_live_submission"] + ) + self.assertEqual(first_adapter.mutation_calls, []) + self.assertEqual( + first_adapter.calls, + [ + ("orders", False), + "asset", + "positions", + "trades", + ("orders", False), + ], + ) + paths = write_qmt_shadow_plan(first, root / "shadow") + verified = verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertTrue(verified["ok"]) + self.assertTrue(verified["read_only"]) + self.assertEqual(verified["status"], "SHADOW_READY") + observation = json.loads( + Path(paths["broker_observation"]).read_text( + encoding="utf-8" + ) + ) + self.assertNotIn("account-fixture", json.dumps(observation)) + self.assertEqual( + observation["asset"]["account_hash"], + fixture_account_hash(), + ) + broker_snapshot = json.loads( + Path(paths["broker_snapshot"]).read_text( + encoding="utf-8" + ) + ) + self.assertEqual( + broker_snapshot["signal_as_of"], + decision_time.isoformat(), + ) + self.assertEqual( + broker_snapshot["observation_as_of"], + observed.isoformat(), + ) + self.assertEqual( + broker_snapshot["next_trading_session"], + "2026-07-27", + ) + self.assertEqual( + broker_snapshot["first_executable_window"]["boundary"], + "[start,end)", + ) + + def test_low_and_high_current_equity_drift_fail_closed(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + assets = ( + { + "account_id": "account-fixture", + "cash": 10_000, + "market_value": 10_000, + "total_asset": 20_000, + }, + { + "account_id": "account-fixture", + "cash": 990_000, + "market_value": 10_000, + "total_asset": 1_000_000, + }, + ) + for asset in assets: + with self.subTest(total_asset=asset["total_asset"]), tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(asset=asset), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = { + item["code"] for item in result["plan"]["blockers"] + } + self.assertIn("DECISION_EQUITY_DRIFT", codes) + self.assertEqual(result["plan"]["status"], "BLOCKED") + self.assertFalse( + result["plan"]["portfolio"]["equity_consistency"][ + "within_tolerance" + ] + ) + self.assertTrue( + all( + row["proposed_delta"] == 0 + for row in result["plan"]["target_diffs"] + ) + ) + # The current observation can still be trustworthy and is + # deliberately emitted so the operator can rebuild targets. + self.assertIsNotNone(result["broker_snapshot"]) + + def test_equity_tolerance_boundary_is_inclusive_and_fixed(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + cases = ( + (100_010.0, True), + (100_010.01, False), + ) + for total_asset, expected_ready in cases: + with self.subTest(total_asset=total_asset), tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter( + asset={ + "account_id": "account-fixture", + "cash": total_asset - 10_000, + "market_value": 10_000, + "total_asset": total_asset, + } + ), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + contract = result["plan"]["portfolio"][ + "equity_consistency" + ] + self.assertEqual(contract["allowed_difference"], 10.0) + self.assertIs( + contract["within_tolerance"], + expected_ready, + ) + self.assertEqual( + result["plan"]["status"], + "SHADOW_READY" if expected_ready else "BLOCKED", + ) + codes = { + item["code"] for item in result["plan"]["blockers"] + } + self.assertIs( + "DECISION_EQUITY_DRIFT" not in codes, + expected_ready, + ) + + def test_invalid_decision_or_current_equity_fails_closed(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + for decision_equity in (None, -1.0, 0.0): + with self.subTest(decision_equity=decision_equity), tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + decision_equity=decision_equity, + ) + adapter = FakeReadOnlyAdapter() + with self.assertRaisesRegex( + QmtShadowPlanError, + "decision equity", + ): + build_qmt_shadow_plan( + decision_manifest=decision, + adapter=adapter, + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertEqual(adapter.calls, []) + + invalid_current_values = (None, -1.0, float("nan")) + for current_total_asset in invalid_current_values: + with self.subTest(current_total_asset=current_total_asset), tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter( + asset={ + "account_id": "account-fixture", + "cash": 90_000, + "market_value": 10_000, + "total_asset": current_total_asset, + } + ), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = { + item["code"] for item in result["plan"]["blockers"] + } + self.assertIn("INVALID_ASSET_FACT", codes) + self.assertEqual(result["plan"]["status"], "BLOCKED") + contract = result["plan"]["portfolio"][ + "equity_consistency" + ] + self.assertFalse(contract["evaluated"]) + self.assertIsNone(contract["current_total_asset"]) + self.assertIsNone(result["broker_snapshot"]) + + def test_position_changes_are_rediffed_from_current_sellable_facts(self): + class ChangedPositionAdapter(FakeReadOnlyAdapter): + def query_positions(self): + self.calls.append("positions") + return [ + { + "account_id": "account-fixture", + "symbol": "600000.XSHG", + "volume": 500, + "sellable": 300, + "market_value": 10_000, + "avg_price": 10, + } + ] + + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=ChangedPositionAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + rows = { + row["symbol"]: row + for row in result["plan"]["target_diffs"] + } + self.assertEqual(result["plan"]["status"], "SHADOW_READY") + self.assertEqual( + rows["600000.XSHG"]["current_quantity"], + 500, + ) + self.assertEqual( + rows["600000.XSHG"]["sellable_quantity"], + 300, + ) + self.assertEqual( + rows["600000.XSHG"]["proposed_delta"], + 0, + ) + self.assertEqual( + result["plan"]["safety"]["position_diff_basis"], + "CURRENT_BROKER_POSITIONS_AND_SELLABLE_QUANTITY", + ) + + def test_zero_volume_position_is_counted_dropped_and_verifiable(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + zero_position = { + "account_id": "account-fixture", + "symbol": "600000.XSHG", + "volume": 0, + "sellable": 0, + "market_value": 0, + "avg_price": 0, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter( + positions=[zero_position], + asset={ + "account_id": "account-fixture", + "cash": 100_000, + "market_value": 0, + "total_asset": 100_000, + }, + ), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertEqual(result["plan"]["status"], "SHADOW_READY") + self.assertEqual( + result["broker_observation"]["semantic_inputs"][ + "raw_position_count" + ], + 1, + ) + self.assertEqual( + result["broker_observation"]["semantic_inputs"][ + "dropped_zero_position_count" + ], + 1, + ) + self.assertEqual( + result["broker_observation"]["positions"], + [], + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + verified = verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertEqual(verified["status"], "SHADOW_READY") + + def test_only_unique_next_session_pre_open_window_is_accepted(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + cases = ( + ( + "weekend", + datetime(2026, 7, 25, 9, 0, tzinfo=UTC8), + "QUERY_WRONG_TRADING_SESSION", + ), + ( + "same_day_after_close", + datetime(2026, 7, 24, 15, 1, tzinfo=UTC8), + "QUERY_WRONG_TRADING_SESSION", + ), + ( + "before_window", + datetime(2026, 7, 27, 8, 59, 59, tzinfo=UTC8), + "QUERY_BEFORE_EXECUTABLE_WINDOW", + ), + ( + "exclusive_end", + datetime(2026, 7, 27, 9, 30, tzinfo=UTC8), + "QUERY_MISSED_EXECUTABLE_WINDOW", + ), + ( + "late_multiple_days", + datetime(2026, 7, 30, 9, 0, tzinfo=UTC8), + "QUERY_WRONG_TRADING_SESSION", + ), + ) + for name, observed, expected_code in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = { + item["code"] for item in result["plan"]["blockers"] + } + self.assertIn(expected_code, codes) + self.assertEqual(result["plan"]["status"], "BLOCKED") + self.assertIsNone(result["broker_snapshot"]) + self.assertTrue( + all( + row["proposed_delta"] == 0 + for row in result["plan"]["target_diffs"] + ) + ) + + def test_national_day_gap_uses_declared_calendar_not_day_count(self): + decision_time = datetime(2024, 9, 30, 15, 0, tzinfo=UTC8) + next_session = date(2024, 10, 8) + observed = datetime(2024, 10, 8, 9, 29, 59, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + next_session=next_session, + broker_observed_at=datetime( + 2024, + 10, + 8, + 9, + 0, + tzinfo=UTC8, + ), + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertEqual(result["plan"]["status"], "SHADOW_READY") + self.assertEqual( + result["plan"]["decision"]["next_trading_session"], + "2024-10-08", + ) + self.assertIsNotNone(result["broker_snapshot"]) + + def test_unfinished_or_unknown_order_blocks_every_proposed_delta(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + for state, code in ( + ("ACCEPTED", "UNFINISHED_ORDER"), + ("UNKNOWN", "UNKNOWN_ORDER_STATE"), + ): + with self.subTest(state=state), tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", decision_time + ) + order = { + "account_id": "account-fixture", + "broker_order_id": 88, + "client_order_id": "decision-001", + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "filled_quantity": 0, + "limit_price": 10.5, + "state": state, + } + adapter = FakeReadOnlyAdapter(orders=[order]) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=adapter, + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertEqual(result["plan"]["status"], "BLOCKED") + self.assertIn( + code, + {item["code"] for item in result["plan"]["blockers"]}, + ) + self.assertTrue( + all( + row["proposed_delta"] == 0 + for row in result["plan"]["target_diffs"] + ) + ) + self.assertEqual(adapter.mutation_calls, []) + + def test_order_change_and_wrong_next_session_fail_closed(self): + decision_time = datetime(2026, 7, 1, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 26, 9, 0, tzinfo=UTC8) + changed = { + "account_id": "account-fixture", + "broker_order_id": 99, + "client_order_id": None, + "symbol": "600000.XSHG", + "side": "SELL", + "quantity": 100, + "filled_quantity": 0, + "limit_price": 10, + "state": "CANCELLED", + } + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + next_session=date(2026, 7, 2), + ) + adapter = FakeReadOnlyAdapter( + orders=[], + second_orders=[changed], + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=adapter, + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = {item["code"] for item in result["plan"]["blockers"]} + self.assertIn("ORDERS_CHANGED_DURING_QUERY", codes) + self.assertIn("QUERY_WRONG_TRADING_SESSION", codes) + self.assertEqual(result["plan"]["status"], "BLOCKED") + self.assertIsNone(result["broker_snapshot"]) + self.assertEqual(adapter.mutation_calls, []) + + def test_bound_broker_observation_after_current_query_is_blocked(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + broker_observed_at=observed + timedelta(seconds=6), + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = {item["code"] for item in result["plan"]["blockers"]} + self.assertIn("DECISION_OBSERVATION_FROM_FUTURE", codes) + self.assertIn("DECISION_SOURCE_FROM_FUTURE", codes) + self.assertEqual(result["plan"]["status"], "BLOCKED") + + def test_live_enabled_or_degraded_adapter_is_rejected_before_query(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", decision_time + ) + for state, live, message in ( + ("READY", True, "allow_live_orders"), + ("DEGRADED", False, "READY state"), + ): + with self.subTest(state=state, live=live): + adapter = FakeReadOnlyAdapter() + adapter.state = state + adapter.allow_live_orders = live + with self.assertRaisesRegex(QmtShadowPlanError, message): + build_qmt_shadow_plan( + decision_manifest=decision, + adapter=adapter, + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertEqual(adapter.calls, []) + self.assertEqual(adapter.mutation_calls, []) + + def test_policy_thresholds_cannot_be_widened(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + cases = ( + ( + { + "max_query_duration_seconds": ( + MAX_SHADOW_QUERY_DURATION_SECONDS + 1 + ) + }, + "max_query_duration_seconds cannot exceed", + ), + ( + { + "market_value_tolerance": ( + MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE + 0.01 + ) + }, + "market_value_tolerance cannot exceed", + ), + ) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", decision_time + ) + for kwargs, expected in cases: + with self.subTest(kwargs=kwargs): + adapter = FakeReadOnlyAdapter() + with self.assertRaisesRegex( + QmtShadowPlanError, expected + ): + build_qmt_shadow_plan( + decision_manifest=decision, + adapter=adapter, + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + **kwargs, + ) + self.assertEqual(adapter.calls, []) + + def test_resealed_semantic_policy_widening_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + for field, widened, expected in ( + ( + "max_query_duration_seconds", + MAX_SHADOW_QUERY_DURATION_SECONDS + 1, + "query duration policy", + ), + ( + "market_value_tolerance", + MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE + 0.01, + "asset tolerance", + ), + ): + with self.subTest(field=field), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan( + result, root / "shadow" + ) + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + broker_observation = json.loads( + Path(paths["broker_observation"]).read_text( + encoding="utf-8" + ) + ) + broker_snapshot = json.loads( + Path(paths["broker_snapshot"]).read_text( + encoding="utf-8" + ) + ) + broker_observation["semantic_inputs"][field] = widened + observation_hash = payload_sha256(broker_observation) + plan["observation"]["observed_facts_sha256"] = ( + observation_hash + ) + broker_snapshot["metadata"][ + "observed_facts_sha256" + ] = observation_hash + broker_snapshot["snapshot_id"] = "" + broker_snapshot["snapshot_id"] = ( + "QMT-SHADOW-" + + payload_sha256(broker_snapshot)[:16] + ) + plan["broker_snapshot_sha256"] = payload_sha256( + broker_snapshot + ) + reseal_shadow_artifacts( + paths, + plan, + broker_observation=broker_observation, + broker_snapshot=broker_snapshot, + evidence_hmac_key=ACCOUNT_HASH_KEY, + ) + with self.assertRaisesRegex( + QmtShadowPlanError, expected + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_tampered_shadow_plan_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact(root / "decision", decision_time) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan_path = Path(paths["plan"]) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + plan["safety"]["ready_for_live_submission"] = True + plan_path.write_text(json.dumps(plan), encoding="utf-8") + with self.assertRaisesRegex( + QmtShadowPlanError, "artifact hash mismatch" + ): + verify_qmt_shadow_plan(paths["manifest"]) + + def test_authenticated_evidence_rejects_wrong_key(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + with self.assertRaisesRegex( + QmtShadowPlanError, + "authenticated shadow evidence HMAC", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=b"x" * 32, + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "account_hash_key is required", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + ) + + def test_authenticated_evidence_rejects_unsigned_observation_rewrite(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + broker_observation = json.loads( + Path(paths["broker_observation"]).read_text( + encoding="utf-8" + ) + ) + broker_snapshot = json.loads( + Path(paths["broker_snapshot"]).read_text( + encoding="utf-8" + ) + ) + broker_observation["semantic_inputs"][ + "max_query_duration_seconds" + ] = 9.0 + observation_hash = payload_sha256(broker_observation) + plan["observation"]["observed_facts_sha256"] = ( + observation_hash + ) + broker_snapshot["metadata"][ + "observed_facts_sha256" + ] = observation_hash + broker_snapshot["snapshot_id"] = "" + broker_snapshot["snapshot_id"] = ( + "QMT-SHADOW-" + + payload_sha256(broker_snapshot)[:16] + ) + plan["broker_snapshot_sha256"] = payload_sha256( + broker_snapshot + ) + reseal_shadow_artifacts( + paths, + plan, + broker_observation=broker_observation, + broker_snapshot=broker_snapshot, + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "authenticated shadow evidence HMAC", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_authenticated_evidence_binds_original_decision(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + original_decision = decision_artifact( + root / "original-decision", decision_time + ) + substituted_decision = decision_artifact( + root / "substituted-decision", + decision_time, + first_target_quantity=2000, + ) + result = build_qmt_shadow_plan( + decision_manifest=original_decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + substituted_manifest_sha256 = hashlib.sha256( + Path(substituted_decision).read_bytes() + ).hexdigest() + plan["decision"][ + "decision_manifest_sha256" + ] = substituted_manifest_sha256 + row = next( + item + for item in plan["target_diffs"] + if item["symbol"] == "000001.XSHE" + ) + row.update( + { + "target_quantity": 2000, + "desired_delta": 2000, + "board_lot_feasible_delta": 2000, + "proposed_delta": 2000, + "proposed_side": "BUY", + "proposed_quantity": 2000, + } + ) + reseal_shadow_artifacts( + paths, + plan, + manifest_updates={ + "decision_manifest_sha256": ( + substituted_manifest_sha256 + ) + }, + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "authenticated shadow evidence HMAC", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=substituted_decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_payload_reformat_with_resealed_byte_hash_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan_path = Path(paths["plan"]) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + plan_path.write_text( + json.dumps( + plan, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=False, + ), + encoding="utf-8", + ) + manifest_path = Path(paths["manifest"]) + manifest = json.loads( + manifest_path.read_text(encoding="utf-8") + ) + manifest["artifact_sha256"]["shadow_plan.json"] = ( + hashlib.sha256(plan_path.read_bytes()).hexdigest() + ) + manifest_path.write_text( + json.dumps( + manifest, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "shadow_plan.json is not in deterministic published encoding", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_manifest_reformat_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + manifest_path = Path(paths["manifest"]) + manifest = json.loads( + manifest_path.read_text(encoding="utf-8") + ) + manifest_path.write_text( + json.dumps( + manifest, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=False, + ), + encoding="utf-8", + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "manifest is not in deterministic published encoding", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_operator_cli_is_bound_into_planner_source_manifest(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertIn( + "tools/qmt_shadow_plan.py", + result["manifest"]["planner_sources"], + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + tampered_sources = dict( + result["manifest"]["planner_sources"] + ) + tampered_sources["tools/qmt_shadow_plan.py"] = "0" * 64 + tampered_source_sha256 = payload_sha256( + tampered_sources + ) + with patch( + "quant60.qmt_shadow._source_manifest", + return_value=( + tampered_source_sha256, + tampered_sources, + ), + ): + with self.assertRaisesRegex( + QmtShadowPlanError, + "current shadow planner sources have changed", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_resealed_equity_contract_tampering_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + for tamper in ("result", "tolerance"): + with self.subTest(tamper=tamper), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan( + result, + root / "shadow", + ) + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + contract = plan["portfolio"]["equity_consistency"] + if tamper == "result": + contract["within_tolerance"] = False + expected = "tolerance result is inconsistent" + else: + contract["absolute_tolerance"] = 100.0 + contract["allowed_difference"] = 100.0 + expected = "tolerance constants/rule were changed" + reseal_shadow_plan(paths, plan) + with self.assertRaisesRegex( + QmtShadowPlanError, + expected, + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_resealed_target_and_feasible_tampering_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + for tamper in ( + "target_quantity", + "target_weight", + "lot_size", + "feasible_delta", + ): + with self.subTest(tamper=tamper), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan( + result, root / "shadow" + ) + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + row = next( + item + for item in plan["target_diffs"] + if item["symbol"] == "000001.XSHE" + ) + if tamper == "target_quantity": + row["target_quantity"] = 100_000 + row["desired_delta"] = 100_000 + row["board_lot_feasible_delta"] = 100_000 + row["proposed_delta"] = 100_000 + row["proposed_side"] = "BUY" + row["proposed_quantity"] = 100_000 + elif tamper == "target_weight": + row["target_weight"] = 0.90 + plan["portfolio"]["target_gross_weight"] = 0.95 + elif tamper == "lot_size": + row["lot_size"] = 200 + else: + row["board_lot_feasible_delta"] = 1_100 + row["proposed_delta"] = 1_100 + row["proposed_side"] = "BUY" + row["proposed_quantity"] = 1_100 + reseal_shadow_plan(paths, plan) + with self.assertRaisesRegex( + QmtShadowPlanError, + "target/order projection|cash/portfolio projection", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_resealed_cross_account_blocker_removal_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", + decision_time, + broker_account_hash="0" * 64, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertIn( + "DECISION_ACCOUNT_MISMATCH", + { + item["code"] + for item in result["plan"]["blockers"] + }, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + force_ready_projection(plan) + reseal_shadow_plan(paths, plan) + with self.assertRaisesRegex( + QmtShadowPlanError, + "blockers do not match semantic replay", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_resealed_blocker_removal_matrix_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + accepted = { + "account_id": "account-fixture", + "broker_order_id": 88, + "client_order_id": "decision-001", + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "filled_quantity": 0, + "limit_price": 10, + "state": "ACCEPTED", + } + unknown = dict(accepted, state="UNKNOWN") + cancelled = dict(accepted, state="CANCELLED") + filled = dict( + accepted, + state="FILLED", + filled_quantity=100, + ) + mismatched_trade = { + "account_id": "account-fixture", + "trade_id": "trade-88", + "broker_order_id": 88, + "symbol": "000001.XSHE", + "side": "BUY", + "quantity": 100, + "price": 10, + "amount": 1000, + } + cases = ( + ( + "unfinished_order", + "UNFINISHED_ORDER", + lambda: FakeReadOnlyAdapter(orders=[accepted]), + lambda: fixed_clock(observed), + ), + ( + "unknown_order", + "UNKNOWN_ORDER_STATE", + lambda: FakeReadOnlyAdapter(orders=[unknown]), + lambda: fixed_clock(observed), + ), + ( + "orders_changed", + "ORDERS_CHANGED_DURING_QUERY", + lambda: FakeReadOnlyAdapter( + orders=[], second_orders=[cancelled] + ), + lambda: fixed_clock(observed), + ), + ( + "live_mode_changed", + "LIVE_MODE_CHANGED_DURING_QUERY", + lambda: FakeReadOnlyAdapter( + flip_live_after_query=True + ), + lambda: fixed_clock(observed), + ), + ( + "adapter_state_changed", + "ADAPTER_NOT_READY_AFTER_QUERY", + lambda: FakeReadOnlyAdapter( + flip_state_after_query=True + ), + lambda: fixed_clock(observed), + ), + ( + "adapter_errors", + "ADAPTER_ERRORS_PRESENT", + lambda: FakeReadOnlyAdapter(errors=["callback error"]), + lambda: fixed_clock(observed), + ), + ( + "live_authorization_history", + "LIVE_AUTHORIZATION_HISTORY_PRESENT", + lambda: FakeReadOnlyAdapter( + live_authorization_audit=[{"event": "attempt"}] + ), + lambda: fixed_clock(observed), + ), + ( + "query_window_too_long", + "QUERY_WINDOW_TOO_LONG", + lambda: FakeReadOnlyAdapter(), + lambda: iter( + (observed, observed + timedelta(seconds=11)) + ).__next__, + ), + ( + "position_market_value", + "POSITION_MARKET_VALUE_MISMATCH", + lambda: FakeReadOnlyAdapter( + asset={ + "account_id": "account-fixture", + "cash": 89_000, + "market_value": 11_000, + "total_asset": 100_000, + } + ), + lambda: fixed_clock(observed), + ), + ( + "asset_identity", + "ASSET_IDENTITY_MISMATCH", + lambda: FakeReadOnlyAdapter( + asset={ + "account_id": "account-fixture", + "cash": 80_000, + "market_value": 10_000, + "total_asset": 100_000, + } + ), + lambda: fixed_clock(observed), + ), + ( + "trade_order_identity", + "TRADE_ORDER_IDENTITY_MISMATCH", + lambda: FakeReadOnlyAdapter( + orders=[filled], + trades=[mismatched_trade], + ), + lambda: fixed_clock(observed), + ), + ) + for name, expected_code, adapter_factory, clock_factory in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=adapter_factory(), + clock=clock_factory(), + account_hash_key=ACCOUNT_HASH_KEY, + ) + self.assertIn( + expected_code, + { + item["code"] + for item in result["plan"]["blockers"] + }, + ) + paths = write_qmt_shadow_plan( + result, root / "shadow" + ) + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + force_ready_projection(plan) + reseal_shadow_plan(paths, plan) + with self.assertRaisesRegex( + QmtShadowPlanError, + "blockers do not match semantic replay", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_resealed_ready_plan_cannot_drop_broker_snapshot(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + plan["broker_snapshot_sha256"] = None + plan["observation"]["broker_snapshot_valid"] = False + reseal_shadow_artifacts( + paths, + plan, + broker_snapshot=None, + evidence_hmac_key=ACCOUNT_HASH_KEY, + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "observation projection|broker snapshot", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_resealed_order_observation_injection_is_replayed(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + broker_observation = json.loads( + Path(paths["broker_observation"]).read_text( + encoding="utf-8" + ) + ) + broker_snapshot = json.loads( + Path(paths["broker_snapshot"]).read_text( + encoding="utf-8" + ) + ) + broker_observation["orders_after"].append( + { + "broker_order_id": "88", + "client_order_id": "decision-001", + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "filled_quantity": 0, + "limit_price": 10.0, + "state": "ACCEPTED", + } + ) + broker_observation["semantic_inputs"][ + "raw_order_after_count" + ] = 1 + observation_hash = payload_sha256(broker_observation) + plan["observation"]["observed_facts_sha256"] = ( + observation_hash + ) + plan["observation"]["order_count"] = 1 + broker_snapshot["metadata"][ + "observed_facts_sha256" + ] = observation_hash + broker_snapshot["snapshot_id"] = "" + broker_snapshot["snapshot_id"] = ( + "QMT-SHADOW-" + + payload_sha256(broker_snapshot)[:16] + ) + plan["broker_snapshot_sha256"] = payload_sha256( + broker_snapshot + ) + reseal_shadow_artifacts( + paths, + plan, + broker_observation=broker_observation, + broker_snapshot=broker_snapshot, + evidence_hmac_key=ACCOUNT_HASH_KEY, + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "blockers do not match semantic replay", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_resealed_invalid_order_state_fill_is_rejected(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + broker_observation = json.loads( + Path(paths["broker_observation"]).read_text( + encoding="utf-8" + ) + ) + broker_snapshot = json.loads( + Path(paths["broker_snapshot"]).read_text( + encoding="utf-8" + ) + ) + impossible_filled_order = { + "broker_order_id": "88", + "client_order_id": "decision-001", + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "filled_quantity": 0, + "limit_price": 10.0, + "state": "FILLED", + } + broker_observation["orders_before"] = [ + impossible_filled_order + ] + broker_observation["orders_after"] = [ + impossible_filled_order + ] + broker_observation["semantic_inputs"].update( + { + "raw_order_before_count": 1, + "raw_order_after_count": 1, + } + ) + observation_hash = payload_sha256(broker_observation) + plan["observation"].update( + { + "observed_facts_sha256": observation_hash, + "order_count": 1, + } + ) + broker_snapshot["metadata"].update( + { + "observed_facts_sha256": observation_hash, + "terminal_order_count": 1, + } + ) + broker_snapshot["snapshot_id"] = "" + broker_snapshot["snapshot_id"] = ( + "QMT-SHADOW-" + + payload_sha256(broker_snapshot)[:16] + ) + plan["broker_snapshot_sha256"] = payload_sha256( + broker_snapshot + ) + reseal_shadow_artifacts( + paths, + plan, + broker_observation=broker_observation, + broker_snapshot=broker_snapshot, + evidence_hmac_key=ACCOUNT_HASH_KEY, + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "filled order 88 is incomplete", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_resealed_asset_observation_fuzz_is_replayed(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + decision = decision_artifact( + root / "decision", decision_time + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + paths = write_qmt_shadow_plan(result, root / "shadow") + plan = json.loads( + Path(paths["plan"]).read_text(encoding="utf-8") + ) + broker_observation = json.loads( + Path(paths["broker_observation"]).read_text( + encoding="utf-8" + ) + ) + broker_snapshot = json.loads( + Path(paths["broker_snapshot"]).read_text( + encoding="utf-8" + ) + ) + broker_observation["asset"].update( + { + "cash": 0.0, + "market_value": 100_000.0, + "total_asset": 100_000.0, + } + ) + observation_hash = payload_sha256(broker_observation) + plan["observation"]["observed_facts_sha256"] = ( + observation_hash + ) + broker_snapshot["metadata"][ + "observed_facts_sha256" + ] = observation_hash + broker_snapshot["snapshot_id"] = "" + broker_snapshot["snapshot_id"] = ( + "QMT-SHADOW-" + + payload_sha256(broker_snapshot)[:16] + ) + plan["broker_snapshot_sha256"] = payload_sha256( + broker_snapshot + ) + reseal_shadow_artifacts( + paths, + plan, + broker_observation=broker_observation, + broker_snapshot=broker_snapshot, + evidence_hmac_key=ACCOUNT_HASH_KEY, + ) + with self.assertRaisesRegex( + QmtShadowPlanError, + "blockers do not match semantic replay|cash/portfolio", + ): + verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=decision, + account_hash_key=ACCOUNT_HASH_KEY, + ) + + def test_unbound_decision_emits_bootstrap_snapshot_and_blocks(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + observed = datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + bind_account=False, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock(observed), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = {item["code"] for item in result["plan"]["blockers"]} + self.assertIn("DECISION_NOT_ACCOUNT_BOUND", codes) + self.assertEqual(result["plan"]["status"], "BLOCKED") + self.assertIsNotNone(result["broker_snapshot"]) + self.assertEqual( + result["broker_snapshot"]["account_hash"], + fixture_account_hash(), + ) + self.assertTrue( + all( + row["proposed_delta"] == 0 + for row in result["plan"]["target_diffs"] + ) + ) + + def test_cross_account_decision_is_blocked(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + broker_account_hash="0" * 64, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter(), + clock=fixed_clock( + datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + ), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = {item["code"] for item in result["plan"]["blockers"]} + self.assertIn("DECISION_ACCOUNT_MISMATCH", codes) + self.assertEqual(result["plan"]["status"], "BLOCKED") + self.assertTrue( + all( + row["proposed_delta"] == 0 + for row in result["plan"]["target_diffs"] + ) + ) + + def test_trade_order_identity_mismatch_invalidates_broker_snapshot(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + order = { + "account_id": "account-fixture", + "broker_order_id": 88, + "client_order_id": "decision-001", + "symbol": "600000.XSHG", + "side": "BUY", + "quantity": 100, + "filled_quantity": 100, + "limit_price": 10, + "state": "FILLED", + } + trade = { + "account_id": "account-fixture", + "trade_id": "trade-88", + "broker_order_id": 88, + "symbol": "000001.XSHE", + "side": "BUY", + "quantity": 100, + "price": 10, + "amount": 1000, + } + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter( + orders=[order], + trades=[trade], + ), + clock=fixed_clock( + datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + ), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = {item["code"] for item in result["plan"]["blockers"]} + self.assertIn("TRADE_ORDER_IDENTITY_MISMATCH", codes) + self.assertEqual(result["plan"]["status"], "BLOCKED") + self.assertIsNone(result["broker_snapshot"]) + + def test_asset_identity_mismatch_invalidates_broker_snapshot(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter( + asset={ + "account_id": "account-fixture", + "cash": 80_000, + "market_value": 20_000, + "total_asset": 100_000, + } + ), + clock=fixed_clock( + datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + ), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = {item["code"] for item in result["plan"]["blockers"]} + self.assertIn("POSITION_MARKET_VALUE_MISMATCH", codes) + self.assertIsNone(result["broker_snapshot"]) + + def test_live_mode_change_during_queries_blocks_and_invalidates_snapshot(self): + decision_time = datetime(2026, 7, 24, 15, 0, tzinfo=UTC8) + with tempfile.TemporaryDirectory() as directory: + decision = decision_artifact( + Path(directory) / "decision", + decision_time, + ) + result = build_qmt_shadow_plan( + decision_manifest=decision, + adapter=FakeReadOnlyAdapter( + flip_live_after_query=True, + ), + clock=fixed_clock( + datetime(2026, 7, 27, 9, 0, tzinfo=UTC8) + ), + account_hash_key=ACCOUNT_HASH_KEY, + ) + codes = {item["code"] for item in result["plan"]["blockers"]} + self.assertIn("LIVE_MODE_CHANGED_DURING_QUERY", codes) + self.assertIsNone(result["broker_snapshot"]) + self.assertEqual( + result["plan"]["observation"][ + "adapter_allow_live_orders_after_query" + ], + True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_qmt_shadow_tool.py b/tests/test_qmt_shadow_tool.py new file mode 100644 index 0000000..b1fe8f3 --- /dev/null +++ b/tests/test_qmt_shadow_tool.py @@ -0,0 +1,102 @@ +import os +import stat +import tempfile +import unittest +from contextlib import redirect_stderr +from io import StringIO +from pathlib import Path + +from tools.qmt_shadow_plan import ( + _load_existing_account_hash_key, + _load_or_create_account_hash_key, + build_parser, +) + + +class QmtShadowToolTests(unittest.TestCase): + def test_account_hash_key_is_created_once_with_private_permissions(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "private" / "account-hash.key" + first = _load_or_create_account_hash_key(path) + second = _load_or_create_account_hash_key(path) + self.assertEqual(first, second) + self.assertEqual(len(first), 32) + if os.name != "nt": + self.assertEqual( + stat.S_IMODE(path.stat().st_mode), + 0o600, + ) + + def test_short_account_hash_key_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "account-hash.key" + path.write_bytes(b"too-short") + path.chmod(0o600) + with self.assertRaisesRegex(ValueError, "at least 32 bytes"): + _load_or_create_account_hash_key(path) + + def test_verify_key_loader_never_creates_missing_key(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "missing-account-hash.key" + with self.assertRaisesRegex( + ValueError, "does not exist" + ): + _load_existing_account_hash_key(path) + self.assertFalse(path.exists()) + + def test_existing_parent_permissions_are_not_changed(self): + if os.name == "nt": + self.skipTest("POSIX permission invariant") + with tempfile.TemporaryDirectory() as directory: + parent = Path(directory) / "shared-parent" + parent.mkdir(mode=0o755) + parent.chmod(0o755) + _load_or_create_account_hash_key(parent / "account-hash.key") + self.assertEqual( + stat.S_IMODE(parent.stat().st_mode), + 0o755, + ) + + def test_cli_rejects_widened_safety_thresholds(self): + base = [ + "plan", + "--decision", + "decision-manifest.json", + "--output", + "shadow-output", + ] + for option, value in ( + ("--max-query-duration-seconds", "10.0001"), + ("--market-value-tolerance", "1.0001"), + ): + with self.subTest(option=option), redirect_stderr(StringIO()): + with self.assertRaises(SystemExit): + build_parser().parse_args(base + [option, value]) + + def test_cli_accepts_only_equal_or_stricter_thresholds(self): + parsed = build_parser().parse_args( + [ + "plan", + "--decision", + "decision-manifest.json", + "--output", + "shadow-output", + "--max-query-duration-seconds", + "5", + "--market-value-tolerance", + "0.5", + ] + ) + self.assertEqual(parsed.max_query_duration_seconds, 5.0) + self.assertEqual(parsed.market_value_tolerance, 0.5) + + def test_cli_verify_requires_original_decision(self): + with redirect_stderr(StringIO()): + with self.assertRaises(SystemExit): + build_parser().parse_args( + ["verify", "--manifest", "shadow-manifest.json"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_research_risk.py b/tests/test_research_risk.py new file mode 100644 index 0000000..329be91 --- /dev/null +++ b/tests/test_research_risk.py @@ -0,0 +1,99 @@ +import math +import unittest + +from quant60.research import ( + DeterministicRidge, + NumpyRidge, + pearson_correlation, + purged_walk_forward, + spearman_correlation, +) +from quant60.risk import check_target_weights, diagonal_variance + + +class ResearchRiskTests(unittest.TestCase): + def test_purged_walk_forward_is_time_ordered(self): + folds = list( + purged_walk_forward( + 40, + train_size=20, + test_size=5, + purge=2, + embargo=1, + ) + ) + self.assertTrue(folds) + for fold in folds: + self.assertLess(max(fold.train), min(fold.purged)) + self.assertLess(max(fold.purged), min(fold.test)) + self.assertTrue(set(fold.train).isdisjoint(fold.test)) + + def test_risk_check_rejects_concentration(self): + result = check_target_weights( + {"600000.SH": 0.6, "000001.SZ": 0.2}, + max_single_weight=0.5, + max_gross_weight=0.9, + ) + self.assertFalse(result.approved) + self.assertIn("SINGLE_NAME_CAP", {item.code for item in result.violations}) + + def test_diagonal_variance_floor(self): + result = diagonal_variance( + {"600000.SH": [0.01], "000001.SZ": [0.0, 0.0, 0.0]} + ) + self.assertGreater(result["600000.XSHG"], 0) + self.assertGreater(result["000001.XSHE"], 0) + + def test_non_finite_weights_fail_closed(self): + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(value=value): + result = check_target_weights( + {"600000.SH": value}, + max_single_weight=0.5, + max_gross_weight=0.9, + ) + self.assertFalse(result.approved) + self.assertIn( + "NON_FINITE_WEIGHT", + {item.code for item in result.violations}, + ) + self.assertTrue(math.isinf(result.gross_weight)) + self.assertTrue(math.isinf(result.one_way_turnover)) + + def test_non_finite_returns_are_rejected(self): + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "returns must be finite"): + diagonal_variance({"600000.SH": [0.01, value]}) + + def test_optional_numpy_ridge_when_numpy_is_present(self): + try: + model = NumpyRidge(alpha=0.1).fit([[0], [1], [2]], [1, 3, 5]) + except Exception as exc: + if type(exc).__name__ == "ResearchDependencyUnavailable": + self.skipTest(str(exc)) + raise + prediction = model.predict([[3]]) + self.assertGreater(float(prediction[0]), 6.0) + + def test_dependency_free_ridge_fits_linear_relationship(self): + model = DeterministicRidge(alpha=0.001).fit( + [[0], [1], [2], [3]], + [1, 3, 5, 7], + ) + prediction = model.predict([[4]])[0] + self.assertAlmostEqual(prediction, 9, places=2) + + def test_rank_and_linear_correlation(self): + self.assertAlmostEqual( + pearson_correlation([1, 2, 3], [2, 4, 6]), + 1.0, + ) + self.assertAlmostEqual( + spearman_correlation([10, 30, 20], [1, 3, 2]), + 1.0, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_snapshot_pipeline.py b/tests/test_snapshot_pipeline.py new file mode 100644 index 0000000..b8fe796 --- /dev/null +++ b/tests/test_snapshot_pipeline.py @@ -0,0 +1,544 @@ +import hashlib +import json +import tempfile +import unittest +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import patch + +import quant60.snapshot_backtest as snapshot_backtest_module +from quant60.config import BacktestConfig, StrategyConfig +from quant60.backtest import verify_artifact_manifest +from quant60.data_snapshot import ( + CanonicalBarRecord, + IndexMembershipRecord, + build_snapshot_payload, + write_data_snapshot, +) +from quant60.snapshot_pipeline import ( + SnapshotDecisionError, + build_snapshot_decision, + first_executable_window, + verify_snapshot_decision, + write_snapshot_decision, +) +from quant60.snapshot_backtest import run_snapshot_backtest + + +def business_days(start: date, count: int) -> list[date]: + values = [] + current = start + while len(values) < count: + if current.weekday() < 5: + values.append(current) + current += timedelta(days=1) + return values + + +def build_provider_snapshot( + root: Path, + *, + st_symbol_on_last_session: str | None = None, +) -> tuple[str, date]: + frozen_calendar = business_days(date(2024, 1, 2), 31) + sessions = frozen_calendar[:-1] + next_trading_session = frozen_calendar[-1] + bars = [] + memberships = [] + symbols = ("600000.XSHG", "000001.XSHE") + for index, session in enumerate(sessions): + for symbol in symbols: + if symbol == "600000.XSHG": + raw = 10.0 * (1.005**index) + factor = 1.0 + else: + unsplit = 20.0 * (1.01**index) + raw = unsplit if index < 15 else unsplit / 2.0 + factor = 1.0 if index < 15 else 2.0 + bars.append( + CanonicalBarRecord( + trading_date=session, + symbol=symbol, + open=raw * 0.999, + high=raw * 1.01, + low=raw * 0.99, + close=raw, + volume=1_000_000, + money=raw * 1_000_000, + paused=False, + source="fixture-provider", + is_st=( + symbol == st_symbol_on_last_session + and session == sessions[-1] + ), + adjustment="none", + adjustment_factor=factor, + previous_close=raw / 1.005, + limit_up=raw * 1.10, + limit_down=raw * 0.90, + ) + ) + memberships.append( + IndexMembershipRecord( + effective_date=session, + index_symbol="000905.XSHG", + member_symbol=symbol, + source="fixture-provider", + ) + ) + payload = build_snapshot_payload( + bars=bars, + memberships=memberships, + provider="fixture-provider", + provider_version="1", + retrieved_at=datetime(2024, 3, 1, tzinfo=timezone.utc), + next_trading_session=next_trading_session, + query={ + "index_symbol": "000905.XSHG", + "start_date": sessions[0].isoformat(), + "end_date": sessions[-1].isoformat(), + "trading_sessions": [ + item.isoformat() for item in frozen_calendar + ], + "adjustment": "none", + }, + ) + paths = write_data_snapshot(payload, root) + return paths["manifest"], sessions[-1] + + +class SnapshotPipelineTests(unittest.TestCase): + @staticmethod + def _broker_snapshot( + as_of: date, + *, + observed_at: datetime | None = None, + next_session: date = date(2024, 2, 13), + ) -> dict: + signal_timestamp = f"{as_of.isoformat()}T15:00:00+08:00" + executable_window = first_executable_window(next_session) + observation_timestamp = ( + observed_at.isoformat() + if observed_at is not None + else executable_window["start"] + ) + return { + "schema_version": "1.0", + "snapshot_id": "broker-fixture", + "signal_as_of": signal_timestamp, + "next_trading_session": next_session.isoformat(), + "first_executable_window": executable_window, + "observation_as_of": observation_timestamp, + "as_of": observation_timestamp, + "source_time": observation_timestamp, + "broker": "fixture", + "account_hash": "fixture-account-hash", + "cash": 1_000_000, + "total_asset": 1_000_000, + "market_value": 0, + "positions": [], + "open_orders": [], + "metadata": {}, + } + + def test_next_day_broker_observation_binds_to_prior_close_signal(self): + config = BacktestConfig( + symbols=("600000.XSHG", "000001.XSHE"), + strategy=StrategyConfig(lookback=5, top_n=1), + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot_manifest, as_of = build_provider_snapshot( + root / "snapshot" + ) + observed_at = datetime( + as_of.year, + as_of.month, + as_of.day, + 9, + 0, + tzinfo=timezone(timedelta(hours=8)), + ) + timedelta(days=1) + broker = self._broker_snapshot( + as_of, + observed_at=observed_at, + ) + result = build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + broker_snapshot=broker, + ) + self.assertEqual( + result["decision"]["signal_as_of"], + f"{as_of.isoformat()}T15:00:00+08:00", + ) + self.assertEqual( + result["decision"]["broker_snapshot"][ + "observation_as_of" + ], + observed_at.isoformat(), + ) + paths = write_snapshot_decision( + result, + root / "decision", + ) + verified = verify_snapshot_decision(paths["manifest"]) + self.assertEqual( + verified["signal_as_of"], + f"{as_of.isoformat()}T15:00:00+08:00", + ) + + def test_verified_snapshot_builds_deterministic_portable_target(self): + config = BacktestConfig( + symbols=("600000.XSHG", "000001.XSHE"), + strategy=StrategyConfig( + lookback=5, + top_n=1, + max_weight=0.50, + gross_target=0.50, + cash_buffer=0.10, + ), + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot_manifest, as_of = build_provider_snapshot( + root / "snapshot" + ) + first = build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + equity=1_000_000, + ) + second = build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + equity=1_000_000, + ) + self.assertEqual(first, second) + self.assertEqual(len(first["signals"]), 2) + self.assertGreater( + next( + row["score"] + for row in first["signals"] + if row["symbol"] == "000001.XSHE" + ), + next( + row["score"] + for row in first["signals"] + if row["symbol"] == "600000.XSHG" + ), + ) + selected = [ + row + for row in first["targets"] + if row["target_weight"] > 0 + ] + self.assertEqual( + [row["symbol"] for row in selected], + ["000001.XSHE"], + ) + paths = write_snapshot_decision(first, root / "decision") + verified = verify_snapshot_decision(paths["manifest"]) + self.assertTrue(verified["ok"]) + self.assertEqual(verified["signal_count"], 2) + self.assertEqual(verified["target_count"], 2) + + def test_pit_st_member_is_excluded_before_portable_ranking(self): + config = BacktestConfig( + symbols=("600000.XSHG", "000001.XSHE"), + strategy=StrategyConfig( + lookback=5, + top_n=1, + max_weight=0.50, + gross_target=0.50, + cash_buffer=0.10, + ), + ) + with tempfile.TemporaryDirectory() as directory: + snapshot_manifest, as_of = build_provider_snapshot( + Path(directory) / "snapshot", + st_symbol_on_last_session="000001.XSHE", + ) + result = build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + equity=1_000_000, + ) + self.assertEqual( + result["decision"]["universe"]["000001.XSHE"]["state"], + "excluded", + ) + self.assertIn( + "ST_EXCLUDED", + result["decision"]["universe"]["000001.XSHE"]["reasons"], + ) + self.assertNotIn( + "000001.XSHE", + result["decision"]["targets"], + ) + + def test_missing_exact_membership_fails_closed(self): + config = BacktestConfig( + symbols=("600000.XSHG", "000001.XSHE"), + strategy=StrategyConfig(lookback=5, top_n=1), + ) + with tempfile.TemporaryDirectory() as directory: + snapshot_manifest, as_of = build_provider_snapshot( + Path(directory) / "snapshot" + ) + with self.assertRaisesRegex( + SnapshotDecisionError, + "no next session", + ): + build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of + timedelta(days=1), + equity=1_000_000, + ) + + def test_provider_snapshot_runs_local_event_backtest(self): + config = BacktestConfig( + symbols=("600000.XSHG", "000001.XSHE"), + strategy=StrategyConfig( + lookback=5, + top_n=1, + max_weight=0.50, + gross_target=0.50, + cash_buffer=0.10, + ), + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot_manifest, unused_as_of = build_provider_snapshot( + root / "snapshot" + ) + del unused_as_of + with patch.object( + snapshot_backtest_module, + "load_verified_data_snapshot", + wraps=( + snapshot_backtest_module.load_verified_data_snapshot + ), + ) as snapshot_loader: + result = run_snapshot_backtest( + snapshot_manifest=snapshot_manifest, + config=config, + ) + self.assertEqual(snapshot_loader.call_count, 1) + self.assertGreater(result.report["decision_count"], 0) + self.assertEqual( + result.report["evidence_class"], + "provider-snapshot-local-backtest", + ) + self.assertFalse(result.report["real_platform_backtest"]) + self.assertFalse(result.report["investment_value_claim"]) + self.assertGreater(len(result.signals), 0) + paths = result.write_artifacts(root / "backtest") + self.assertTrue( + verify_artifact_manifest(paths["manifest"])["ok"] + ) + + def test_decision_tampering_is_rejected(self): + config = BacktestConfig( + symbols=("600000.XSHG", "000001.XSHE"), + strategy=StrategyConfig(lookback=5, top_n=1), + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot_manifest, as_of = build_provider_snapshot( + root / "snapshot" + ) + result = build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + equity=1_000_000, + ) + paths = write_snapshot_decision(result, root / "decision") + decision_path = Path(paths["decision"]) + decision = json.loads(decision_path.read_text(encoding="utf-8")) + decision["orders"] = {} + decision_path.write_text(json.dumps(decision), encoding="utf-8") + with self.assertRaisesRegex( + SnapshotDecisionError, + "artifact hash mismatch", + ): + verify_snapshot_decision(paths["manifest"]) + + def test_copied_input_calendar_cannot_be_rebound_by_artifact_hash(self): + config = BacktestConfig( + symbols=("600000.XSHG", "000001.XSHE"), + strategy=StrategyConfig(lookback=5, top_n=1), + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot_manifest, as_of = build_provider_snapshot( + root / "snapshot" + ) + result = build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + equity=1_000_000, + ) + paths = write_snapshot_decision(result, root / "decision") + input_path = Path(paths["input_data_manifest"]) + copied = json.loads(input_path.read_text(encoding="utf-8")) + copied["query"]["trading_sessions"][-1] = "2024-02-14" + input_path.write_text( + json.dumps( + copied, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + manifest_path = Path(paths["manifest"]) + manifest = json.loads( + manifest_path.read_text(encoding="utf-8") + ) + manifest["artifact_sha256"][ + "input_data_manifest.json" + ] = hashlib.sha256(input_path.read_bytes()).hexdigest() + manifest_path.write_text( + json.dumps( + manifest, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex( + SnapshotDecisionError, + "calendar manifest hash binding mismatch", + ): + verify_snapshot_decision(manifest_path) + + def test_wrong_window_stale_source_or_duplicate_broker_facts_fail_closed(self): + config = BacktestConfig( + symbols=("600000.XSHG", "000001.XSHE"), + strategy=StrategyConfig(lookback=5, top_n=1), + ) + with tempfile.TemporaryDirectory() as directory: + snapshot_manifest, as_of = build_provider_snapshot( + Path(directory) / "snapshot" + ) + observed_at = datetime( + as_of.year, + as_of.month, + as_of.day, + 9, + 2, + tzinfo=timezone(timedelta(hours=8)), + ) + timedelta(days=1) + for source_time, message in ( + ( + (observed_at - timedelta(seconds=61)).isoformat(), + "more than 60s stale", + ), + ( + (observed_at + timedelta(seconds=6)).isoformat(), + "more than 5s after", + ), + ): + with self.subTest(source_time=source_time): + broker = self._broker_snapshot( + as_of, + observed_at=observed_at, + ) + broker["source_time"] = source_time + with self.assertRaisesRegex( + SnapshotDecisionError, + message, + ): + build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + broker_snapshot=broker, + ) + signal_time = datetime( + as_of.year, + as_of.month, + as_of.day, + 15, + 0, + tzinfo=timezone(timedelta(hours=8)), + ) + for observed_at, message in ( + ( + signal_time + timedelta(minutes=1), + "outside the unique next session", + ), + ( + observed_at.replace(hour=9, minute=30), + "outside the unique next session", + ), + ( + observed_at + timedelta(days=3), + "outside the unique next session", + ), + ): + with self.subTest(observed_at=observed_at): + broker = self._broker_snapshot( + as_of, + observed_at=observed_at, + ) + with self.assertRaisesRegex( + SnapshotDecisionError, + message, + ): + build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + broker_snapshot=broker, + ) + for field in ("signal_as_of", "observation_as_of"): + with self.subTest(missing=field): + broker = self._broker_snapshot(as_of) + del broker[field] + with self.assertRaisesRegex( + SnapshotDecisionError, + rf"missing required fields.*{field}", + ): + build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + broker_snapshot=broker, + ) + duplicate = self._broker_snapshot(as_of) + position = { + "symbol": "600000.XSHG", + "quantity": 100, + "sellable_quantity": 100, + "average_cost": 10, + "market_value": 1000, + } + duplicate["positions"] = [position, dict(position)] + with self.assertRaisesRegex( + SnapshotDecisionError, + "duplicate position", + ): + build_snapshot_decision( + snapshot_manifest=snapshot_manifest, + config=config, + as_of=as_of, + broker_snapshot=duplicate, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..c4ddf08 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ +"""Build and environment inspection tools for quant60.""" diff --git a/tools/build_qlib_tiny_fixture.py b/tools/build_qlib_tiny_fixture.py new file mode 100644 index 0000000..74abc2d --- /dev/null +++ b/tools/build_qlib_tiny_fixture.py @@ -0,0 +1,164 @@ +"""Build a tiny synthetic Qlib binary provider with only the standard library.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import math +import struct +from pathlib import Path +from typing import Dict, Iterable, Mapping, Optional, Sequence + + +def _qlib_name(symbol: str) -> str: + value = str(symbol).strip().upper() + if len(value) != 8 or value[:2] not in {"SH", "SZ", "BJ"}: + raise ValueError("fixture symbols must use Qlib SH600000 form") + if not value[2:].isdigit(): + raise ValueError("invalid Qlib symbol") + return value + + +def _write_lines(path: Path, lines: Iterable[str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _write_feature(path: Path, start_index: int, values: Sequence[float]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + numeric = [float(start_index)] + [float(value) for value in values] + path.write_bytes(struct.pack("<" + "f" * len(numeric), *numeric)) + + +def write_qlib_fixture( + root: Path, + *, + calendar: Sequence[str], + features: Mapping[str, Mapping[str, Sequence[float]]], + markets: Optional[Mapping[str, Sequence[str]]] = None, +) -> Dict[str, object]: + """Write calendar/instruments/features in Qlib's documented bin layout.""" + output = root.expanduser().resolve() + if not calendar: + raise ValueError("calendar must not be empty") + if len(set(calendar)) != len(calendar): + raise ValueError("calendar contains duplicates") + ordered_calendar = sorted(str(value) for value in calendar) + if ordered_calendar != list(calendar): + raise ValueError("calendar must be sorted") + _write_lines(output / "calendars/day.txt", ordered_calendar) + + normalized: Dict[str, Mapping[str, Sequence[float]]] = {} + for raw_symbol, fields in features.items(): + symbol = _qlib_name(raw_symbol) + if not fields: + raise ValueError(f"{symbol} has no fields") + for field, values in fields.items(): + if len(values) != len(calendar): + raise ValueError( + f"{symbol}/{field} has {len(values)} values; " + f"expected {len(calendar)}" + ) + for value in values: + if not math.isfinite(float(value)): + raise ValueError(f"{symbol}/{field} contains non-finite data") + _write_feature( + output + / "features" + / symbol.lower() + / (str(field).lower().lstrip("$") + ".day.bin"), + 0, + values, + ) + normalized[symbol] = fields + + start, end = calendar[0], calendar[-1] + all_lines = [ + f"{symbol}\t{start}\t{end}" for symbol in sorted(normalized) + ] + _write_lines(output / "instruments/all.txt", all_lines) + for market, symbols in (markets or {}).items(): + selected = [_qlib_name(symbol) for symbol in symbols] + unknown = set(selected).difference(normalized) + if unknown: + raise ValueError(f"{market} contains missing symbols: {sorted(unknown)}") + _write_lines( + output / f"instruments/{market.lower()}.txt", + [f"{symbol}\t{start}\t{end}" for symbol in sorted(selected)], + ) + return { + "provider_uri": str(output), + "calendar_count": len(calendar), + "instruments": sorted(normalized), + "markets": sorted((markets or {}).keys()), + "format_note": ( + "each *.day.bin is little-endian float32; element 0 is calendar " + "start index, followed by field values" + ), + } + + +def _business_days(start: dt.date, count: int) -> list[str]: + output = [] + current = start + while len(output) < count: + if current.weekday() < 5: + output.append(current.isoformat()) + current += dt.timedelta(days=1) + return output + + +def _fields(closes: Sequence[float], volume: float) -> Dict[str, Sequence[float]]: + change = [0.0] + for previous, current in zip(closes, closes[1:]): + change.append(float(current) / float(previous) - 1.0) + return { + "open": [value * 0.998 for value in closes], + "high": [value * 1.01 for value in closes], + "low": [value * 0.99 for value in closes], + "close": list(closes), + "volume": [volume] * len(closes), + "factor": [1.0] * len(closes), + "change": change, + } + + +def build_default_fixture(root: Path, days: int = 80) -> Dict[str, object]: + """Create a deterministic two-stock market and one benchmark.""" + if int(days) < 30: + raise ValueError("default fixture needs at least 30 business days") + calendar = _business_days(dt.date(2024, 1, 2), int(days)) + closes_up = [10.0 * (1.004 ** index) for index in range(len(calendar))] + closes_down = [20.0 * (0.999 ** index) for index in range(len(calendar))] + closes_bench = [100.0 * (1.001 ** index) for index in range(len(calendar))] + return write_qlib_fixture( + root, + calendar=calendar, + features={ + "SH600000": _fields(closes_up, 2_000_000.0), + "SZ000001": _fields(closes_down, 1_500_000.0), + "SH000300": _fields(closes_bench, 100_000_000.0), + }, + markets={"csi300": ["SH600000", "SZ000001"]}, + ) + + +def _main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output_dir", type=Path) + parser.add_argument("--days", type=int, default=80) + args = parser.parse_args(argv) + print( + json.dumps( + build_default_fixture(args.output_dir, args.days), + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/tools/bundle_platforms.py b/tools/bundle_platforms.py new file mode 100644 index 0000000..e875a81 --- /dev/null +++ b/tools/bundle_platforms.py @@ -0,0 +1,276 @@ +"""Inline one portable core into JoinQuant and QMT single-file artifacts.""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import pprint +from pathlib import Path +from typing import Any, Dict, Optional + + +START = "# __PORTABLE_CORE_BUNDLE_START__" +END = "# __PORTABLE_CORE_BUNDLE_END__" +CONFIG_START = "# __BASELINE_CONFIG_START__" +CONFIG_END = "# __BASELINE_CONFIG_END__" + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _portable_body(source: str) -> str: + """Remove the core module docstring and future imports before inlining.""" + tree = ast.parse(source) + lines = source.splitlines() + first_line = 0 + if ( + tree.body + and isinstance(tree.body[0], ast.Expr) + and isinstance(tree.body[0].value, ast.Constant) + and isinstance(tree.body[0].value.value, str) + ): + first_line = int(tree.body[0].end_lineno or 0) + body = lines[first_line:] + body = [ + line + for line in body + if not line.lstrip().startswith("from __future__ import ") + ] + return "\n".join(body).strip() + "\n" + + +def inline_core(wrapper_source: str, core_source: str) -> str: + if wrapper_source.count(START) != 1 or wrapper_source.count(END) != 1: + raise ValueError("wrapper must contain exactly one portable-core marker pair") + start = wrapper_source.index(START) + end = wrapper_source.index(END, start) + len(END) + replacement = ( + START + + "\n# Inlined by tools/bundle_platforms.py; do not edit this region.\n" + + _portable_body(core_source) + + END + ) + return wrapper_source[:start] + replacement + wrapper_source[end:] + + +def _qmt_symbol(symbol: str) -> str: + suffixes = { + ".XSHG": ".SH", + ".XSHE": ".SZ", + ".XBSE": ".BJ", + } + for suffix, replacement in suffixes.items(): + if symbol.endswith(suffix): + return symbol[: -len(suffix)] + replacement + raise ValueError(f"unsupported canonical symbol in baseline config: {symbol}") + + +def effective_platform_config( + baseline: Dict[str, Any], + platform: str, +) -> Dict[str, Any]: + strategy = baseline["strategy"] + execution = baseline["execution"] + fees = baseline["fees"] + universe_config = baseline["universe"] + symbols = [str(symbol) for symbol in baseline["symbols"]] + if platform == "qmt_builtin": + universe = [_qmt_symbol(symbol) for symbol in symbols] + elif platform == "joinquant": + universe = symbols + else: + raise ValueError(f"unsupported platform config: {platform}") + common: Dict[str, Any] = { + "universe_mode": str(universe_config["mode"]), + "index_symbol": str(universe_config["index_symbol"]), + "qmt_sector_name": str(universe_config["qmt_sector_name"]), + "dedicated_account_required": bool( + universe_config["dedicated_account_required"] + ), + "exclude_st": bool(universe_config["exclude_st"]), + "universe": universe, + "lookback": int(strategy["lookback"]), + "skip": int(strategy["skip"]), + "top_n": int(strategy["top_n"]), + "max_weight": float(strategy["max_weight"]), + "gross_target": float(strategy["gross_target"]), + "cash_buffer": float(strategy["cash_buffer"]), + "lot_size": int(execution["lot_size"]), + "participation_rate": float(execution["participation_rate"]), + "slippage_bps": float(execution["slippage_bps"]), + "commission_rate": float(fees["commission_rate"]), + "minimum_commission": float(fees["minimum_commission"]), + "stamp_duty_rate": float(fees["stamp_duty_rate"]), + "transfer_fee_rate": float(fees["transfer_fee_rate"]), + "rebalance_schedule": str(strategy["rebalance_schedule"]), + } + if platform == "qmt_builtin": + return { + "account_id": "test", + **common, + } + return common + + +def inline_config(wrapper_source: str, config: Dict[str, Any]) -> str: + if ( + wrapper_source.count(CONFIG_START) != 1 + or wrapper_source.count(CONFIG_END) != 1 + ): + raise ValueError("wrapper must contain exactly one baseline-config marker pair") + start = wrapper_source.index(CONFIG_START) + end = wrapper_source.index(CONFIG_END, start) + len(CONFIG_END) + body = pprint.pformat(config, width=88, sort_dicts=False) + # QMT's built-in strategy is shipped with a GBK coding declaration but + # kept byte-for-byte ASCII for portability. Preserve non-ASCII config + # values (for example a Chinese sector name) as Python unicode escapes. + body = body.encode("ascii", "backslashreplace").decode("ascii") + replacement = ( + CONFIG_START + + "\n# Generated from configs/baseline.json; do not edit this region.\n" + + "CONFIG = " + + body + + "\n" + + CONFIG_END + ) + return wrapper_source[:start] + replacement + wrapper_source[end:] + + +def bundle_one( + *, + core_path: Path, + wrapper_path: Path, + output_path: Path, + effective_config: Dict[str, Any], + require_ascii: bool = False, +) -> Dict[str, str]: + core_bytes = core_path.read_bytes() + wrapper_bytes = wrapper_path.read_bytes() + core_source = core_bytes.decode("utf-8") + wrapper_source = wrapper_bytes.decode("utf-8") + bundled = inline_config( + inline_core(wrapper_source, core_source), + effective_config, + ) + output_bytes = bundled.encode("utf-8") + if require_ascii: + try: + bundled.encode("ascii") + except UnicodeEncodeError as exc: + raise ValueError( + "QMT #coding:gbk bundle must remain ASCII-only" + ) from exc + # The built-in QMT runtime is documented as Python 3.6. Parse against + # that grammar explicitly; compiling only on the build host would miss + # accidental modern syntax. + ast.parse(bundled, filename=str(output_path), feature_version=(3, 6)) + compile(bundled, str(output_path), "exec") + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(output_bytes) + return { + "core": str(core_path), + "core_sha256": sha256_bytes(core_bytes), + "wrapper": str(wrapper_path), + "wrapper_sha256": sha256_bytes(wrapper_bytes), + "effective_config_sha256": sha256_bytes( + ( + json.dumps( + effective_config, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ).encode("ascii") + ), + "output": str(output_path), + "output_sha256": sha256_bytes(output_bytes), + } + + +def bundle_all( + project_root: Path, + output_dir: Optional[Path] = None, +) -> Dict[str, object]: + root = project_root.expanduser().resolve() + output = ( + output_dir.expanduser().resolve() + if output_dir + else root / "dist" + ) + core = root / "src" / "quant60" / "portable_core.py" + if not core.is_file(): + raise FileNotFoundError(core) + config_path = root / "configs" / "baseline.json" + baseline_bytes = config_path.read_bytes() + baseline = json.loads(baseline_bytes.decode("utf-8")) + if not isinstance(baseline, dict): + raise ValueError("configs/baseline.json must contain an object") + entries = { + "joinquant": bundle_one( + core_path=core, + wrapper_path=root / "platforms" / "joinquant_strategy.py", + output_path=output / "joinquant_strategy.py", + effective_config=effective_platform_config(baseline, "joinquant"), + ), + "qmt_builtin": bundle_one( + core_path=core, + wrapper_path=root / "platforms" / "qmt_builtin_strategy.py", + output_path=output / "qmt_builtin_strategy.py", + effective_config=effective_platform_config(baseline, "qmt_builtin"), + require_ascii=True, + ), + } + # Paths in a committed manifest must be portable and must not expose the + # local account/home directory. Hashes remain byte-for-byte identities. + logical_outputs = { + "joinquant": "dist/joinquant_strategy.py", + "qmt_builtin": "dist/qmt_builtin_strategy.py", + } + for platform_name, entry in entries.items(): + entry["core"] = "src/quant60/portable_core.py" + entry["wrapper"] = ( + "platforms/joinquant_strategy.py" + if "joinquant" in entry["wrapper"] + else "platforms/qmt_builtin_strategy.py" + ) + # The manifest describes the committed logical artifact, not the + # caller's temporary build directory. This keeps clean-room builds + # byte-for-byte comparable with committed ``dist``. + entry["output"] = logical_outputs[platform_name] + manifest = { + "schema_version": 1, + "baseline_config": "configs/baseline.json", + "baseline_config_sha256": sha256_bytes(baseline_bytes), + "portable_core_sha256": entries["joinquant"]["core_sha256"], + "artifacts": entries, + } + manifest_path = output / "bundle_manifest.json" + encoded = ( + json.dumps(manifest, ensure_ascii=True, indent=2, sort_keys=True) + "\n" + ).encode("ascii") + manifest_path.write_bytes(encoded) + try: + manifest["manifest"] = str(manifest_path.relative_to(root)) + except ValueError: + manifest["manifest"] = manifest_path.name + manifest["manifest_sha256"] = sha256_bytes(encoded) + return manifest + + +def _main(argv: Optional[list[str]] = None) -> int: + default_root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project-root", type=Path, default=default_root) + parser.add_argument("--output-dir", type=Path) + args = parser.parse_args(argv) + manifest = bundle_all(args.project_root, args.output_dir) + print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/tools/capability_probe.py b/tools/capability_probe.py new file mode 100644 index 0000000..02f51e9 --- /dev/null +++ b/tools/capability_probe.py @@ -0,0 +1,209 @@ +"""Report static artifacts and local optional-runtime availability.""" + +from __future__ import annotations + +import argparse +import importlib +import importlib.metadata +import importlib.util +import json +import platform +import sys +from pathlib import Path +from typing import Any, Dict, Optional + + +CAPABILITY_MATRIX: Dict[str, Dict[str, Any]] = { + "quant_os_research": { + "backtest": True, + "hosted": False, + "live_orders": False, + "safe_default": True, + "market_scope": ( + "synthetic research plus verified authorized-provider " + "PIT snapshot decisions/backtests" + ), + }, + "portable_local": { + "backtest": True, + "hosted": False, + "live_orders": False, + "safe_default": True, + "market_scope": "SH/SZ/BJ core math", + }, + "joinquant_hosted": { + "backtest": True, + "hosted": True, + "live_orders": False, + "safe_default": True, + "market_scope": "configured SH/SZ wrapper universe", + }, + "qmt_builtin": { + "backtest": True, + "hosted": True, + "live_orders": False, + "safe_default": True, + "market_scope": "configured SH/SZ wrapper universe; backtest-only", + }, + "qmt_research": { + "backtest": True, + "hosted": False, + "live_orders": False, + "safe_default": True, + "market_scope": "QMT entitlement; runner is backtest/history only", + }, + "qmt_shadow": { + "backtest": False, + "hosted": False, + "live_orders": False, + "safe_default": True, + "market_scope": ( + "licensed XtTrader SH/SZ read-only account queries; " + "account-bound inert target diff only" + ), + }, + "xttrader": { + "backtest": False, + "hosted": False, + "live_orders": "explicit opt-in plus fresh structured policy guard", + "safe_default": True, + "market_scope": "SH/SZ; adapter rejects BJ explicitly", + }, + "qlib_0_9_7": { + "backtest": True, + "hosted": False, + "live_orders": False, + "safe_default": True, + "market_scope": ( + "provider-defined; portable signal research bridge, " + "not target/order parity" + ), + }, +} + + +def _module_probe(name: str, deep: bool) -> Dict[str, Any]: + try: + available = importlib.util.find_spec(name) is not None + except (ImportError, ModuleNotFoundError, ValueError): + available = False + result: Dict[str, Any] = { + "available": available, + "import_checked": bool(deep), + "import_ok": None, + "error": None, + } + top_level = name.split(".", 1)[0] + try: + result["version"] = importlib.metadata.version( + "pyqlib" if top_level == "qlib" else top_level + ) + except importlib.metadata.PackageNotFoundError: + result["version"] = None + if available and deep: + try: + importlib.import_module(name) + result["import_ok"] = True + except Exception as exc: + result["import_ok"] = False + result["error"] = f"{type(exc).__name__}: {exc}" + return result + + +def probe_capabilities( + project_root: Optional[Path] = None, + *, + deep: bool = False, +) -> Dict[str, Any]: + root = ( + project_root.expanduser().resolve() + if project_root + else Path(__file__).resolve().parents[1] + ) + generated = root / "dist" + return { + "schema_version": 1, + "python": { + "version": platform.python_version(), + "implementation": platform.python_implementation(), + "executable": sys.executable, + "platform": platform.platform(), + }, + "artifacts": { + "portable_core": (root / "src/quant60/portable_core.py").is_file(), + "joinquant_wrapper": (root / "platforms/joinquant_strategy.py").is_file(), + "qmt_builtin_wrapper": ( + root / "platforms/qmt_builtin_strategy.py" + ).is_file(), + "joinquant_bundle": ( + generated / "joinquant_strategy.py" + ).is_file(), + "qmt_builtin_bundle": ( + generated / "qmt_builtin_strategy.py" + ).is_file(), + "bundle_manifest": ( + generated / "bundle_manifest.json" + ).is_file(), + "colab_notebook": ( + root / "notebooks/quant_os_colab.ipynb" + ).is_file(), + "jqdata_snapshot_adapter": ( + root / "adapters/jqdata_local.py" + ).is_file(), + "snapshot_decision_pipeline": ( + root / "src/quant60/snapshot_pipeline.py" + ).is_file(), + "snapshot_backtest": ( + root / "src/quant60/snapshot_backtest.py" + ).is_file(), + "mock_parity_exporter": ( + root / "tools/export_mock_parity.py" + ).is_file(), + "qlib_momentum_and_alpha158_cli": ( + root / "platforms/qlib_runner.py" + ).is_file(), + "qmt_shadow_planner": ( + root / "src/quant60/qmt_shadow.py" + ).is_file(), + "qmt_shadow_cli": ( + root / "tools/qmt_shadow_plan.py" + ).is_file(), + }, + "optional_runtimes": { + "jqdatasdk": _module_probe("jqdatasdk", deep), + "cvxpy": _module_probe("cvxpy", deep), + "qlib": _module_probe("qlib", deep), + "xtquant": _module_probe("xtquant", deep), + "xtquant.qmttools": _module_probe("xtquant.qmttools", deep), + "xtquant.xttrader": _module_probe("xtquant.xttrader", deep), + }, + "matrix": CAPABILITY_MATRIX, + "verification_boundary": ( + "artifact presence and optional import probes only; no external " + "platform backtest or broker entitlement is claimed" + ), + } + + +def _main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project-root", type=Path) + parser.add_argument( + "--deep", + action="store_true", + help="Import optional runtimes; still performs no broker mutation.", + ) + args = parser.parse_args(argv) + print( + json.dumps( + probe_capabilities(args.project_root, deep=args.deep), + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/tools/export_mock_parity.py b/tools/export_mock_parity.py new file mode 100644 index 0000000..f5283fc --- /dev/null +++ b/tools/export_mock_parity.py @@ -0,0 +1,208 @@ +"""Export and verify the deterministic JoinQuant/QMT wrapper parity fixture. + +This evidence proves only the local wrapper contract. It never represents a +real hosted-platform, licensed QMT, broker, or investment-performance pass. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +from platforms import joinquant_strategy, qmt_builtin_strategy +from platforms.fake_joinquant_harness import FakeJoinQuantHarness +from platforms.fake_qmt_harness import FakeQmtContext, FakeQmtHarness + + +PROJECT = Path(__file__).resolve().parents[1] + + +def _canonical_json(payload: Any) -> str: + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _sha256_payload(payload: Any) -> str: + return _sha256_bytes(_canonical_json(payload).encode("utf-8")) + + +def _price_histories() -> dict[str, list[float]]: + return { + "600000.XSHG": [15.0 - index * 0.02 for index in range(21)], + "000001.XSHE": [10.0 + index * 0.10 for index in range(21)], + "300750.XSHE": [100.0 + index * 1.5 for index in range(21)], + "000333.XSHE": [20.0 + index * 0.05 for index in range(21)], + } + + +def _source_hashes() -> dict[str, str]: + paths = ( + PROJECT / "platforms/joinquant_strategy.py", + PROJECT / "platforms/qmt_builtin_strategy.py", + PROJECT / "src/quant60/portable_core.py", + PROJECT / "configs/baseline.json", + ) + return { + str(path.relative_to(PROJECT)): _sha256_bytes(path.read_bytes()) + for path in paths + } + + +def build_report() -> dict[str, Any]: + histories = _price_histories() + jq = FakeJoinQuantHarness(histories) + jq.initialize(joinquant_strategy) + jq.context.previous_date = dt.date(2026, 7, 24) + jq.context.current_dt = dt.datetime(2026, 7, 27, 9, 30) + jq_plan = joinquant_strategy.compute_plan(jq.context) + + qmt_histories = { + symbol.replace(".XSHE", ".SZ").replace(".XSHG", ".SH"): values + for symbol, values in histories.items() + } + qmt_context = FakeQmtContext( + qmt_histories, + bar_time=dt.datetime(2026, 7, 24, 15, 0), + ) + FakeQmtHarness(qmt_context).install(qmt_builtin_strategy) + qmt_builtin_strategy.init(qmt_context) + qmt_plan = qmt_builtin_strategy.compute_plan( + qmt_context, + dt.datetime(2026, 7, 24, 15, 0), + ) + + sources = _source_hashes() + body = { + "schema_version": "1.0", + "evidence_class": "mock_contract_only", + "real_platform_pass": False, + "investment_value_claim": False, + "fixture": { + "as_of": "2026-07-24T15:00:00+08:00", + "history_length": 21, + "symbols": sorted(histories), + "price_history_sha256": _sha256_payload(histories), + }, + "tolerances": { + "score_absolute": 1e-12, + "target_weight_absolute": 1e-10, + "target_quantity": "exact_integer", + "order_delta": "exact_integer", + }, + "joinquant_plan": jq_plan, + "qmt_plan": qmt_plan, + "comparisons": { + "whole_plan_exact": jq_plan == qmt_plan, + "score_exact": jq_plan["scores"] == qmt_plan["scores"], + "target_weight_exact": ( + jq_plan["weights"] == qmt_plan["weights"] + ), + "target_quantity_exact": ( + jq_plan["targets"] == qmt_plan["targets"] + ), + "order_delta_exact": jq_plan["orders"] == qmt_plan["orders"], + }, + "source_sha256": sources, + "source_aggregate_sha256": _sha256_payload(sources), + "gate_credit": [], + "required_next_evidence": [ + "JoinQuant hosted export", + "licensed QMT backtest export", + "canonical clock/data/fee/fill difference report", + ], + } + if not all(body["comparisons"].values()): + raise RuntimeError("local JoinQuant/QMT wrapper parity failed") + body["report_sha256"] = _sha256_payload(body) + return body + + +def write_report(path: Path) -> dict[str, Any]: + report = build_report() + output = path.expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + encoded = ( + json.dumps( + report, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + descriptor, temporary = tempfile.mkstemp( + prefix=f".{output.name}.", + suffix=".tmp", + dir=str(output.parent), + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, output) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + return report + + +def verify_report(path: Path) -> dict[str, Any]: + report = json.loads(path.read_text(encoding="utf-8")) + saved_hash = report.pop("report_sha256", None) + if not isinstance(saved_hash, str) or _sha256_payload(report) != saved_hash: + raise ValueError("mock parity report hash mismatch") + expected = build_report() + if saved_hash != expected["report_sha256"]: + raise ValueError("mock parity report does not match current sources") + return { + "ok": True, + "evidence_class": report["evidence_class"], + "report_sha256": saved_hash, + "whole_plan_exact": report["comparisons"]["whole_plan_exact"], + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + export_parser = subparsers.add_parser("export") + export_parser.add_argument("--output", required=True, type=Path) + verify_parser = subparsers.add_parser("verify") + verify_parser.add_argument("report", type=Path) + args = parser.parse_args(argv) + if args.command == "export": + result = write_report(args.output) + summary = { + "ok": True, + "output": str(args.output.expanduser().resolve()), + "evidence_class": result["evidence_class"], + "report_sha256": result["report_sha256"], + } + else: + summary = verify_report(args.report) + print(json.dumps(summary, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/jqdata_snapshot.py b/tools/jqdata_snapshot.py new file mode 100644 index 0000000..4ffca19 --- /dev/null +++ b/tools/jqdata_snapshot.py @@ -0,0 +1,37 @@ +"""Authenticate interactively and build a canonical JQData index snapshot.""" + +from __future__ import annotations + +import argparse +import json +from datetime import date + +from adapters.jqdata_local import ( + authenticate, + fetch_index_daily_snapshot, + load_jqdata, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--index", default="000905.XSHG") + parser.add_argument("--start", required=True, type=date.fromisoformat) + parser.add_argument("--end", required=True, type=date.fromisoformat) + parser.add_argument("--output", required=True) + args = parser.parse_args() + sdk = load_jqdata() + authenticate(sdk) + paths = fetch_index_daily_snapshot( + sdk, + index_symbol=args.index, + start_date=args.start, + end_date=args.end, + output_dir=args.output, + ) + print(json.dumps({"ok": True, "artifacts": paths}, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/qmt_shadow_plan.py b/tools/qmt_shadow_plan.py new file mode 100644 index 0000000..2c6bd30 --- /dev/null +++ b/tools/qmt_shadow_plan.py @@ -0,0 +1,255 @@ +"""Run or verify the strictly read-only QMT/XtTrader shadow preflight. + +Environment fallbacks: + QMT_USERDATA_PATH QMT_ACCOUNT_ID QMT_SESSION_ID + QMT_ACCOUNT_HASH_KEY_FILE + +The runtime adapter is always constructed with ``allow_live_orders=False``. +No command in this tool submits or cancels an order. +""" + +from __future__ import annotations + +import argparse +import json +import os +import secrets +import stat +import sys +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +for import_root in (PROJECT_ROOT, PROJECT_ROOT / "src"): + if str(import_root) not in sys.path: + sys.path.insert(0, str(import_root)) + +from adapters.xttrader_live import XtTraderLiveAdapter +from quant60.qmt_shadow import ( + MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE, + MAX_SHADOW_QUERY_DURATION_SECONDS, + build_qmt_shadow_plan, + verify_qmt_shadow_plan, + write_qmt_shadow_plan, +) + + +def _required(value: str | None, name: str) -> str: + if value is None or not value.strip(): + raise ValueError(f"{name} is required") + return value.strip() + + +def _bounded_non_negative_float( + *, + name: str, + maximum: float, +): + def parse(value: str) -> float: + try: + parsed = float(value) + except ValueError as exc: + raise argparse.ArgumentTypeError( + f"{name} must be a number" + ) from exc + if not 0 <= parsed <= maximum: + raise argparse.ArgumentTypeError( + f"{name} must be between 0 and {maximum:g}" + ) + return parsed + + return parse + + +def _load_or_create_account_hash_key(path: str | Path) -> bytes: + """Load a stable local HMAC key without ever exposing it in artifacts.""" + + key_path = Path(path).expanduser() + parent_existed = key_path.parent.exists() + key_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if not parent_existed: + try: + key_path.parent.chmod(0o700) + except OSError: + pass + + create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + create_flags |= getattr(os, "O_CLOEXEC", 0) + try: + descriptor = os.open(str(key_path), create_flags, 0o600) + except FileExistsError: + descriptor = None + if descriptor is not None: + key = secrets.token_bytes(32) + with os.fdopen(descriptor, "wb") as handle: + handle.write(key) + handle.flush() + os.fsync(handle.fileno()) + + return _load_existing_account_hash_key(key_path) + + +def _load_existing_account_hash_key(path: str | Path) -> bytes: + key_path = Path(path).expanduser() + if not key_path.is_file(): + raise ValueError( + "QMT account hash key file does not exist; verification never " + "creates a replacement key" + ) + read_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + read_flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(str(key_path), read_flags) + with os.fdopen(descriptor, "rb") as handle: + key = handle.read() + if len(key) < 32: + raise ValueError("QMT account hash key must contain at least 32 bytes") + try: + mode = stat.S_IMODE(key_path.stat().st_mode) + if os.name != "nt" and mode & 0o077: + raise ValueError( + "QMT account hash key file must not be group/world accessible" + ) + except OSError as exc: + raise ValueError("cannot validate QMT account hash key file") from exc + return key + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + plan = subparsers.add_parser("plan", help="query QMT and write a shadow plan") + plan.add_argument("--decision", required=True) + plan.add_argument("--output", required=True) + plan.add_argument( + "--userdata-path", default=os.environ.get("QMT_USERDATA_PATH") + ) + plan.add_argument("--account-id", default=os.environ.get("QMT_ACCOUNT_ID")) + plan.add_argument( + "--session-id", + type=int, + default=( + int(os.environ["QMT_SESSION_ID"]) + if os.environ.get("QMT_SESSION_ID") + else None + ), + ) + plan.add_argument( + "--max-query-duration-seconds", + type=_bounded_non_negative_float( + name="max-query-duration-seconds", + maximum=MAX_SHADOW_QUERY_DURATION_SECONDS, + ), + default=MAX_SHADOW_QUERY_DURATION_SECONDS, + help=( + "query-window ceiling; may be tightened but cannot exceed " + f"{MAX_SHADOW_QUERY_DURATION_SECONDS:g}s" + ), + ) + plan.add_argument( + "--market-value-tolerance", + type=_bounded_non_negative_float( + name="market-value-tolerance", + maximum=MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE, + ), + default=MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE, + help=( + "asset identity tolerance; may be tightened but cannot exceed " + f"{MAX_BROKER_ASSET_ABSOLUTE_TOLERANCE:g} CNY" + ), + ) + plan.add_argument( + "--account-hash-key-file", + default=os.environ.get( + "QMT_ACCOUNT_HASH_KEY_FILE", + str(Path.home() / ".config" / "quant-os" / "account-hash.key"), + ), + help=( + "local 0600 HMAC key file; generated once when absent and never " + "written to artifacts" + ), + ) + + verify = subparsers.add_parser( + "verify", help="verify a completed shadow-plan artifact set" + ) + verify.add_argument("--manifest", required=True) + verify.add_argument("--decision", required=True) + verify.add_argument( + "--account-hash-key-file", + default=os.environ.get( + "QMT_ACCOUNT_HASH_KEY_FILE", + str(Path.home() / ".config" / "quant-os" / "account-hash.key"), + ), + help=( + "existing local 0600 HMAC key used to authenticate the broker " + "observation; verify never creates a key" + ), + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "verify": + account_hash_key = _load_existing_account_hash_key( + args.account_hash_key_file + ) + payload = verify_qmt_shadow_plan( + args.manifest, + decision_manifest=args.decision, + account_hash_key=account_hash_key, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + userdata_path = _required(args.userdata_path, "QMT userdata path") + account_id = _required(args.account_id, "QMT account id") + session_id = ( + args.session_id + if args.session_id is not None + else secrets.randbelow(2_000_000_000) + 1 + ) + account_hash_key = _load_or_create_account_hash_key( + args.account_hash_key_file + ) + adapter = XtTraderLiveAdapter.from_xtquant( + userdata_path=userdata_path, + session_id=session_id, + account_id=account_id, + allow_live_orders=False, + strategy_name="quant-os-shadow", + ) + try: + adapter.connect() + result = build_qmt_shadow_plan( + decision_manifest=args.decision, + adapter=adapter, + max_query_duration_seconds=args.max_query_duration_seconds, + market_value_tolerance=args.market_value_tolerance, + account_hash_key=account_hash_key, + ) + paths = write_qmt_shadow_plan(result, args.output) + verification = verify_qmt_shadow_plan( + paths["manifest"], + decision_manifest=args.decision, + account_hash_key=account_hash_key, + ) + ready = result["plan"]["status"] == "SHADOW_READY" + payload = { + "ok": ready, + "artifact_written": True, + "artifact_verified": verification["ok"], + "plan_id": result["plan"]["plan_id"], + "status": result["plan"]["status"], + "blockers": result["plan"]["blockers"], + "artifacts": paths, + "read_only": True, + } + finally: + adapter.close() + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload["ok"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/run_colab_notebook.py b/tools/run_colab_notebook.py new file mode 100644 index 0000000..5b2a030 --- /dev/null +++ b/tools/run_colab_notebook.py @@ -0,0 +1,74 @@ +"""Validate or execute the plain-Python cells in the Quant60 Colab notebook.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def load_notebook(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("nbformat") != 4 or not isinstance(payload.get("cells"), list): + raise ValueError("expected a valid nbformat 4 notebook") + return payload + + +def code_cells( + payload: dict[str, Any], + *, + include_optional: bool, +) -> list[tuple[int, str]]: + output: list[tuple[int, str]] = [] + for index, cell in enumerate(payload["cells"]): + if cell.get("cell_type") != "code": + continue + tags = cell.get("metadata", {}).get("tags", []) + if "optional" in tags and not include_optional: + continue + source = cell.get("source") + if not isinstance(source, list) or any( + not isinstance(line, str) for line in source + ): + raise ValueError(f"cell {index} source must be a string array") + output.append((index, "".join(source))) + return output + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("path", type=Path) + parser.add_argument("--execute", action="store_true") + parser.add_argument("--include-optional", action="store_true") + args = parser.parse_args(argv) + + payload = load_notebook(args.path.resolve()) + cells = code_cells(payload, include_optional=args.include_optional) + for index, source in cells: + compile(source, f"{args.path}:cell-{index}", "exec") + if args.execute: + namespace: dict[str, Any] = {"__name__": "__quant60_notebook__"} + for index, source in cells: + print(f"[quant60-notebook] executing cell {index}") + exec( + compile(source, f"{args.path}:cell-{index}", "exec"), + namespace, + namespace, + ) + print( + json.dumps( + { + "ok": True, + "code_cells": len(cells), + "executed": bool(args.execute), + "optional_included": bool(args.include_optional), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())