feat: operationalize local Tushare Qlib research
This commit is contained in:
+1
-1
@@ -4,7 +4,7 @@
|
||||
|
||||
## 目标位置
|
||||
|
||||
- 本地：`/Users/zhangbiao/boat-workspace/Code/quant-os`
|
||||
- 本地：用户选定的独立 checkout（本文记为 `$QUANT_OS_ROOT`）
|
||||
- 远端：`https://git.gomars.fun/boat/quant-os.git`
|
||||
- 公共状态：`https://gomars.fun/quant-os/status/`
|
||||
- 原仓：`https://git.gomars.fun/boat/quants-strategies.git`
|
||||
|
||||
@@ -3,8 +3,11 @@ PROJECT_PYTHONPATH := src:.
|
||||
SMOKE_DIR := artifacts/local-smoke
|
||||
RESEARCH_DIR := artifacts/research-smoke
|
||||
PARITY_REPORT := artifacts/mock-parity/report.json
|
||||
TUSHARE_PROVIDER ?= data/qlib/tushare-csi500-2018-2025-v2
|
||||
TUSHARE_SNAPSHOT ?= artifacts/local-tushare/scoped-source-manifest.json
|
||||
TUSHARE_EVIDENCE_DIR ?= artifacts/tushare
|
||||
|
||||
.PHONY: test architecture standard bundle smoke verify research research-verify parity parity-verify doctor secret-scan status status-check notebook-check notebook-smoke local
|
||||
.PHONY: test architecture standard bundle smoke verify research research-verify parity parity-verify doctor secret-scan status status-check notebook-check notebook-smoke tushare-check-root tushare-snapshot tushare-build tushare-verify tushare-lineage tushare-backtest tushare-replay local
|
||||
|
||||
test:
|
||||
PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m unittest discover -s tests -p '*.py' -q
|
||||
@@ -56,4 +59,69 @@ notebook-check:
|
||||
notebook-smoke:
|
||||
$(PYTHON) tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb --execute
|
||||
|
||||
tushare-check-root:
|
||||
@test -n "$(TUSHARE_MIRROR_ROOT)" || (echo "set TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror" >&2; exit 2)
|
||||
|
||||
tushare-snapshot: tushare-check-root
|
||||
$(PYTHON) tools/tushare_snapshot.py \
|
||||
--mirror-root "$(TUSHARE_MIRROR_ROOT)" \
|
||||
--start 2018-01-01 \
|
||||
--end 2025-12-31 \
|
||||
--apis daily adj_factor trade_cal stock_basic index_daily index_weight \
|
||||
--index-codes 000905.SH \
|
||||
--output "$(TUSHARE_SNAPSHOT)"
|
||||
|
||||
tushare-build: tushare-check-root
|
||||
$(PYTHON) tools/tushare_qlib.py build \
|
||||
--mirror-root "$(TUSHARE_MIRROR_ROOT)" \
|
||||
--output-dir "$(TUSHARE_PROVIDER)" \
|
||||
--start 2018-01-01 \
|
||||
--end 2025-12-31 \
|
||||
--market-name tushare_csi500 \
|
||||
--benchmark-index-code 000905.SH \
|
||||
--universe-index-code 000905.SH \
|
||||
--minimum-observations 60 \
|
||||
--ohlc-policy fail \
|
||||
--output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-build.json"
|
||||
|
||||
tushare-verify:
|
||||
$(PYTHON) tools/tushare_qlib.py verify "$(TUSHARE_PROVIDER)" \
|
||||
--output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-verify.json"
|
||||
|
||||
tushare-lineage:
|
||||
$(PYTHON) tools/tushare_lineage.py \
|
||||
--source-manifest "$(TUSHARE_SNAPSHOT)" \
|
||||
--provider-dir "$(TUSHARE_PROVIDER)" \
|
||||
--output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-lineage.json"
|
||||
|
||||
tushare-backtest:
|
||||
PYTHONHASHSEED=0 PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m platforms.qlib_runner \
|
||||
--provider-uri "$(TUSHARE_PROVIDER)" \
|
||||
--market tushare_csi500 \
|
||||
--benchmark SH000905 \
|
||||
--start 2019-01-02 \
|
||||
--end 2025-12-30 \
|
||||
--feature-start 2018-01-02 \
|
||||
--lookback 20 \
|
||||
--topk 50 \
|
||||
--n-drop 5 \
|
||||
--rebalance weekly \
|
||||
--output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-momentum-run-1.json"
|
||||
|
||||
tushare-replay: tushare-backtest
|
||||
PYTHONHASHSEED=0 PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m platforms.qlib_runner \
|
||||
--provider-uri "$(TUSHARE_PROVIDER)" \
|
||||
--market tushare_csi500 \
|
||||
--benchmark SH000905 \
|
||||
--start 2019-01-02 \
|
||||
--end 2025-12-30 \
|
||||
--feature-start 2018-01-02 \
|
||||
--lookback 20 \
|
||||
--topk 50 \
|
||||
--n-drop 5 \
|
||||
--rebalance weekly \
|
||||
--output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-momentum-run-2.json"
|
||||
cmp "$(TUSHARE_EVIDENCE_DIR)/csi500-momentum-run-1.json" \
|
||||
"$(TUSHARE_EVIDENCE_DIR)/csi500-momentum-run-2.json"
|
||||
|
||||
local: architecture standard test bundle smoke verify research research-verify parity parity-verify doctor secret-scan status-check notebook-check notebook-smoke
|
||||
|
||||
@@ -70,7 +70,7 @@ python3 tools/build_public_status.py --check
|
||||
| 聚宽 hosted | 默认动量 smoke；显式注入 TargetPackage 后只消费本地 post-risk 权重并绑定平台账户/价格 | 动量模式已有一次 2024 全年 run；一个 synthetic-research TargetPackage 已真实命中、绑定 `day_open` 并产生 3 笔委托/成交 | 只观察 execution consumer；无授权真实长样本/OOS、上游四层 hosted 执行、QMT peer 或逐层真实数据 parity |
|
||||
| QMT built-in/qmttools | 默认动量 smoke；TargetPackage 模式为 Python 3.6 backtest-only 消费器 | Python 3.6 语法、TargetPackage/fake/qmttools 合约 | 需要已授权 QMT 客户端、历史数据权限与 TargetPackage 实跑导出 |
|
||||
| Qlib 0.9.7 | native momentum backtest；Alpha158/LightGBM CLI/Recorder 工作流 | Python 3.12 两条 fixture 已真实跑通，并保存表级 hash、行列/时间边界和重算指标 | synthetic 两股票没有投资价值；无真实 OOS、L3/L4 parity |
|
||||
| Tushare → Qlib | 只读盘点 completed Parquet、校验 checksum、构建/验证不可变 provider、运行本地回测 | 93 个 completed 文件已验证；早期未复权 provider 与两次 byte-identical momentum run | 下载仅到 1993-10；缺复权、真实 benchmark、历史成分/ST/停牌/涨跌停，不能进入生产 decision |
|
||||
| Tushare → Qlib | 只读盘点 live mirror、冻结 scoped release、构建/验证不可变 provider、运行本地回测 | 2018—2025 v2 verifier 通过；1,111 instruments、1,942 sessions；97 observed/96 effective 月末 snapshots 严格 500 成分、次日生效；lineage 221 jobs×6 fields mismatch 0、converter current true；双次 run byte-identical | event-time/保守次日生效 PIT 近似，不是严格 knowledge-time PIT；不同 start 的首观测锚 provider 不可直接拼接；research-only、`gate_credit=[]`、ST/统一 9.5% 仍缺 |
|
||||
| XtTrader shadow | 只读账户查询、HMAC 账户绑定、broker observation/snapshot、零下单 target-diff 计划与完整 verifier | fake broker/QMT 合约；所有 broker mutation 均不可达 | 需要授权 QMT 环境的账户合同探针、恢复/对账和券商确认；Baseline 60 至少 20 个交易日，Production 80 当前缺口为同一冻结候选至少 60 个交易日 |
|
||||
|
||||
`portable-momentum-v1` 只是当前真实跑过聚宽的跨引擎连通性 smoke，不是
|
||||
@@ -477,37 +477,70 @@ python -c "import qlib; assert qlib.__version__ == '0.9.7'"
|
||||
|
||||
### 使用本地 Tushare 镜像
|
||||
|
||||
`tools/tushare_qlib.py` 不调用 Tushare、不读取 token，也不修改镜像。它只接受
|
||||
SQLite 中状态为 `completed` 且文件大小、SHA-256、Parquet 行数一致的分区；
|
||||
生成新 Qlib provider 时保存 source job、build parameter、converter hash、
|
||||
tree hash 与可重算 `data_version`，并拒绝覆盖已有目录。
|
||||
`tools/tushare_snapshot.py` 和 `tools/tushare_qlib.py` 不调用 Tushare、不读取
|
||||
token，也不修改镜像。先从仍可变化的 raw mirror 冻结一个日期/API/指数范围
|
||||
明确的 source manifest，再生成新 Qlib provider。provider 保存 source job、
|
||||
build parameter、converter hash、tree hash 与可重算 `data_version`，并拒绝
|
||||
覆盖已有目录。当前 build 不直接读取 scoped manifest，所以验收时还要确认
|
||||
两份 manifest 的 source job/file hashes 完全一致。
|
||||
|
||||
推荐使用 Makefile 固定同一组路径和参数：
|
||||
|
||||
```bash
|
||||
export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror
|
||||
export TUSHARE_PROVIDER=data/qlib/tushare-csi500-2018-2025-next
|
||||
export TUSHARE_SNAPSHOT=artifacts/local-tushare/scoped-source-manifest.json
|
||||
|
||||
make tushare-snapshot
|
||||
make tushare-build
|
||||
make tushare-verify
|
||||
make tushare-lineage
|
||||
make tushare-backtest
|
||||
make tushare-replay
|
||||
```
|
||||
|
||||
`TUSHARE_PROVIDER` 指向的 build 输出目录必须不存在；工具拒绝覆盖已有
|
||||
provider。`tushare-replay` 会先生成第一份 run，再生成第二份并用 `cmp`
|
||||
检查逐 byte 重放。
|
||||
|
||||
```bash
|
||||
export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror
|
||||
|
||||
PYTHONPATH=src:. python tools/tushare_qlib.py inventory \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--skip-file-verification \
|
||||
--output-json artifacts/tushare-inventory.json
|
||||
|
||||
python tools/tushare_snapshot.py \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--start 2018-01-01 \
|
||||
--end 2025-12-31 \
|
||||
--apis daily adj_factor trade_cal stock_basic index_daily index_weight \
|
||||
--index-codes 000905.SH \
|
||||
--output artifacts/local-tushare-20260731/scoped-source-manifest.json
|
||||
|
||||
PYTHONPATH=src:. python tools/tushare_qlib.py build \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--output-dir data/qlib/tushare-early-v1 \
|
||||
--start 1990-12-19 --end 1993-10-29 \
|
||||
--output-dir data/qlib/tushare-csi500-2018-2025-v2 \
|
||||
--start 2018-01-01 --end 2025-12-31 \
|
||||
--market-name tushare_csi500 \
|
||||
--benchmark-index-code 000905.SH \
|
||||
--universe-index-code 000905.SH \
|
||||
--minimum-observations 60 \
|
||||
--allow-unadjusted \
|
||||
--ohlc-policy expand-range \
|
||||
--ohlc-policy fail \
|
||||
--output-json artifacts/tushare-qlib-build.json
|
||||
|
||||
PYTHONPATH=src:. python tools/tushare_qlib.py verify \
|
||||
data/qlib/tushare-early-v1
|
||||
data/qlib/tushare-csi500-2018-2025-v2
|
||||
```
|
||||
|
||||
当前镜像只有 1990-12—1993-10 日线，且没有 `adj_factor`、真实指数行情、
|
||||
历史成分、ST、停牌和涨跌停数据。因此 `--allow-unadjusted` 是显式技术
|
||||
smoke 降级，provider 固定 `production_eligible=false`、
|
||||
`investment_value_claim=false`、`gate_credit=[]`。盘点、字段口径、完整命令、
|
||||
v4 hash 与两次本地回测结果见
|
||||
[`docs/TUSHARE_LOCAL_DATA.md`](docs/TUSHARE_LOCAL_DATA.md)。
|
||||
当前镜像已有 1990-12—2026-07 日线和复权因子、真实中证 500 指数行情与历史
|
||||
权重、停牌和涨跌停数据；但 raw mirror 是移动目标，且历史 `stock_st` 权限被
|
||||
拒。v2 provider verifier 和两次 byte-identical 真实本地 run 已完成，但
|
||||
结果仍为 `production_ready=false`、`investment_value_claim=false`、
|
||||
`gate_credit=[]`。完整盘点、字段口径和本机
|
||||
命令见 [`docs/TUSHARE_LOCAL_DATA.md`](docs/TUSHARE_LOCAL_DATA.md) 与
|
||||
[`docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md`](docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md)。
|
||||
|
||||
native momentum smoke：
|
||||
|
||||
|
||||
+1022
-101
File diff suppressed because it is too large
Load Diff
+21
-8
@@ -66,12 +66,13 @@ XtTrader:
|
||||
|
||||
## Data plane and point-in-time semantics
|
||||
|
||||
本地 Tushare 镜像是独立只读输入边界。adapter 只信任 SQLite 中 completed
|
||||
job，并重新核对文件大小、SHA-256、Parquet 行数/footer；Qlib provider
|
||||
发布到新目录，manifest 绑定 source job、build parameter、converter source、
|
||||
完整 provider tree 和可重算 `data_version`。当前镜像未完成且缺复权、
|
||||
真实 benchmark、历史成分/ST/停牌/涨跌停，因此只能进入 Qlib 技术 smoke，
|
||||
不能直接进入 canonical production decision。完整合同与实跑证据见
|
||||
本地 Tushare live mirror 是独立只读输入边界。scoped snapshot 在一次 SQLite
|
||||
读事务中选择目标日期/API/指数的 canonical completed job，并重新核对文件
|
||||
大小、SHA-256、Parquet 行数/footer；Qlib provider 发布到新目录，manifest
|
||||
绑定 source job、build parameter、converter source、完整 provider tree 和
|
||||
可重算 `data_version`。当前已有复权因子、真实 `000905.SH` benchmark/历史
|
||||
权重、停牌和涨跌停数据，但 live mirror 仍可变化，且 `stock_st` 权限被拒；
|
||||
因此必须先冻结、再验证，不能直接进入完整 canonical production decision。完整合同见
|
||||
[`TUSHARE_LOCAL_DATA.md`](TUSHARE_LOCAL_DATA.md)。
|
||||
|
||||
JQData adapter 的输入合同是逐交易日历史指数成员和 `get_bars` 日线字段：
|
||||
@@ -190,8 +191,20 @@ Qlib 路径固定为 `0.9.7`。CPython 3.12 实际 fixture smoke 已得到：
|
||||
重算的有限值指标，明确 `investment_value_claim=false`；
|
||||
- local MLflow file store 使用时，runner 以
|
||||
`os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true")` 显式确认。
|
||||
- 本地 Tushare completed 分区已生成 managed provider；早期未复权动量
|
||||
smoke 在 `PYTHONHASHSEED=0` 下连续两次得到逐 byte 相同的 evidence JSON。
|
||||
- 旧 Tushare 早期未复权 managed provider/momentum smoke 在
|
||||
`PYTHONHASHSEED=0` 下连续两次得到逐 byte 相同的 evidence JSON；它只证明
|
||||
旧转换器和运行时可重放；
|
||||
- 2018—2025 v2 managed provider 已使用真实复权、`SH000905` benchmark 和
|
||||
PIT index-weight universe，通过 verifier；同一 momentum run 两次 JSON
|
||||
byte-identical。月末权重保守地从下一 provider 交易日生效，
|
||||
`same_session_membership_use=false`；97 个 observed snapshots 形成 96 个
|
||||
effective rosters，每个严格 500 成分、weight sum 100±0.5，最大间隔 36 天。
|
||||
live DB/WAL 可被无关下载任务修改，但 221 个 scoped jobs double verified、
|
||||
验证后仍为 latest；独立 lineage 工具对 221 jobs 的 6 个身份字段得到
|
||||
mismatch 0，并确认 converter source 为 current。它仍是
|
||||
event-time/保守次日生效 PIT 近似，不是严格 knowledge-time PIT；不同
|
||||
`start` 的首观测锚 provider 不可直接拼接。它仍是 research-only、
|
||||
`gate_credit=[]`，不表达历史 ST 和逐股逐日涨跌停。
|
||||
|
||||
该 fixture 只有两只股票和一个 benchmark，性能没有投资意义。Qlib 的单一
|
||||
`limit_threshold` 也不能表达逐日板块/ST 规则，所以它只参与 L1/L2 和诊断
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
| XtTrader read-only shadow | 账户绑定 target-diff 与 broker observation | MiniQMT/QMT、账户查询权限、既有本地 HMAC key | HMAC 账户绑定 + authenticated evidence envelope、decision/observation semantic replay、资产/持仓/委托/成交恒等式、fail-closed exact verifier 与 fake broker 合同 | HMAC 不是 broker attestation；仍缺官方 query 失败/空结果合同及 callback/restart；Baseline 60 至少 20 个交易日，Production 80 当前缺口为同一冻结候选至少 60 个交易日；not live-ready |
|
||||
| Qlib native momentum | signal/model research | CPython 3.12 + exactly `pyqlib==0.9.7` | 真实本地 fixture 成功，24 signal；保存 signal/report hash、表边界和重算 portfolio 指标 | synthetic 两股票；无真实数据/OOS，无 L3/L4 parity |
|
||||
| Qlib Alpha158/LightGBM | model fit + Recorder workflow | 同上 + LightGBM/MLflow stack | 真实 fixture 完成 model/Recorder；读取 portfolio report，保存表 hash、边界、重算指标与 artifact path | synthetic 两股票性能无意义；真实 provider/OOS 和冻结模型缺失 |
|
||||
| Tushare local → Qlib | completed Parquet 的只读盘点、不可变 provider 与本地研究 smoke | 同上 + `pyarrow==24.0.0`；本地镜像 | 93 个 completed 文件全部通过 size/SHA/row-count；v4 provider verifier 成功；同一动量命令连续两次 evidence JSON 逐 byte 相同 | `daily` 仅完成约 8.2%，缺复权、真实 benchmark、PIT 成分/ST/停牌/涨跌停；`production_ready=false` |
|
||||
| Tushare local → Qlib | live mirror 只读盘点、scoped release、不可变 provider 与本地研究 | 同上 + `pyarrow==24.0.0`；本地镜像 | selected jobs double verified、仍 latest；独立 lineage 验证 221 jobs×6 fields、mismatch 0、converter current true；v2 verifier 通过：1,111 instruments、1,942 sessions、1,975,455 rows；97 observed/96 effective snapshots 严格 500 成分、次日生效；双次 JSON byte-identical | event-time/保守次日生效 PIT 近似，不是严格 knowledge-time PIT；不同 start 的首观测锚 provider 不可直接拼接；research-only、`gate_credit=[]`；ST/统一 9.5% 仍缺 |
|
||||
| Mock parity exporter | JoinQuant/QMT wrapper 的 L2—L4 合同回归 | Python ≥3.10 | `mock_contract_only`、`real_platform_pass=false`、`gate_credit=[]` | 不计 G9；必须换成真实平台导出 |
|
||||
|
||||
G1—G10 当前都未通过。JoinQuant TargetPackage run 支持且只支持
|
||||
|
||||
@@ -8,3 +8,5 @@
|
||||
- 当前实现架构：[`../docs/ARCHITECTURE.md`](ARCHITECTURE.md)
|
||||
- 平台矩阵：[`PLATFORM_MATRIX.md`](PLATFORM_MATRIX.md)
|
||||
- Tushare 数据：[`TUSHARE_LOCAL_DATA.md`](TUSHARE_LOCAL_DATA.md)
|
||||
- Tushare → Qlib 本机运行：
|
||||
[`runbooks/TUSHARE_QLIB_LOCAL_RUN.md`](runbooks/TUSHARE_QLIB_LOCAL_RUN.md)
|
||||
|
||||
+276
-172
@@ -1,225 +1,329 @@
|
||||
# Tushare 本地数据盘点、Qlib 接入与回测手册
|
||||
# Tushare 本地数据盘点、冻结边界与 Qlib 接入
|
||||
|
||||
截至 2026-07-26，Quant OS 已能只读盘点另一个任务生成的 Tushare
|
||||
Parquet 镜像，校验已完成分区，并转换为不可变的 Qlib 0.9.7 provider。
|
||||
这条路径已经完成两次 byte-identical 的本地动量回测。
|
||||
|
||||
当前证据的边界是：
|
||||
点时证据：2026-07-31 00:40 CST。原先“93 个 completed 文件、日线只到
|
||||
1993-10”的结论已经失效，不得再用于判断当前镜像。当前镜像约 7.7 GB，
|
||||
SQLite job ledger 在该点时已有 378,707 条 `completed` 版本记录、约
|
||||
1.80 亿行；同时仍有与本次中证 500 研究无关的指数任务处于 `running`。
|
||||
|
||||
```text
|
||||
completed_partitions_only_not_full_mirror
|
||||
tushare_daily_completed_partitions = 428
|
||||
tushare_daily_planned_partitions = 428
|
||||
tushare_daily_completed_rows = 18,041,386
|
||||
tushare_daily_market_data_from = 1990-12-19
|
||||
tushare_daily_market_data_through = 2026-07-28
|
||||
production_ready = false
|
||||
investment_value_claim = false
|
||||
gate_credit = []
|
||||
```
|
||||
|
||||
它证明“本地 Tushare 数据可以进入 Qlib 并可重放”，不证明镜像已经下载完整，
|
||||
也不证明策略收益有效或 Quant OS 已达到 `BASELINE_60`。
|
||||
这意味着数据已经足够做一条真实的 2018—2025 本地 Qlib 研究链，但不能把
|
||||
“live raw mirror 很大”直接等同于“已经有可复现、可用于发布结论的数据集”。
|
||||
正确边界是三层：
|
||||
|
||||
## 1. 2026-07-26 点时盘点
|
||||
| 层 | 本机位置/产物 | 用途 | 是否可变 |
|
||||
| --- | --- | --- | --- |
|
||||
| live raw mirror | `$TUSHARE_MIRROR_ROOT` | 下载器持续落盘的 Parquet、权限探测和 job ledger | 是；不能直接作为可重放 run 的唯一引用 |
|
||||
| scoped frozen release | `tools/tushare_snapshot.py` 生成的范围化 manifest | 固定日期、API、指数代码、所选 job/file hash 和选择规则 | manifest 不变；引用的 raw bytes 仍需留存 |
|
||||
| Qlib provider | `data/qlib/...` + provider manifest | Qlib 0.9.7 可直接读取的 calendar/instruments/features | 否；是 frozen release 的派生物，不是原始数据备份 |
|
||||
|
||||
镜像中共有 93 个状态为 `completed` 的 Parquet，均通过 SQLite 记录的文件大小、
|
||||
SHA-256 和 Parquet row count 校验；另有 1 个遗留 `running` job。
|
||||
`tools/tushare_qlib.py` 和冻结工具都只读镜像，不调用 Tushare，也不读取或保存
|
||||
token。snapshot 工具不复制 Parquet；manifest 能在源文件被覆盖时检测不一致，
|
||||
但若要长期从 raw bytes 重建，还需要后续接入 content-addressed archive。
|
||||
原始 Parquet、绝对路径、账号信息和本地运行 artifacts 不进入 Git。
|
||||
|
||||
| 表 | 文件数 | 行数 | 当前范围/说明 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `daily` | 35 | 28,731 | 1990-12-19—1993-10-29，138 个代码、731 个交易日 |
|
||||
| `trade_cal` | 37 | 13,004 | SSE 1990—2026 日历，8,689 个开市日 |
|
||||
| `stock_basic` | 4 | 5,869 | 当前上市/退市/暂停上市等基础快照 |
|
||||
| `stock_company` | 3 | 6,294 | SSE/SZSE/BSE 公司资料 |
|
||||
| `index_basic` | 7 | 9,643 | CSI/SW/SSE/SZSE 等指数基础资料 |
|
||||
| `index_classify` | 6 | 870 | 申万 2014/2021 分类 |
|
||||
| `bse_mapping` | 1 | 248 | 北交所新旧代码映射 |
|
||||
| **合计** | **93** | **64,659** | **5,734,035 bytes Parquet** |
|
||||
## 1. 当前真实覆盖
|
||||
|
||||
`daily` 原计划覆盖 1990-12—2026-07，共 428 个月；当前完成 35 个月，约
|
||||
8.2%。下载进程已停止，SQLite 仍遗留
|
||||
`daily/month=1993-11` 的 `running` 状态。日志显示首先发生 DNS 解析失败，
|
||||
错误处理阶段又发生 `sqlite3.OperationalError: unable to open database
|
||||
file`。因此不能把这个状态解释为仍在后台下载，也不要在 Quant OS
|
||||
转换过程中修改或自动恢复原下载器。
|
||||
下面同时区分“自然分区文件”和 SQLite 的“completed 版本记录”。下载器在
|
||||
2026-07 月内多次刷新同一个分区，所以 `daily` 和 `adj_factor` 都是 428 个
|
||||
自然月文件，却各有 430 条 completed 版本记录。job 行数求和会重复计算旧版
|
||||
2026-07，不能当作当前文件行数。
|
||||
|
||||
尚未落盘的生产关键表包括：
|
||||
| 数据 | 当前自然文件 | 当前行数 | 实际可见范围 | 可直接支持什么 |
|
||||
| --- | ---: | ---: | --- | --- |
|
||||
| `daily` | 428 | 18,041,386 | 1990-12-19—2026-07-28 | OHLC、前收、成交量额、收益与本地撮合输入 |
|
||||
| `adj_factor` | 428 | 18,868,457 | 1990-12—2026-07；最大 `trade_date=20260729` | 区间内累计复权 |
|
||||
| `trade_cal` | 39 completed 版本 | 13,423 | SSE 1990—2026 | 交易日历与完整性检查 |
|
||||
| `stock_basic` | 4 | 5,869 | 当前上市、退市、暂停上市等快照 | 代码、上市/退市日期；不是 PIT 成分表 |
|
||||
| `daily_basic` | 331 | 17,335,584 | 1999-01-04—2026-07-28 | 市值、换手、估值与横截面特征 |
|
||||
| `stk_limit` | 235 | 17,878,521 | 2007-01-04—2026-07-29 | 逐股逐日真实涨跌停价格 |
|
||||
| `suspend_d` | 37 | 641,379 | 非空记录 1999-05-04—2026-07-29 | 停复牌与可交易性 |
|
||||
| `index_daily`，`000905.SH` | 37 年 job | 5,239 | 2004-12-31—2026-07-29 | 真实中证 500 benchmark |
|
||||
| `index_weight`，`000905.SH` | 37 年 job | 129,000 | 2005-01-31—2026-06-30 | 中证 500 的历史定期成分/权重快照 |
|
||||
| `index_member_all` | 1 | 3,000 | 申万行业层级及进出日期 | 行业分类；不能冒充中证 500 历史成分 |
|
||||
|
||||
- `adj_factor`、`index_daily`；
|
||||
- `index_member_all`、`index_weight`；
|
||||
- `stk_limit`、`suspend_d`、`stock_st`；
|
||||
- `daily_basic`。
|
||||
`daily` 原计划覆盖 1990-12—2026-07，共 428 个月；当前完成
|
||||
428 个月，约 100.0%。这里的
|
||||
“完成”只指自然月文件覆盖，不代表 live mirror 已冻结，也不代表
|
||||
production-ready。
|
||||
|
||||
其中历史 `stock_st` 权限探测被拒绝。即使其他下载完成，历史 ST 仍需要新的
|
||||
授权数据源或明确的降级门禁。
|
||||
另外已有 `income`、`balancesheet`、`cashflow`、`fina_indicator`、
|
||||
`disclosure_date`、`moneyflow` 等数据。它们可以用于下一阶段的基本面、质量、
|
||||
资金流特征，但财务报表必须按公告/披露可得日做 as-of join，不能直接按报告期
|
||||
回填到历史日期。
|
||||
|
||||
## 2. 已确认的数据质量
|
||||
### 仍然存在的硬缺口
|
||||
|
||||
- 已完成 Parquet 自然键无重复，核心日线字段无空值；
|
||||
- 无负成交量或负成交额，日线日期与同期 SSE 开市日完全对应；
|
||||
- 17 条早期 OHLC 包络异常；默认拒绝，只有显式
|
||||
`--ohlc-policy expand-range` 才按原始 `open/high/low/close` 四价的
|
||||
min/max 修复，并记录数量与异常键 hash；
|
||||
- 2 条 `pct_chg` 与保存精度下重算结果的偏差超过 0.02 个百分点；
|
||||
- 日线代码 `000022.SZ` 不在当前 `stock_basic` 快照；
|
||||
- `stock_basic` 存在遗留非标准代码 `T600018.SH`，转换器不会把它当成
|
||||
A 股代码；
|
||||
- `index_basic.base_point` 存在分区类型漂移，后续 canonical 合并时必须
|
||||
显式 cast。
|
||||
- 2026-07-26 的权限探测明确显示 `stock_st` 为 `denied`。在获得授权历史
|
||||
ST 源之前，不能声称完整复现 ST 股票的逐日交易限制；
|
||||
- live ledger 在本次点时仍有 24 个 `running`：19 个
|
||||
`931632CNY04.CSI` 的 `index_daily` 年任务和 5 个 `931632.CSI` 的
|
||||
`index_weight` 年任务。它们与 `000905.SH` 的 2018—2025 release 无关，
|
||||
但说明全镜像仍是移动目标；
|
||||
- `daily` 的最新 completed 请求元数据包含 `20260729`，但当前 Parquet
|
||||
的最大实际日期是 `20260728`；任何冻结工具都必须验证 Parquet 内容，而不能
|
||||
只相信请求参数或 job 状态；
|
||||
- 多个 completed 版本可指向同一个最终路径。冻结时必须按明确规则选择当前
|
||||
canonical 版本并复核 size/SHA-256/Parquet row count，不能把所有
|
||||
`completed` 盲目拼接；
|
||||
- `stock_basic` 是当前快照，不得用它反向构造历史指数 universe。
|
||||
|
||||
Tushare 的 `daily` 是未复权行情，停牌期间不返回记录；`vol` 单位为手，
|
||||
`amount` 单位为千元。转换器固定执行：
|
||||
## 2. 数据如何进入 Quant OS
|
||||
|
||||
第一条可用研究路径是中证 500 横截面研究：
|
||||
|
||||
```text
|
||||
daily + adj_factor + trade_cal
|
||||
+ index_daily(000905.SH)
|
||||
+ index_weight(000905.SH)
|
||||
-> scoped frozen release (2018-01-01—2025-12-31)
|
||||
-> managed Qlib provider
|
||||
-> momentum / Alpha158 / LightGBM
|
||||
-> evidence JSON + provider verifier
|
||||
```
|
||||
|
||||
第二条是本地事件回测输入：
|
||||
|
||||
```text
|
||||
raw daily/pre_close + stk_limit + suspend_d
|
||||
+ PIT universe + signal/target
|
||||
-> Quant OS reference execution
|
||||
-> ledger / replay / platform parity
|
||||
```
|
||||
|
||||
两条路径不能混为一谈。Qlib 负责因子、模型和组合研究；Quant OS 的
|
||||
Signal/Target/Order/Broker 合同、涨跌停、停牌、T+1、成交和对账仍由本地
|
||||
事件链负责。历史 ST 未补齐时，本地执行链必须保留显式缺口或 fail-closed，
|
||||
不能静默按普通股票处理后再把结果标为完整市场仿真。
|
||||
|
||||
### 复权口径
|
||||
|
||||
新 provider 对每个 symbol 使用 release 区间内第一个有效 `adj_factor` 作为
|
||||
锚点：
|
||||
|
||||
```text
|
||||
factor_t = adj_factor_t / first_valid_adj_factor
|
||||
adjusted_OHLC_t = raw_OHLC_t * factor_t
|
||||
adjusted_volume_t = raw_volume_in_shares_t / factor_t
|
||||
money_t = raw_amount_in_CNY_t
|
||||
```
|
||||
|
||||
这是“首个有效因子锚定的区间累计复权”，文档和证据中不要把它简写成
|
||||
`qfq`。不用区间末因子做归一化，是为了避免研究区间末端信息参与早期价格的
|
||||
缩放。停牌缺口保留为 `NaN`，不会伪造 K 线。
|
||||
|
||||
固定单位仍为：
|
||||
|
||||
```text
|
||||
600000.SH -> SH600000
|
||||
000001.SZ -> SZ000001
|
||||
430047.BJ -> BJ430047
|
||||
volume = vol * 100
|
||||
money = amount * 1000
|
||||
factor = 1
|
||||
raw_volume_in_shares = Tushare vol * 100
|
||||
raw_amount_in_CNY = Tushare amount * 1000
|
||||
```
|
||||
|
||||
未上市、退市后和停牌缺口在 Qlib feature 中保留为 `NaN`。字段语义来源见
|
||||
[Tushare 日线接口](https://tushare.pro/document/2?doc_id=27)与
|
||||
[复权因子接口](https://tushare.pro/document/2?doc_id=28)。
|
||||
### universe 与 benchmark 口径
|
||||
|
||||
## 3. Quant OS 如何使用这些数据
|
||||
- benchmark 使用 `index_daily(000905.SH)`，Qlib symbol 为 `SH000905`；
|
||||
- universe 使用 `index_weight(000905.SH)` 的历史快照，并按“只向未来
|
||||
carry-forward、绝不从未来 backfill、月末权重下一交易日生效”生成时变
|
||||
成分区间。这是 event-time / 保守次日生效 PIT 近似；源数据没有单独验证的
|
||||
`published_at`，不能声称严格 knowledge-time PIT；
|
||||
- `index_member_all` 是申万行业数据，不参与中证 500 universe；
|
||||
- 研究报告必须分别披露 benchmark、universe、价格复权和可交易性过滤来源。
|
||||
|
||||
当前已经实现的链路是研究/引擎验证路径：
|
||||
## 3. 现有代码基线与 v2 接口
|
||||
|
||||
```text
|
||||
Tushare mirror
|
||||
-> 只读取 SQLite completed jobs
|
||||
-> 文件大小 + SHA-256 + Parquet row count/footer 校验
|
||||
-> 新目录原子发布 Qlib binary provider
|
||||
-> provider tree hash + data_version + manifest verifier
|
||||
-> Qlib momentum / Alpha158 research runner
|
||||
```
|
||||
### 现有基线
|
||||
|
||||
正式 Quant OS 还需要第二条生产数据路径：
|
||||
|
||||
```text
|
||||
完整 Tushare/JQData PIT 数据
|
||||
-> canonical immutable snapshot
|
||||
-> Quant OS 本地事件回测 / decision
|
||||
-> 聚宽、QMT、Qlib 的同输入分层比较
|
||||
```
|
||||
|
||||
Qlib 适合因子、模型和组合研究，不拥有原始数据，也不替代 Quant OS 的
|
||||
Signal/Target/Order/Broker 合同。当前 Tushare adapter 只实现第一条链路；
|
||||
在复权、真实 benchmark、历史成分、停牌、涨跌停和 ST 数据补齐前，不会把它
|
||||
接入生产 decision。
|
||||
|
||||
## 4. 可运行命令
|
||||
|
||||
从 Quant OS 根目录执行。镜像路径只放在当前 shell 环境变量中，不写入 Git：
|
||||
仓库当前基线的 `build` 只消费 `daily/trade_cal/stock_basic`，必须显式传
|
||||
`--allow-unadjusted`，会生成 synthetic equal-weight benchmark，并按整个
|
||||
研究区间的最少观测数筛选股票。它只能用于转换器/运行时技术验证：
|
||||
|
||||
```bash
|
||||
export QUANT_OS_ROOT=/path/to/quant-os
|
||||
export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror
|
||||
cd "$QUANT_OS_ROOT"
|
||||
|
||||
python3.12 -m venv .venv-qlib312
|
||||
source .venv-qlib312/bin/activate
|
||||
python -m pip install -r requirements/research-py312.txt
|
||||
export PYTHONPATH=src:.
|
||||
```
|
||||
|
||||
盘点并验证全部 completed 文件：
|
||||
|
||||
```bash
|
||||
python tools/tushare_qlib.py inventory \
|
||||
PYTHONPATH=src:. python tools/tushare_qlib.py build \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--output-json artifacts/tushare-inventory.json
|
||||
```
|
||||
|
||||
当前未复权数据只能显式构建技术验证 provider；每次必须使用新目录，工具拒绝
|
||||
覆盖已有 provider：
|
||||
|
||||
```bash
|
||||
python tools/tushare_qlib.py build \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--output-dir data/qlib/tushare-early-v1 \
|
||||
--start 1990-12-19 \
|
||||
--end 1993-10-29 \
|
||||
--output-dir data/qlib/tushare-legacy-2018-2025 \
|
||||
--start 2018-01-01 \
|
||||
--end 2025-12-31 \
|
||||
--minimum-observations 60 \
|
||||
--allow-unadjusted \
|
||||
--ohlc-policy expand-range \
|
||||
--output-json artifacts/tushare-qlib-build.json
|
||||
|
||||
python tools/tushare_qlib.py verify \
|
||||
data/qlib/tushare-early-v1 \
|
||||
--output-json artifacts/tushare-qlib-verify.json
|
||||
--ohlc-policy fail \
|
||||
--output-json artifacts/tushare/legacy-build.json
|
||||
```
|
||||
|
||||
运行 Qlib 0.9.7 本地回测：
|
||||
不要用这个结果回答“中证 500 策略是否有效”，也不要给它任何发布门禁分数。
|
||||
|
||||
### 已验证的 v2 接口
|
||||
|
||||
v2 已消费 `adj_factor`、真实指数行情和历史指数权重，并完成 provider
|
||||
构建、验证和两次本地 run。实际构建命令为：
|
||||
|
||||
```bash
|
||||
PYTHONHASHSEED=0 python -m platforms.qlib_runner \
|
||||
--provider-uri data/qlib/tushare-early-v1 \
|
||||
--market tushare_a \
|
||||
--benchmark SH999999 \
|
||||
--start 1992-01-02 \
|
||||
--end 1993-10-28 \
|
||||
--feature-start 1991-01-02 \
|
||||
--lookback 20 \
|
||||
--topk 10 \
|
||||
--n-drop 2 \
|
||||
--rebalance weekly \
|
||||
--output-json artifacts/tushare-qlib-momentum.json
|
||||
PYTHONPATH=src:. python tools/tushare_qlib.py build \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--output-dir data/qlib/tushare-csi500-2018-2025-v2 \
|
||||
--start 2018-01-01 \
|
||||
--end 2025-12-31 \
|
||||
--market-name tushare_csi500 \
|
||||
--benchmark-index-code 000905.SH \
|
||||
--universe-index-code 000905.SH \
|
||||
--minimum-observations 60 \
|
||||
--ohlc-policy fail \
|
||||
--output-json artifacts/tushare/csi500-2018-2025-build.json
|
||||
```
|
||||
|
||||
Qlib simulator 会消费结束日后的下一 provider session，所以 backtest `end`
|
||||
必须早于 provider 最后一个交易日。Quant OS 对受管 provider 会在 Qlib
|
||||
初始化前检查 market、benchmark、起始边界和这个终点条件。
|
||||
这条命令不再传 `--allow-unadjusted`。provider manifest 必须记录复权口径、
|
||||
真实 benchmark、时变 universe、所有 source job/file hash、转换器 hash、
|
||||
质量统计和可重算 `data_version`。
|
||||
|
||||
`artifacts/` 与 `data/qlib/` 均被 Git 忽略。原始 Parquet、token、绝对镜像
|
||||
路径和回测临时产物不得提交到仓库。
|
||||
`tools/tushare_snapshot.py` 是独立的 raw scoped freeze 入口。Qlib 核心输入
|
||||
可以这样冻结：
|
||||
|
||||
## 5. 本次真实本地运行证据
|
||||
```bash
|
||||
python tools/tushare_snapshot.py \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--start 2018-01-01 \
|
||||
--end 2025-12-31 \
|
||||
--apis daily adj_factor trade_cal stock_basic index_daily index_weight \
|
||||
--index-codes 000905.SH \
|
||||
--output artifacts/local-tushare-20260731/scoped-source-manifest.json
|
||||
```
|
||||
|
||||
最终 v4 provider：
|
||||
默认会验证 size、SHA-256 和 Parquet row count。需要研究
|
||||
`daily_basic/stk_limit/suspend_d` 时应再冻结一个明确包含它们的扩展
|
||||
release；不要把未列入 manifest 的表暗中加入既有 run。当前 `build` 仍从
|
||||
mirror 重新选择 source jobs，并不接受这份 manifest 作为参数，因此 run
|
||||
验收还必须比较 provider manifest 与 scoped manifest 的 source
|
||||
job/file hashes。
|
||||
|
||||
| 证据 | 值 |
|
||||
snapshot 工具在请求 `index_weight` 时会自动包含 `start.year-1` 的 pre-start
|
||||
PIT anchor。本次 scoped manifest 与 provider 均为 221 个 source jobs，
|
||||
全字段比对零差异，已经覆盖有效 roster 日期 `2017-12-29`：
|
||||
|
||||
```text
|
||||
scoped_manifest_sha256 = 744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2
|
||||
scoped_selection_sha256 = 66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5
|
||||
scoped_source_jobs = 221
|
||||
provider_source_jobs = 221
|
||||
source_job_full_field_diff = 0
|
||||
provider_source_jobs_before_read = 221
|
||||
provider_source_jobs_after_read = 221
|
||||
provider_selected_jobs_still_latest = true
|
||||
provider_post_read_file_verification = true
|
||||
```
|
||||
|
||||
冻结期间 SQLite 主文件/WAL 仍被无关下载任务更新：
|
||||
`changed_through_file_verification=true`。这不是 selected scope 未冻结：
|
||||
SQLite 事务读取期间 `changed_while_reading=false`，221 个 selected jobs
|
||||
均完成两次 size/SHA-256/Parquet row-count 校验，验证后仍是各自 partition
|
||||
的 latest，并与 provider 的 221 个 source jobs 全字段零差异。这里的证据边界
|
||||
是“scoped jobs 稳定”，不是“整个 live DB/WAL 静止”。
|
||||
|
||||
scoped manifest 与 provider 的正式 lineage 由独立工具核对：
|
||||
|
||||
```bash
|
||||
python tools/tushare_lineage.py \
|
||||
--source-manifest artifacts/local-tushare-20260731/scoped-source-manifest.json \
|
||||
--provider-dir data/qlib/tushare-csi500-2018-2025-v2 \
|
||||
--output-json artifacts/tushare/csi500-2018-2025-lineage.json
|
||||
```
|
||||
|
||||
结果为 221 个 jobs、6 个逐 job 字段
|
||||
`id/path/row_count/byte_count/sha256/request_sha256` 全部一致，
|
||||
`mismatch_count=0`、`converter_source_matches_current=true`。lineage artifact
|
||||
同样明确 `investment_value_claim=false`、`gate_credit=[]`。
|
||||
|
||||
## 4. 2018—2025 provider 与双次回测实证
|
||||
|
||||
managed provider 已构建在新目录，并由独立 verifier 通过：
|
||||
|
||||
| 证据 | 实测值 |
|
||||
| --- | --- |
|
||||
| adapter source SHA-256 | `2228c23af176a0f2fcf311e88dca9bdc5d63fd36fa5a3c2d3e42808a405e7243` |
|
||||
| data version | `3378aa75a5601bde476ad07bea90418966a66a037ca59195dec93d77b41cc3f8` |
|
||||
| provider tree SHA-256 | `dc85b8daf692a66641afef64398700264c6054f1dac84b1211cb258a06b62948` |
|
||||
| manifest SHA-256 | `862e7435bb0be7c8a3c66180e49761c110056c1a3cc02c1618e70d9eb566d945` |
|
||||
| provider 文件/大小 | 975 / 1,085,886 bytes |
|
||||
| calendar | 1990-12-19—1993-10-29，731 sessions |
|
||||
| market | 107 个至少有 60 条记录的 A 股代码 |
|
||||
| selected rows | 27,998 |
|
||||
| data version | `5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963` |
|
||||
| provider tree SHA-256 | `f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f` |
|
||||
| provider manifest SHA-256 | `27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0` |
|
||||
| provider files / bytes | 10,011 / 71,707,194 |
|
||||
| source stability | 221/221 jobs 在消费前后稳定；读取后重新校验文件且仍为 latest |
|
||||
| calendar | 2018-01-02—2025-12-31，1,942 sessions |
|
||||
| event-time PIT 近似 market | 1,111 个曾进入 `000905.SH` roster 的 instruments |
|
||||
| snapshot coverage | 97 observed / 96 effective；最大日历间隔 36 天 |
|
||||
| roster quality | 每个 observed snapshot 严格 500 constituents；weight sum 容差 ±0.5 |
|
||||
| availability | 月末权重从下一 provider 交易日生效；`same_session_membership_use=false` |
|
||||
| selected daily rows | 1,975,455 |
|
||||
| price adjustment | 完整；0 个缺失 daily/adj keys；首个有效因子锚定 |
|
||||
| OHLC quality | `--ohlc-policy fail`；0 个 envelope anomalies |
|
||||
| benchmark | 真实 `index_daily(000905.SH)` / `SH000905` |
|
||||
| release flags | `production_ready=false`、`investment_value_claim=false`、`gate_credit=[]` |
|
||||
|
||||
回测参数为 1992-01-02—1993-10-28、20 日动量、周频、Top 10、每期 drop 2，
|
||||
初始资金 1,000 万，Qlib 0.9.7、Python 3.12.13、`PYTHONHASHSEED=0`。
|
||||
相同命令连续执行两次，结果 JSON 逐 byte 相同：
|
||||
周频 20 日动量参数为 2019-01-02—2025-12-30、Top 50、每期 drop 5、
|
||||
初始资金 1,000 万、开仓费 0.03%、平仓费 0.08%、最低费用 5 元、
|
||||
`PYTHONHASHSEED=0`。相同命令连续执行两次，两个 JSON 逐 byte 相同：
|
||||
|
||||
| 证据/指标 | 值 |
|
||||
| 证据/指标 | 实测值 |
|
||||
| --- | ---: |
|
||||
| result JSON SHA-256 | `92ff9b8cea746e9c89ddf62fcfe3feb21248ca9112d9e10d24e0639058a86020` |
|
||||
| runner source SHA-256 | `fa8118819252c55e99e67de356cc961d8a4a22f82256b4d62579a45426d96167` |
|
||||
| signal | 4,700 行；hash `cfad4087d68b7f71e33f0f46fc5bb97db985e02ac2515f01857e2ab6813a5909` |
|
||||
| portfolio report | 466 行；hash `5f8606fb64963d3e2618f66503e6ae94048c24b62c5d8e1806ff44ec57558b04` |
|
||||
| strategy cumulative return | -52.5251% |
|
||||
| max drawdown | -87.7604% |
|
||||
| synthetic benchmark cumulative return | 196,302.0822% |
|
||||
| total cost / turnover | 0.016013 / 29.567521 |
|
||||
| run JSON SHA-256 | `803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3` |
|
||||
| signal rows | 176,615 |
|
||||
| portfolio report rows | 1,698 |
|
||||
| strategy cumulative return | 0.253151986388197（25.3152%） |
|
||||
| benchmark cumulative return | 0.789559383530678（78.9559%） |
|
||||
| max drawdown | -0.6041641365538415（-60.4164%） |
|
||||
| total cost / turnover | 0.03579348112198462 / 65.51393782463258 |
|
||||
|
||||
这些收益数字没有投资解释。原因包括未复权早期行情、没有真实指数 benchmark、
|
||||
没有历史成分/ST/停牌/涨跌停、按全区间至少 60 条观测筛选带来的非 PIT
|
||||
偏差，以及 1992—1993 特殊市场阶段。极端 synthetic benchmark 恰好说明
|
||||
为什么“程序跑完”不能等价为“回测有效”。
|
||||
这组数字证明真实 Tushare → event-time/保守次日生效 PIT 近似 universe →
|
||||
复权 Qlib provider →
|
||||
momentum runner 的链路可重放，不证明策略可投资。策略显著跑输 benchmark，
|
||||
且最大回撤超过 60%，不能把正累计收益单独拿出来宣传。
|
||||
|
||||
## 6. 何时可以升级为正式研究数据
|
||||
最终身份变化来自 verifier/转换器源码绑定；provider tree 与回测数值未变。
|
||||
verifier 已确认 recorded/current converter SHA-256 一致。由于 run evidence
|
||||
绑定 provider identity，两个 run JSON 的 SHA-256 随之更新。
|
||||
|
||||
至少完成以下步骤后，才能新建 production-eligible provider：
|
||||
## 5. 验证顺序
|
||||
|
||||
1. 修复下载器错误状态并完成目标日期的 `daily`；
|
||||
2. 补齐 `adj_factor`，冻结复权基准日和公式；
|
||||
3. 使用 `index_daily` 替换合成 benchmark；
|
||||
4. 指数策略补齐 `index_member_all/index_weight`；
|
||||
5. 补齐 `suspend_d/stk_limit`，为历史 `stock_st` 找到授权来源或保持硬阻断；
|
||||
6. 冻结下载 job、请求 hash、Parquet hash、转换器 hash 和新 `data_version`；
|
||||
7. 运行真实长样本 walk-forward/OOS，而不是复用本次早期 smoke；
|
||||
8. 同一冻结输入进入 Quant OS 本地回测、聚宽和可用 QMT，生成 L1—L4
|
||||
差异报告。
|
||||
1. 对 live mirror 做快速元数据盘点，确认目标 API 和 `000905.SH` 范围；
|
||||
2. 生成 2018—2025 scoped frozen release，并验证所选文件的 size、
|
||||
SHA-256、Parquet footer/row count、自然键和日期边界；
|
||||
3. 在冻结后从同一 source state 构建新 provider，绝不覆盖已有目录，并核对
|
||||
provider/scoped manifest 的 source hashes；
|
||||
4. 运行 `tools/tushare_qlib.py verify`，核对 manifest、tree hash、
|
||||
calendar、instrument 和 feature 文件；
|
||||
5. 运行同一条 Qlib momentum 命令两次，比较 evidence JSON 的关键输入、
|
||||
signal/report hash 和重算指标；
|
||||
6. 再进入 walk-forward、Alpha158/LightGBM、成本/风险扰动和本地事件回测。
|
||||
|
||||
Qlib provider 的目录格式和缺失值/复权约定参见
|
||||
[Qlib Data Layer 文档](https://qlib.readthedocs.io/en/latest/component/data.html)。
|
||||
完整可复制命令见
|
||||
[`runbooks/TUSHARE_QLIB_LOCAL_RUN.md`](runbooks/TUSHARE_QLIB_LOCAL_RUN.md)。
|
||||
|
||||
## 6. 结果边界
|
||||
|
||||
2018—2025 的真实行情、复权、指数行情和历史权重会让回测比旧的
|
||||
1990—1993 technical smoke 有意义得多，但仍只是一条研究证据：
|
||||
|
||||
- momentum smoke 的收益不能证明投资价值；
|
||||
- Qlib 的成交模型不能替代 A 股逐股涨跌停、停牌、T+1 和真实容量仿真；
|
||||
- 本次 Qlib run 使用统一 `limit_threshold=0.095`，只是 9.5% 市场级近似；
|
||||
它没有消费已有 `stk_limit` 的逐股逐日价格，也不能表达不同板块、日期和
|
||||
ST 股票的真实限制；
|
||||
- `stock_st` 缺口没有因为已有 `namechange` 或 `stk_limit` 就自动消失；
|
||||
- `index_weight` 没有单独验证的 `published_at`，因此这里不是严格
|
||||
knowledge-time PIT；
|
||||
- 复权以每个 symbol 在 provider 区间内的首个有效因子为锚。不同 `start`
|
||||
构建的 provider 归一化基准不同，不能直接拼接其价格 level；需要统一起点
|
||||
重建或显式重新归一化；
|
||||
- 必须做时间切分、walk-forward/OOS、参数稳定性、成本压力和跨引擎差异；
|
||||
- 本文不据此声称 Quant OS 已达到 60 分或 80 分标准。
|
||||
|
||||
相关字段语义参见
|
||||
[Tushare 日线接口](https://tushare.pro/document/2?doc_id=27)、
|
||||
[复权因子接口](https://tushare.pro/document/2?doc_id=28)和
|
||||
[Qlib Data Layer](https://qlib.readthedocs.io/en/latest/component/data.html)。
|
||||
|
||||
@@ -19,13 +19,14 @@ Investment value claim: false
|
||||
独立仓：
|
||||
|
||||
```text
|
||||
~/boat-workspace/Code/quant-os
|
||||
$QUANT_OS_ROOT
|
||||
```
|
||||
|
||||
推荐 Python 3.12：
|
||||
|
||||
```bash
|
||||
cd ~/boat-workspace/Code/quant-os
|
||||
export QUANT_OS_ROOT=/path/to/quant-os
|
||||
cd "$QUANT_OS_ROOT"
|
||||
python3.12 -m venv .venv-core
|
||||
source .venv-core/bin/activate
|
||||
python -m pip install -e .
|
||||
@@ -36,7 +37,7 @@ python -m pip install -e .
|
||||
`quant-os` 应在 checkout 内运行；从其他目录调用新命令时显式写：
|
||||
|
||||
```bash
|
||||
quant-os --project-root ~/boat-workspace/Code/quant-os doctor
|
||||
quant-os --project-root "$QUANT_OS_ROOT" doctor
|
||||
```
|
||||
|
||||
wheel 不是独立的数据/runtime 分发包。
|
||||
@@ -117,16 +118,29 @@ quant-os verify-research-manifest artifacts/research-smoke/manifest.json
|
||||
|
||||
## 5. Tushare 数据与 Qlib
|
||||
|
||||
状态：`[已实现到技术 smoke｜真实镜像仍不完整]`
|
||||
状态：`[v2 provider/run 已验证｜research-only]`
|
||||
|
||||
先阅读 [`../TUSHARE_LOCAL_DATA.md`](../TUSHARE_LOCAL_DATA.md)。当前盘点表明
|
||||
completed Parquet 能校验并转换为 managed Qlib provider，但缺复权、真实
|
||||
benchmark、历史成分、ST、停牌和涨跌停，不能进入生产 decision。
|
||||
先阅读 [`../TUSHARE_LOCAL_DATA.md`](../TUSHARE_LOCAL_DATA.md) 和
|
||||
[`../runbooks/TUSHARE_QLIB_LOCAL_RUN.md`](../runbooks/TUSHARE_QLIB_LOCAL_RUN.md)。
|
||||
当前镜像已有复权因子、真实中证 500 benchmark/历史权重、停牌和涨跌停；
|
||||
raw mirror 仍可变化，历史 `stock_st` 权限被拒，所以必须先冻结 scoped
|
||||
release，再构建、验证 provider。2018—2025 provider 和双次 momentum run
|
||||
已经完成；月末指数权重保守地从下一 provider 交易日生效，不做 same-session
|
||||
使用，97 个 observed snapshots 形成 96 个 effective rosters，均严格 500
|
||||
成分、weight sum 在 100±0.5 内、最大间隔 36 天。但统一 9.5% 涨跌停近似和
|
||||
ST 缺口仍使其不能进入完整 production decision，`gate_credit=[]`。
|
||||
冻结期间全 DB/WAL 可能因无关下载任务变化；验收看的是 221 个 scoped jobs
|
||||
double verified、验证后仍为 latest；再用 `tools/tushare_lineage.py` 核对
|
||||
221 jobs 的 6 个身份字段，要求 mismatch 0 且 converter current true。
|
||||
历史权重没有单独验证的 `published_at`，所以 universe 是 event-time/保守
|
||||
次日生效 PIT 近似，不是严格 knowledge-time PIT。复权使用首观测锚；不同
|
||||
`start` 构建的 provider 不能直接拼接价格 level。
|
||||
|
||||
典型流程：
|
||||
|
||||
```bash
|
||||
PYTHONPATH=src:. python tools/tushare_qlib.py inventory --help
|
||||
python tools/tushare_snapshot.py --help
|
||||
PYTHONPATH=src:. python tools/tushare_qlib.py build --help
|
||||
PYTHONPATH=src:. python tools/tushare_qlib.py verify --help
|
||||
PYTHONPATH=src:. python -m platforms.qlib_runner --help
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
# 本机 Tushare → 冻结 release → Qlib 回测
|
||||
|
||||
目标：只使用本机已有数据，冻结 `000905.SH` 的 2018—2025 输入，构建
|
||||
managed Qlib provider，验证后运行一条可重放的周频动量 smoke。
|
||||
|
||||
## 1. 固定本机路径与 Python 环境
|
||||
|
||||
```bash
|
||||
export QUANT_OS_ROOT=/path/to/quant-os
|
||||
export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror
|
||||
cd "$QUANT_OS_ROOT"
|
||||
|
||||
python3.12 -m venv .venv-qlib312
|
||||
source .venv-qlib312/bin/activate
|
||||
python -m pip install -r requirements/research-py312.txt
|
||||
python -c "import qlib; assert qlib.__version__ == '0.9.7'"
|
||||
export PYTHONPATH=src:.
|
||||
```
|
||||
|
||||
当前正式项目目录内应新建自己的 `.venv-qlib312`。不要复用
|
||||
`quant-os-migration-backup-20260730` 下的迁移遗留环境。
|
||||
|
||||
### 推荐的 Makefile 全链调用
|
||||
|
||||
```bash
|
||||
export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror
|
||||
export TUSHARE_PROVIDER=data/qlib/tushare-csi500-2018-2025-next
|
||||
export TUSHARE_SNAPSHOT=artifacts/local-tushare/scoped-source-manifest.json
|
||||
|
||||
make tushare-snapshot
|
||||
make tushare-build
|
||||
make tushare-verify
|
||||
make tushare-lineage
|
||||
make tushare-backtest
|
||||
make tushare-replay
|
||||
```
|
||||
|
||||
每次 build 必须使用一个尚不存在的 `TUSHARE_PROVIDER` 目录，不能覆盖本页记录的
|
||||
正式 provider。`make tushare-replay` 依赖 `tushare-backtest`，随后运行第二次
|
||||
并用 `cmp` 要求两个 evidence JSON 逐 byte 相同。下面保留展开命令，便于审计
|
||||
每一个参数。
|
||||
|
||||
## 2. 先盘点 live mirror
|
||||
|
||||
全镜像仍可能下载指数数据。先做快速 job-ledger 盘点：
|
||||
|
||||
```bash
|
||||
python tools/tushare_qlib.py inventory \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--skip-file-verification \
|
||||
--output-json artifacts/tushare/live-inventory.json
|
||||
```
|
||||
|
||||
对几十万文件做全镜像 SHA 校验成本较高，也不能解决“扫描过程中镜像变化”的
|
||||
问题。正式 run 应在下一步冻结范围后，只校验 release 选中的文件。
|
||||
|
||||
## 3. 冻结 2018—2025 scoped release
|
||||
|
||||
本次 Qlib 核心 release 包含：
|
||||
|
||||
```text
|
||||
daily,adj_factor,trade_cal,stock_basic,index_daily,index_weight
|
||||
index code = 000905.SH
|
||||
date = 2018-01-01 ... 2025-12-31
|
||||
```
|
||||
|
||||
独立入口为 `tools/tushare_snapshot.py`。先确认接口，再生成 manifest：
|
||||
|
||||
```bash
|
||||
python tools/tushare_snapshot.py --help
|
||||
|
||||
python tools/tushare_snapshot.py \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--start 2018-01-01 \
|
||||
--end 2025-12-31 \
|
||||
--apis daily adj_factor trade_cal stock_basic index_daily index_weight \
|
||||
--index-codes 000905.SH \
|
||||
--output artifacts/local-tushare-20260731/scoped-source-manifest.json
|
||||
```
|
||||
|
||||
默认开启文件验证，manifest 记录选择规则、目标 API/指数/日期、source job、
|
||||
实际 Parquet 路径、size、row count、SHA-256 和 release hash。生成后不要
|
||||
继续用 live ledger 代替这份 manifest。若下一条执行仿真需要
|
||||
`daily_basic/stk_limit/suspend_d`，另建包含这些 API 的 release，不要静默
|
||||
扩充本次 Qlib 输入。该 manifest 是 hash 锁定证据，不复制 raw Parquet；
|
||||
需要长期 raw 重建时还要保留对应内容寻址副本。
|
||||
|
||||
## 4. 构建并验证 v2 provider
|
||||
|
||||
先确认 v2 参数：
|
||||
|
||||
```bash
|
||||
python tools/tushare_qlib.py build --help
|
||||
```
|
||||
|
||||
当前已验证命令如下：
|
||||
|
||||
```bash
|
||||
python tools/tushare_qlib.py build \
|
||||
--mirror-root "$TUSHARE_MIRROR_ROOT" \
|
||||
--output-dir data/qlib/tushare-csi500-2018-2025-v2 \
|
||||
--start 2018-01-01 \
|
||||
--end 2025-12-31 \
|
||||
--market-name tushare_csi500 \
|
||||
--benchmark-index-code 000905.SH \
|
||||
--universe-index-code 000905.SH \
|
||||
--minimum-observations 60 \
|
||||
--ohlc-policy fail \
|
||||
--output-json artifacts/tushare/csi500-2018-2025-build.json
|
||||
|
||||
python tools/tushare_qlib.py verify \
|
||||
data/qlib/tushare-csi500-2018-2025-v2 \
|
||||
--output-json artifacts/tushare/csi500-2018-2025-verify.json
|
||||
|
||||
python tools/tushare_lineage.py \
|
||||
--source-manifest artifacts/local-tushare-20260731/scoped-source-manifest.json \
|
||||
--provider-dir data/qlib/tushare-csi500-2018-2025-v2 \
|
||||
--output-json artifacts/tushare/csi500-2018-2025-lineage.json
|
||||
```
|
||||
|
||||
输出目录必须是新目录。失败时保留错误和 inventory/release 证据，不要删除门禁
|
||||
或回退到 synthetic benchmark。v2 的复权是每个 symbol 以区间首个有效因子
|
||||
为锚：`factor_t=adj_t/first_adj`；OHLC 乘 factor、share volume 除 factor、
|
||||
money 不变。不同 `start` 构建的 provider 具有不同归一化锚点，价格 level
|
||||
不能直接拼接；需要以统一起点重建或显式重新归一化。
|
||||
|
||||
当前 `build` 会自行从 mirror 选择 source jobs，并把它们写进 provider
|
||||
manifest；它不会直接读取上一步的 scoped manifest。验收时必须比较两份
|
||||
manifest 的 source job/file hashes。snapshot 工具会为 `index_weight` 自动
|
||||
包含 `start.year-1` 的 pre-start anchor。本次两边均为 221 个 jobs，全字段
|
||||
diff 为 0；scoped manifest SHA-256 为
|
||||
`744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2`，
|
||||
selection SHA-256 为
|
||||
`66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5`。
|
||||
provider 对 221 个 jobs 在消费前完成校验，消费后重新读取 SQLite 并再次校验
|
||||
221 个文件；`selected_jobs_still_latest=true`、
|
||||
`post_read_file_verification=true`。
|
||||
|
||||
live 下载器在整个验证窗口内继续写入无关任务，所以全 DB/WAL 的
|
||||
`changed_through_file_verification=true`。scoped 证据仍成立：事务读取期间
|
||||
`changed_while_reading=false`，221 个 jobs 均 double verified、验证后仍为
|
||||
latest，且与 build 的 221 个 source jobs 全字段零差异。不要把“全库有变化”
|
||||
误写成“本次 scoped inputs 有变化”。
|
||||
|
||||
lineage 结果还必须满足：`source_job_count=221`，匹配字段严格为
|
||||
`id/path/row_count/byte_count/sha256/request_sha256`，
|
||||
`mismatch_count=0`，并且 `converter_source_matches_current=true`。
|
||||
|
||||
## 5. 运行本地 momentum smoke
|
||||
|
||||
provider 以 2025-12-31 结束；Qlib simulator 会访问回测结束日之后的下一个
|
||||
provider session，因此回测结束日使用 2025-12-30：
|
||||
|
||||
```bash
|
||||
PYTHONHASHSEED=0 python -m platforms.qlib_runner \
|
||||
--provider-uri data/qlib/tushare-csi500-2018-2025-v2 \
|
||||
--market tushare_csi500 \
|
||||
--benchmark SH000905 \
|
||||
--start 2019-01-02 \
|
||||
--end 2025-12-30 \
|
||||
--feature-start 2018-01-02 \
|
||||
--lookback 20 \
|
||||
--topk 50 \
|
||||
--n-drop 5 \
|
||||
--rebalance weekly \
|
||||
--output-json artifacts/tushare/csi500-momentum-run-1.json
|
||||
```
|
||||
|
||||
原参数不变再运行一次，只改输出文件名为
|
||||
`csi500-momentum-run-2.json`。比较输入/provider identity、signal hash、
|
||||
portfolio report hash、表边界和重算指标；`created_at` 等非决定性元数据若
|
||||
存在，应由 verifier 的语义比较处理，而不是只看终端是否打印成功。
|
||||
|
||||
本次两个 run JSON 实际逐 byte 相同，SHA-256 均为
|
||||
`803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3`。
|
||||
|
||||
## 6. 已获得的正式研究证据
|
||||
|
||||
| 项目 | 实测值 |
|
||||
| --- | --- |
|
||||
| data version | `5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963` |
|
||||
| provider tree SHA-256 | `f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f` |
|
||||
| provider manifest SHA-256 | `27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0` |
|
||||
| lineage | 221 jobs × 6 fields；mismatch 0；converter current true |
|
||||
| provider files / bytes | 10,011 / 71,707,194 |
|
||||
| source stability | 221/221 jobs before/after stable；post-read file verification 通过 |
|
||||
| instruments / sessions | 1,111 / 1,942 |
|
||||
| selected rows | 1,975,455 |
|
||||
| observed / effective snapshots | 97 / 96 |
|
||||
| universe quality | 每个 snapshot 严格 500 constituents；weight sum ±0.5；最大间隔 36 天 |
|
||||
| membership availability | 月末权重下一 provider 交易日生效；same-session use 为 false |
|
||||
| signal / report rows | 176,615 / 1,698 |
|
||||
| strategy cumulative return | 0.253151986388197 |
|
||||
| benchmark cumulative return | 0.789559383530678 |
|
||||
| max drawdown | -0.6041641365538415 |
|
||||
| evidence flags | `production_ready=false`、`investment_value_claim=false`、`gate_credit=[]` |
|
||||
|
||||
## 7. 如何读结果
|
||||
|
||||
先回答四个工程问题：
|
||||
|
||||
1. frozen release 与 provider 是否通过 hash/语义验证；
|
||||
2. benchmark 是否确为 `SH000905`，universe 是否来自历史权重并从下一交易日
|
||||
生效；
|
||||
3. 两次 run 的关键表和指标是否可重放；
|
||||
4. 日期、股票数、缺失率、换手、成本和极端收益是否合理。
|
||||
|
||||
然后才看年化收益、最大回撤、信息比率等研究指标。该 momentum run 只是数据桥
|
||||
和研究运行时 smoke，不是选股结论；它尚未完整表达历史 ST、逐股成交约束、
|
||||
冲击成本、容量和券商执行，也不自动获得 60/80 标准中的任何门禁分数。
|
||||
|
||||
尤其注意，本次 Qlib run 的 `limit_threshold=0.095` 是统一 9.5% 近似，没有
|
||||
使用镜像内 `stk_limit` 的逐股逐日价格，无法表达主板/创业板/科创板、规则变更
|
||||
和 ST 股票的不同涨跌停限制；历史 `stock_st` 权限仍被拒。因此这些结果严格是
|
||||
research-only，不能升级为 production-ready。
|
||||
|
||||
`index_weight` 源没有单独验证的 `published_at`，所以该 universe 只能称为
|
||||
event-time / 保守次日生效 PIT 近似，不能称为严格 knowledge-time PIT。
|
||||
|
||||
更完整的数据边界与当前覆盖见
|
||||
[`../TUSHARE_LOCAL_DATA.md`](../TUSHARE_LOCAL_DATA.md)。
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"record_type": "quant_os_tushare_qlib_research_run",
|
||||
"run_id": "5bf19d2d",
|
||||
"audit_as_of": "2026-07-31",
|
||||
"scope": {
|
||||
"index_code": "000905.SH",
|
||||
"start": "2018-01-01",
|
||||
"end": "2025-12-31",
|
||||
"daily_market_data_from": "2018-01-02",
|
||||
"daily_market_data_through": "2025-12-31",
|
||||
"daily_partitions_completed": 96,
|
||||
"daily_partitions_planned": 96,
|
||||
"daily_rows": 8813885
|
||||
},
|
||||
"snapshot": {
|
||||
"selected_jobs": 221,
|
||||
"verified_jobs": 221,
|
||||
"double_verified_jobs": 221,
|
||||
"manifest_sha256": "744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2",
|
||||
"selection_sha256": "66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5"
|
||||
},
|
||||
"provider": {
|
||||
"verified": true,
|
||||
"data_version": "5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963",
|
||||
"tree_sha256": "f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f",
|
||||
"manifest_sha256": "27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0",
|
||||
"file_count": 10011,
|
||||
"byte_count": 71707194,
|
||||
"calendar_sessions": 1942,
|
||||
"instrument_count": 1111,
|
||||
"selected_daily_rows": 1975455,
|
||||
"observed_membership_snapshots": 97,
|
||||
"effective_membership_snapshots": 96,
|
||||
"same_session_membership_use": false,
|
||||
"price_adjustment_complete": true,
|
||||
"benchmark": "SH000905",
|
||||
"converter_source_matches_current": true
|
||||
},
|
||||
"lineage": {
|
||||
"verified": true,
|
||||
"source_job_count": 221,
|
||||
"matched_fields": [
|
||||
"id",
|
||||
"path",
|
||||
"row_count",
|
||||
"byte_count",
|
||||
"sha256",
|
||||
"request_sha256"
|
||||
],
|
||||
"mismatch_count": 0,
|
||||
"converter_source_matches_current": true
|
||||
},
|
||||
"source_stability": {
|
||||
"state_db_changed_through_file_verification": true,
|
||||
"state_db_change_scope": "unrelated live downloader jobs",
|
||||
"snapshot_jobs_still_latest": true,
|
||||
"selected_scope_stable": true,
|
||||
"post_read_file_verification": true,
|
||||
"provider_source_jobs": 221,
|
||||
"snapshot_source_jobs": 221,
|
||||
"source_job_full_field_diff": 0
|
||||
},
|
||||
"reproducibility": {
|
||||
"runs": 2,
|
||||
"byte_identical": true,
|
||||
"run_sha256": "803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3",
|
||||
"signal_rows": 176615,
|
||||
"portfolio_days": 1698
|
||||
},
|
||||
"performance": {
|
||||
"strategy_cumulative_return": 0.2531519864,
|
||||
"benchmark_cumulative_return": 0.7895593835,
|
||||
"max_drawdown": -0.6041641366
|
||||
},
|
||||
"limitations": {
|
||||
"stock_st": "denied",
|
||||
"qlib_limit_threshold": 0.095,
|
||||
"limit_rule_fidelity": "market-level approximation; not per-symbol daily limit prices",
|
||||
"research_only": true
|
||||
},
|
||||
"production_ready": false,
|
||||
"investment_value_claim": false,
|
||||
"gate_credit": []
|
||||
}
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"system": "Quant OS / quant60 baseline kernel",
|
||||
"audit_as_of": "2026-07-30",
|
||||
"audit_as_of": "2026-07-31",
|
||||
"claim_policy": {
|
||||
"baseline_threshold": 60,
|
||||
"all_gates_required": true,
|
||||
@@ -35,7 +35,7 @@
|
||||
"status": "not_scored",
|
||||
"reason": "Quant OS 已接通本地 synthetic 五层候选链，能冻结 Ridge ModelBundle、逐层 trace 和无股数 TargetPackageV1；一个 synthetic-research TargetPackage 已在真实 JoinQuant hosted 环境完成消费、账户/开盘价绑定、3 笔委托和 3 笔成交，因此 execution 层已有真实平台观察。但真实 provider-data 长样本/OOS 尚未接入，JoinQuant 没有重跑上游 Universe/Alpha/Portfolio/Risk，QMT peer 与逐层真实数据 parity 仍缺失，factor risk、CVXPY 和 guarded TWAP/POV 也尚未进入当前主链。因此它仍是 NOT_BASELINE_60，G9 未通过。",
|
||||
"verified_supporting_facts": [
|
||||
"The independent-repository standard-library suite completed 315 tests with OK and 6 optional-runtime skips; successful local tests do not prove real-platform, broker or compliance gates.",
|
||||
"The independent-repository standard-library suite completed 325 tests with OK and 1 optional-runtime skip; successful local tests do not prove real-platform, broker or compliance gates.",
|
||||
"The local synthetic research entrypoint now executes one causally linked Universe → Ridge Alpha → cost/risk-aware Portfolio → post-risk gate → reference Execution path, freezes a replayable ModelBundle, and publishes a hash-linked TargetPackageV1 and five-layer trace. This is structural synthetic evidence only.",
|
||||
"The connected candidate currently uses 60-session diagonal realized variance, explicit spread/fee/square-root-impact cost forecasts, the deterministic constrained optimizer, a post-target risk gate, and reference lot/T+1 order deltas. factor_risk.py, the optional CVXPY solver mode, and guarded TWAP/POV are not selected by the current main slice.",
|
||||
"JoinQuant and QMT target-package modes verify an exact embedded package tape, match the declared dual clock, bind platform equity/positions/sellable quantities/prices, and reject missing, duplicate or tampered packages without falling back to momentum. Both modes have local harness/contract evidence; JoinQuant execution alone now also has one narrow real hosted smoke.",
|
||||
@@ -44,7 +44,7 @@
|
||||
"The local snapshot can drive snapshot-backtest and snapshot-decision with manifest verification; it freezes a cross-checked provider trading calendar, and a T-close signal can bind only to a broker observation in the unique next session's Asia/Shanghai [09:00,09:30) window without collapsing the two clocks.",
|
||||
"CPython 3.12 with pyqlib 0.9.7 completed a native momentum fixture smoke with 24 signals and saved audited portfolio-table hashes, bounds and recomputed metrics.",
|
||||
"Qlib Alpha158, LightGBM and Recorder completed a real local fixture smoke with audited portfolio output; the deterministic two-stock synthetic fixture has no investment-value meaning.",
|
||||
"The local Tushare inventory verified all 93 completed Parquet files against recorded size, SHA-256 and row count. An immutable managed Qlib provider and the same early-history momentum command produced byte-identical evidence JSON twice with PYTHONHASHSEED=0. The mirror is only about 8.2% complete for daily partitions and lacks adjustment, real benchmark, point-in-time membership, ST, suspension and limit data, so production_ready and investment_value_claim remain false and no gate credit is earned.",
|
||||
"The local Tushare CSI500 2018-2025 scoped release double-verified 221/221 selected jobs and confirmed that all remained latest; the live SQLite ledger changed only because of unrelated downloader jobs. The dedicated lineage verifier matched all 221 converter source jobs on id, path, row_count, byte_count, sha256 and request_sha256 with mismatch_count=0, so converter_source_matches_current=true. The verified Qlib provider has data version 5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963, manifest SHA-256 27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0, 10,011 files, 71,707,194 bytes, 1,942 sessions, 1,111 instruments, adjusted prices, a real SH000905 benchmark and next-session-effective historical membership. Two identical runs produced SHA-256 803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3, 176,615 signals and 1,698 portfolio days. Historical stock_st remains unavailable and the Qlib 9.5% limit threshold is only a market-level approximation, so production_ready and investment_value_claim remain false and gate_credit remains empty.",
|
||||
"A real JoinQuant daily hosted backtest completed for 2024-01-02 through 2024-12-31 with CNY 1,000,000; the visible completed-run log had zero ERROR/Traceback/order-rejection entries. That run used portable_momentum_smoke, not the Ridge/TargetPackage Baseline candidate, and earns no candidate or cross-engine gate credit.",
|
||||
"The QMT shadow CLI is strictly read-only and produces HMAC account-bound broker observation, broker snapshot and target-diff artifacts. QMT_SHADOW_EVIDENCE_V1 authenticates semantic content, deterministic published bytes, the original decision, planner/engine/operator source and fixed policy; semantic replay verifies exact derived state. Independent local QA completed 58 deterministic scenarios, 500 malformed fuzz cases and 300 legal-observation fuzz cases with no P0/P1. This is Quant OS publisher integrity, not broker attestation, and it still has only fake-broker evidence.",
|
||||
"JoinQuant/QMT mock parity is mock_contract_only, real_platform_pass is false and it earns no G9 gate credit."
|
||||
|
||||
+83
-35
@@ -1,6 +1,6 @@
|
||||
// Generated by tools/build_public_status.py; do not edit by hand.
|
||||
window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"auditAsOf": "2026-07-30",
|
||||
"auditAsOf": "2026-07-31",
|
||||
"bundles": {
|
||||
"artifacts": [
|
||||
{
|
||||
@@ -198,6 +198,12 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"label": "Tushare inventory and Qlib bridge record",
|
||||
"path": "docs/TUSHARE_LOCAL_DATA.md"
|
||||
},
|
||||
{
|
||||
"id": "tushare-csi500-research-run",
|
||||
"kind": "machine_record",
|
||||
"label": "Tushare CSI500 2018-2025 reproducible research run",
|
||||
"path": "evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json"
|
||||
},
|
||||
{
|
||||
"id": "platform-deployment",
|
||||
"kind": "runbook",
|
||||
@@ -213,7 +219,7 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
],
|
||||
"freshness": [
|
||||
{
|
||||
"asOf": "2026-07-30",
|
||||
"asOf": "2026-07-31",
|
||||
"evidenceLinks": [
|
||||
"scorecard",
|
||||
"audit"
|
||||
@@ -222,7 +228,7 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"label": "项目证据审计",
|
||||
"semantics": "证据盘点日期；不是部署时间，也不是行情截止日。",
|
||||
"status": "recorded",
|
||||
"value": "2026-07-30"
|
||||
"value": "2026-07-31"
|
||||
},
|
||||
{
|
||||
"evidenceLinks": [
|
||||
@@ -257,23 +263,58 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"value": "2024-03-11 — 2024-03-13; 3 orders / 3 fills"
|
||||
},
|
||||
{
|
||||
"completedPartitions": 35,
|
||||
"completedRows": 28731,
|
||||
"completionLabel": "35/428（约 8.2%）",
|
||||
"completionRatio": 0.081776,
|
||||
"benchmarkCumulativeReturn": 0.7895593835,
|
||||
"calendarSessions": 1942,
|
||||
"completedPartitions": 96,
|
||||
"completedRows": 8813885,
|
||||
"completionLabel": "96/96（约 100%）",
|
||||
"completionRatio": 1.0,
|
||||
"converterSourceMatchesCurrent": true,
|
||||
"effectiveMembershipSnapshots": 96,
|
||||
"evidenceLinks": [
|
||||
"tushare-csi500-research-run",
|
||||
"tushare-data",
|
||||
"scorecard"
|
||||
],
|
||||
"gateCredit": [],
|
||||
"id": "tushare-daily-market-data",
|
||||
"label": "Tushare daily 行情覆盖",
|
||||
"marketDataFrom": "1990-12-19",
|
||||
"marketDataThrough": "1993-10-29",
|
||||
"plannedPartitions": 428,
|
||||
"instrumentCount": 1111,
|
||||
"investmentValueClaim": false,
|
||||
"label": "Tushare CSI500 scoped research release",
|
||||
"lineageMatchedFields": [
|
||||
"id",
|
||||
"path",
|
||||
"row_count",
|
||||
"byte_count",
|
||||
"sha256",
|
||||
"request_sha256"
|
||||
],
|
||||
"lineageMismatchCount": 0,
|
||||
"marketDataFrom": "2018-01-02",
|
||||
"marketDataThrough": "2025-12-31",
|
||||
"maxDrawdown": -0.6041641366,
|
||||
"observedMembershipSnapshots": 97,
|
||||
"plannedPartitions": 96,
|
||||
"portfolioDays": 1698,
|
||||
"productionReady": false,
|
||||
"semantics": "marketDataThrough 仅取自 daily 行情分区；绝不使用覆盖更晚的 trade_cal 日期。",
|
||||
"status": "partial",
|
||||
"value": "35/428（约 8.2%），行情截至 1993-10-29"
|
||||
"providerBytes": 71707194,
|
||||
"providerDataVersion": "5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963",
|
||||
"providerFiles": 10011,
|
||||
"providerManifestSha256": "27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0",
|
||||
"providerTreeSha256": "f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f",
|
||||
"runSha256": "803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3",
|
||||
"sameSessionMembershipUse": false,
|
||||
"selectedDailyRows": 1975455,
|
||||
"semantics": "96/96 只表示 2018—2025 scoped daily 输入完整且 221/221 source jobs 经双重校验；全局 ledger 因无关下载任务发生变化，但 scoped jobs 仍为 latest；lineage 工具按 id/path/row_count/byte_count/sha256/request_sha256 验证 converter 221/221 零差异。universe 成分次一交易日生效。该 Qlib run 未使用逐股涨跌停，stock_st 不可用，不代表 production-ready 或投资价值。",
|
||||
"signalRows": 176615,
|
||||
"snapshotJobs": 221,
|
||||
"sourceJobCount": 221,
|
||||
"sourceStable": true,
|
||||
"stateDbChangedDuringVerification": true,
|
||||
"status": "research_only",
|
||||
"strategyCumulativeReturn": 0.2531519864,
|
||||
"value": "96/96 scoped daily 分区；221/221 source jobs；行情 2018-01-02 — 2025-12-31",
|
||||
"verifiedSnapshotJobs": 221
|
||||
},
|
||||
{
|
||||
"evidenceLinks": [
|
||||
@@ -525,24 +566,25 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"status": "not_passed"
|
||||
}
|
||||
],
|
||||
"generatedAt": "2026-07-30T00:00:00Z",
|
||||
"generatedAt": "2026-07-31T00:00:00Z",
|
||||
"generatedAtSource": "audit_as_of",
|
||||
"layers": [
|
||||
{
|
||||
"capabilities": [
|
||||
"JQData raw/factor/limits/paused/PIT membership/ST snapshot",
|
||||
"Tushare completed-Parquet inventory and immutable Qlib provider",
|
||||
"Tushare scoped source release and verified immutable Qlib provider",
|
||||
"semantic manifest and data-version verification"
|
||||
],
|
||||
"claim": "不可变快照、PIT 字段合同和 Tushare→Qlib 技术链路已有本地合同/运行证据；这些 provider-data 路径尚未接成当前 Ridge 五层候选。",
|
||||
"evidenceLinks": [
|
||||
"architecture",
|
||||
"tushare-data",
|
||||
"tushare-csi500-research-run",
|
||||
"scorecard"
|
||||
],
|
||||
"id": "data",
|
||||
"limitations": [
|
||||
"尚无授权 JQData 真实快照与许可 lineage；Tushare 镜像仅部分完成且缺生产关键字段；当前五层候选仍使用 synthetic 数据。"
|
||||
"Tushare 2018—2025 中证 500 scoped research 链已验证，但 live raw 仍可变、stock_st 不可用、9.5% 涨跌停为市场级近似，且尚未接入当前 Ridge 五层候选；JQData 授权快照与许可 lineage 仍缺失。"
|
||||
],
|
||||
"name": "数据与时点",
|
||||
"releaseVerified": false,
|
||||
@@ -1439,15 +1481,16 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"status": "real_runtime"
|
||||
},
|
||||
{
|
||||
"claim": "93 个 completed 文件全部通过 size/SHA/row-count；v4 provider verifier 成功；同一动量命令连续两次 evidence JSON 逐 byte 相同",
|
||||
"claim": "selected jobs double verified、仍 latest；独立 lineage 验证 221 jobs×6 fields、mismatch 0、converter current true；v2 verifier 通过：1,111 instruments、1,942 sessions、1,975,455 rows；97 observed/96 effective snapshots 严格 500 成分、次日生效；双次 JSON byte-identical",
|
||||
"evidenceLinks": [
|
||||
"platform-matrix",
|
||||
"tushare-data"
|
||||
"tushare-data",
|
||||
"tushare-csi500-research-run"
|
||||
],
|
||||
"id": "tushare-local-to-qlib",
|
||||
"intendedUse": "completed Parquet 的只读盘点、不可变 provider 与本地研究 smoke",
|
||||
"intendedUse": "live mirror 只读盘点、scoped release、不可变 provider 与本地研究",
|
||||
"limitations": [
|
||||
"daily 仅完成约 8.2%，缺复权、真实 benchmark、PIT 成分/ST/停牌/涨跌停；production_ready=false"
|
||||
"event-time/保守次日生效 PIT 近似，不是严格 knowledge-time PIT；不同 start 的首观测锚 provider 不可直接拼接；research-only、gate_credit=[]；ST/统一 9.5% 仍缺"
|
||||
],
|
||||
"liveReady": false,
|
||||
"name": "Tushare local → Qlib",
|
||||
@@ -1475,18 +1518,18 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"provenance": {
|
||||
"baselineConfig": "configs/baseline.json",
|
||||
"bundleManifest": "dist/bundle_manifest.json",
|
||||
"payloadSha256": "2582133135aa956bb567fd009e73d31c3242ffb4f89b8cf70d2932027cc5d763",
|
||||
"payloadSha256": "c607f08ef7450714084a562a39afe8d98783550882fbb26b4a844c7161f5bfe1",
|
||||
"payloadSha256Semantics": "SHA-256 of canonical payload before payloadSha256 is added.",
|
||||
"platformMatrix": "docs/PLATFORM_MATRIX.md",
|
||||
"production80Assessment": "profiles/production-80/current-assessment.json",
|
||||
"production80Standard": "profiles/production-80/standard.json",
|
||||
"releaseReachability": "releases/quant60-research-candidate-v1/reachability.json",
|
||||
"scorecard": "gate_scorecard.json",
|
||||
"sourceSetSha256": "92485e439b1ecfb75166f8750e26b4d392d44ff614755295635e7761fa255c5c",
|
||||
"sourceSetSha256": "de518af50036a250cf75ea53e0b6804b14043f55abd19125e279e4117c833f87",
|
||||
"sources": [
|
||||
{
|
||||
"path": "gate_scorecard.json",
|
||||
"sha256": "2cc29f8d3670cd2e1e90ebe10a7d4d5a1ab9fc6a6b08612df2bbad485cd19655"
|
||||
"sha256": "8723af0d0a29ea540f91bb9f82fbdd982f31903bc8d0959dfa96d8740564d925"
|
||||
},
|
||||
{
|
||||
"path": "profiles/baseline-60/profile.json",
|
||||
@@ -1512,17 +1555,21 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"path": "evidence/joinquant/TP-20240311-795dcfba/2457a7c39a276e09e0fabf99e28978e1/run.json",
|
||||
"sha256": "9956c5e61bdc1c82bc24f20f24c1d65a201bab2f71c2c9e4e23eb5bd40eca6ce"
|
||||
},
|
||||
{
|
||||
"path": "evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json",
|
||||
"sha256": "6374776487a4a10c6351e2844eba74f7af4ab1db5696dc4d13d5672da9daab76"
|
||||
},
|
||||
{
|
||||
"path": "configs/baseline.json",
|
||||
"sha256": "939a7df386b06bc82869940572c35544e9361913b94d9e03027411e51b1006e1"
|
||||
},
|
||||
{
|
||||
"path": "README.md",
|
||||
"sha256": "0bf0562ae9247a88a5b0248f66b9b314d7ab0ea435bc8f8e5efa78f7d2f785bb"
|
||||
"sha256": "e67e9fb71550cedbf3f8436c06ceafc5d1fc50e34ba0feaf66310aee8ec5ea78"
|
||||
},
|
||||
{
|
||||
"path": "docs/ARCHITECTURE.md",
|
||||
"sha256": "8aae46f1b371651ac6f72905b7bcb3d72fdfda2a793e5505ab890352921c3219"
|
||||
"sha256": "a2f9be07b138d587ffa1cb01608e8a469af8a951ef7e6ac371752eb8cdd2ae2a"
|
||||
},
|
||||
{
|
||||
"path": "docs/architecture/ARCHITECTURE_80.md",
|
||||
@@ -1542,11 +1589,11 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
},
|
||||
{
|
||||
"path": "docs/getting-started/GETTING_STARTED_80.md",
|
||||
"sha256": "5c18a558bd2f9e1640144a1729309bf38ada3185188c985476bd609b28c4479e"
|
||||
"sha256": "14339c6324b142d176813bd274b70cd55b4532d7bfc8f5a39408cc33737df7ba"
|
||||
},
|
||||
{
|
||||
"path": "docs/PLATFORM_MATRIX.md",
|
||||
"sha256": "9c7952b019c92efd1055488b58838045ee30979ccc84008cd1218fab2a6d6717"
|
||||
"sha256": "90c8936b2c51d9e8375e0934f1e9894e9c6fc9c2f29eb2920a8ed226869ba392"
|
||||
},
|
||||
{
|
||||
"path": "docs/CROSS_ENGINE_CONSISTENCY.md",
|
||||
@@ -1562,11 +1609,11 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
},
|
||||
{
|
||||
"path": "docs/TUSHARE_LOCAL_DATA.md",
|
||||
"sha256": "611b120ba59b59b443b5cd9237356c509f2f2699063784a4685506e88a95adea"
|
||||
"sha256": "560d5e039e19393b64aee1d2b1b213ef32c6d67734688627c261da138a860246"
|
||||
},
|
||||
{
|
||||
"path": "MIGRATION.md",
|
||||
"sha256": "2b86f9f4ced687de61c4a3268d57bb38cde4fb1fac75be61ec8cd18c12ccdd26"
|
||||
"sha256": "5f16fc828879a33a686b6d10c30129938c29b2669b0da2808d889b4542d4165e"
|
||||
},
|
||||
{
|
||||
"path": "ops/README.md",
|
||||
@@ -1602,7 +1649,7 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
},
|
||||
{
|
||||
"path": "tools/build_public_status.py",
|
||||
"sha256": "ec24dd12a6383ab1210a475c6f982b911f6ce45a7c205b9474bb2bb70665bc2a"
|
||||
"sha256": "17d78b2fbe8e3373ebb283483a4f00f5fae0b0ec2e93a3f9a751c1d69f2004e5"
|
||||
},
|
||||
{
|
||||
"path": "tools/check_tracked_secrets.py",
|
||||
@@ -1612,7 +1659,8 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"path": "tools/inventory_local_assets.py",
|
||||
"sha256": "f52a89b2e42d0f3aed7e3c79e0110c37cab779794f550658116e07b9e338160b"
|
||||
}
|
||||
]
|
||||
],
|
||||
"tushareResearchRun": "evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json"
|
||||
},
|
||||
"schemaVersion": 2,
|
||||
"summary": {
|
||||
@@ -1621,17 +1669,17 @@ window.QUANT_OS_PUBLIC_STATUS = {
|
||||
"candidateRealPlatformObserved": true,
|
||||
"gateCoverageLabel": "0/10 passed",
|
||||
"investmentValueClaim": false,
|
||||
"optionalSkipsRecorded": 6,
|
||||
"optionalSkipsRecorded": 1,
|
||||
"productionReady": false,
|
||||
"realRuntimeSmokes": [
|
||||
"JoinQuant TargetPackage execution consumer: one synthetic-research package, 3 orders / 3 fills (not Baseline/G9 evidence)",
|
||||
"JoinQuant portable_momentum_smoke hosted daily backtest (not Baseline/TargetPackage evidence)",
|
||||
"Qlib native momentum fixture",
|
||||
"Qlib Alpha158 + LightGBM fixture",
|
||||
"Tushare completed-partition Qlib momentum smoke"
|
||||
"Tushare CSI500 2018—2025 scoped Qlib momentum reproducibility run"
|
||||
],
|
||||
"testEvidenceQualifier": "Scorecard-recorded run; tests do not prove real-platform or broker gates.",
|
||||
"testsRecorded": 315
|
||||
"testsRecorded": 325
|
||||
},
|
||||
"system": "Quant OS / quant60 baseline kernel"
|
||||
};
|
||||
|
||||
+324
-6
@@ -13,9 +13,11 @@ sys.path.insert(0, str(ROOT))
|
||||
from adapters.tushare_local import (
|
||||
TushareMirrorError,
|
||||
build_qlib_provider,
|
||||
inventory_mirror,
|
||||
tushare_to_qlib_symbol,
|
||||
verify_qlib_provider,
|
||||
)
|
||||
from tools.tushare_qlib import _assert_json_outside_provider
|
||||
|
||||
|
||||
class TushareSymbolTest(unittest.TestCase):
|
||||
@@ -26,6 +28,20 @@ class TushareSymbolTest(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
tushare_to_qlib_symbol("T600018.SH")
|
||||
|
||||
def test_provider_json_output_cannot_mutate_provider_tree(self):
|
||||
with self.assertRaisesRegex(
|
||||
ValueError,
|
||||
"outside the immutable provider",
|
||||
):
|
||||
_assert_json_outside_provider(
|
||||
"/tmp/provider/evidence.json",
|
||||
"/tmp/provider",
|
||||
)
|
||||
_assert_json_outside_provider(
|
||||
"/tmp/evidence.json",
|
||||
"/tmp/provider",
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
importlib.util.find_spec("pandas") is not None
|
||||
@@ -33,7 +49,14 @@ class TushareSymbolTest(unittest.TestCase):
|
||||
"pandas and pyarrow are optional outside the Qlib research environment",
|
||||
)
|
||||
class TushareProviderTest(unittest.TestCase):
|
||||
def _mirror(self, root: Path, *, bad_ohlc: bool = False) -> Path:
|
||||
def _mirror(
|
||||
self,
|
||||
root: Path,
|
||||
*,
|
||||
bad_ohlc: bool = False,
|
||||
advanced: bool = False,
|
||||
shadowed_daily: bool = False,
|
||||
) -> Path:
|
||||
import pandas
|
||||
|
||||
mirror = root / "mirror"
|
||||
@@ -43,7 +66,10 @@ class TushareProviderTest(unittest.TestCase):
|
||||
(parquet_root / "stock_basic").mkdir(parents=True)
|
||||
dates = pandas.bdate_range("2024-01-02", periods=8)
|
||||
daily_rows = []
|
||||
for symbol, base in (("600000.SH", 10.0), ("000001.SZ", 20.0)):
|
||||
symbols = [("600000.SH", 10.0), ("000001.SZ", 20.0)]
|
||||
if advanced:
|
||||
symbols.append(("000002.SZ", 30.0))
|
||||
for symbol, base in symbols:
|
||||
previous = base
|
||||
for index, value in enumerate(dates):
|
||||
close = base * (1.01 ** index)
|
||||
@@ -78,11 +104,121 @@ class TushareProviderTest(unittest.TestCase):
|
||||
),
|
||||
"stock_basic/list_status=L.parquet": pandas.DataFrame(
|
||||
{
|
||||
"ts_code": ["600000.SH", "000001.SZ"],
|
||||
"curr_type": ["CNY", "CNY"],
|
||||
"ts_code": [value[0] for value in symbols],
|
||||
"curr_type": ["CNY"] * len(symbols),
|
||||
}
|
||||
),
|
||||
}
|
||||
partitions = {}
|
||||
if advanced:
|
||||
adj_rows = []
|
||||
for symbol, _ in symbols:
|
||||
for index, value in enumerate(dates):
|
||||
factor = 2.0 if symbol == "600000.SH" and index >= 4 else 1.0
|
||||
adj_rows.append(
|
||||
{
|
||||
"ts_code": symbol,
|
||||
"trade_date": value.strftime("%Y%m%d"),
|
||||
"adj_factor": factor,
|
||||
}
|
||||
)
|
||||
index_rows = []
|
||||
previous = 1000.0
|
||||
for index, value in enumerate(dates):
|
||||
close = 1000.0 + index * 10.0
|
||||
index_rows.append(
|
||||
{
|
||||
"ts_code": "000905.SH",
|
||||
"trade_date": value.strftime("%Y%m%d"),
|
||||
"open": close - 2.0,
|
||||
"high": close + 5.0,
|
||||
"low": close - 5.0,
|
||||
"close": close,
|
||||
"pre_close": previous,
|
||||
"change": close - previous,
|
||||
"pct_chg": (close / previous - 1.0) * 100.0,
|
||||
"vol": 2000.0,
|
||||
"amount": close * 200.0,
|
||||
}
|
||||
)
|
||||
previous = close
|
||||
first_snapshot = dates[0].strftime("%Y%m%d")
|
||||
second_snapshot = dates[4].strftime("%Y%m%d")
|
||||
anchor_weight_rows = [
|
||||
{
|
||||
"index_code": "000905.SH",
|
||||
"con_code": "600000.SH",
|
||||
"trade_date": "20231229",
|
||||
"weight": 50.0,
|
||||
},
|
||||
{
|
||||
"index_code": "000905.SH",
|
||||
"con_code": "000001.SZ",
|
||||
"trade_date": "20231229",
|
||||
"weight": 50.0,
|
||||
},
|
||||
]
|
||||
weight_rows = [
|
||||
{
|
||||
"index_code": "000905.SH",
|
||||
"con_code": "600000.SH",
|
||||
"trade_date": first_snapshot,
|
||||
"weight": 50.0,
|
||||
},
|
||||
{
|
||||
"index_code": "000905.SH",
|
||||
"con_code": "000001.SZ",
|
||||
"trade_date": first_snapshot,
|
||||
"weight": 50.0,
|
||||
},
|
||||
{
|
||||
"index_code": "000905.SH",
|
||||
"con_code": "000001.SZ",
|
||||
"trade_date": second_snapshot,
|
||||
"weight": 55.0,
|
||||
},
|
||||
{
|
||||
"index_code": "000905.SH",
|
||||
"con_code": "000002.SZ",
|
||||
"trade_date": second_snapshot,
|
||||
"weight": 45.0,
|
||||
},
|
||||
]
|
||||
frames.update(
|
||||
{
|
||||
"adj_factor/month=2024-01.parquet": pandas.DataFrame(
|
||||
adj_rows
|
||||
),
|
||||
(
|
||||
"index_daily/"
|
||||
"ts_code=000905_SH_year=2024.parquet"
|
||||
): pandas.DataFrame(index_rows),
|
||||
(
|
||||
"index_weight/"
|
||||
"index_code=000905_SH_year=2023.parquet"
|
||||
): pandas.DataFrame(anchor_weight_rows),
|
||||
(
|
||||
"index_weight/"
|
||||
"index_code=000905_SH_year=2024.parquet"
|
||||
): pandas.DataFrame(weight_rows),
|
||||
}
|
||||
)
|
||||
partitions.update(
|
||||
{
|
||||
(
|
||||
"index_daily/"
|
||||
"ts_code=000905_SH_year=2024.parquet"
|
||||
): "ts_code=000905_SH/year=2024",
|
||||
(
|
||||
"index_weight/"
|
||||
"index_code=000905_SH_year=2023.parquet"
|
||||
): "index_code=000905_SH/year=2023",
|
||||
(
|
||||
"index_weight/"
|
||||
"index_code=000905_SH_year=2024.parquet"
|
||||
): "index_code=000905_SH/year=2024",
|
||||
}
|
||||
)
|
||||
state = mirror / "data/state.sqlite3"
|
||||
connection = sqlite3.connect(state)
|
||||
connection.execute(
|
||||
@@ -117,13 +253,24 @@ class TushareProviderTest(unittest.TestCase):
|
||||
(
|
||||
index,
|
||||
api,
|
||||
path.stem,
|
||||
partitions.get(relative, path.stem),
|
||||
len(frame),
|
||||
path.stat().st_size,
|
||||
hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
str(path),
|
||||
),
|
||||
)
|
||||
if shadowed_daily:
|
||||
daily_path = parquet_root / "daily/month=2024-01.parquet"
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO jobs VALUES
|
||||
(0, 'daily', 'month=2024-01', 'completed', 1, 1,
|
||||
'obsolete-checksum', ?, NULL,
|
||||
'2024-01-01T00:00:00Z', NULL)
|
||||
""",
|
||||
(str(daily_path),),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO jobs VALUES
|
||||
@@ -150,7 +297,9 @@ class TushareProviderTest(unittest.TestCase):
|
||||
)
|
||||
self.assertFalse(result["investment_value_claim"])
|
||||
self.assertEqual(result["market"]["instrument_count"], 2)
|
||||
self.assertTrue(verify_qlib_provider(output)["ok"])
|
||||
verified = verify_qlib_provider(output)
|
||||
self.assertTrue(verified["ok"])
|
||||
self.assertTrue(verified["converter_source_matches_current"])
|
||||
manifest = json.loads(
|
||||
(output / "quant_os_tushare_manifest.json").read_text()
|
||||
)
|
||||
@@ -225,6 +374,175 @@ class TushareProviderTest(unittest.TestCase):
|
||||
self.assertAlmostEqual(high[1], 10.0)
|
||||
self.assertAlmostEqual(low[1], 8.0)
|
||||
|
||||
def test_inventory_ignores_shadowed_completed_file_version(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
mirror = self._mirror(
|
||||
Path(temporary),
|
||||
shadowed_daily=True,
|
||||
)
|
||||
result = inventory_mirror(mirror)
|
||||
self.assertEqual(result["shadowed_completed_versions"], 1)
|
||||
self.assertEqual(
|
||||
result["shadowed_completed_versions_by_api"],
|
||||
{"daily": 1},
|
||||
)
|
||||
self.assertEqual(result["tables"]["daily"]["files"], 1)
|
||||
self.assertEqual(result["verified_completed_files"], 3)
|
||||
|
||||
def test_adjusted_real_benchmark_and_pit_universe(self):
|
||||
from array import array
|
||||
import math
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
mirror = self._mirror(root, advanced=True)
|
||||
output = root / "provider"
|
||||
result = build_qlib_provider(
|
||||
mirror,
|
||||
output,
|
||||
start="2024-01-02",
|
||||
end="2024-01-11",
|
||||
minimum_observations=999,
|
||||
benchmark_index_code="000905.SH",
|
||||
universe_index_code="000905.SH",
|
||||
expected_constituent_count=2,
|
||||
)
|
||||
self.assertEqual(result["schema_version"], 2)
|
||||
self.assertEqual(
|
||||
result["price_adjustment"]["mode"],
|
||||
"cumulative_adjusted_first_observation_anchor",
|
||||
)
|
||||
self.assertTrue(result["price_adjustment"]["component_complete"])
|
||||
self.assertFalse(result["price_adjustment"]["production_eligible"])
|
||||
self.assertEqual(
|
||||
result["benchmark"]["kind"],
|
||||
"tushare_index_daily",
|
||||
)
|
||||
self.assertTrue(result["market"]["point_in_time"])
|
||||
self.assertFalse(result["market"]["minimum_observations_applied"])
|
||||
self.assertFalse(result["market"]["same_session_membership_use"])
|
||||
self.assertEqual(
|
||||
result["market"]["availability_contract"],
|
||||
"conservative next-session effectiveness because the source "
|
||||
"does not expose a separately verified published_at timestamp",
|
||||
)
|
||||
self.assertEqual(result["market"]["instrument_count"], 3)
|
||||
self.assertTrue(
|
||||
result["source_stability"]["post_read_file_verification"]
|
||||
)
|
||||
self.assertEqual(
|
||||
{item["api_name"] for item in result["source_jobs"]},
|
||||
{
|
||||
"adj_factor",
|
||||
"daily",
|
||||
"index_daily",
|
||||
"index_weight",
|
||||
"stock_basic",
|
||||
"trade_cal",
|
||||
},
|
||||
)
|
||||
|
||||
factor = array("f")
|
||||
factor.frombytes(
|
||||
(output / "features/sh600000/factor.day.bin").read_bytes()
|
||||
)
|
||||
close = array("f")
|
||||
close.frombytes(
|
||||
(output / "features/sh600000/close.day.bin").read_bytes()
|
||||
)
|
||||
volume = array("f")
|
||||
volume.frombytes(
|
||||
(output / "features/sh600000/volume.day.bin").read_bytes()
|
||||
)
|
||||
change = array("f")
|
||||
change.frombytes(
|
||||
(output / "features/sh600000/change.day.bin").read_bytes()
|
||||
)
|
||||
self.assertAlmostEqual(factor[1], 1.0)
|
||||
self.assertAlmostEqual(factor[5], 2.0)
|
||||
raw_close = 10.0 * (1.01 ** 4)
|
||||
self.assertAlmostEqual(close[5] / factor[5], raw_close, places=5)
|
||||
self.assertAlmostEqual(volume[5], 50_000.0, places=3)
|
||||
self.assertTrue(math.isnan(change[1]))
|
||||
self.assertAlmostEqual(
|
||||
change[5],
|
||||
(raw_close * 2.0) / (10.0 * (1.01 ** 3)) - 1.0,
|
||||
places=5,
|
||||
)
|
||||
|
||||
index_close = array("f")
|
||||
index_close.frombytes(
|
||||
(output / "features/sh000905/close.day.bin").read_bytes()
|
||||
)
|
||||
self.assertAlmostEqual(index_close[1], 1000.0)
|
||||
market_lines = (
|
||||
output / "instruments/tushare_a.txt"
|
||||
).read_text(encoding="utf-8").splitlines()
|
||||
self.assertEqual(
|
||||
market_lines,
|
||||
[
|
||||
"SZ000001\t2024-01-02\t2024-01-11",
|
||||
"SZ000002\t2024-01-09\t2024-01-11",
|
||||
"SH600000\t2024-01-02\t2024-01-08",
|
||||
],
|
||||
)
|
||||
self.assertTrue(verify_qlib_provider(output)["ok"])
|
||||
|
||||
def test_pit_universe_fails_closed_on_missing_anchor_or_bad_roster(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
mirror = self._mirror(root, advanced=True)
|
||||
state = mirror / "data/state.sqlite3"
|
||||
connection = sqlite3.connect(state)
|
||||
connection.execute(
|
||||
"DELETE FROM jobs WHERE partition_key = ?",
|
||||
("index_code=000905_SH/year=2023",),
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
with self.assertRaisesRegex(
|
||||
TushareMirrorError,
|
||||
"missing required partitions",
|
||||
):
|
||||
build_qlib_provider(
|
||||
mirror,
|
||||
root / "provider-missing-anchor",
|
||||
start="2024-01-02",
|
||||
end="2024-01-11",
|
||||
benchmark_index_code="000905.SH",
|
||||
universe_index_code="000905.SH",
|
||||
expected_constituent_count=2,
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
mirror = self._mirror(root, advanced=True)
|
||||
with self.assertRaisesRegex(
|
||||
TushareMirrorError,
|
||||
"constituents; expected 3",
|
||||
):
|
||||
build_qlib_provider(
|
||||
mirror,
|
||||
root / "provider-bad-roster",
|
||||
start="2024-01-02",
|
||||
end="2024-01-11",
|
||||
benchmark_index_code="000905.SH",
|
||||
universe_index_code="000905.SH",
|
||||
expected_constituent_count=3,
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
ValueError,
|
||||
"requires benchmark_index_code",
|
||||
):
|
||||
build_qlib_provider(
|
||||
mirror,
|
||||
root / "provider-synthetic-benchmark",
|
||||
start="2024-01-02",
|
||||
end="2024-01-11",
|
||||
universe_index_code="000905.SH",
|
||||
expected_constituent_count=2,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -76,6 +76,8 @@ class PublicStatusTests(unittest.TestCase):
|
||||
self.assertFalse(payload["marketScope"]["bseTradingEnabled"])
|
||||
self.assertIn("北交所", payload["marketScope"]["statement"])
|
||||
self.assertIn("fail closed", payload["marketScope"]["statement"])
|
||||
self.assertEqual(payload["summary"]["testsRecorded"], 325)
|
||||
self.assertEqual(payload["summary"]["optionalSkipsRecorded"], 1)
|
||||
|
||||
def test_public_payload_is_deliberately_non_secret_and_non_local(self):
|
||||
payload = _read_payload()
|
||||
@@ -99,6 +101,25 @@ class PublicStatusTests(unittest.TestCase):
|
||||
self.assertFalse(Path(evidence["path"]).is_absolute())
|
||||
self.assertNotIn("..", Path(evidence["path"]).parts)
|
||||
|
||||
def test_public_tushare_claims_cannot_regress_to_early_mirror(self):
|
||||
payload = _read_payload()
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True)
|
||||
scorecard = (PROJECT / "gate_scorecard.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
stale_claims = (
|
||||
"93 个",
|
||||
"8.2%",
|
||||
"1993-10-29",
|
||||
"CSI500-2018-2025/de9657cd",
|
||||
"de5b96253ff765fa4e4be9736ddc2fad528f72c7721964313956d44b02969173",
|
||||
"3c550c0ba986f7eb736b421ac318fdc2a58eacaa65cfbdeaf6b3f920b5288304",
|
||||
"aed22a7b83e4defd45c4874e568d6aeafdabe43987cea301e0c1d09543dccb7c",
|
||||
)
|
||||
for stale_claim in stale_claims:
|
||||
self.assertNotIn(stale_claim, encoded)
|
||||
self.assertNotIn(stale_claim, scorecard)
|
||||
|
||||
def test_static_page_has_fail_closed_fallback_and_no_local_paths(self):
|
||||
page = INDEX_PATH.read_text(encoding="utf-8")
|
||||
self.assertIn('src="./status-data.js"', page)
|
||||
@@ -272,6 +293,10 @@ class PublicStatusTests(unittest.TestCase):
|
||||
evidence["joinquant-target-package-evidence"]["path"],
|
||||
"docs/JOINQUANT_TARGET_PACKAGE_EVIDENCE_2026-07-26.md",
|
||||
)
|
||||
self.assertEqual(
|
||||
evidence["tushare-csi500-research-run"]["path"],
|
||||
"evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json",
|
||||
)
|
||||
|
||||
gaps = payload["deliveryGaps"]
|
||||
self.assertGreaterEqual(len(gaps["internal"]), 3)
|
||||
@@ -374,7 +399,7 @@ class PublicStatusTests(unittest.TestCase):
|
||||
def test_freshness_uses_daily_bars_not_the_later_trade_calendar(self):
|
||||
payload = _read_payload()
|
||||
freshness = {item["id"]: item for item in payload["freshness"]}
|
||||
self.assertEqual(freshness["project-audit"]["value"], "2026-07-30")
|
||||
self.assertEqual(freshness["project-audit"]["value"], "2026-07-31")
|
||||
self.assertEqual(
|
||||
freshness["joinquant-hosted-market-window"]["marketDataThrough"],
|
||||
"2024-12-31",
|
||||
@@ -394,17 +419,67 @@ class PublicStatusTests(unittest.TestCase):
|
||||
self.assertIn("G9", target_package["semantics"])
|
||||
|
||||
tushare = freshness["tushare-daily-market-data"]
|
||||
self.assertEqual(tushare["completedPartitions"], 35)
|
||||
self.assertEqual(tushare["plannedPartitions"], 428)
|
||||
self.assertAlmostEqual(tushare["completionRatio"], 35 / 428, places=6)
|
||||
self.assertEqual(tushare["marketDataThrough"], "1993-10-29")
|
||||
self.assertEqual(tushare["status"], "research_only")
|
||||
self.assertEqual(tushare["completedPartitions"], 96)
|
||||
self.assertEqual(tushare["plannedPartitions"], 96)
|
||||
self.assertEqual(tushare["completionRatio"], 1.0)
|
||||
self.assertEqual(tushare["completedRows"], 8_813_885)
|
||||
self.assertEqual(tushare["marketDataFrom"], "2018-01-02")
|
||||
self.assertEqual(tushare["marketDataThrough"], "2025-12-31")
|
||||
self.assertEqual(
|
||||
(tushare["verifiedSnapshotJobs"], tushare["snapshotJobs"]),
|
||||
(221, 221),
|
||||
)
|
||||
self.assertEqual(tushare["providerFiles"], 10_011)
|
||||
self.assertEqual(tushare["providerBytes"], 71_707_194)
|
||||
self.assertEqual(tushare["calendarSessions"], 1_942)
|
||||
self.assertEqual(tushare["instrumentCount"], 1_111)
|
||||
self.assertEqual(tushare["selectedDailyRows"], 1_975_455)
|
||||
self.assertEqual(tushare["observedMembershipSnapshots"], 97)
|
||||
self.assertEqual(tushare["effectiveMembershipSnapshots"], 96)
|
||||
self.assertFalse(tushare["sameSessionMembershipUse"])
|
||||
self.assertTrue(tushare["sourceStable"])
|
||||
self.assertTrue(tushare["stateDbChangedDuringVerification"])
|
||||
self.assertTrue(tushare["converterSourceMatchesCurrent"])
|
||||
self.assertEqual(
|
||||
tushare["providerDataVersion"],
|
||||
"5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963",
|
||||
)
|
||||
self.assertEqual(
|
||||
tushare["providerManifestSha256"],
|
||||
"27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0",
|
||||
)
|
||||
self.assertEqual(tushare["sourceJobCount"], 221)
|
||||
self.assertEqual(
|
||||
tushare["lineageMatchedFields"],
|
||||
[
|
||||
"id",
|
||||
"path",
|
||||
"row_count",
|
||||
"byte_count",
|
||||
"sha256",
|
||||
"request_sha256",
|
||||
],
|
||||
)
|
||||
self.assertEqual(tushare["lineageMismatchCount"], 0)
|
||||
self.assertEqual(
|
||||
tushare["runSha256"],
|
||||
"803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3",
|
||||
)
|
||||
self.assertEqual(tushare["signalRows"], 176_615)
|
||||
self.assertEqual(tushare["portfolioDays"], 1_698)
|
||||
self.assertFalse(tushare["productionReady"])
|
||||
self.assertFalse(tushare["investmentValueClaim"])
|
||||
self.assertEqual(tushare["gateCredit"], [])
|
||||
source = (PROJECT / "docs" / "TUSHARE_LOCAL_DATA.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertRegex(source, r"`trade_cal`.*2026")
|
||||
self.assertNotIn("2026", tushare["marketDataThrough"])
|
||||
self.assertIn("daily", tushare["semantics"])
|
||||
self.assertIn("221/221", tushare["semantics"])
|
||||
self.assertIn(
|
||||
"tushare-csi500-research-run",
|
||||
tushare["evidenceLinks"],
|
||||
)
|
||||
|
||||
self.assertEqual(freshness["jqdata-real-snapshot"]["value"], "N/A")
|
||||
self.assertEqual(freshness["jqdata-real-snapshot"]["status"], "blocked")
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from tools.tushare_lineage import (
|
||||
TushareLineageError,
|
||||
_compare_source_jobs,
|
||||
_validate_snapshot,
|
||||
)
|
||||
|
||||
|
||||
def _canonical(value):
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
|
||||
|
||||
class TushareLineageTest(unittest.TestCase):
|
||||
def _snapshot(self):
|
||||
job = {
|
||||
"api_name": "daily",
|
||||
"byte_count": 123,
|
||||
"completed_at": "2026-07-31T00:00:00Z",
|
||||
"id": 7,
|
||||
"observed": {
|
||||
"row_count": 11,
|
||||
"sha256": "a" * 64,
|
||||
"size_bytes": 123,
|
||||
},
|
||||
"partition_key": "month=2024-01",
|
||||
"path": "data/parquet/daily/month=2024-01.parquet",
|
||||
"request_sha256": "b" * 64,
|
||||
"row_count": 11,
|
||||
"sha256": "a" * 64,
|
||||
"shadowed_completed_count": 0,
|
||||
"verified": True,
|
||||
}
|
||||
selection_identity = {
|
||||
key: value
|
||||
for key, value in job.items()
|
||||
if key not in {"observed", "verified"}
|
||||
}
|
||||
snapshot = {
|
||||
"format": "quant-os.tushare-scoped-snapshot/v2",
|
||||
"selected_jobs": [job],
|
||||
"selection_sha256": hashlib.sha256(
|
||||
_canonical({"selected_jobs": [selection_identity]})
|
||||
).hexdigest(),
|
||||
"verification": {
|
||||
"double_verified_jobs": 1,
|
||||
"selected_jobs_still_latest_after_verification": True,
|
||||
"verified_jobs": 1,
|
||||
},
|
||||
}
|
||||
snapshot["manifest_sha256"] = hashlib.sha256(
|
||||
_canonical(snapshot)
|
||||
).hexdigest()
|
||||
return snapshot
|
||||
|
||||
def test_exact_source_lineage_and_tamper_detection(self):
|
||||
snapshot = self._snapshot()
|
||||
jobs = _validate_snapshot(snapshot)
|
||||
provider_job = {
|
||||
"api_name": "daily",
|
||||
"byte_count": 123,
|
||||
"job_id": 7,
|
||||
"partition_key": "month=2024-01",
|
||||
"path": "data/parquet/daily/month=2024-01.parquet",
|
||||
"request_sha256": "b" * 64,
|
||||
"row_count": 11,
|
||||
"sha256": "a" * 64,
|
||||
}
|
||||
_compare_source_jobs(jobs, [provider_job])
|
||||
provider_job["sha256"] = "c" * 64
|
||||
with self.assertRaisesRegex(
|
||||
TushareLineageError,
|
||||
"source job fields differ",
|
||||
):
|
||||
_compare_source_jobs(jobs, [provider_job])
|
||||
|
||||
snapshot["selected_jobs"][0]["path"] = "../escape.parquet"
|
||||
snapshot_without_hash = dict(snapshot)
|
||||
snapshot_without_hash.pop("manifest_sha256")
|
||||
snapshot["manifest_sha256"] = hashlib.sha256(
|
||||
_canonical(snapshot_without_hash)
|
||||
).hexdigest()
|
||||
with self.assertRaisesRegex(
|
||||
TushareLineageError,
|
||||
"stay relative",
|
||||
):
|
||||
_validate_snapshot(snapshot)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,379 @@
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from tools.tushare_snapshot import (
|
||||
TushareSnapshotError,
|
||||
create_snapshot,
|
||||
write_manifest,
|
||||
)
|
||||
|
||||
|
||||
def _manifest_hash(manifest):
|
||||
payload = dict(manifest)
|
||||
payload.pop("manifest_sha256")
|
||||
encoded = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
class TushareSnapshotTest(unittest.TestCase):
|
||||
def _new_mirror(self, root: Path):
|
||||
mirror = root / "mirror"
|
||||
(mirror / "data/parquet").mkdir(parents=True)
|
||||
connection = sqlite3.connect(mirror / "data/state.sqlite3")
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE jobs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
api_name TEXT NOT NULL,
|
||||
partition_key TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
row_count INTEGER,
|
||||
byte_count INTEGER,
|
||||
sha256 TEXT,
|
||||
file_path TEXT,
|
||||
completed_at TEXT,
|
||||
params_json TEXT,
|
||||
fields TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
return mirror, connection
|
||||
|
||||
def _completed(
|
||||
self,
|
||||
mirror: Path,
|
||||
connection: sqlite3.Connection,
|
||||
*,
|
||||
job_id: int,
|
||||
api: str,
|
||||
partition: str,
|
||||
content: bytes = b"not-a-parquet",
|
||||
path_suffix: str = "",
|
||||
row_count: int = 1,
|
||||
):
|
||||
safe_partition = (
|
||||
partition.replace("/", "_").replace("=", "-") + path_suffix
|
||||
)
|
||||
path = mirror / "data/parquet" / api / f"{safe_partition}.parquet"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(content)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO jobs
|
||||
(id, api_name, partition_key, status, row_count, byte_count,
|
||||
sha256, file_path, completed_at, params_json, fields)
|
||||
VALUES (?, ?, ?, 'completed', ?, ?, ?, ?, ?, '[]', NULL)
|
||||
""",
|
||||
(
|
||||
job_id,
|
||||
api,
|
||||
partition,
|
||||
row_count,
|
||||
path.stat().st_size,
|
||||
hashlib.sha256(content).hexdigest(),
|
||||
str(path),
|
||||
f"2018-03-{min(job_id, 28):02d}T00:00:00Z",
|
||||
),
|
||||
)
|
||||
return path
|
||||
|
||||
def _core_fixture(self, root: Path):
|
||||
mirror, connection = self._new_mirror(root)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=1,
|
||||
api="daily",
|
||||
partition="month=2018-01",
|
||||
path_suffix="-old",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=2,
|
||||
api="daily",
|
||||
partition="month=2018-01",
|
||||
content=b"newer",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=3,
|
||||
api="daily",
|
||||
partition="month=2018-02",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=4,
|
||||
api="daily",
|
||||
partition="month=2018-03",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=5,
|
||||
api="adj_factor",
|
||||
partition="month=2018-01",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=6,
|
||||
api="adj_factor",
|
||||
partition="month=2018-02",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=7,
|
||||
api="trade_cal",
|
||||
partition="exchange=SSE/year=2018",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=8,
|
||||
api="stock_basic",
|
||||
partition="list_status=L",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=9,
|
||||
api="index_daily",
|
||||
partition="ts_code=000905_SH/year=2018",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=10,
|
||||
api="index_weight",
|
||||
partition="index_code=000905_SH/year=2018",
|
||||
)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=20,
|
||||
api="index_weight",
|
||||
partition="index_code=000905_SH/year=2017",
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO jobs
|
||||
(id, api_name, partition_key, status, row_count, byte_count,
|
||||
sha256, file_path, completed_at)
|
||||
VALUES
|
||||
(11, 'index_daily', 'ts_code=931643_CNI/year=2018', 'running',
|
||||
NULL, NULL, NULL, NULL, NULL)
|
||||
"""
|
||||
)
|
||||
connection.commit()
|
||||
return mirror, connection
|
||||
|
||||
def test_scoped_selection_shadows_old_completed_and_excludes_other_index(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
mirror, connection = self._core_fixture(root)
|
||||
connection.close()
|
||||
|
||||
manifest = create_snapshot(
|
||||
mirror,
|
||||
start="2018-01-15",
|
||||
end="2018-02-03",
|
||||
verify=False,
|
||||
)
|
||||
|
||||
self.assertEqual(manifest["summary"]["selected_jobs"], 9)
|
||||
self.assertEqual(
|
||||
manifest["summary"]["shadowed_completed_jobs"],
|
||||
1,
|
||||
)
|
||||
selected_ids = {job["id"] for job in manifest["selected_jobs"]}
|
||||
self.assertIn(2, selected_ids)
|
||||
self.assertNotIn(1, selected_ids)
|
||||
self.assertNotIn(4, selected_ids)
|
||||
self.assertIn(20, selected_ids)
|
||||
self.assertEqual(
|
||||
manifest["scope"]["index_weight_anchor_year"],
|
||||
2017,
|
||||
)
|
||||
self.assertEqual(
|
||||
manifest["excluded_live_jobs"][0]["reason"],
|
||||
"different_index_code",
|
||||
)
|
||||
self.assertEqual(
|
||||
manifest["verification"]["mode"],
|
||||
"not_performed",
|
||||
)
|
||||
self.assertTrue(
|
||||
manifest["verification"][
|
||||
"selected_jobs_still_latest_after_verification"
|
||||
]
|
||||
)
|
||||
self.assertRegex(
|
||||
manifest["selected_jobs"][0]["request_sha256"],
|
||||
r"^[0-9a-f]{64}$",
|
||||
)
|
||||
self.assertNotIn(
|
||||
"params_json",
|
||||
manifest["selected_jobs"][0],
|
||||
)
|
||||
self.assertEqual(
|
||||
manifest["manifest_sha256"],
|
||||
_manifest_hash(manifest),
|
||||
)
|
||||
serialized = json.dumps(manifest)
|
||||
self.assertNotIn(str(root), serialized)
|
||||
self.assertTrue(
|
||||
all(
|
||||
not Path(job["path"]).is_absolute()
|
||||
for job in manifest["selected_jobs"]
|
||||
)
|
||||
)
|
||||
|
||||
output = root / "snapshot.json"
|
||||
self.assertEqual(write_manifest(output, manifest), output.resolve())
|
||||
self.assertEqual(
|
||||
json.loads(output.read_text())["manifest_sha256"],
|
||||
manifest["manifest_sha256"],
|
||||
)
|
||||
|
||||
def test_in_scope_running_job_fails_closed(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
mirror, connection = self._core_fixture(root)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO jobs
|
||||
(id, api_name, partition_key, status, row_count, byte_count,
|
||||
sha256, file_path, completed_at)
|
||||
VALUES
|
||||
(12, 'daily', 'month=2018-02', 'running',
|
||||
NULL, NULL, NULL, NULL, NULL)
|
||||
"""
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
with self.assertRaisesRegex(
|
||||
TushareSnapshotError,
|
||||
"contains running/deferred",
|
||||
):
|
||||
create_snapshot(
|
||||
mirror,
|
||||
start="2018-01-15",
|
||||
end="2018-02-03",
|
||||
verify=False,
|
||||
)
|
||||
|
||||
def test_missing_partition_fails_closed(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
mirror, connection = self._new_mirror(root)
|
||||
self._completed(
|
||||
mirror,
|
||||
connection,
|
||||
job_id=1,
|
||||
api="daily",
|
||||
partition="month=2018-01",
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
with self.assertRaisesRegex(
|
||||
TushareSnapshotError,
|
||||
"missing daily:month=2018-02",
|
||||
):
|
||||
create_snapshot(
|
||||
mirror,
|
||||
start="2018-01-01",
|
||||
end="2018-02-28",
|
||||
apis=("daily",),
|
||||
verify=False,
|
||||
)
|
||||
|
||||
@unittest.skipUnless(
|
||||
importlib.util.find_spec("pyarrow") is not None,
|
||||
"pyarrow is needed for Parquet metadata verification",
|
||||
)
|
||||
def test_default_verification_detects_file_tampering(self):
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
mirror, connection = self._new_mirror(root)
|
||||
table = pa.table(
|
||||
{
|
||||
"ts_code": ["600000.SH", "000001.SZ"],
|
||||
"trade_date": ["20180102", "20180102"],
|
||||
}
|
||||
)
|
||||
parquet_path = (
|
||||
mirror / "data/parquet/daily/month-2018-01.parquet"
|
||||
)
|
||||
parquet_path.parent.mkdir(parents=True)
|
||||
pq.write_table(table, parquet_path)
|
||||
content = parquet_path.read_bytes()
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO jobs
|
||||
(id, api_name, partition_key, status, row_count, byte_count,
|
||||
sha256, file_path, completed_at, params_json, fields)
|
||||
VALUES
|
||||
(1, 'daily', 'month=2018-01', 'completed', 2, ?, ?, ?,
|
||||
'2018-02-01T00:00:00Z', '[]',
|
||||
'ts_code,trade_date')
|
||||
""",
|
||||
(
|
||||
len(content),
|
||||
hashlib.sha256(content).hexdigest(),
|
||||
str(parquet_path),
|
||||
),
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
manifest = create_snapshot(
|
||||
mirror,
|
||||
start="2018-01-01",
|
||||
end="2018-01-31",
|
||||
apis=("daily",),
|
||||
)
|
||||
self.assertEqual(manifest["verification"]["verified_jobs"], 1)
|
||||
self.assertEqual(
|
||||
manifest["verification"]["double_verified_jobs"],
|
||||
1,
|
||||
)
|
||||
self.assertEqual(
|
||||
manifest["selected_jobs"][0]["observed"]["row_count"],
|
||||
2,
|
||||
)
|
||||
|
||||
parquet_path.write_bytes(content + b"tampered")
|
||||
with self.assertRaisesRegex(TushareSnapshotError, "size mismatch"):
|
||||
create_snapshot(
|
||||
mirror,
|
||||
start="2018-01-01",
|
||||
end="2018-01-31",
|
||||
apis=("daily",),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+391
-43
@@ -9,7 +9,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Mapping, Sequence, Tuple
|
||||
|
||||
@@ -33,6 +33,9 @@ JOINQUANT_TARGET_PACKAGE_RUN_PATH = (
|
||||
JOINQUANT_TARGET_PACKAGE_DOC_PATH = (
|
||||
"docs/JOINQUANT_TARGET_PACKAGE_EVIDENCE_2026-07-26.md"
|
||||
)
|
||||
TUSHARE_RESEARCH_RUN_PATH = (
|
||||
"evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json"
|
||||
)
|
||||
JOINQUANT_TARGET_PACKAGE_RUN_ID = "2457a7c39a276e09e0fabf99e28978e1"
|
||||
JOINQUANT_TARGET_PACKAGE_OBSERVED_AT = "2026-07-26T08:50:15Z"
|
||||
JOINQUANT_TARGET_PACKAGE_BUNDLE_SHA256 = (
|
||||
@@ -47,6 +50,7 @@ SOURCE_PATHS: Tuple[str, ...] = (
|
||||
"dist/bundle_manifest.json",
|
||||
REACHABILITY_PATH,
|
||||
JOINQUANT_TARGET_PACKAGE_RUN_PATH,
|
||||
TUSHARE_RESEARCH_RUN_PATH,
|
||||
"configs/baseline.json",
|
||||
"README.md",
|
||||
"docs/ARCHITECTURE.md",
|
||||
@@ -566,11 +570,209 @@ def _validate_joinquant_target_package_run(
|
||||
}
|
||||
|
||||
|
||||
def _validate_tushare_research_run(
|
||||
project: Path,
|
||||
run: Mapping[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
expected_scope = {
|
||||
"index_code": "000905.SH",
|
||||
"start": "2018-01-01",
|
||||
"end": "2025-12-31",
|
||||
"daily_market_data_from": "2018-01-02",
|
||||
"daily_market_data_through": "2025-12-31",
|
||||
"daily_partitions_completed": 96,
|
||||
"daily_partitions_planned": 96,
|
||||
"daily_rows": 8_813_885,
|
||||
}
|
||||
expected_snapshot = {
|
||||
"selected_jobs": 221,
|
||||
"verified_jobs": 221,
|
||||
"double_verified_jobs": 221,
|
||||
"manifest_sha256": (
|
||||
"744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2"
|
||||
),
|
||||
"selection_sha256": (
|
||||
"66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5"
|
||||
),
|
||||
}
|
||||
expected_provider = {
|
||||
"verified": True,
|
||||
"data_version": (
|
||||
"5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963"
|
||||
),
|
||||
"tree_sha256": (
|
||||
"f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f"
|
||||
),
|
||||
"manifest_sha256": (
|
||||
"27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0"
|
||||
),
|
||||
"file_count": 10_011,
|
||||
"byte_count": 71_707_194,
|
||||
"calendar_sessions": 1_942,
|
||||
"instrument_count": 1_111,
|
||||
"selected_daily_rows": 1_975_455,
|
||||
"observed_membership_snapshots": 97,
|
||||
"effective_membership_snapshots": 96,
|
||||
"same_session_membership_use": False,
|
||||
"price_adjustment_complete": True,
|
||||
"benchmark": "SH000905",
|
||||
"converter_source_matches_current": True,
|
||||
}
|
||||
expected_lineage = {
|
||||
"verified": True,
|
||||
"source_job_count": 221,
|
||||
"matched_fields": [
|
||||
"id",
|
||||
"path",
|
||||
"row_count",
|
||||
"byte_count",
|
||||
"sha256",
|
||||
"request_sha256",
|
||||
],
|
||||
"mismatch_count": 0,
|
||||
"converter_source_matches_current": True,
|
||||
}
|
||||
expected_stability = {
|
||||
"state_db_changed_through_file_verification": True,
|
||||
"state_db_change_scope": "unrelated live downloader jobs",
|
||||
"snapshot_jobs_still_latest": True,
|
||||
"selected_scope_stable": True,
|
||||
"post_read_file_verification": True,
|
||||
"provider_source_jobs": 221,
|
||||
"snapshot_source_jobs": 221,
|
||||
"source_job_full_field_diff": 0,
|
||||
}
|
||||
expected_reproducibility = {
|
||||
"runs": 2,
|
||||
"byte_identical": True,
|
||||
"run_sha256": (
|
||||
"803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3"
|
||||
),
|
||||
"signal_rows": 176_615,
|
||||
"portfolio_days": 1_698,
|
||||
}
|
||||
expected_performance = {
|
||||
"strategy_cumulative_return": 0.2531519864,
|
||||
"benchmark_cumulative_return": 0.7895593835,
|
||||
"max_drawdown": -0.6041641366,
|
||||
}
|
||||
expected_limitations = {
|
||||
"stock_st": "denied",
|
||||
"qlib_limit_threshold": 0.095,
|
||||
"limit_rule_fidelity": (
|
||||
"market-level approximation; not per-symbol daily limit prices"
|
||||
),
|
||||
"research_only": True,
|
||||
}
|
||||
if (
|
||||
run.get("schema_version") != 1
|
||||
or run.get("record_type")
|
||||
!= "quant_os_tushare_qlib_research_run"
|
||||
or run.get("run_id") != "5bf19d2d"
|
||||
or run.get("audit_as_of") != "2026-07-31"
|
||||
or run.get("scope") != expected_scope
|
||||
or run.get("snapshot") != expected_snapshot
|
||||
or run.get("provider") != expected_provider
|
||||
or run.get("lineage") != expected_lineage
|
||||
or run.get("source_stability") != expected_stability
|
||||
or run.get("reproducibility") != expected_reproducibility
|
||||
or run.get("performance") != expected_performance
|
||||
or run.get("limitations") != expected_limitations
|
||||
or run.get("production_ready") is not False
|
||||
or run.get("investment_value_claim") is not False
|
||||
or run.get("gate_credit") != []
|
||||
):
|
||||
raise PublicStatusError(
|
||||
"Tushare CSI500 research machine record drifted"
|
||||
)
|
||||
encoded = json.dumps(run, ensure_ascii=False, sort_keys=True)
|
||||
if ABSOLUTE_PATH_PATTERN.search(encoded):
|
||||
raise PublicStatusError(
|
||||
"Tushare CSI500 research record contains an absolute local path"
|
||||
)
|
||||
path = _safe_repo_path(
|
||||
TUSHARE_RESEARCH_RUN_PATH,
|
||||
label="Tushare CSI500 research run",
|
||||
)
|
||||
if not (project / path).is_file():
|
||||
raise PublicStatusError(
|
||||
"tracked Tushare CSI500 research run is missing"
|
||||
)
|
||||
return {
|
||||
"runId": str(run["run_id"]),
|
||||
"auditAsOf": str(run["audit_as_of"]),
|
||||
"indexCode": str(expected_scope["index_code"]),
|
||||
"scopeStart": str(expected_scope["start"]),
|
||||
"scopeEnd": str(expected_scope["end"]),
|
||||
"marketDataFrom": str(expected_scope["daily_market_data_from"]),
|
||||
"marketDataThrough": str(
|
||||
expected_scope["daily_market_data_through"]
|
||||
),
|
||||
"completedPartitions": int(
|
||||
expected_scope["daily_partitions_completed"]
|
||||
),
|
||||
"plannedPartitions": int(
|
||||
expected_scope["daily_partitions_planned"]
|
||||
),
|
||||
"completedRows": int(expected_scope["daily_rows"]),
|
||||
"snapshotJobs": int(expected_snapshot["selected_jobs"]),
|
||||
"verifiedSnapshotJobs": int(expected_snapshot["verified_jobs"]),
|
||||
"snapshotManifestSha256": str(
|
||||
expected_snapshot["manifest_sha256"]
|
||||
),
|
||||
"snapshotSelectionSha256": str(
|
||||
expected_snapshot["selection_sha256"]
|
||||
),
|
||||
"providerDataVersion": str(expected_provider["data_version"]),
|
||||
"providerTreeSha256": str(expected_provider["tree_sha256"]),
|
||||
"providerManifestSha256": str(
|
||||
expected_provider["manifest_sha256"]
|
||||
),
|
||||
"sourceJobCount": int(expected_lineage["source_job_count"]),
|
||||
"lineageMatchedFields": list(expected_lineage["matched_fields"]),
|
||||
"lineageMismatchCount": int(expected_lineage["mismatch_count"]),
|
||||
"providerFiles": int(expected_provider["file_count"]),
|
||||
"providerBytes": int(expected_provider["byte_count"]),
|
||||
"calendarSessions": int(expected_provider["calendar_sessions"]),
|
||||
"instrumentCount": int(expected_provider["instrument_count"]),
|
||||
"selectedDailyRows": int(
|
||||
expected_provider["selected_daily_rows"]
|
||||
),
|
||||
"observedMembershipSnapshots": int(
|
||||
expected_provider["observed_membership_snapshots"]
|
||||
),
|
||||
"effectiveMembershipSnapshots": int(
|
||||
expected_provider["effective_membership_snapshots"]
|
||||
),
|
||||
"sameSessionMembershipUse": False,
|
||||
"sourceStable": True,
|
||||
"stateDbChangedDuringVerification": True,
|
||||
"converterSourceMatchesCurrent": True,
|
||||
"runSha256": str(expected_reproducibility["run_sha256"]),
|
||||
"signalRows": int(expected_reproducibility["signal_rows"]),
|
||||
"portfolioDays": int(expected_reproducibility["portfolio_days"]),
|
||||
"strategyCumulativeReturn": float(
|
||||
expected_performance["strategy_cumulative_return"]
|
||||
),
|
||||
"benchmarkCumulativeReturn": float(
|
||||
expected_performance["benchmark_cumulative_return"]
|
||||
),
|
||||
"maxDrawdown": float(expected_performance["max_drawdown"]),
|
||||
"stockStAvailable": False,
|
||||
"limitThreshold": float(
|
||||
expected_limitations["qlib_limit_threshold"]
|
||||
),
|
||||
"productionReady": False,
|
||||
"investmentValueClaim": False,
|
||||
"gateCredit": [],
|
||||
}
|
||||
|
||||
|
||||
def _extract_test_summary(facts: Sequence[Any]) -> Tuple[int | None, int | None]:
|
||||
for fact in facts:
|
||||
text = str(fact)
|
||||
match = re.search(
|
||||
r"completed\s+(\d+)\s+tests.*?(\d+)\s+optional-runtime skips",
|
||||
r"completed\s+(\d+)\s+tests.*?(\d+)\s+optional-runtime skips?",
|
||||
text,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
@@ -618,6 +820,8 @@ def _platform_evidence_links(name: str) -> List[str]:
|
||||
)
|
||||
if "Qlib" in name or "Tushare" in name:
|
||||
links.append("tushare-data")
|
||||
if "Tushare" in name:
|
||||
links.append("tushare-csi500-research-run")
|
||||
if "QMT" in name or "XtTrader" in name:
|
||||
links.append("platform-deployment")
|
||||
return list(dict.fromkeys(links))
|
||||
@@ -719,6 +923,12 @@ def _build_evidence(project: Path) -> List[Dict[str, Any]]:
|
||||
"docs/TUSHARE_LOCAL_DATA.md",
|
||||
"runtime_evidence",
|
||||
),
|
||||
(
|
||||
"tushare-csi500-research-run",
|
||||
"Tushare CSI500 2018-2025 reproducible research run",
|
||||
TUSHARE_RESEARCH_RUN_PATH,
|
||||
"machine_record",
|
||||
),
|
||||
(
|
||||
"platform-deployment",
|
||||
"Platform deployment runbook",
|
||||
@@ -788,13 +998,20 @@ def _build_layers(
|
||||
"claim": "不可变快照、PIT 字段合同和 Tushare→Qlib 技术链路已有本地合同/运行证据；这些 provider-data 路径尚未接成当前 Ridge 五层候选。",
|
||||
"capabilities": [
|
||||
"JQData raw/factor/limits/paused/PIT membership/ST snapshot",
|
||||
"Tushare completed-Parquet inventory and immutable Qlib provider",
|
||||
"Tushare scoped source release and verified immutable Qlib provider",
|
||||
"semantic manifest and data-version verification",
|
||||
],
|
||||
"limitations": [
|
||||
"尚无授权 JQData 真实快照与许可 lineage；Tushare 镜像仅部分完成且缺生产关键字段；当前五层候选仍使用 synthetic 数据。"
|
||||
"Tushare 2018—2025 中证 500 scoped research 链已验证，但 live raw "
|
||||
"仍可变、stock_st 不可用、9.5% 涨跌停为市场级近似，且尚未接入"
|
||||
"当前 Ridge 五层候选；JQData 授权快照与许可 lineage 仍缺失。"
|
||||
],
|
||||
"evidenceLinks": [
|
||||
"architecture",
|
||||
"tushare-data",
|
||||
"tushare-csi500-research-run",
|
||||
"scorecard",
|
||||
],
|
||||
"evidenceLinks": ["architecture", "tushare-data", "scorecard"],
|
||||
"releaseVerified": False,
|
||||
},
|
||||
{
|
||||
@@ -1046,6 +1263,7 @@ def _build_freshness(
|
||||
scorecard: Mapping[str, Any],
|
||||
joinquant_markdown: str,
|
||||
joinquant_target_package_run: Mapping[str, Any],
|
||||
tushare_research_run: Mapping[str, Any],
|
||||
tushare_markdown: str,
|
||||
baseline_shadow_days: int,
|
||||
production_shadow_days: int,
|
||||
@@ -1061,49 +1279,100 @@ def _build_freshness(
|
||||
)
|
||||
joinquant_start, joinquant_end = period_match.groups()
|
||||
|
||||
daily_match = re.search(
|
||||
r"(?m)^\|\s*`daily`\s*\|\s*([\d,]+)\s*\|\s*([\d,]+)\s*\|\s*"
|
||||
r"(\d{4}-\d{2}-\d{2})[—-](\d{4}-\d{2}-\d{2})",
|
||||
tushare_markdown,
|
||||
)
|
||||
plan_match = re.search(
|
||||
r"`daily`\s*原计划覆盖.*?共\s*(\d+)\s*个月；当前完成\s*"
|
||||
r"(\d+)\s*个月，约\s*([\d.]+)%",
|
||||
tushare_markdown,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
tushare_fields = {
|
||||
key: re.search(
|
||||
rf"(?m)^{key}\s*=\s*([^\s]+)\s*$",
|
||||
tushare_markdown,
|
||||
)
|
||||
for key in (
|
||||
"tushare_daily_completed_partitions",
|
||||
"tushare_daily_planned_partitions",
|
||||
"tushare_daily_completed_rows",
|
||||
"tushare_daily_market_data_from",
|
||||
"tushare_daily_market_data_through",
|
||||
)
|
||||
}
|
||||
ready_match = re.search(
|
||||
r"(?m)^production_ready\s*=\s*(true|false)\s*$",
|
||||
tushare_markdown,
|
||||
)
|
||||
if daily_match is None or plan_match is None or ready_match is None:
|
||||
if any(match is None for match in tushare_fields.values()) or ready_match is None:
|
||||
raise PublicStatusError(
|
||||
"Tushare evidence must declare daily coverage and production readiness"
|
||||
)
|
||||
(
|
||||
daily_partitions_text,
|
||||
daily_rows_text,
|
||||
tushare_start,
|
||||
tushare_end,
|
||||
) = daily_match.groups()
|
||||
planned_months_text, completed_months_text, percentage_text = plan_match.groups()
|
||||
completed_partitions = int(daily_partitions_text.replace(",", ""))
|
||||
completed_months = int(completed_months_text)
|
||||
planned_months = int(planned_months_text)
|
||||
if completed_partitions != completed_months:
|
||||
live_completed_partitions = int(
|
||||
tushare_fields["tushare_daily_completed_partitions"]
|
||||
.group(1)
|
||||
.replace(",", "")
|
||||
)
|
||||
live_planned_months = int(
|
||||
tushare_fields["tushare_daily_planned_partitions"]
|
||||
.group(1)
|
||||
.replace(",", "")
|
||||
)
|
||||
live_daily_rows = int(
|
||||
tushare_fields["tushare_daily_completed_rows"]
|
||||
.group(1)
|
||||
.replace(",", "")
|
||||
)
|
||||
live_tushare_start = tushare_fields[
|
||||
"tushare_daily_market_data_from"
|
||||
].group(1)
|
||||
live_tushare_end = tushare_fields[
|
||||
"tushare_daily_market_data_through"
|
||||
].group(1)
|
||||
if (
|
||||
live_planned_months <= 0
|
||||
or live_completed_partitions > live_planned_months
|
||||
or live_daily_rows <= 0
|
||||
):
|
||||
raise PublicStatusError(
|
||||
"Tushare daily partition count differs from completed-month statement"
|
||||
"Tushare daily partition coverage is invalid"
|
||||
)
|
||||
calculated_ratio = completed_months / planned_months
|
||||
recorded_percentage = float(percentage_text)
|
||||
if abs(calculated_ratio * 100.0 - recorded_percentage) > 0.1:
|
||||
for label, value in (
|
||||
("market_data_from", live_tushare_start),
|
||||
("market_data_through", live_tushare_end),
|
||||
):
|
||||
try:
|
||||
date.fromisoformat(value)
|
||||
except ValueError as exc:
|
||||
raise PublicStatusError(
|
||||
f"Tushare {label} must be an ISO date"
|
||||
) from exc
|
||||
if live_tushare_start > live_tushare_end:
|
||||
raise PublicStatusError(
|
||||
"Tushare recorded coverage percentage does not match month counts"
|
||||
"Tushare daily market-data start must not follow its end"
|
||||
)
|
||||
completed_partitions = int(
|
||||
tushare_research_run["completedPartitions"]
|
||||
)
|
||||
planned_months = int(tushare_research_run["plannedPartitions"])
|
||||
daily_rows = int(tushare_research_run["completedRows"])
|
||||
tushare_start = str(tushare_research_run["marketDataFrom"])
|
||||
tushare_end = str(tushare_research_run["marketDataThrough"])
|
||||
if (
|
||||
completed_partitions != planned_months
|
||||
or completed_partitions > live_completed_partitions
|
||||
or daily_rows > live_daily_rows
|
||||
or date.fromisoformat(tushare_start)
|
||||
< date.fromisoformat(live_tushare_start)
|
||||
or date.fromisoformat(tushare_end)
|
||||
> date.fromisoformat(live_tushare_end)
|
||||
):
|
||||
raise PublicStatusError(
|
||||
"Tushare scoped research record is inconsistent with live inventory"
|
||||
)
|
||||
calculated_ratio = completed_partitions / planned_months
|
||||
recorded_percentage = calculated_ratio * 100.0
|
||||
production_ready = ready_match.group(1) == "true"
|
||||
if production_ready:
|
||||
if (
|
||||
production_ready
|
||||
or tushare_research_run["productionReady"] is not False
|
||||
or tushare_research_run["investmentValueClaim"] is not False
|
||||
or tushare_research_run["gateCredit"] != []
|
||||
):
|
||||
raise PublicStatusError(
|
||||
"partial Tushare source must not be publicized as production ready"
|
||||
"current Tushare research bridge must not be publicized as production ready"
|
||||
)
|
||||
|
||||
audit_as_of = str(scorecard.get("audit_as_of"))
|
||||
@@ -1159,24 +1428,94 @@ def _build_freshness(
|
||||
},
|
||||
{
|
||||
"id": "tushare-daily-market-data",
|
||||
"label": "Tushare daily 行情覆盖",
|
||||
"status": "partial",
|
||||
"label": "Tushare CSI500 scoped research release",
|
||||
"status": "research_only",
|
||||
"marketDataFrom": tushare_start,
|
||||
"marketDataThrough": tushare_end,
|
||||
"completedPartitions": completed_partitions,
|
||||
"plannedPartitions": planned_months,
|
||||
"completedRows": int(daily_rows_text.replace(",", "")),
|
||||
"completedRows": daily_rows,
|
||||
"completionRatio": round(calculated_ratio, 6),
|
||||
"completionLabel": (
|
||||
f"{completed_partitions}/{planned_months}（约 {recorded_percentage:g}%）"
|
||||
),
|
||||
"value": (
|
||||
f"{completed_partitions}/{planned_months}（约 {recorded_percentage:g}%），"
|
||||
f"行情截至 {tushare_end}"
|
||||
f"{completed_partitions}/{planned_months} scoped daily 分区；"
|
||||
f"{tushare_research_run['verifiedSnapshotJobs']}/"
|
||||
f"{tushare_research_run['snapshotJobs']} source jobs；"
|
||||
f"行情 {tushare_start} — {tushare_end}"
|
||||
),
|
||||
"productionReady": production_ready,
|
||||
"semantics": "marketDataThrough 仅取自 daily 行情分区；绝不使用覆盖更晚的 trade_cal 日期。",
|
||||
"evidenceLinks": ["tushare-data", "scorecard"],
|
||||
"investmentValueClaim": False,
|
||||
"gateCredit": [],
|
||||
"snapshotJobs": int(tushare_research_run["snapshotJobs"]),
|
||||
"verifiedSnapshotJobs": int(
|
||||
tushare_research_run["verifiedSnapshotJobs"]
|
||||
),
|
||||
"providerDataVersion": str(
|
||||
tushare_research_run["providerDataVersion"]
|
||||
),
|
||||
"providerTreeSha256": str(
|
||||
tushare_research_run["providerTreeSha256"]
|
||||
),
|
||||
"providerManifestSha256": str(
|
||||
tushare_research_run["providerManifestSha256"]
|
||||
),
|
||||
"sourceJobCount": int(
|
||||
tushare_research_run["sourceJobCount"]
|
||||
),
|
||||
"lineageMatchedFields": list(
|
||||
tushare_research_run["lineageMatchedFields"]
|
||||
),
|
||||
"lineageMismatchCount": int(
|
||||
tushare_research_run["lineageMismatchCount"]
|
||||
),
|
||||
"providerFiles": int(tushare_research_run["providerFiles"]),
|
||||
"providerBytes": int(tushare_research_run["providerBytes"]),
|
||||
"calendarSessions": int(
|
||||
tushare_research_run["calendarSessions"]
|
||||
),
|
||||
"instrumentCount": int(
|
||||
tushare_research_run["instrumentCount"]
|
||||
),
|
||||
"selectedDailyRows": int(
|
||||
tushare_research_run["selectedDailyRows"]
|
||||
),
|
||||
"observedMembershipSnapshots": int(
|
||||
tushare_research_run["observedMembershipSnapshots"]
|
||||
),
|
||||
"effectiveMembershipSnapshots": int(
|
||||
tushare_research_run["effectiveMembershipSnapshots"]
|
||||
),
|
||||
"sameSessionMembershipUse": False,
|
||||
"sourceStable": True,
|
||||
"stateDbChangedDuringVerification": True,
|
||||
"converterSourceMatchesCurrent": True,
|
||||
"runSha256": str(tushare_research_run["runSha256"]),
|
||||
"signalRows": int(tushare_research_run["signalRows"]),
|
||||
"portfolioDays": int(tushare_research_run["portfolioDays"]),
|
||||
"strategyCumulativeReturn": float(
|
||||
tushare_research_run["strategyCumulativeReturn"]
|
||||
),
|
||||
"benchmarkCumulativeReturn": float(
|
||||
tushare_research_run["benchmarkCumulativeReturn"]
|
||||
),
|
||||
"maxDrawdown": float(tushare_research_run["maxDrawdown"]),
|
||||
"semantics": (
|
||||
"96/96 只表示 2018—2025 scoped daily 输入完整且 221/221 "
|
||||
"source jobs 经双重校验；全局 ledger 因无关下载任务发生变化，"
|
||||
"但 scoped jobs 仍为 latest；lineage 工具按 id/path/row_count/"
|
||||
"byte_count/sha256/request_sha256 验证 converter 221/221 "
|
||||
"零差异。"
|
||||
"universe 成分次一交易日生效。该 Qlib "
|
||||
"run 未使用逐股涨跌停，stock_st 不可用，不代表 production-ready "
|
||||
"或投资价值。"
|
||||
),
|
||||
"evidenceLinks": [
|
||||
"tushare-csi500-research-run",
|
||||
"tushare-data",
|
||||
"scorecard",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "jqdata-real-snapshot",
|
||||
@@ -1413,6 +1752,9 @@ def build_public_status(
|
||||
joinquant_target_package_run_raw = _load_json(
|
||||
project / JOINQUANT_TARGET_PACKAGE_RUN_PATH
|
||||
)
|
||||
tushare_research_run_raw = _load_json(
|
||||
project / TUSHARE_RESEARCH_RUN_PATH
|
||||
)
|
||||
baseline = _load_json(project / "configs" / "baseline.json")
|
||||
platform_markdown = (project / "docs" / "PLATFORM_MATRIX.md").read_text(
|
||||
encoding="utf-8"
|
||||
@@ -1501,6 +1843,10 @@ def build_public_status(
|
||||
project,
|
||||
joinquant_target_package_run_raw,
|
||||
)
|
||||
tushare_research_run = _validate_tushare_research_run(
|
||||
project,
|
||||
tushare_research_run_raw,
|
||||
)
|
||||
maturity80 = _build_maturity80(
|
||||
project,
|
||||
production_80_standard,
|
||||
@@ -1582,7 +1928,7 @@ def build_public_status(
|
||||
"JoinQuant portable_momentum_smoke hosted daily backtest (not Baseline/TargetPackage evidence)",
|
||||
"Qlib native momentum fixture",
|
||||
"Qlib Alpha158 + LightGBM fixture",
|
||||
"Tushare completed-partition Qlib momentum smoke",
|
||||
"Tushare CSI500 2018—2025 scoped Qlib momentum reproducibility run",
|
||||
],
|
||||
"gateCoverageLabel": f"{passed}/{len(gates)} passed",
|
||||
"productionReady": bool(maturity80["qualified"]),
|
||||
@@ -1630,6 +1976,7 @@ def build_public_status(
|
||||
scorecard=scorecard,
|
||||
joinquant_markdown=joinquant_markdown,
|
||||
joinquant_target_package_run=joinquant_target_package_run,
|
||||
tushare_research_run=tushare_research_run,
|
||||
tushare_markdown=tushare_markdown,
|
||||
baseline_shadow_days=baseline_shadow_days,
|
||||
production_shadow_days=production_shadow_days,
|
||||
@@ -1672,6 +2019,7 @@ def build_public_status(
|
||||
"production80Assessment": PRODUCTION_80_ASSESSMENT_PATH,
|
||||
"bundleManifest": "dist/bundle_manifest.json",
|
||||
"releaseReachability": REACHABILITY_PATH,
|
||||
"tushareResearchRun": TUSHARE_RESEARCH_RUN_PATH,
|
||||
"platformMatrix": "docs/PLATFORM_MATRIX.md",
|
||||
"baselineConfig": "configs/baseline.json",
|
||||
"sourceSetSha256": source_set_sha,
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify exact lineage from a Tushare scoped manifest to a Qlib provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any, Mapping, Optional, Sequence
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from adapters.tushare_local import MANIFEST_NAME, verify_qlib_provider
|
||||
|
||||
|
||||
MATCHED_FIELDS = (
|
||||
("id", "job_id"),
|
||||
("path", "path"),
|
||||
("row_count", "row_count"),
|
||||
("byte_count", "byte_count"),
|
||||
("sha256", "sha256"),
|
||||
("request_sha256", "request_sha256"),
|
||||
)
|
||||
|
||||
|
||||
class TushareLineageError(RuntimeError):
|
||||
"""Raised when a scoped manifest and provider do not match exactly."""
|
||||
|
||||
|
||||
def _canonical_json(value: Mapping[str, Any]) -> bytes:
|
||||
return json.dumps(
|
||||
dict(value),
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _load_object(path: Path, *, label: str) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise TushareLineageError(f"cannot read {label}: {path}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise TushareLineageError(f"{label} must be a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_snapshot(
|
||||
snapshot: Mapping[str, Any],
|
||||
) -> Sequence[Mapping[str, Any]]:
|
||||
if snapshot.get("format") != "quant-os.tushare-scoped-snapshot/v2":
|
||||
raise TushareLineageError("source snapshot must use scoped format v2")
|
||||
manifest_sha = str(snapshot.get("manifest_sha256") or "")
|
||||
without_manifest_sha = dict(snapshot)
|
||||
without_manifest_sha.pop("manifest_sha256", None)
|
||||
computed_manifest_sha = hashlib.sha256(
|
||||
_canonical_json(without_manifest_sha)
|
||||
).hexdigest()
|
||||
if manifest_sha != computed_manifest_sha:
|
||||
raise TushareLineageError("source snapshot manifest SHA-256 mismatch")
|
||||
|
||||
jobs = snapshot.get("selected_jobs")
|
||||
if not isinstance(jobs, list) or not jobs:
|
||||
raise TushareLineageError("source snapshot has no selected_jobs")
|
||||
selection_identity = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for raw_job in jobs:
|
||||
if not isinstance(raw_job, Mapping):
|
||||
raise TushareLineageError("source snapshot job must be an object")
|
||||
key = (
|
||||
str(raw_job.get("api_name") or ""),
|
||||
str(raw_job.get("partition_key") or ""),
|
||||
)
|
||||
if not all(key) or key in seen:
|
||||
raise TushareLineageError(
|
||||
"source snapshot has empty or duplicate job keys"
|
||||
)
|
||||
seen.add(key)
|
||||
path = Path(str(raw_job.get("path") or ""))
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise TushareLineageError(
|
||||
"source snapshot job paths must stay relative to the mirror"
|
||||
)
|
||||
observed = raw_job.get("observed")
|
||||
if raw_job.get("verified") is not True or not isinstance(
|
||||
observed,
|
||||
Mapping,
|
||||
):
|
||||
raise TushareLineageError(
|
||||
"every source snapshot job must carry verified observations"
|
||||
)
|
||||
if (
|
||||
observed.get("row_count") != raw_job.get("row_count")
|
||||
or observed.get("size_bytes") != raw_job.get("byte_count")
|
||||
or observed.get("sha256") != raw_job.get("sha256")
|
||||
):
|
||||
raise TushareLineageError(
|
||||
"source snapshot observed file evidence is inconsistent"
|
||||
)
|
||||
selection_identity.append(
|
||||
{
|
||||
field: value
|
||||
for field, value in raw_job.items()
|
||||
if field not in {"observed", "verified"}
|
||||
}
|
||||
)
|
||||
computed_selection_sha = hashlib.sha256(
|
||||
_canonical_json({"selected_jobs": selection_identity})
|
||||
).hexdigest()
|
||||
if computed_selection_sha != snapshot.get("selection_sha256"):
|
||||
raise TushareLineageError("source snapshot selection SHA-256 mismatch")
|
||||
|
||||
verification = snapshot.get("verification")
|
||||
if (
|
||||
not isinstance(verification, Mapping)
|
||||
or verification.get("verified_jobs") != len(jobs)
|
||||
or verification.get("double_verified_jobs") != len(jobs)
|
||||
or verification.get(
|
||||
"selected_jobs_still_latest_after_verification"
|
||||
)
|
||||
is not True
|
||||
):
|
||||
raise TushareLineageError(
|
||||
"source snapshot lacks closed double-verification evidence"
|
||||
)
|
||||
return jobs
|
||||
|
||||
|
||||
def _compare_source_jobs(
|
||||
snapshot_jobs: Sequence[Mapping[str, Any]],
|
||||
provider_jobs: Sequence[Mapping[str, Any]],
|
||||
) -> None:
|
||||
def key(job: Mapping[str, Any]) -> tuple[str, str]:
|
||||
return (
|
||||
str(job.get("api_name") or ""),
|
||||
str(job.get("partition_key") or ""),
|
||||
)
|
||||
|
||||
snapshot_by_key = {key(job): job for job in snapshot_jobs}
|
||||
provider_by_key = {key(job): job for job in provider_jobs}
|
||||
if len(snapshot_by_key) != len(snapshot_jobs):
|
||||
raise TushareLineageError("source snapshot has duplicate job keys")
|
||||
if len(provider_by_key) != len(provider_jobs):
|
||||
raise TushareLineageError("provider manifest has duplicate job keys")
|
||||
if set(snapshot_by_key) != set(provider_by_key):
|
||||
missing = sorted(set(snapshot_by_key).difference(provider_by_key))
|
||||
extra = sorted(set(provider_by_key).difference(snapshot_by_key))
|
||||
raise TushareLineageError(
|
||||
"source job sets differ: "
|
||||
f"missing_from_provider={missing[:5]}, "
|
||||
f"extra_in_provider={extra[:5]}"
|
||||
)
|
||||
mismatches: list[str] = []
|
||||
for job_key in sorted(snapshot_by_key):
|
||||
source = snapshot_by_key[job_key]
|
||||
provider = provider_by_key[job_key]
|
||||
for source_field, provider_field in MATCHED_FIELDS:
|
||||
if source.get(source_field) != provider.get(provider_field):
|
||||
mismatches.append(
|
||||
f"{job_key[0]}:{job_key[1]}:{source_field}"
|
||||
)
|
||||
if mismatches:
|
||||
raise TushareLineageError(
|
||||
"source job fields differ: " + ", ".join(mismatches[:12])
|
||||
)
|
||||
|
||||
|
||||
def verify_lineage(
|
||||
source_manifest: str | Path,
|
||||
provider_dir: str | Path,
|
||||
) -> dict[str, Any]:
|
||||
source_path = Path(source_manifest).expanduser().resolve()
|
||||
provider = Path(provider_dir).expanduser().resolve()
|
||||
snapshot = _load_object(source_path, label="source snapshot")
|
||||
snapshot_jobs = _validate_snapshot(snapshot)
|
||||
provider_verification = verify_qlib_provider(provider)
|
||||
provider_manifest = _load_object(
|
||||
provider / MANIFEST_NAME,
|
||||
label="provider manifest",
|
||||
)
|
||||
provider_jobs = provider_manifest.get("source_jobs")
|
||||
if not isinstance(provider_jobs, list):
|
||||
raise TushareLineageError("provider manifest has no source_jobs")
|
||||
_compare_source_jobs(snapshot_jobs, provider_jobs)
|
||||
return {
|
||||
"artifact_type": "quant_os_tushare_source_provider_lineage",
|
||||
"ok": True,
|
||||
"snapshot_format": snapshot["format"],
|
||||
"snapshot_manifest_sha256": snapshot["manifest_sha256"],
|
||||
"snapshot_selection_sha256": snapshot["selection_sha256"],
|
||||
"source_job_count": len(snapshot_jobs),
|
||||
"matched_fields": [
|
||||
source_field for source_field, _ in MATCHED_FIELDS
|
||||
],
|
||||
"mismatch_count": 0,
|
||||
"provider": provider_verification,
|
||||
"tool_source_sha256": hashlib.sha256(
|
||||
Path(__file__).resolve().read_bytes()
|
||||
).hexdigest(),
|
||||
"gate_credit": [],
|
||||
"investment_value_claim": False,
|
||||
}
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: Mapping[str, Any]) -> None:
|
||||
destination = path.expanduser().resolve()
|
||||
if destination.exists() and destination.is_dir():
|
||||
raise ValueError("output points to a directory")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
provider_bytes = (
|
||||
json.dumps(
|
||||
dict(payload),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
).encode("utf-8")
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{destination.name}.",
|
||||
suffix=".tmp",
|
||||
dir=destination.parent,
|
||||
)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(provider_bytes)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
if temporary.exists():
|
||||
temporary.unlink()
|
||||
|
||||
|
||||
def _main(argv: Optional[list[str]] = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-manifest", required=True)
|
||||
parser.add_argument("--provider-dir", required=True)
|
||||
parser.add_argument("--output-json")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
result = verify_lineage(
|
||||
args.source_manifest,
|
||||
args.provider_dir,
|
||||
)
|
||||
if args.output_json:
|
||||
output = Path(args.output_json).expanduser().resolve()
|
||||
provider = Path(args.provider_dir).expanduser().resolve()
|
||||
if output == provider or provider in output.parents:
|
||||
raise ValueError(
|
||||
"output-json must be outside the immutable provider "
|
||||
"directory"
|
||||
)
|
||||
_write_json(output, result)
|
||||
except (OSError, ValueError, TushareLineageError) as exc:
|
||||
parser.error(str(exc))
|
||||
print(
|
||||
json.dumps(
|
||||
result,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(_main())
|
||||
@@ -8,6 +8,10 @@ from pathlib import Path
|
||||
import sys
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from adapters.tushare_local import (
|
||||
TushareMirrorError,
|
||||
build_qlib_provider,
|
||||
@@ -16,6 +20,20 @@ from adapters.tushare_local import (
|
||||
)
|
||||
|
||||
|
||||
def _assert_json_outside_provider(
|
||||
output_json: Optional[str],
|
||||
provider_dir: str,
|
||||
) -> None:
|
||||
if output_json is None:
|
||||
return
|
||||
destination = Path(output_json).expanduser().resolve()
|
||||
provider = Path(provider_dir).expanduser().resolve()
|
||||
if destination == provider or provider in destination.parents:
|
||||
raise ValueError(
|
||||
"output-json must be outside the immutable provider directory"
|
||||
)
|
||||
|
||||
|
||||
def _write_json(path: Optional[str], payload: Mapping[str, Any]) -> None:
|
||||
if path is None:
|
||||
return
|
||||
@@ -52,6 +70,22 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
build.add_argument("--end", required=True)
|
||||
build.add_argument("--market-name", default="tushare_a")
|
||||
build.add_argument("--benchmark-symbol", default="SH999999")
|
||||
build.add_argument(
|
||||
"--benchmark-index-code",
|
||||
help="use real Tushare index_daily data, for example 000905.SH",
|
||||
)
|
||||
build.add_argument(
|
||||
"--universe-index-code",
|
||||
help="build point-in-time market intervals from index_weight snapshots",
|
||||
)
|
||||
build.add_argument(
|
||||
"--expected-constituent-count",
|
||||
type=int,
|
||||
help=(
|
||||
"required for unknown index codes; known indices such as "
|
||||
"000905.SH use a built-in exact count"
|
||||
),
|
||||
)
|
||||
build.add_argument("--minimum-observations", type=int, default=60)
|
||||
build.add_argument(
|
||||
"--allow-unadjusted",
|
||||
@@ -81,6 +115,10 @@ def _main(argv: Optional[list[str]] = None) -> int:
|
||||
verify_files=not args.skip_file_verification,
|
||||
)
|
||||
elif args.command == "build":
|
||||
_assert_json_outside_provider(
|
||||
args.output_json,
|
||||
args.output_dir,
|
||||
)
|
||||
result = build_qlib_provider(
|
||||
args.mirror_root,
|
||||
args.output_dir,
|
||||
@@ -91,8 +129,15 @@ def _main(argv: Optional[list[str]] = None) -> int:
|
||||
minimum_observations=args.minimum_observations,
|
||||
allow_unadjusted=args.allow_unadjusted,
|
||||
expand_range=args.ohlc_policy == "expand-range",
|
||||
benchmark_index_code=args.benchmark_index_code,
|
||||
universe_index_code=args.universe_index_code,
|
||||
expected_constituent_count=args.expected_constituent_count,
|
||||
)
|
||||
else:
|
||||
_assert_json_outside_provider(
|
||||
args.output_json,
|
||||
args.provider_dir,
|
||||
)
|
||||
result = verify_qlib_provider(args.provider_dir)
|
||||
_write_json(args.output_json, result)
|
||||
except (OSError, ValueError, TushareMirrorError) as exc:
|
||||
|
||||
@@ -0,0 +1,977 @@
|
||||
"""Create an auditable, scoped manifest from a live local Tushare mirror.
|
||||
|
||||
The command is intentionally read-only with respect to the mirror. It records
|
||||
the latest completed job for every selected ``(api_name, partition_key)`` in one
|
||||
SQLite read transaction, then optionally verifies the immutable Parquet files.
|
||||
It never reads Tushare credentials and never copies source data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import date, datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any, Iterable, Mapping, Optional, Sequence
|
||||
|
||||
|
||||
TOOL_VERSION = "2"
|
||||
DEFAULT_APIS = (
|
||||
"daily",
|
||||
"adj_factor",
|
||||
"trade_cal",
|
||||
"stock_basic",
|
||||
"index_daily",
|
||||
"index_weight",
|
||||
)
|
||||
DEFAULT_INDEX_CODES = ("000905.SH",)
|
||||
BLOCKING_LIVE_STATUSES = frozenset(("running", "deferred"))
|
||||
_MONTH_RE = re.compile(r"(?:^|[/_])month=(\d{4}-\d{2})(?:$|[/_])")
|
||||
_YEAR_RE = re.compile(r"(?:^|[/_])year=(\d{4})(?:$|[/_])")
|
||||
_INDEX_CODE_RE = re.compile(
|
||||
r"(?:^|/)(?:ts_code|index_code)=([^/]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_API_RE = re.compile(r"^[A-Za-z0-9_]+$")
|
||||
_INDEX_INPUT_RE = re.compile(r"^([A-Za-z0-9]+)[._]([A-Za-z0-9]+)$")
|
||||
|
||||
|
||||
class TushareSnapshotError(RuntimeError):
|
||||
"""Raised when a requested snapshot cannot be frozen safely."""
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace(
|
||||
"+00:00",
|
||||
"Z",
|
||||
)
|
||||
|
||||
|
||||
def _canonical_json(payload: Mapping[str, Any]) -> bytes:
|
||||
return json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _normalize_index_code(raw: str) -> str:
|
||||
value = raw.strip().upper()
|
||||
match = _INDEX_INPUT_RE.fullmatch(value)
|
||||
if match is None:
|
||||
raise ValueError(
|
||||
f"invalid index code {raw!r}; expected a value such as 000905.SH"
|
||||
)
|
||||
return f"{match.group(1)}.{match.group(2)}"
|
||||
|
||||
|
||||
def _partition_index_code(partition_key: str) -> Optional[str]:
|
||||
match = _INDEX_CODE_RE.search(partition_key)
|
||||
if match is None:
|
||||
return None
|
||||
raw = match.group(1)
|
||||
try:
|
||||
return _normalize_index_code(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _partition_month(partition_key: str) -> Optional[str]:
|
||||
match = _MONTH_RE.search(partition_key)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def _partition_year(partition_key: str) -> Optional[int]:
|
||||
match = _YEAR_RE.search(partition_key)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def _month_keys(start: date, end: date) -> list[str]:
|
||||
year = start.year
|
||||
month = start.month
|
||||
result: list[str] = []
|
||||
while (year, month) <= (end.year, end.month):
|
||||
result.append(f"{year:04d}-{month:02d}")
|
||||
month += 1
|
||||
if month == 13:
|
||||
year += 1
|
||||
month = 1
|
||||
return result
|
||||
|
||||
|
||||
def _parse_values(values: Sequence[str]) -> list[str]:
|
||||
result: list[str] = []
|
||||
for value in values:
|
||||
result.extend(part.strip() for part in value.split(",") if part.strip())
|
||||
return result
|
||||
|
||||
|
||||
def _validate_apis(apis: Iterable[str]) -> tuple[str, ...]:
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in apis:
|
||||
api = raw.strip()
|
||||
if not _API_RE.fullmatch(api):
|
||||
raise ValueError(f"invalid API name: {raw!r}")
|
||||
if api not in seen:
|
||||
seen.add(api)
|
||||
result.append(api)
|
||||
if not result:
|
||||
raise ValueError("at least one API must be requested")
|
||||
return tuple(result)
|
||||
|
||||
|
||||
def _scope_reason(
|
||||
api_name: str,
|
||||
partition_key: str,
|
||||
*,
|
||||
apis: set[str],
|
||||
months: set[str],
|
||||
years: set[int],
|
||||
index_weight_years: set[int],
|
||||
index_codes: set[str],
|
||||
) -> tuple[bool, str]:
|
||||
if api_name not in apis:
|
||||
return False, "api_not_requested"
|
||||
|
||||
if api_name in {"daily", "adj_factor"}:
|
||||
month = _partition_month(partition_key)
|
||||
if month is None:
|
||||
return False, "unrecognized_partition"
|
||||
return (True, "in_scope") if month in months else (
|
||||
False,
|
||||
"outside_date_range",
|
||||
)
|
||||
|
||||
if api_name == "trade_cal":
|
||||
year = _partition_year(partition_key)
|
||||
if year is None:
|
||||
return False, "unrecognized_partition"
|
||||
return (True, "in_scope") if year in years else (
|
||||
False,
|
||||
"outside_date_range",
|
||||
)
|
||||
|
||||
if api_name in {"index_daily", "index_weight"}:
|
||||
code = _partition_index_code(partition_key)
|
||||
if code not in index_codes:
|
||||
return False, "different_index_code"
|
||||
year = _partition_year(partition_key)
|
||||
if year is None:
|
||||
return False, "unrecognized_partition"
|
||||
accepted_years = (
|
||||
index_weight_years if api_name == "index_weight" else years
|
||||
)
|
||||
return (True, "in_scope") if year in accepted_years else (
|
||||
False,
|
||||
"outside_date_range",
|
||||
)
|
||||
|
||||
if api_name == "stock_basic":
|
||||
return True, "in_scope"
|
||||
|
||||
month = _partition_month(partition_key)
|
||||
if month is not None:
|
||||
return (True, "in_scope") if month in months else (
|
||||
False,
|
||||
"outside_date_range",
|
||||
)
|
||||
year = _partition_year(partition_key)
|
||||
if year is not None:
|
||||
return (True, "in_scope") if year in years else (
|
||||
False,
|
||||
"outside_date_range",
|
||||
)
|
||||
return True, "in_scope"
|
||||
|
||||
|
||||
def _state_file_identity(path: Path) -> dict[str, Any]:
|
||||
stat = path.stat()
|
||||
return {
|
||||
"device": stat.st_dev,
|
||||
"inode": stat.st_ino,
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"path": "data/state.sqlite3",
|
||||
"size_bytes": stat.st_size,
|
||||
}
|
||||
|
||||
|
||||
def _wal_identity(state_path: Path) -> Optional[dict[str, int]]:
|
||||
wal_path = state_path.with_name(state_path.name + "-wal")
|
||||
if not wal_path.exists():
|
||||
return None
|
||||
stat = wal_path.stat()
|
||||
return {
|
||||
"inode": stat.st_ino,
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"size_bytes": stat.st_size,
|
||||
}
|
||||
|
||||
|
||||
def _candidate_query(
|
||||
connection: sqlite3.Connection,
|
||||
api_name: str,
|
||||
*,
|
||||
months: Sequence[str],
|
||||
years: Sequence[int],
|
||||
index_codes: Sequence[str],
|
||||
) -> list[sqlite3.Row]:
|
||||
columns = (
|
||||
"id, api_name, partition_key, status, row_count, byte_count, "
|
||||
"sha256, file_path, completed_at, params_json, fields"
|
||||
)
|
||||
where = ["api_name = ?", "status = 'completed'"]
|
||||
parameters: list[Any] = [api_name]
|
||||
|
||||
if api_name in {"daily", "adj_factor"}:
|
||||
placeholders = ",".join("?" for _ in months)
|
||||
where.append(f"partition_key IN ({placeholders})")
|
||||
parameters.extend(f"month={month}" for month in months)
|
||||
elif api_name == "trade_cal":
|
||||
clauses = []
|
||||
for year in years:
|
||||
clauses.append("partition_key LIKE ?")
|
||||
parameters.append(f"%year={year}%")
|
||||
where.append("(" + " OR ".join(clauses) + ")")
|
||||
elif api_name in {"index_daily", "index_weight"}:
|
||||
code_clauses = []
|
||||
for code in index_codes:
|
||||
code_clauses.append("partition_key LIKE ?")
|
||||
parameters.append(f"%={code.replace('.', '_')}%")
|
||||
year_clauses = []
|
||||
for year in years:
|
||||
year_clauses.append("partition_key LIKE ?")
|
||||
parameters.append(f"%year={year}%")
|
||||
where.append("(" + " OR ".join(code_clauses) + ")")
|
||||
where.append("(" + " OR ".join(year_clauses) + ")")
|
||||
|
||||
query = f"SELECT {columns} FROM jobs WHERE " + " AND ".join(where)
|
||||
return list(connection.execute(query, parameters))
|
||||
|
||||
|
||||
def _assert_scope_complete(
|
||||
selected: Sequence[sqlite3.Row],
|
||||
*,
|
||||
apis: Sequence[str],
|
||||
months: Sequence[str],
|
||||
years: Sequence[int],
|
||||
index_weight_years: Sequence[int],
|
||||
index_codes: Sequence[str],
|
||||
) -> None:
|
||||
by_api: dict[str, list[sqlite3.Row]] = defaultdict(list)
|
||||
for row in selected:
|
||||
by_api[str(row["api_name"])].append(row)
|
||||
|
||||
missing: list[str] = []
|
||||
for api in apis:
|
||||
rows = by_api.get(api, [])
|
||||
if api in {"daily", "adj_factor"}:
|
||||
found = {
|
||||
value
|
||||
for value in (
|
||||
_partition_month(str(row["partition_key"])) for row in rows
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
missing.extend(
|
||||
f"{api}:month={month}" for month in months if month not in found
|
||||
)
|
||||
elif api == "trade_cal":
|
||||
found = {
|
||||
value
|
||||
for value in (
|
||||
_partition_year(str(row["partition_key"])) for row in rows
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
missing.extend(
|
||||
f"{api}:year={year}" for year in years if year not in found
|
||||
)
|
||||
elif api in {"index_daily", "index_weight"}:
|
||||
found = {
|
||||
(
|
||||
_partition_index_code(str(row["partition_key"])),
|
||||
_partition_year(str(row["partition_key"])),
|
||||
)
|
||||
for row in rows
|
||||
}
|
||||
required_years = (
|
||||
index_weight_years if api == "index_weight" else years
|
||||
)
|
||||
missing.extend(
|
||||
f"{api}:{code}/year={year}"
|
||||
for code in index_codes
|
||||
for year in required_years
|
||||
if (code, year) not in found
|
||||
)
|
||||
elif not rows:
|
||||
missing.append(f"{api}:no_completed_partition")
|
||||
|
||||
if missing:
|
||||
preview = ", ".join(missing[:12])
|
||||
suffix = "" if len(missing) <= 12 else f" (+{len(missing) - 12} more)"
|
||||
raise TushareSnapshotError(
|
||||
f"requested scope is incomplete; missing {preview}{suffix}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_source_file(mirror_root: Path, raw_path: str) -> tuple[Path, str]:
|
||||
path = Path(raw_path)
|
||||
if not path.is_absolute():
|
||||
path = mirror_root / path
|
||||
resolved = path.expanduser().resolve(strict=True)
|
||||
root = mirror_root.resolve(strict=True)
|
||||
try:
|
||||
relative = resolved.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise TushareSnapshotError(
|
||||
"source file escapes the declared mirror root"
|
||||
) from exc
|
||||
return resolved, relative.as_posix()
|
||||
|
||||
|
||||
def _parquet_row_count(path: Path) -> int:
|
||||
try:
|
||||
import pyarrow.parquet as parquet
|
||||
except ImportError as exc:
|
||||
raise TushareSnapshotError(
|
||||
"Parquet row verification requires pyarrow; install the Qlib "
|
||||
"research requirements or use --no-verify explicitly"
|
||||
) from exc
|
||||
try:
|
||||
metadata = parquet.ParquetFile(path).metadata
|
||||
except Exception as exc:
|
||||
raise TushareSnapshotError(
|
||||
f"cannot read Parquet metadata for {path.name}: {exc}"
|
||||
) from exc
|
||||
return int(metadata.num_rows)
|
||||
|
||||
|
||||
def _verify_job_file(
|
||||
mirror_root: Path,
|
||||
row: sqlite3.Row,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
raw_path = row["file_path"]
|
||||
if not isinstance(raw_path, str) or not raw_path:
|
||||
raise TushareSnapshotError(
|
||||
f"completed job {row['id']} has no source file path"
|
||||
)
|
||||
path, relative = _resolve_source_file(mirror_root, raw_path)
|
||||
stat = path.stat()
|
||||
observed_size = int(stat.st_size)
|
||||
expected_size = row["byte_count"]
|
||||
if expected_size is None or observed_size != int(expected_size):
|
||||
raise TushareSnapshotError(
|
||||
f"size mismatch for job {row['id']} ({relative}): "
|
||||
f"expected {expected_size}, observed {observed_size}"
|
||||
)
|
||||
|
||||
expected_sha = str(row["sha256"] or "").lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", expected_sha):
|
||||
raise TushareSnapshotError(
|
||||
f"completed job {row['id']} has no valid SHA-256"
|
||||
)
|
||||
observed_sha = _sha256_file(path)
|
||||
if observed_sha != expected_sha:
|
||||
raise TushareSnapshotError(
|
||||
f"SHA-256 mismatch for job {row['id']} ({relative})"
|
||||
)
|
||||
|
||||
expected_rows = row["row_count"]
|
||||
observed_rows = _parquet_row_count(path)
|
||||
if expected_rows is None or observed_rows != int(expected_rows):
|
||||
raise TushareSnapshotError(
|
||||
f"row-count mismatch for job {row['id']} ({relative}): "
|
||||
f"expected {expected_rows}, observed {observed_rows}"
|
||||
)
|
||||
return relative, {
|
||||
"row_count": observed_rows,
|
||||
"sha256": observed_sha,
|
||||
"size_bytes": observed_size,
|
||||
}
|
||||
|
||||
|
||||
def _relative_job_path(mirror_root: Path, row: sqlite3.Row) -> str:
|
||||
raw_path = row["file_path"]
|
||||
if not isinstance(raw_path, str) or not raw_path:
|
||||
raise TushareSnapshotError(
|
||||
f"completed job {row['id']} has no source file path"
|
||||
)
|
||||
_, relative = _resolve_source_file(mirror_root, raw_path)
|
||||
return relative
|
||||
|
||||
|
||||
def _expected_job_metadata(row: sqlite3.Row) -> tuple[int, int, str]:
|
||||
try:
|
||||
row_count = int(row["row_count"])
|
||||
byte_count = int(row["byte_count"])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TushareSnapshotError(
|
||||
f"completed job {row['id']} has incomplete count metadata"
|
||||
) from exc
|
||||
if row_count < 0 or byte_count < 0:
|
||||
raise TushareSnapshotError(
|
||||
f"completed job {row['id']} has negative count metadata"
|
||||
)
|
||||
source_sha = str(row["sha256"] or "").lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", source_sha):
|
||||
raise TushareSnapshotError(
|
||||
f"completed job {row['id']} has no valid SHA-256"
|
||||
)
|
||||
return row_count, byte_count, source_sha
|
||||
|
||||
|
||||
def _build_table_summaries(
|
||||
jobs: Sequence[Mapping[str, Any]],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
grouped: dict[str, list[Mapping[str, Any]]] = defaultdict(list)
|
||||
for job in jobs:
|
||||
grouped[str(job["api_name"])].append(job)
|
||||
summaries: dict[str, dict[str, Any]] = {}
|
||||
for api_name in sorted(grouped):
|
||||
rows = grouped[api_name]
|
||||
partitions = sorted(str(row["partition_key"]) for row in rows)
|
||||
summaries[api_name] = {
|
||||
"bytes": sum(int(row["byte_count"]) for row in rows),
|
||||
"first_partition": partitions[0],
|
||||
"last_partition": partitions[-1],
|
||||
"partitions": len(rows),
|
||||
"rows": sum(int(row["row_count"]) for row in rows),
|
||||
"shadowed_completed_jobs": sum(
|
||||
int(row["shadowed_completed_count"]) for row in rows
|
||||
),
|
||||
}
|
||||
return summaries
|
||||
|
||||
|
||||
def _request_sha256(row: sqlite3.Row) -> str:
|
||||
raw_params = row["params_json"]
|
||||
if not isinstance(raw_params, str):
|
||||
raise TushareSnapshotError(
|
||||
f"completed job {row['id']} has no params_json provenance"
|
||||
)
|
||||
try:
|
||||
params = json.loads(raw_params)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise TushareSnapshotError(
|
||||
f"completed job {row['id']} has invalid params_json provenance"
|
||||
) from exc
|
||||
request = {
|
||||
"api_name": str(row["api_name"]),
|
||||
"partition_key": str(row["partition_key"]),
|
||||
"fields": str(row["fields"] or ""),
|
||||
"params": params,
|
||||
}
|
||||
return hashlib.sha256(_canonical_json(request)).hexdigest()
|
||||
|
||||
|
||||
def _latest_completed_ids(
|
||||
state_path: Path,
|
||||
) -> dict[tuple[str, str], int]:
|
||||
connection = sqlite3.connect(
|
||||
state_path.as_uri() + "?mode=ro",
|
||||
uri=True,
|
||||
timeout=30,
|
||||
)
|
||||
try:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT api_name, partition_key, MAX(id)
|
||||
FROM jobs
|
||||
WHERE status = 'completed'
|
||||
GROUP BY api_name, partition_key
|
||||
"""
|
||||
)
|
||||
return {
|
||||
(str(api_name), str(partition_key)): int(job_id)
|
||||
for api_name, partition_key, job_id in rows
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def create_snapshot(
|
||||
mirror_root: str | Path,
|
||||
*,
|
||||
start: str | date,
|
||||
end: str | date,
|
||||
apis: Sequence[str] = DEFAULT_APIS,
|
||||
index_codes: Sequence[str] = DEFAULT_INDEX_CODES,
|
||||
verify: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Read and verify one deterministic Tushare mirror selection."""
|
||||
|
||||
root = Path(mirror_root).expanduser().resolve(strict=True)
|
||||
state_path = root / "data" / "state.sqlite3"
|
||||
if not state_path.is_file():
|
||||
raise TushareSnapshotError(
|
||||
"mirror is missing its data/state.sqlite3 catalog"
|
||||
)
|
||||
|
||||
start_date = (
|
||||
start if isinstance(start, date) else date.fromisoformat(str(start))
|
||||
)
|
||||
end_date = end if isinstance(end, date) else date.fromisoformat(str(end))
|
||||
if start_date > end_date:
|
||||
raise ValueError("start must be on or before end")
|
||||
|
||||
requested_apis = _validate_apis(apis)
|
||||
requested_codes = tuple(
|
||||
dict.fromkeys(_normalize_index_code(code) for code in index_codes)
|
||||
)
|
||||
if (
|
||||
{"index_daily", "index_weight"} & set(requested_apis)
|
||||
and not requested_codes
|
||||
):
|
||||
raise ValueError("index APIs require at least one --index-codes value")
|
||||
|
||||
months = _month_keys(start_date, end_date)
|
||||
years = list(range(start_date.year, end_date.year + 1))
|
||||
index_weight_years = (
|
||||
[start_date.year - 1, *years]
|
||||
if "index_weight" in requested_apis
|
||||
else years
|
||||
)
|
||||
month_set = set(months)
|
||||
year_set = set(years)
|
||||
index_weight_year_set = set(index_weight_years)
|
||||
code_set = set(requested_codes)
|
||||
api_set = set(requested_apis)
|
||||
|
||||
identity_before = _state_file_identity(state_path)
|
||||
wal_before = _wal_identity(state_path)
|
||||
read_started_at = _utc_now()
|
||||
connection = sqlite3.connect(
|
||||
state_path.as_uri() + "?mode=ro",
|
||||
uri=True,
|
||||
timeout=30,
|
||||
)
|
||||
connection.row_factory = sqlite3.Row
|
||||
try:
|
||||
connection.execute("BEGIN")
|
||||
required_columns = {
|
||||
"id",
|
||||
"api_name",
|
||||
"partition_key",
|
||||
"status",
|
||||
"row_count",
|
||||
"byte_count",
|
||||
"sha256",
|
||||
"file_path",
|
||||
"completed_at",
|
||||
"fields",
|
||||
"params_json",
|
||||
}
|
||||
available_columns = {
|
||||
str(row["name"])
|
||||
for row in connection.execute("PRAGMA table_info(jobs)")
|
||||
}
|
||||
missing_columns = required_columns - available_columns
|
||||
if missing_columns:
|
||||
raise TushareSnapshotError(
|
||||
"jobs table is missing required columns: "
|
||||
+ ", ".join(sorted(missing_columns))
|
||||
)
|
||||
|
||||
sqlite_identity = {
|
||||
"data_version": int(
|
||||
connection.execute("PRAGMA data_version").fetchone()[0]
|
||||
),
|
||||
"journal_mode": str(
|
||||
connection.execute("PRAGMA journal_mode").fetchone()[0]
|
||||
),
|
||||
"page_count": int(
|
||||
connection.execute("PRAGMA page_count").fetchone()[0]
|
||||
),
|
||||
"page_size": int(
|
||||
connection.execute("PRAGMA page_size").fetchone()[0]
|
||||
),
|
||||
"schema_version": int(
|
||||
connection.execute("PRAGMA schema_version").fetchone()[0]
|
||||
),
|
||||
"user_version": int(
|
||||
connection.execute("PRAGMA user_version").fetchone()[0]
|
||||
),
|
||||
}
|
||||
|
||||
scoped_completed: list[sqlite3.Row] = []
|
||||
for api_name in requested_apis:
|
||||
api_years = (
|
||||
index_weight_years
|
||||
if api_name == "index_weight"
|
||||
else years
|
||||
)
|
||||
rows = _candidate_query(
|
||||
connection,
|
||||
api_name,
|
||||
months=months,
|
||||
years=api_years,
|
||||
index_codes=requested_codes,
|
||||
)
|
||||
scoped_completed.extend(
|
||||
row
|
||||
for row in rows
|
||||
if _scope_reason(
|
||||
str(row["api_name"]),
|
||||
str(row["partition_key"]),
|
||||
apis=api_set,
|
||||
months=month_set,
|
||||
years=year_set,
|
||||
index_weight_years=index_weight_year_set,
|
||||
index_codes=code_set,
|
||||
)[0]
|
||||
)
|
||||
|
||||
live_rows = list(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT id, api_name, partition_key, status
|
||||
FROM jobs
|
||||
WHERE status IN ('running', 'deferred')
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
)
|
||||
connection.commit()
|
||||
finally:
|
||||
connection.close()
|
||||
read_finished_at = _utc_now()
|
||||
identity_after = _state_file_identity(state_path)
|
||||
wal_after = _wal_identity(state_path)
|
||||
|
||||
excluded_live_jobs: list[dict[str, Any]] = []
|
||||
blocking_live_jobs: list[sqlite3.Row] = []
|
||||
for row in live_rows:
|
||||
in_scope, reason = _scope_reason(
|
||||
str(row["api_name"]),
|
||||
str(row["partition_key"]),
|
||||
apis=api_set,
|
||||
months=month_set,
|
||||
years=year_set,
|
||||
index_weight_years=index_weight_year_set,
|
||||
index_codes=code_set,
|
||||
)
|
||||
if in_scope and str(row["status"]) in BLOCKING_LIVE_STATUSES:
|
||||
blocking_live_jobs.append(row)
|
||||
else:
|
||||
excluded_live_jobs.append(
|
||||
{
|
||||
"api_name": str(row["api_name"]),
|
||||
"id": int(row["id"]),
|
||||
"partition_key": str(row["partition_key"]),
|
||||
"reason": reason,
|
||||
"status": str(row["status"]),
|
||||
}
|
||||
)
|
||||
if blocking_live_jobs:
|
||||
details = ", ".join(
|
||||
f"{row['api_name']}:{row['partition_key']}#{row['id']}"
|
||||
f"[{row['status']}]"
|
||||
for row in blocking_live_jobs[:12]
|
||||
)
|
||||
raise TushareSnapshotError(
|
||||
"requested scope contains running/deferred jobs; retry after they "
|
||||
f"reach a terminal state: {details}"
|
||||
)
|
||||
|
||||
grouped: dict[tuple[str, str], list[sqlite3.Row]] = defaultdict(list)
|
||||
for row in scoped_completed:
|
||||
grouped[(str(row["api_name"]), str(row["partition_key"]))].append(row)
|
||||
latest_rows: list[sqlite3.Row] = []
|
||||
shadow_counts: dict[tuple[str, str], int] = {}
|
||||
for key, rows in grouped.items():
|
||||
ordered = sorted(rows, key=lambda row: int(row["id"]))
|
||||
latest_rows.append(ordered[-1])
|
||||
shadow_counts[key] = len(ordered) - 1
|
||||
latest_rows.sort(
|
||||
key=lambda row: (
|
||||
str(row["api_name"]),
|
||||
str(row["partition_key"]),
|
||||
int(row["id"]),
|
||||
)
|
||||
)
|
||||
|
||||
_assert_scope_complete(
|
||||
latest_rows,
|
||||
apis=requested_apis,
|
||||
months=months,
|
||||
years=years,
|
||||
index_weight_years=index_weight_years,
|
||||
index_codes=requested_codes,
|
||||
)
|
||||
|
||||
selected_jobs: list[dict[str, Any]] = []
|
||||
for row in latest_rows:
|
||||
key = (str(row["api_name"]), str(row["partition_key"]))
|
||||
row_count, byte_count, expected_sha = _expected_job_metadata(row)
|
||||
if verify:
|
||||
relative_path, observed = _verify_job_file(root, row)
|
||||
else:
|
||||
relative_path = _relative_job_path(root, row)
|
||||
observed = None
|
||||
job: dict[str, Any] = {
|
||||
"api_name": key[0],
|
||||
"byte_count": byte_count,
|
||||
"completed_at": row["completed_at"],
|
||||
"id": int(row["id"]),
|
||||
"partition_key": key[1],
|
||||
"path": relative_path,
|
||||
"row_count": row_count,
|
||||
"request_sha256": _request_sha256(row),
|
||||
"sha256": expected_sha,
|
||||
"shadowed_completed_count": shadow_counts[key],
|
||||
"verified": verify,
|
||||
}
|
||||
if observed is not None:
|
||||
job["observed"] = observed
|
||||
selected_jobs.append(job)
|
||||
|
||||
latest_ids_after_verification = _latest_completed_ids(state_path)
|
||||
stale_jobs = [
|
||||
{
|
||||
"api_name": job["api_name"],
|
||||
"partition_key": job["partition_key"],
|
||||
"selected_job_id": job["id"],
|
||||
"latest_job_id": latest_ids_after_verification.get(
|
||||
(job["api_name"], job["partition_key"])
|
||||
),
|
||||
}
|
||||
for job in selected_jobs
|
||||
if latest_ids_after_verification.get(
|
||||
(job["api_name"], job["partition_key"])
|
||||
)
|
||||
!= job["id"]
|
||||
]
|
||||
if stale_jobs:
|
||||
raise TushareSnapshotError(
|
||||
"selected jobs changed during file verification: "
|
||||
f"{stale_jobs[:12]}"
|
||||
)
|
||||
if verify:
|
||||
for row, job in zip(latest_rows, selected_jobs):
|
||||
relative_path, observed = _verify_job_file(root, row)
|
||||
if (
|
||||
relative_path != job["path"]
|
||||
or observed != job.get("observed")
|
||||
):
|
||||
raise TushareSnapshotError(
|
||||
"selected source bytes changed during double verification"
|
||||
)
|
||||
identity_after_verification = _state_file_identity(state_path)
|
||||
wal_after_verification = _wal_identity(state_path)
|
||||
|
||||
selection_identity = [
|
||||
{
|
||||
key: value
|
||||
for key, value in job.items()
|
||||
if key not in {"observed", "verified"}
|
||||
}
|
||||
for job in selected_jobs
|
||||
]
|
||||
selection_sha = hashlib.sha256(
|
||||
_canonical_json({"selected_jobs": selection_identity})
|
||||
).hexdigest()
|
||||
changed_while_reading = (
|
||||
identity_before != identity_after or wal_before != wal_after
|
||||
)
|
||||
status_counts = Counter(str(row["status"]) for row in live_rows)
|
||||
manifest: dict[str, Any] = {
|
||||
"as_of": read_started_at,
|
||||
"excluded_live_jobs": excluded_live_jobs,
|
||||
"format": "quant-os.tushare-scoped-snapshot/v2",
|
||||
"hash_algorithms": {
|
||||
"canonical_json": (
|
||||
"UTF-8 JSON; keys sorted; separators ',' and ':'; "
|
||||
"ensure_ascii=false; allow_nan=false"
|
||||
),
|
||||
"manifest_sha256": (
|
||||
"SHA-256 of canonical_json(the complete manifest excluding "
|
||||
"the manifest_sha256 member)"
|
||||
),
|
||||
"request_sha256": (
|
||||
"SHA-256 of canonical_json({api_name, partition_key, fields, "
|
||||
"params}), with params_json parsed as JSON and without "
|
||||
"embedding it in this manifest"
|
||||
),
|
||||
"selection_sha256": (
|
||||
"SHA-256 of canonical_json({selected_jobs: [...]}), excluding "
|
||||
"each job's observed and verified members so verify/no-verify "
|
||||
"runs over the same SQLite selection have the same identity"
|
||||
),
|
||||
},
|
||||
"mirror": {
|
||||
"label": root.name,
|
||||
"state_db": {
|
||||
"changed_while_reading": changed_while_reading,
|
||||
"changed_through_file_verification": (
|
||||
identity_before != identity_after_verification
|
||||
or wal_before != wal_after_verification
|
||||
),
|
||||
"file_after": identity_after,
|
||||
"file_after_verification": identity_after_verification,
|
||||
"file_before": identity_before,
|
||||
"read_finished_at": read_finished_at,
|
||||
"read_started_at": read_started_at,
|
||||
"sqlite_snapshot": sqlite_identity,
|
||||
"wal_after": wal_after,
|
||||
"wal_after_verification": wal_after_verification,
|
||||
"wal_before": wal_before,
|
||||
},
|
||||
},
|
||||
"scope": {
|
||||
"apis": list(requested_apis),
|
||||
"end": end_date.isoformat(),
|
||||
"index_codes": list(requested_codes),
|
||||
"partition_policy": (
|
||||
"calendar partitions overlapping [start,end]; timeless "
|
||||
"stock_basic partitions; exact requested index codes; "
|
||||
"index_weight also includes start.year-1 as the pre-start "
|
||||
"point-in-time membership anchor"
|
||||
),
|
||||
"index_weight_anchor_year": (
|
||||
start_date.year - 1
|
||||
if "index_weight" in requested_apis
|
||||
else None
|
||||
),
|
||||
"start": start_date.isoformat(),
|
||||
},
|
||||
"selected_jobs": selected_jobs,
|
||||
"selection_sha256": selection_sha,
|
||||
"summary": {
|
||||
"excluded_live_job_status_counts": dict(sorted(status_counts.items())),
|
||||
"excluded_live_jobs": len(excluded_live_jobs),
|
||||
"selected_bytes": sum(job["byte_count"] for job in selected_jobs),
|
||||
"selected_jobs": len(selected_jobs),
|
||||
"selected_rows": sum(job["row_count"] for job in selected_jobs),
|
||||
"shadowed_completed_jobs": sum(
|
||||
job["shadowed_completed_count"] for job in selected_jobs
|
||||
),
|
||||
"tables": _build_table_summaries(selected_jobs),
|
||||
},
|
||||
"tool": {
|
||||
"name": "tools/tushare_snapshot.py",
|
||||
"version": TOOL_VERSION,
|
||||
},
|
||||
"verification": {
|
||||
"mode": "size+sha256+parquet_row_count" if verify else "not_performed",
|
||||
"verified_jobs": len(selected_jobs) if verify else 0,
|
||||
"double_verified_jobs": len(selected_jobs) if verify else 0,
|
||||
"selected_jobs_still_latest_after_verification": True,
|
||||
},
|
||||
}
|
||||
manifest["manifest_sha256"] = hashlib.sha256(
|
||||
_canonical_json(manifest)
|
||||
).hexdigest()
|
||||
return manifest
|
||||
|
||||
|
||||
def write_manifest(path: str | Path, manifest: Mapping[str, Any]) -> Path:
|
||||
destination = Path(path).expanduser().resolve()
|
||||
if destination.exists() and destination.is_dir():
|
||||
raise ValueError(f"output points to a directory: {destination}")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = (
|
||||
json.dumps(
|
||||
manifest,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{destination.name}.",
|
||||
suffix=".tmp",
|
||||
dir=destination.parent,
|
||||
)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
if temporary.exists():
|
||||
temporary.unlink()
|
||||
return destination
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--mirror-root", required=True)
|
||||
parser.add_argument("--start", required=True)
|
||||
parser.add_argument("--end", required=True)
|
||||
parser.add_argument(
|
||||
"--apis",
|
||||
nargs="+",
|
||||
default=list(DEFAULT_APIS),
|
||||
help="space- or comma-separated API names",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--index-codes",
|
||||
nargs="+",
|
||||
default=list(DEFAULT_INDEX_CODES),
|
||||
help="space- or comma-separated Tushare index codes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verify",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="verify file size, SHA-256 and Parquet row count (default: true)",
|
||||
)
|
||||
parser.add_argument("--output", "--output-json", dest="output", required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def _main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
manifest = create_snapshot(
|
||||
args.mirror_root,
|
||||
start=args.start,
|
||||
end=args.end,
|
||||
apis=_parse_values(args.apis),
|
||||
index_codes=_parse_values(args.index_codes),
|
||||
verify=args.verify,
|
||||
)
|
||||
output = write_manifest(args.output, manifest)
|
||||
except (OSError, ValueError, sqlite3.Error, TushareSnapshotError) as exc:
|
||||
parser.error(str(exc))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"manifest_sha256": manifest["manifest_sha256"],
|
||||
"output": str(output),
|
||||
"selected_jobs": manifest["summary"]["selected_jobs"],
|
||||
"verified_jobs": manifest["verification"]["verified_jobs"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(_main())
|
||||
Reference in New Issue
Block a user