From 6275370afc14ac5dec3b834647e8166c3e9724e3 Mon Sep 17 00:00:00 2001 From: zebzhang Date: Fri, 31 Jul 2026 01:26:49 +0800 Subject: [PATCH] feat: operationalize local Tushare Qlib research --- MIGRATION.md | 2 +- Makefile | 70 +- README.md | 65 +- adapters/tushare_local.py | 1123 +++++++++++++++-- docs/ARCHITECTURE.md | 29 +- docs/PLATFORM_MATRIX.md | 2 +- docs/README.md | 2 + docs/TUSHARE_LOCAL_DATA.md | 448 ++++--- docs/getting-started/GETTING_STARTED_80.md | 28 +- docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md | 221 ++++ .../CSI500-2018-2025/5bf19d2d/run.json | 85 ++ gate_scorecard.json | 6 +- status/status-data.js | 118 +- tests/test_data_tushare.py | 330 ++++- tests/test_public_status.py | 89 +- tests/test_tushare_lineage.py | 105 ++ tests/test_tushare_snapshot.py | 379 ++++++ tools/build_public_status.py | 434 ++++++- tools/tushare_lineage.py | 281 +++++ tools/tushare_qlib.py | 45 + tools/tushare_snapshot.py | 977 ++++++++++++++ 21 files changed, 4438 insertions(+), 401 deletions(-) create mode 100644 docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md create mode 100644 evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json create mode 100644 tests/test_tushare_lineage.py create mode 100644 tests/test_tushare_snapshot.py create mode 100644 tools/tushare_lineage.py create mode 100644 tools/tushare_snapshot.py diff --git a/MIGRATION.md b/MIGRATION.md index d9119a6..f10047c 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -4,7 +4,7 @@ ## 目标位置 -- 本地：`/Users/zhangbiao/boat-workspace/Code/quant-os` +- 本地：用户选定的独立 checkout（本文记为 `$QUANT_OS_ROOT`） - 远端：`https://git.gomars.fun/boat/quant-os.git` - 公共状态：`https://gomars.fun/quant-os/status/` - 原仓：`https://git.gomars.fun/boat/quants-strategies.git` diff --git a/Makefile b/Makefile index 0410166..f3c3c1d 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,11 @@ PROJECT_PYTHONPATH := src:. SMOKE_DIR := artifacts/local-smoke RESEARCH_DIR := artifacts/research-smoke PARITY_REPORT := artifacts/mock-parity/report.json +TUSHARE_PROVIDER ?= data/qlib/tushare-csi500-2018-2025-v2 +TUSHARE_SNAPSHOT ?= artifacts/local-tushare/scoped-source-manifest.json +TUSHARE_EVIDENCE_DIR ?= artifacts/tushare -.PHONY: test architecture standard bundle smoke verify research research-verify parity parity-verify doctor secret-scan status status-check notebook-check notebook-smoke local +.PHONY: test architecture standard bundle smoke verify research research-verify parity parity-verify doctor secret-scan status status-check notebook-check notebook-smoke tushare-check-root tushare-snapshot tushare-build tushare-verify tushare-lineage tushare-backtest tushare-replay local test: PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m unittest discover -s tests -p '*.py' -q @@ -56,4 +59,69 @@ notebook-check: notebook-smoke: $(PYTHON) tools/run_colab_notebook.py notebooks/quant_os_colab.ipynb --execute +tushare-check-root: + @test -n "$(TUSHARE_MIRROR_ROOT)" || (echo "set TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror" >&2; exit 2) + +tushare-snapshot: tushare-check-root + $(PYTHON) tools/tushare_snapshot.py \ + --mirror-root "$(TUSHARE_MIRROR_ROOT)" \ + --start 2018-01-01 \ + --end 2025-12-31 \ + --apis daily adj_factor trade_cal stock_basic index_daily index_weight \ + --index-codes 000905.SH \ + --output "$(TUSHARE_SNAPSHOT)" + +tushare-build: tushare-check-root + $(PYTHON) tools/tushare_qlib.py build \ + --mirror-root "$(TUSHARE_MIRROR_ROOT)" \ + --output-dir "$(TUSHARE_PROVIDER)" \ + --start 2018-01-01 \ + --end 2025-12-31 \ + --market-name tushare_csi500 \ + --benchmark-index-code 000905.SH \ + --universe-index-code 000905.SH \ + --minimum-observations 60 \ + --ohlc-policy fail \ + --output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-build.json" + +tushare-verify: + $(PYTHON) tools/tushare_qlib.py verify "$(TUSHARE_PROVIDER)" \ + --output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-verify.json" + +tushare-lineage: + $(PYTHON) tools/tushare_lineage.py \ + --source-manifest "$(TUSHARE_SNAPSHOT)" \ + --provider-dir "$(TUSHARE_PROVIDER)" \ + --output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-lineage.json" + +tushare-backtest: + PYTHONHASHSEED=0 PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m platforms.qlib_runner \ + --provider-uri "$(TUSHARE_PROVIDER)" \ + --market tushare_csi500 \ + --benchmark SH000905 \ + --start 2019-01-02 \ + --end 2025-12-30 \ + --feature-start 2018-01-02 \ + --lookback 20 \ + --topk 50 \ + --n-drop 5 \ + --rebalance weekly \ + --output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-momentum-run-1.json" + +tushare-replay: tushare-backtest + PYTHONHASHSEED=0 PYTHONPATH=$(PROJECT_PYTHONPATH) $(PYTHON) -m platforms.qlib_runner \ + --provider-uri "$(TUSHARE_PROVIDER)" \ + --market tushare_csi500 \ + --benchmark SH000905 \ + --start 2019-01-02 \ + --end 2025-12-30 \ + --feature-start 2018-01-02 \ + --lookback 20 \ + --topk 50 \ + --n-drop 5 \ + --rebalance weekly \ + --output-json "$(TUSHARE_EVIDENCE_DIR)/csi500-momentum-run-2.json" + cmp "$(TUSHARE_EVIDENCE_DIR)/csi500-momentum-run-1.json" \ + "$(TUSHARE_EVIDENCE_DIR)/csi500-momentum-run-2.json" + local: architecture standard test bundle smoke verify research research-verify parity parity-verify doctor secret-scan status-check notebook-check notebook-smoke diff --git a/README.md b/README.md index 18b67d6..1713f60 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ python3 tools/build_public_status.py --check | 聚宽 hosted | 默认动量 smoke；显式注入 TargetPackage 后只消费本地 post-risk 权重并绑定平台账户/价格 | 动量模式已有一次 2024 全年 run；一个 synthetic-research TargetPackage 已真实命中、绑定 `day_open` 并产生 3 笔委托/成交 | 只观察 execution consumer；无授权真实长样本/OOS、上游四层 hosted 执行、QMT peer 或逐层真实数据 parity | | QMT built-in/qmttools | 默认动量 smoke；TargetPackage 模式为 Python 3.6 backtest-only 消费器 | Python 3.6 语法、TargetPackage/fake/qmttools 合约 | 需要已授权 QMT 客户端、历史数据权限与 TargetPackage 实跑导出 | | Qlib 0.9.7 | native momentum backtest；Alpha158/LightGBM CLI/Recorder 工作流 | Python 3.12 两条 fixture 已真实跑通，并保存表级 hash、行列/时间边界和重算指标 | synthetic 两股票没有投资价值；无真实 OOS、L3/L4 parity | -| Tushare → Qlib | 只读盘点 completed Parquet、校验 checksum、构建/验证不可变 provider、运行本地回测 | 93 个 completed 文件已验证；早期未复权 provider 与两次 byte-identical momentum run | 下载仅到 1993-10；缺复权、真实 benchmark、历史成分/ST/停牌/涨跌停，不能进入生产 decision | +| Tushare → Qlib | 只读盘点 live mirror、冻结 scoped release、构建/验证不可变 provider、运行本地回测 | 2018—2025 v2 verifier 通过；1,111 instruments、1,942 sessions；97 observed/96 effective 月末 snapshots 严格 500 成分、次日生效；lineage 221 jobs×6 fields mismatch 0、converter current true；双次 run byte-identical | event-time/保守次日生效 PIT 近似，不是严格 knowledge-time PIT；不同 start 的首观测锚 provider 不可直接拼接；research-only、`gate_credit=[]`、ST/统一 9.5% 仍缺 | | XtTrader shadow | 只读账户查询、HMAC 账户绑定、broker observation/snapshot、零下单 target-diff 计划与完整 verifier | fake broker/QMT 合约；所有 broker mutation 均不可达 | 需要授权 QMT 环境的账户合同探针、恢复/对账和券商确认；Baseline 60 至少 20 个交易日，Production 80 当前缺口为同一冻结候选至少 60 个交易日 | `portable-momentum-v1` 只是当前真实跑过聚宽的跨引擎连通性 smoke，不是 @@ -477,37 +477,70 @@ python -c "import qlib; assert qlib.__version__ == '0.9.7'" ### 使用本地 Tushare 镜像 -`tools/tushare_qlib.py` 不调用 Tushare、不读取 token，也不修改镜像。它只接受 -SQLite 中状态为 `completed` 且文件大小、SHA-256、Parquet 行数一致的分区； -生成新 Qlib provider 时保存 source job、build parameter、converter hash、 -tree hash 与可重算 `data_version`，并拒绝覆盖已有目录。 +`tools/tushare_snapshot.py` 和 `tools/tushare_qlib.py` 不调用 Tushare、不读取 +token，也不修改镜像。先从仍可变化的 raw mirror 冻结一个日期/API/指数范围 +明确的 source manifest，再生成新 Qlib provider。provider 保存 source job、 +build parameter、converter hash、tree hash 与可重算 `data_version`，并拒绝 +覆盖已有目录。当前 build 不直接读取 scoped manifest，所以验收时还要确认 +两份 manifest 的 source job/file hashes 完全一致。 + +推荐使用 Makefile 固定同一组路径和参数： + +```bash +export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror +export TUSHARE_PROVIDER=data/qlib/tushare-csi500-2018-2025-next +export TUSHARE_SNAPSHOT=artifacts/local-tushare/scoped-source-manifest.json + +make tushare-snapshot +make tushare-build +make tushare-verify +make tushare-lineage +make tushare-backtest +make tushare-replay +``` + +`TUSHARE_PROVIDER` 指向的 build 输出目录必须不存在；工具拒绝覆盖已有 +provider。`tushare-replay` 会先生成第一份 run，再生成第二份并用 `cmp` +检查逐 byte 重放。 ```bash export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror PYTHONPATH=src:. python tools/tushare_qlib.py inventory \ --mirror-root "$TUSHARE_MIRROR_ROOT" \ + --skip-file-verification \ --output-json artifacts/tushare-inventory.json +python tools/tushare_snapshot.py \ + --mirror-root "$TUSHARE_MIRROR_ROOT" \ + --start 2018-01-01 \ + --end 2025-12-31 \ + --apis daily adj_factor trade_cal stock_basic index_daily index_weight \ + --index-codes 000905.SH \ + --output artifacts/local-tushare-20260731/scoped-source-manifest.json + PYTHONPATH=src:. python tools/tushare_qlib.py build \ --mirror-root "$TUSHARE_MIRROR_ROOT" \ - --output-dir data/qlib/tushare-early-v1 \ - --start 1990-12-19 --end 1993-10-29 \ + --output-dir data/qlib/tushare-csi500-2018-2025-v2 \ + --start 2018-01-01 --end 2025-12-31 \ + --market-name tushare_csi500 \ + --benchmark-index-code 000905.SH \ + --universe-index-code 000905.SH \ --minimum-observations 60 \ - --allow-unadjusted \ - --ohlc-policy expand-range \ + --ohlc-policy fail \ --output-json artifacts/tushare-qlib-build.json PYTHONPATH=src:. python tools/tushare_qlib.py verify \ - data/qlib/tushare-early-v1 + data/qlib/tushare-csi500-2018-2025-v2 ``` -当前镜像只有 1990-12—1993-10 日线，且没有 `adj_factor`、真实指数行情、 -历史成分、ST、停牌和涨跌停数据。因此 `--allow-unadjusted` 是显式技术 -smoke 降级，provider 固定 `production_eligible=false`、 -`investment_value_claim=false`、`gate_credit=[]`。盘点、字段口径、完整命令、 -v4 hash 与两次本地回测结果见 -[`docs/TUSHARE_LOCAL_DATA.md`](docs/TUSHARE_LOCAL_DATA.md)。 +当前镜像已有 1990-12—2026-07 日线和复权因子、真实中证 500 指数行情与历史 +权重、停牌和涨跌停数据；但 raw mirror 是移动目标，且历史 `stock_st` 权限被 +拒。v2 provider verifier 和两次 byte-identical 真实本地 run 已完成，但 +结果仍为 `production_ready=false`、`investment_value_claim=false`、 +`gate_credit=[]`。完整盘点、字段口径和本机 +命令见 [`docs/TUSHARE_LOCAL_DATA.md`](docs/TUSHARE_LOCAL_DATA.md) 与 +[`docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md`](docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md)。 native momentum smoke： diff --git a/adapters/tushare_local.py b/adapters/tushare_local.py index d30449d..d126d06 100644 --- a/adapters/tushare_local.py +++ b/adapters/tushare_local.py @@ -57,6 +57,12 @@ MISSING_PRODUCTION_TABLES = ( "stock_st", "suspend_d", ) +KNOWN_INDEX_CONSTITUENT_COUNTS = { + "000016.SH": 50, + "000300.SH": 300, + "000905.SH": 500, + "000852.SH": 1000, +} REQUIRED_PRODUCTION_TABLES = ( "daily", "trade_cal", @@ -69,19 +75,20 @@ class TushareMirrorError(RuntimeError): """Raised when mirror or provider evidence fails closed.""" -def _canonical_bytes(value: Mapping[str, Any]) -> bytes: - return ( - json.dumps( - dict(value), - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - + "\n" +def _canonical_json_bytes(value: Mapping[str, Any]) -> bytes: + return json.dumps( + dict(value), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, ).encode("utf-8") +def _canonical_bytes(value: Mapping[str, Any]) -> bytes: + return _canonical_json_bytes(value) + b"\n" + + def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -105,6 +112,59 @@ def _yyyymmdd(value: str) -> str: return value.replace("-", "") +def _month_keys(start_iso: str, end_iso: str) -> list[str]: + current = dt.date.fromisoformat(start_iso).replace(day=1) + end_month = dt.date.fromisoformat(end_iso).replace(day=1) + result: list[str] = [] + while current <= end_month: + result.append(current.strftime("%Y-%m")) + if current.month == 12: + current = current.replace( + year=current.year + 1, + month=1, + ) + else: + current = current.replace(month=current.month + 1) + return result + + +def _partition_year(partition_key: str) -> Optional[int]: + match = re.search( + r"(?:^|[/_])year=(\d{4})(?:$|[/_])", + partition_key, + ) + return int(match.group(1)) if match else None + + +def _partition_month(partition_key: str) -> Optional[str]: + match = re.search( + r"(?:^|[/_])month=(\d{4}-\d{2})(?:$|[/_])", + partition_key, + ) + return match.group(1) if match else None + + +def _require_partition_values( + jobs: Sequence[Mapping[str, Any]], + *, + api_name: str, + expected: Sequence[Any], + extractor: Any, +) -> None: + observed = { + value + for value in ( + extractor(str(job["partition_key"])) for job in jobs + ) + if value is not None + } + missing = [value for value in expected if value not in observed] + if missing: + raise TushareMirrorError( + f"{api_name} is missing required partitions: {missing[:12]}" + ) + + def tushare_to_qlib_symbol(value: str) -> str: """Convert ``600000.SH`` into Qlib's ``SH600000`` spelling.""" @@ -125,7 +185,9 @@ def _looks_like_a_share(value: str) -> bool: if exchange == "SH": return code.startswith(("600", "601", "603", "605", "688", "689")) if exchange == "SZ": - return code.startswith(("000", "001", "002", "003", "300", "301")) + return code.startswith( + ("000", "001", "002", "003", "300", "301", "302") + ) return code[0] in {"4", "8", "9"} @@ -155,10 +217,23 @@ def _read_jobs(root: Path) -> list[Dict[str, Any]]: connection.row_factory = sqlite3.Row try: connection.execute("BEGIN") + columns = { + str(row["name"]) + for row in connection.execute("PRAGMA table_info(jobs)").fetchall() + } + optional_columns = ", ".join( + ( + name + if name in columns + else f"NULL AS {name}" + ) + for name in ("params_json", "fields") + ) rows = connection.execute( - """ + f""" SELECT id, api_name, partition_key, status, row_count, byte_count, - sha256, file_path, started_at, completed_at, error_code + sha256, file_path, started_at, completed_at, error_code, + {optional_columns} FROM jobs ORDER BY id """ @@ -168,6 +243,40 @@ def _read_jobs(root: Path) -> list[Dict[str, Any]]: return [dict(row) for row in rows] +def _latest_completed_jobs( + jobs: Sequence[Mapping[str, Any]], +) -> tuple[list[Dict[str, Any]], list[Dict[str, Any]]]: + """Select the newest completed version of every logical partition. + + The mirror replaces a partition file in place when it is refreshed. Older + completed rows can therefore carry a perfectly valid historical checksum + for bytes which are no longer present at ``file_path``. Treating every + completed database row as a live file produces false corruption failures. + """ + + newest: Dict[tuple[str, str], Mapping[str, Any]] = {} + completed = [ + job for job in jobs if str(job.get("status")) == "completed" + ] + for job in completed: + key = (str(job["api_name"]), str(job["partition_key"])) + incumbent = newest.get(key) + if incumbent is None or int(job["id"]) > int(incumbent["id"]): + newest[key] = job + selected_ids = {int(job["id"]) for job in newest.values()} + selected = [ + dict(job) + for job in completed + if int(job["id"]) in selected_ids + ] + shadowed = [ + dict(job) + for job in completed + if int(job["id"]) not in selected_ids + ] + return selected, shadowed + + def _job_path(root: Path, job: Mapping[str, Any]) -> Path: raw = str(job.get("file_path") or "").strip() if not raw: @@ -267,7 +376,25 @@ def _verify_completed_job( raise TushareMirrorError( f"invalid Parquet query_sha256 for {path.name}" ) + raw_params = job.get("params_json") + try: + params = json.loads(str(raw_params)) if raw_params is not None else None + except json.JSONDecodeError as exc: + raise TushareMirrorError( + f"invalid params_json for completed job {job.get('id')}" + ) from exc + request_sha256 = hashlib.sha256( + _canonical_json_bytes( + { + "api_name": str(job["api_name"]), + "partition_key": str(job["partition_key"]), + "fields": str(job.get("fields") or ""), + "params": params, + } + ) + ).hexdigest() return { + "job_id": int(job["id"]), "api_name": str(job["api_name"]), "partition_key": str(job["partition_key"]), "path": _relative_job_path(root, path), @@ -275,6 +402,7 @@ def _verify_completed_job( "byte_count": actual_bytes, "sha256": actual_sha, "query_sha256": query_sha256, + "request_sha256": request_sha256, "downloaded_at": footer.get("downloaded_at"), "completed_at": job.get("completed_at"), } @@ -289,13 +417,13 @@ def inventory_mirror( root = _mirror_root(mirror_root) jobs = _read_jobs(root) + completed, shadowed = _latest_completed_jobs(jobs) statuses: Dict[str, int] = {} tables: Dict[str, Dict[str, Any]] = {} for job in jobs: status = str(job["status"]) statuses[status] = statuses.get(status, 0) + 1 - if status != "completed": - continue + for job in completed: api = str(job["api_name"]) table = tables.setdefault( api, @@ -322,10 +450,9 @@ def inventory_mirror( verified = 0 if verify_files: parquet = _load_parquet_module() - for job in jobs: - if job["status"] == "completed": - _verify_completed_job(root, job, parquet=parquet) - verified += 1 + for job in completed: + _verify_completed_job(root, job, parquet=parquet) + verified += 1 running = [ { "api_name": str(job["api_name"]), @@ -350,6 +477,18 @@ def inventory_mirror( "job_status_counts": dict(sorted(statuses.items())), "tables": dict(sorted(tables.items())), "verified_completed_files": verified, + "selected_completed_partitions": len(completed), + "shadowed_completed_versions": len(shadowed), + "shadowed_completed_versions_by_api": dict( + sorted( + { + api: sum( + 1 for job in shadowed if str(job["api_name"]) == api + ) + for api in {str(job["api_name"]) for job in shadowed} + }.items() + ) + ), "noncompleted_jobs": running, "missing_production_tables": missing_production_tables, "coverage_claim": "completed_partitions_only_not_full_mirror", @@ -417,24 +556,60 @@ def _year_overlaps(partition: str, start: str, end: str) -> bool: return start[:4] <= year <= end[:4] +def _partition_code_matches( + partition: str, + *, + key: str, + code: str, +) -> bool: + encoded = re.sub(r"[^A-Za-z0-9-]+", "_", code.strip().upper()) + return partition.startswith(f"{key}={encoded}/") + + +def _normalize_index_code(value: str, *, name: str) -> str: + normalized = str(value or "").strip().upper() + if re.fullmatch(r"\d{6}\.(SH|SZ|BJ)", normalized) is None: + raise ValueError(f"{name} must use Tushare 000905.SH form") + return normalized + + +def _source_job_sort_key(job: Mapping[str, Any]) -> tuple[str, str, int]: + return ( + str(job["api_name"]), + str(job["partition_key"]), + int(job["id"]), + ) + + def _data_version_payload( *, source_jobs: Sequence[Mapping[str, Any]], build_parameters: Mapping[str, Any], converter_sha256: str, + schema_version: int, ) -> Dict[str, Any]: + if schema_version == 1: + return { + "source_jobs": [dict(value) for value in source_jobs], + "range": [ + build_parameters["start"], + build_parameters["end"], + ], + "market": build_parameters["market_name"], + "benchmark": build_parameters["benchmark_symbol"], + "minimum_observations": build_parameters["minimum_observations"], + "allow_unadjusted": build_parameters["allow_unadjusted"], + "price_adjustment": "raw_unadjusted_factor_1", + "ohlc_policy": build_parameters["ohlc_policy"], + "converter_sha256": converter_sha256, + } + if schema_version != 2: + raise ValueError( + f"unsupported provider manifest schema_version: {schema_version}" + ) return { "source_jobs": [dict(value) for value in source_jobs], - "range": [ - build_parameters["start"], - build_parameters["end"], - ], - "market": build_parameters["market_name"], - "benchmark": build_parameters["benchmark_symbol"], - "minimum_observations": build_parameters["minimum_observations"], - "allow_unadjusted": build_parameters["allow_unadjusted"], - "price_adjustment": "raw_unadjusted_factor_1", - "ohlc_policy": build_parameters["ohlc_policy"], + "build_parameters": dict(build_parameters), "converter_sha256": converter_sha256, } @@ -463,22 +638,58 @@ def build_qlib_provider( minimum_observations: int = 60, allow_unadjusted: bool = False, expand_range: bool = False, + benchmark_index_code: Optional[str] = None, + universe_index_code: Optional[str] = None, + expected_constituent_count: Optional[int] = None, ) -> Dict[str, Any]: """Freeze completed daily partitions into a Qlib 0.9.7 binary provider.""" - if not allow_unadjusted: - raise TushareMirrorError( - "adj_factor is unavailable; pass allow_unadjusted=True only for " - "an explicitly non-investment technical run" - ) start_iso = _iso_date(start, name="start") end_iso = _iso_date(end, name="end") if start_iso > end_iso: raise ValueError("start must be on or before end") market = _market_name(market_name) - benchmark = str(benchmark_symbol).strip().upper() - if re.fullmatch(r"(SH|SZ|BJ)\d{6}", benchmark) is None: - raise ValueError("benchmark_symbol must use Qlib SH999999 form") + benchmark_code = ( + _normalize_index_code( + benchmark_index_code, + name="benchmark_index_code", + ) + if benchmark_index_code is not None + else None + ) + universe_code = ( + _normalize_index_code( + universe_index_code, + name="universe_index_code", + ) + if universe_index_code is not None + else None + ) + if universe_code is not None and benchmark_code is None: + raise ValueError( + "universe_index_code requires benchmark_index_code; a PIT " + "universe must not use the full-range synthetic benchmark" + ) + if expected_constituent_count is None and universe_code is not None: + resolved_constituent_count = KNOWN_INDEX_CONSTITUENT_COUNTS.get( + universe_code + ) + elif expected_constituent_count is None: + resolved_constituent_count = None + else: + resolved_constituent_count = int(expected_constituent_count) + if resolved_constituent_count < 2: + raise ValueError("expected_constituent_count must be at least 2") + if universe_code is not None and resolved_constituent_count is None: + raise ValueError( + "unknown universe index requires expected_constituent_count" + ) + if benchmark_code is None: + benchmark = str(benchmark_symbol).strip().upper() + if re.fullmatch(r"(SH|SZ|BJ)\d{6}", benchmark) is None: + raise ValueError("benchmark_symbol must use Qlib SH999999 form") + else: + benchmark = tushare_to_qlib_symbol(benchmark_code) if int(minimum_observations) < 2: raise ValueError("minimum_observations must be at least 2") @@ -489,7 +700,7 @@ def build_qlib_provider( output.parent.mkdir(parents=True, exist_ok=True) jobs = _read_jobs(root) - completed = [job for job in jobs if job["status"] == "completed"] + completed, shadowed = _latest_completed_jobs(jobs) daily_jobs = [ job for job in completed @@ -505,20 +716,170 @@ def build_qlib_provider( stock_jobs = [ job for job in completed if job["api_name"] == "stock_basic" ] + adjustment_jobs = [ + job + for job in completed + if job["api_name"] == "adj_factor" + and _month_overlaps(str(job["partition_key"]), start_iso, end_iso) + ] + benchmark_jobs = ( + [ + job + for job in completed + if job["api_name"] == "index_daily" + and _partition_code_matches( + str(job["partition_key"]), + key="ts_code", + code=benchmark_code, + ) + and _year_overlaps(str(job["partition_key"]), start_iso, end_iso) + ] + if benchmark_code is not None + else [] + ) + universe_jobs = ( + [ + job + for job in completed + if job["api_name"] == "index_weight" + and _partition_code_matches( + str(job["partition_key"]), + key="index_code", + code=universe_code, + ) + and ( + (match := re.search( + r"year=(\d{4})", + str(job["partition_key"]), + )) + is not None + and str(int(start_iso[:4]) - 1) + <= match.group(1) + <= end_iso[:4] + ) + ] + if universe_code is not None + else [] + ) if not daily_jobs or not calendar_jobs or not stock_jobs: raise TushareMirrorError( "completed daily, trade_cal and stock_basic jobs are required" ) + if benchmark_code is not None and not benchmark_jobs: + raise TushareMirrorError( + f"no completed index_daily partitions for {benchmark_code}" + ) + if universe_code is not None and not universe_jobs: + raise TushareMirrorError( + f"no completed index_weight partitions for {universe_code}" + ) + expected_months = _month_keys(start_iso, end_iso) + expected_years = list( + range(int(start_iso[:4]), int(end_iso[:4]) + 1) + ) + _require_partition_values( + daily_jobs, + api_name="daily", + expected=expected_months, + extractor=_partition_month, + ) + _require_partition_values( + calendar_jobs, + api_name="trade_cal", + expected=expected_years, + extractor=_partition_year, + ) + if not allow_unadjusted: + _require_partition_values( + adjustment_jobs, + api_name="adj_factor", + expected=expected_months, + extractor=_partition_month, + ) + if benchmark_code is not None: + _require_partition_values( + benchmark_jobs, + api_name="index_daily", + expected=expected_years, + extractor=_partition_year, + ) + if universe_code is not None: + _require_partition_values( + universe_jobs, + api_name="index_weight", + expected=[expected_years[0] - 1, *expected_years], + extractor=_partition_year, + ) parquet = _load_parquet_module() - selected_jobs = daily_jobs + calendar_jobs + stock_jobs + selected_jobs_by_id = { + int(job["id"]): job + for job in ( + daily_jobs + + calendar_jobs + + stock_jobs + + adjustment_jobs + + benchmark_jobs + + universe_jobs + ) + } + selected_jobs = sorted( + selected_jobs_by_id.values(), + key=_source_job_sort_key, + ) source_jobs = [ _verify_completed_job(root, job, parquet=parquet) for job in selected_jobs ] pandas = _load_pandas() + universe_prefilter_codes: Optional[set[str]] = None + if universe_code is not None: + prefilter_frames = [ + pandas.read_parquet( + _job_path(root, job), + columns=["index_code", "con_code", "trade_date"], + ) + for job in universe_jobs + ] + prefilter_weights = pandas.concat( + prefilter_frames, + ignore_index=True, + ) + prefilter_weights["index_code"] = ( + prefilter_weights["index_code"].astype(str).str.upper() + ) + prefilter_weights["con_code"] = ( + prefilter_weights["con_code"].astype(str).str.upper() + ) + prefilter_weights["trade_date"] = ( + prefilter_weights["trade_date"].astype(str) + ) + universe_prefilter_codes = set( + prefilter_weights.loc[ + (prefilter_weights["index_code"] == universe_code) + & (prefilter_weights["trade_date"] <= _yyyymmdd(end_iso)), + "con_code", + ].tolist() + ) + if not universe_prefilter_codes: + raise TushareMirrorError( + f"index_weight has no constituents through {end_iso} " + f"for {universe_code}" + ) + daily_frames = [] + for job in daily_jobs: + frame = pandas.read_parquet(_job_path(root, job)) + if universe_prefilter_codes is not None: + if "ts_code" not in frame.columns: + raise TushareMirrorError( + "daily partitions miss required field: ts_code" + ) + frame = frame[ + frame["ts_code"].astype(str).isin(universe_prefilter_codes) + ] + daily_frames.append(frame) daily = pandas.concat( - [pandas.read_parquet(_job_path(root, job)) for job in daily_jobs], + daily_frames, ignore_index=True, ) missing = REQUIRED_DAILY_FIELDS.difference(daily.columns) @@ -629,16 +990,352 @@ def build_qlib_provider( lambda value: _eligible_a_share(str(value), metadata) ) ].copy() - counts = daily.groupby("ts_code").size() - selected_codes = sorted( - str(value) - for value in counts[counts >= int(minimum_observations)].index - ) + calendar = [ + dt.datetime.strptime(value, "%Y%m%d").date().isoformat() + for value in calendar_raw + ] + calendar_index = {value: index for index, value in enumerate(calendar_raw)} + + membership_lines: list[str] = [] + universe_summary: Dict[str, Any] + if universe_code is None: + counts = daily.groupby("ts_code").size() + selected_codes = sorted( + str(value) + for value in counts[counts >= int(minimum_observations)].index + ) + universe_summary = { + "kind": "full_range_minimum_observations", + "point_in_time": False, + "index_code": None, + "snapshot_count": 0, + "selection_uses_future_range_count": True, + } + else: + weights = pandas.concat( + [ + pandas.read_parquet(_job_path(root, job)) + for job in universe_jobs + ], + ignore_index=True, + ) + required_weight_fields = { + "index_code", + "con_code", + "trade_date", + "weight", + } + missing_weight_fields = required_weight_fields.difference( + weights.columns + ) + if missing_weight_fields: + raise TushareMirrorError( + "index_weight partitions miss required fields: " + f"{sorted(missing_weight_fields)}" + ) + weights["index_code"] = weights["index_code"].astype(str).str.upper() + weights["con_code"] = weights["con_code"].astype(str).str.upper() + weights["trade_date"] = weights["trade_date"].astype(str) + weights = weights[ + (weights["index_code"] == universe_code) + & (weights["trade_date"] <= end_raw) + ].copy() + if weights.empty: + raise TushareMirrorError( + f"index_weight has no snapshots through {end_iso} " + f"for {universe_code}" + ) + weights["weight"] = pandas.to_numeric( + weights["weight"], + errors="coerce", + ) + if ( + weights["weight"].isna().any() + or (~weights["weight"].map(math.isfinite)).any() + or (weights["weight"] <= 0).any() + ): + raise TushareMirrorError( + "index_weight contains null/non-finite/non-positive weights" + ) + if weights.duplicated(["trade_date", "con_code"]).any(): + raise TushareMirrorError( + "index_weight contains duplicate trade_date/con_code keys" + ) + snapshot_dates_all = sorted(set(weights["trade_date"].tolist())) + anchor_candidates = [ + value for value in snapshot_dates_all if value < start_raw + ] + in_range_snapshot_dates = [ + value + for value in snapshot_dates_all + if start_raw <= value <= end_raw + ] + if not anchor_candidates: + raise TushareMirrorError( + "index_weight has no pre-start snapshot anchor; same-session " + "or future backfill is forbidden" + ) + unknown_snapshot_dates = sorted( + set(in_range_snapshot_dates).difference(calendar_raw) + ) + if unknown_snapshot_dates: + raise TushareMirrorError( + "index_weight snapshots are outside the SSE calendar: " + f"{unknown_snapshot_dates[:5]}" + ) + observed_snapshot_dates = [ + max(anchor_candidates), + *in_range_snapshot_dates, + ] + parsed_snapshot_dates = [ + dt.datetime.strptime(value, "%Y%m%d").date() + for value in observed_snapshot_dates + ] + snapshot_gaps = [ + (right - left).days + for left, right in zip( + parsed_snapshot_dates, + parsed_snapshot_dates[1:], + ) + ] + tail_gap = ( + dt.date.fromisoformat(end_iso) - parsed_snapshot_dates[-1] + ).days + max_snapshot_gap = max([*snapshot_gaps, tail_gap], default=tail_gap) + if max_snapshot_gap > 45: + raise TushareMirrorError( + "index_weight snapshot cadence has a gap greater than " + f"45 calendar days: {max_snapshot_gap}" + ) + snapshot_members: Dict[str, list[str]] = {} + snapshot_stats = [] + for snapshot_date in observed_snapshot_dates: + frame = weights[weights["trade_date"] == snapshot_date] + members = sorted(set(frame["con_code"].tolist())) + if not members: + raise TushareMirrorError( + f"index_weight snapshot {snapshot_date} is empty" + ) + if len(members) != resolved_constituent_count: + raise TushareMirrorError( + f"index_weight snapshot {snapshot_date} has " + f"{len(members)} constituents; expected " + f"{resolved_constituent_count}" + ) + weight_sum = float(frame["weight"].sum()) + if abs(weight_sum - 100.0) > 0.5: + raise TushareMirrorError( + f"index_weight snapshot {snapshot_date} weight sum " + f"is {weight_sum}; expected 100 +/- 0.5" + ) + for code in members: + try: + tushare_to_qlib_symbol(code) + except ValueError as exc: + raise TushareMirrorError( + f"invalid index constituent code {code!r}" + ) from exc + snapshot_members[snapshot_date] = members + if snapshot_date < start_raw: + effective_from = calendar[0] + else: + snapshot_index = calendar_index[snapshot_date] + effective_from = ( + calendar[snapshot_index + 1] + if snapshot_index + 1 < len(calendar) + else None + ) + snapshot_stats.append( + { + "trade_date": snapshot_date, + "constituent_count": len(members), + "weight_sum": weight_sum, + "effective_from": effective_from, + } + ) + effective_snapshots: list[tuple[str, int]] = [ + (observed_snapshot_dates[0], 0) + ] + for snapshot_date in in_range_snapshot_dates: + snapshot_index = calendar_index[snapshot_date] + first_index = snapshot_index + 1 + if first_index < len(calendar): + effective_snapshots.append((snapshot_date, first_index)) + if not effective_snapshots: + raise TushareMirrorError( + "index_weight has no roster effective in the requested range" + ) + selected_codes = sorted( + { + code + for snapshot_date, _ in effective_snapshots + for code in snapshot_members[snapshot_date] + } + ) + daily_codes = set(daily["ts_code"].astype(str)) + missing_daily_codes = sorted(set(selected_codes).difference(daily_codes)) + if missing_daily_codes: + raise TushareMirrorError( + "index_weight constituents have no daily rows in requested " + f"range: {missing_daily_codes[:10]}" + ) + intervals: Dict[str, list[tuple[int, int]]] = {} + for position, (snapshot_date, first_index) in enumerate( + effective_snapshots + ): + if position + 1 < len(effective_snapshots): + last_index = effective_snapshots[position + 1][1] - 1 + else: + last_index = len(calendar_raw) - 1 + if last_index < first_index: + continue + for code in snapshot_members[snapshot_date]: + bucket = intervals.setdefault(code, []) + if bucket and first_index <= bucket[-1][1] + 1: + bucket[-1] = (bucket[-1][0], last_index) + else: + bucket.append((first_index, last_index)) + for code in sorted(intervals): + qlib_symbol = tushare_to_qlib_symbol(code) + for first_index, last_index in intervals[code]: + membership_lines.append( + f"{qlib_symbol}\t" + f"{calendar[first_index]}\t{calendar[last_index]}" + ) + universe_summary = { + "kind": "tushare_index_weight_snapshots", + "point_in_time": True, + "index_code": universe_code, + "snapshot_count": len(observed_snapshot_dates), + "effective_snapshot_count": len(effective_snapshots), + "snapshot_constituent_counts": snapshot_stats, + "expected_constituent_count": resolved_constituent_count, + "snapshot_max_gap_calendar_days": max_snapshot_gap, + "weight_sum_tolerance": 0.5, + "selected_code_semantics": "union_of_effective_snapshots", + "membership_interval_semantics": ( + "latest snapshot strictly before requested start is anchored " + "at requested start; every in-range snapshot becomes " + "effective on the next provider trading session and remains " + "effective through the session before the next roster" + ), + "availability_contract": ( + "conservative next-session effectiveness because the source " + "does not expose a separately verified published_at timestamp" + ), + "same_session_membership_use": False, + "coverage_start": calendar[0], + "selection_uses_future_range_count": False, + } if len(selected_codes) < 2: raise TushareMirrorError( - "fewer than two eligible instruments meet minimum_observations" + "fewer than two eligible instruments are available" ) daily = daily[daily["ts_code"].isin(selected_codes)].copy() + + adjustment_complete = False + missing_adjustment_keys = int(len(daily)) + if adjustment_jobs: + adjustment_frames = [] + for job in adjustment_jobs: + frame = pandas.read_parquet(_job_path(root, job)) + if {"ts_code", "trade_date"}.issubset(frame.columns): + frame = frame[ + frame["ts_code"].astype(str).isin(selected_codes) + & (frame["trade_date"].astype(str) >= start_raw) + & (frame["trade_date"].astype(str) <= end_raw) + ] + adjustment_frames.append(frame) + adjustments = pandas.concat( + adjustment_frames, + ignore_index=True, + ) + required_adjustment_fields = {"ts_code", "trade_date", "adj_factor"} + missing_adjustment_fields = required_adjustment_fields.difference( + adjustments.columns + ) + if missing_adjustment_fields: + raise TushareMirrorError( + "adj_factor partitions miss required fields: " + f"{sorted(missing_adjustment_fields)}" + ) + adjustments["ts_code"] = adjustments["ts_code"].astype(str) + adjustments["trade_date"] = adjustments["trade_date"].astype(str) + adjustments = adjustments[ + (adjustments["trade_date"] >= start_raw) + & (adjustments["trade_date"] <= end_raw) + & (adjustments["ts_code"].isin(selected_codes)) + ].copy() + if adjustments.duplicated(["ts_code", "trade_date"]).any(): + raise TushareMirrorError( + "adj_factor contains duplicate ts_code/trade_date keys" + ) + adjustments["adj_factor"] = pandas.to_numeric( + adjustments["adj_factor"], + errors="coerce", + ) + if ( + adjustments["adj_factor"].isna().any() + or (~adjustments["adj_factor"].map(math.isfinite)).any() + or (adjustments["adj_factor"] <= 0).any() + ): + raise TushareMirrorError( + "adj_factor contains null/non-finite/non-positive values" + ) + adjustment_keys = set( + zip( + adjustments["ts_code"], + adjustments["trade_date"], + ) + ) + daily_keys = set(zip(daily["ts_code"], daily["trade_date"])) + missing_adjustment_keys = len(daily_keys.difference(adjustment_keys)) + adjustment_complete = missing_adjustment_keys == 0 + if adjustment_complete: + daily = daily.merge( + adjustments[["ts_code", "trade_date", "adj_factor"]], + on=["ts_code", "trade_date"], + how="left", + validate="one_to_one", + ) + if not adjustment_complete: + if not allow_unadjusted: + raise TushareMirrorError( + "complete adj_factor coverage is unavailable " + f"({missing_adjustment_keys} daily keys missing); pass " + "allow_unadjusted=True only for an explicitly non-investment " + "technical run" + ) + daily["multiplier"] = 1.0 + price_adjustment_mode = "raw_unadjusted" + else: + daily["adj_factor"] = pandas.to_numeric( + daily["adj_factor"], + errors="raise", + ) + daily = daily.sort_values(["ts_code", "trade_date"]).copy() + anchor_factor = daily.groupby("ts_code")["adj_factor"].transform( + "first" + ) + daily["multiplier"] = daily["adj_factor"] / anchor_factor + if ( + daily["multiplier"].isna().any() + or (~daily["multiplier"].map(math.isfinite)).any() + or (daily["multiplier"] <= 0).any() + ): + raise TushareMirrorError( + "computed first-observation-anchored adjustment multipliers " + "are invalid" + ) + price_adjustment_mode = ( + "cumulative_adjusted_first_observation_anchor" + ) + + daily = daily.sort_values(["ts_code", "trade_date"]).copy() + daily["_adjusted_close"] = daily["close"] * daily["multiplier"] + daily["_adjusted_change"] = daily.groupby("ts_code")[ + "_adjusted_close" + ].pct_change(fill_method=None) daily["qlib_symbol"] = daily["ts_code"].map(tushare_to_qlib_symbol) symbol_pairs = ( daily[["ts_code", "qlib_symbol"]].drop_duplicates().sort_values("ts_code") @@ -655,11 +1352,6 @@ def build_qlib_provider( (pct_recomputed - daily["pct_chg"]).abs().gt(0.02).sum() ) missing_metadata = sorted(set(selected_codes).difference(metadata)) - calendar = [ - dt.datetime.strptime(value, "%Y%m%d").date().isoformat() - for value in calendar_raw - ] - calendar_index = {value: index for index, value in enumerate(calendar_raw)} temporary = Path( tempfile.mkdtemp(prefix=f".{output.name}.", dir=str(output.parent)) @@ -682,22 +1374,22 @@ def build_qlib_provider( offset = calendar_index[str(row["trade_date"])] - first_index volume_shares = float(row["vol"]) * 100.0 money_cny = float(row["amount"]) * 1000.0 - vwap = ( + raw_vwap = ( money_cny / volume_shares if volume_shares > 0.0 else float(row["close"]) ) - change = float(row["close"]) / float(row["pre_close"]) - 1.0 + multiplier = float(row["multiplier"]) values = { - "open": float(row["open"]), - "high": float(row["high"]), - "low": float(row["low"]), - "close": float(row["close"]), - "vwap": vwap, - "volume": volume_shares, + "open": float(row["open"]) * multiplier, + "high": float(row["high"]) * multiplier, + "low": float(row["low"]) * multiplier, + "close": float(row["close"]) * multiplier, + "vwap": raw_vwap * multiplier, + "volume": volume_shares / multiplier, "money": money_cny, - "factor": 1.0, - "change": change, + "factor": multiplier, + "change": float(row["_adjusted_change"]), } for field, value in values.items(): fields[field][offset] = value @@ -715,32 +1407,153 @@ def build_qlib_provider( instrument_lines.append(f"{qlib_symbol}\t{first_iso}\t{last_iso}") stock_symbols.append(str(qlib_symbol)) - returns = ( - daily.assign( - _return=daily["close"] / daily["pre_close"] - 1.0 - ) - .groupby("trade_date")["_return"] - .mean() - .to_dict() - ) benchmark_fields = {name: [] for name in QLIB_FIELDS} - level = 100.0 - for raw_date in calendar_raw: - change = float(returns.get(raw_date, 0.0)) - if not math.isfinite(change) or change <= -1.0: + if benchmark_code is None: + returns = ( + daily.groupby("trade_date")["_adjusted_change"] + .mean() + .to_dict() + ) + level = 100.0 + for raw_date in calendar_raw: + change = float(returns.get(raw_date, 0.0)) + if not math.isfinite(change): + change = 0.0 + if change <= -1.0: + raise TushareMirrorError( + f"invalid derived benchmark return on {raw_date}" + ) + level *= 1.0 + change + benchmark_fields["open"].append(level) + benchmark_fields["high"].append(level) + benchmark_fields["low"].append(level) + benchmark_fields["close"].append(level) + benchmark_fields["vwap"].append(level) + benchmark_fields["volume"].append(100_000_000.0) + benchmark_fields["money"].append(level * 100_000_000.0) + benchmark_fields["factor"].append(1.0) + benchmark_fields["change"].append(change) + benchmark_manifest = { + "symbol": benchmark, + "kind": "synthetic_equal_weight", + "method": ( + "daily arithmetic mean of eligible stock cumulative-" + "adjusted " + "close pct_change observations" + ), + "index_code": None, + "real_index_claim": False, + } + else: + index_daily = pandas.concat( + [ + pandas.read_parquet(_job_path(root, job)) + for job in benchmark_jobs + ], + ignore_index=True, + ) + required_index_fields = { + "ts_code", + "trade_date", + "open", + "high", + "low", + "close", + "pre_close", + "vol", + "amount", + } + missing_index_fields = required_index_fields.difference( + index_daily.columns + ) + if missing_index_fields: raise TushareMirrorError( - f"invalid derived benchmark return on {raw_date}" + "index_daily partitions miss required fields: " + f"{sorted(missing_index_fields)}" ) - level *= 1.0 + change - benchmark_fields["open"].append(level) - benchmark_fields["high"].append(level) - benchmark_fields["low"].append(level) - benchmark_fields["close"].append(level) - benchmark_fields["vwap"].append(level) - benchmark_fields["volume"].append(100_000_000.0) - benchmark_fields["money"].append(level * 100_000_000.0) - benchmark_fields["factor"].append(1.0) - benchmark_fields["change"].append(change) + index_daily["ts_code"] = ( + index_daily["ts_code"].astype(str).str.upper() + ) + index_daily["trade_date"] = index_daily["trade_date"].astype(str) + index_daily = index_daily[ + (index_daily["ts_code"] == benchmark_code) + & (index_daily["trade_date"] >= start_raw) + & (index_daily["trade_date"] <= end_raw) + ].copy() + if index_daily.duplicated(["trade_date"]).any(): + raise TushareMirrorError( + "index_daily contains duplicate benchmark trade_date keys" + ) + actual_index_dates = set(index_daily["trade_date"].tolist()) + missing_index_dates = sorted( + set(calendar_raw).difference(actual_index_dates) + ) + unexpected_index_dates = sorted( + actual_index_dates.difference(calendar_raw) + ) + if missing_index_dates or unexpected_index_dates: + raise TushareMirrorError( + "index_daily/trade_cal mismatch: " + f"missing={missing_index_dates[:5]}, " + f"unexpected={unexpected_index_dates[:5]}" + ) + index_numeric_fields = ( + "open", + "high", + "low", + "close", + "pre_close", + "vol", + "amount", + ) + for field in index_numeric_fields: + index_daily[field] = pandas.to_numeric( + index_daily[field], + errors="coerce", + ) + if index_daily[list(index_numeric_fields)].isna().any().any(): + raise TushareMirrorError( + "index_daily contains null/non-numeric core values" + ) + if ( + index_daily[["open", "high", "low", "close", "pre_close"]] + <= 0 + ).any().any(): + raise TushareMirrorError( + "index_daily contains non-positive price values" + ) + if (index_daily[["vol", "amount"]] < 0).any().any(): + raise TushareMirrorError( + "index_daily contains negative volume or amount" + ) + index_daily = index_daily.set_index("trade_date").loc[calendar_raw] + index_change = index_daily["close"].pct_change(fill_method=None) + for position, raw_date in enumerate(calendar_raw): + row = index_daily.loc[raw_date] + volume_shares = float(row["vol"]) * 100.0 + money_cny = float(row["amount"]) * 1000.0 + vwap = ( + money_cny / volume_shares + if volume_shares > 0.0 + else float(row["close"]) + ) + benchmark_fields["open"].append(float(row["open"])) + benchmark_fields["high"].append(float(row["high"])) + benchmark_fields["low"].append(float(row["low"])) + benchmark_fields["close"].append(float(row["close"])) + benchmark_fields["vwap"].append(vwap) + benchmark_fields["volume"].append(volume_shares) + benchmark_fields["money"].append(money_cny) + benchmark_fields["factor"].append(1.0) + change = index_change.iloc[position] + benchmark_fields["change"].append(float(change)) + benchmark_manifest = { + "symbol": benchmark, + "kind": "tushare_index_daily", + "method": "raw index OHLCV from completed index_daily jobs", + "index_code": benchmark_code, + "real_index_claim": True, + } for field, values in benchmark_fields.items(): _write_feature( temporary @@ -753,15 +1566,60 @@ def build_qlib_provider( benchmark_line = ( f"{benchmark}\t{calendar[0]}\t{calendar[-1]}" ) + market_instrument_lines = ( + membership_lines if universe_code is not None else instrument_lines + ) _write_lines( temporary / f"instruments/{market}.txt", - instrument_lines, + market_instrument_lines, ) _write_lines( temporary / "instruments/all.txt", [*instrument_lines, benchmark_line], ) + jobs_after_read = _read_jobs(root) + completed_after_read, _ = _latest_completed_jobs(jobs_after_read) + latest_after_by_key = { + (str(job["api_name"]), str(job["partition_key"])): int(job["id"]) + for job in completed_after_read + } + changed_source_jobs = [ + { + "api_name": str(job["api_name"]), + "partition_key": str(job["partition_key"]), + "selected_job_id": int(job["id"]), + "latest_job_id": latest_after_by_key.get( + ( + str(job["api_name"]), + str(job["partition_key"]), + ) + ), + } + for job in selected_jobs + if latest_after_by_key.get( + ( + str(job["api_name"]), + str(job["partition_key"]), + ) + ) + != int(job["id"]) + ] + if changed_source_jobs: + raise TushareMirrorError( + "selected source jobs changed while the provider was being " + f"built: {changed_source_jobs[:5]}" + ) + source_jobs_after_read = [ + _verify_completed_job(root, job, parquet=parquet) + for job in selected_jobs + ] + if source_jobs_after_read != source_jobs: + raise TushareMirrorError( + "source bytes or provenance changed while the provider was " + "being built" + ) + tree = _tree_evidence(temporary) completed_apis = {str(job["api_name"]) for job in completed} converter_sha256 = _sha256_file(Path(__file__).resolve()) @@ -770,14 +1628,35 @@ def build_qlib_provider( "end": end_iso, "market_name": market, "benchmark_symbol": benchmark, + "benchmark_index_code": benchmark_code, + "benchmark_mode": ( + "tushare_index_daily" + if benchmark_code is not None + else "synthetic_equal_weight" + ), + "universe_index_code": universe_code, + "universe_mode": ( + "tushare_index_weight_pit" + if universe_code is not None + else "full_range_minimum_observations" + ), + "expected_constituent_count": resolved_constituent_count, + "membership_availability": ( + "next_provider_session" + if universe_code is not None + else None + ), "minimum_observations": int(minimum_observations), - "allow_unadjusted": True, + "minimum_observations_applied": universe_code is None, + "allow_unadjusted": bool(allow_unadjusted), + "price_adjustment_mode": price_adjustment_mode, "ohlc_policy": "expand_range" if expand_range else "fail", } version_payload = _data_version_payload( source_jobs=source_jobs, build_parameters=build_parameters, converter_sha256=converter_sha256, + schema_version=2, ) data_version = hashlib.sha256( _canonical_bytes(version_payload) @@ -787,7 +1666,7 @@ def build_qlib_provider( key = str(job["status"]) status_counts[key] = status_counts.get(key, 0) + 1 manifest: Dict[str, Any] = { - "schema_version": 1, + "schema_version": 2, "artifact_type": "quant_os_tushare_qlib_provider", "source": "tushare_local_mirror", "created_at": dt.datetime.now(dt.timezone.utc).isoformat(), @@ -809,25 +1688,34 @@ def build_qlib_provider( "name": market, "instrument_count": len(stock_symbols), "minimum_observations": int(minimum_observations), + "minimum_observations_applied": universe_code is None, "benchmark_excluded": True, + **universe_summary, }, - "benchmark": { - "symbol": benchmark, - "kind": "synthetic_equal_weight", - "method": ( - "daily arithmetic mean of eligible stock " - "close/pre_close-1 observations" - ), - "real_index_claim": False, - }, + "benchmark": benchmark_manifest, "fields": list(QLIB_FIELDS), "units": { "volume": "shares; Tushare vol(hand) multiplied by 100", "money": "CNY; Tushare amount(thousand CNY) multiplied by 1000", }, "price_adjustment": { - "mode": "raw_unadjusted", - "factor": 1.0, + "mode": price_adjustment_mode, + "formula": ( + "multiplier=adj_factor/" + "first_adj_factor_for_symbol_in_requested_interval" + if adjustment_complete + else "factor=1" + ), + "anchor_semantics": ( + "per-symbol first observed adjustment factor is a " + "constant anchor; no future corporate action is used to " + "rescale earlier observations" + if adjustment_complete + else "not_applicable" + ), + "coverage_complete": adjustment_complete, + "missing_daily_keys": missing_adjustment_keys, + "component_complete": adjustment_complete, "production_eligible": False, }, "quality": { @@ -842,13 +1730,27 @@ def build_qlib_provider( "selected_rows": int(len(daily)), "selected_symbols_missing_stock_basic": missing_metadata, "universe_selection": ( - "eligible A-share symbols with at least " - f"{int(minimum_observations)} observations over the full " - "requested range; not point-in-time eligibility" + universe_summary["membership_interval_semantics"] + if universe_code is not None + else ( + "eligible A-share symbols with at least " + f"{int(minimum_observations)} observations over the " + "full requested range; not point-in-time eligibility" + ) ), }, "source_jobs": source_jobs, + "source_stability": { + "post_read_file_verification": True, + "selected_jobs_still_latest": True, + "policy": ( + "verify selected bytes before consumption, then re-read " + "the SQLite ledger and re-verify every selected file " + "after all provider inputs have been consumed" + ), + }, "mirror_job_status_counts_at_build": dict(sorted(status_counts.items())), + "shadowed_completed_versions_at_build": len(shadowed), "missing_production_tables": [ value for value in MISSING_PRODUCTION_TABLES @@ -908,6 +1810,7 @@ def verify_qlib_provider(provider_dir: str | Path) -> Dict[str, Any]: source_jobs=source_jobs, build_parameters=build_parameters, converter_sha256=str(converter["source_sha256"]), + schema_version=int(manifest.get("schema_version", 1)), ) computed_data_version = hashlib.sha256( _canonical_bytes(version_payload) @@ -920,6 +1823,17 @@ def verify_qlib_provider(provider_dir: str | Path) -> Dict[str, Any]: raise TushareMirrorError( "provider data_version does not match source/build evidence" ) + schema_version = int(manifest.get("schema_version", 1)) + if schema_version >= 2: + source_stability = manifest.get("source_stability") + if ( + not isinstance(source_stability, Mapping) + or source_stability.get("post_read_file_verification") is not True + or source_stability.get("selected_jobs_still_latest") is not True + ): + raise TushareMirrorError( + "schema-v2 provider lacks closed source-stability evidence" + ) tree = _tree_evidence(root) for name in ( "provider_tree_sha256", @@ -934,10 +1848,17 @@ def verify_qlib_provider(provider_dir: str | Path) -> Dict[str, Any]: for required in ("calendars/day.txt", "instruments/all.txt"): if not (root / required).is_file(): raise TushareMirrorError(f"provider misses {required}") + current_converter_sha256 = _sha256_file(Path(__file__).resolve()) + recorded_converter_sha256 = str(converter["source_sha256"]) return { "ok": True, "data_version": manifest.get("data_version"), **tree, "manifest_sha256": _sha256_file(manifest_path), + "recorded_converter_sha256": recorded_converter_sha256, + "current_converter_sha256": current_converter_sha256, + "converter_source_matches_current": ( + current_converter_sha256 == recorded_converter_sha256 + ), "investment_value_claim": False, } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f1176ed..8fce1a4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -66,12 +66,13 @@ XtTrader: ## Data plane and point-in-time semantics -本地 Tushare 镜像是独立只读输入边界。adapter 只信任 SQLite 中 completed -job，并重新核对文件大小、SHA-256、Parquet 行数/footer；Qlib provider -发布到新目录，manifest 绑定 source job、build parameter、converter source、 -完整 provider tree 和可重算 `data_version`。当前镜像未完成且缺复权、 -真实 benchmark、历史成分/ST/停牌/涨跌停，因此只能进入 Qlib 技术 smoke， -不能直接进入 canonical production decision。完整合同与实跑证据见 +本地 Tushare live mirror 是独立只读输入边界。scoped snapshot 在一次 SQLite +读事务中选择目标日期/API/指数的 canonical completed job，并重新核对文件 +大小、SHA-256、Parquet 行数/footer；Qlib provider 发布到新目录，manifest +绑定 source job、build parameter、converter source、完整 provider tree 和 +可重算 `data_version`。当前已有复权因子、真实 `000905.SH` benchmark/历史 +权重、停牌和涨跌停数据，但 live mirror 仍可变化，且 `stock_st` 权限被拒； +因此必须先冻结、再验证，不能直接进入完整 canonical production decision。完整合同见 [`TUSHARE_LOCAL_DATA.md`](TUSHARE_LOCAL_DATA.md)。 JQData adapter 的输入合同是逐交易日历史指数成员和 `get_bars` 日线字段： @@ -190,8 +191,20 @@ Qlib 路径固定为 `0.9.7`。CPython 3.12 实际 fixture smoke 已得到： 重算的有限值指标，明确 `investment_value_claim=false`； - local MLflow file store 使用时，runner 以 `os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true")` 显式确认。 -- 本地 Tushare completed 分区已生成 managed provider；早期未复权动量 - smoke 在 `PYTHONHASHSEED=0` 下连续两次得到逐 byte 相同的 evidence JSON。 +- 旧 Tushare 早期未复权 managed provider/momentum smoke 在 + `PYTHONHASHSEED=0` 下连续两次得到逐 byte 相同的 evidence JSON；它只证明 + 旧转换器和运行时可重放； +- 2018—2025 v2 managed provider 已使用真实复权、`SH000905` benchmark 和 + PIT index-weight universe，通过 verifier；同一 momentum run 两次 JSON + byte-identical。月末权重保守地从下一 provider 交易日生效， + `same_session_membership_use=false`；97 个 observed snapshots 形成 96 个 + effective rosters，每个严格 500 成分、weight sum 100±0.5，最大间隔 36 天。 + live DB/WAL 可被无关下载任务修改，但 221 个 scoped jobs double verified、 + 验证后仍为 latest；独立 lineage 工具对 221 jobs 的 6 个身份字段得到 + mismatch 0，并确认 converter source 为 current。它仍是 + event-time/保守次日生效 PIT 近似，不是严格 knowledge-time PIT；不同 + `start` 的首观测锚 provider 不可直接拼接。它仍是 research-only、 + `gate_credit=[]`，不表达历史 ST 和逐股逐日涨跌停。 该 fixture 只有两只股票和一个 benchmark，性能没有投资意义。Qlib 的单一 `limit_threshold` 也不能表达逐日板块/ST 规则，所以它只参与 L1/L2 和诊断 diff --git a/docs/PLATFORM_MATRIX.md b/docs/PLATFORM_MATRIX.md index a43c6bf..97bf33b 100644 --- a/docs/PLATFORM_MATRIX.md +++ b/docs/PLATFORM_MATRIX.md @@ -19,7 +19,7 @@ | XtTrader read-only shadow | 账户绑定 target-diff 与 broker observation | MiniQMT/QMT、账户查询权限、既有本地 HMAC key | HMAC 账户绑定 + authenticated evidence envelope、decision/observation semantic replay、资产/持仓/委托/成交恒等式、fail-closed exact verifier 与 fake broker 合同 | HMAC 不是 broker attestation；仍缺官方 query 失败/空结果合同及 callback/restart；Baseline 60 至少 20 个交易日，Production 80 当前缺口为同一冻结候选至少 60 个交易日；not live-ready | | Qlib native momentum | signal/model research | CPython 3.12 + exactly `pyqlib==0.9.7` | 真实本地 fixture 成功，24 signal；保存 signal/report hash、表边界和重算 portfolio 指标 | synthetic 两股票；无真实数据/OOS，无 L3/L4 parity | | Qlib Alpha158/LightGBM | model fit + Recorder workflow | 同上 + LightGBM/MLflow stack | 真实 fixture 完成 model/Recorder；读取 portfolio report，保存表 hash、边界、重算指标与 artifact path | synthetic 两股票性能无意义；真实 provider/OOS 和冻结模型缺失 | -| Tushare local → Qlib | completed Parquet 的只读盘点、不可变 provider 与本地研究 smoke | 同上 + `pyarrow==24.0.0`；本地镜像 | 93 个 completed 文件全部通过 size/SHA/row-count；v4 provider verifier 成功；同一动量命令连续两次 evidence JSON 逐 byte 相同 | `daily` 仅完成约 8.2%，缺复权、真实 benchmark、PIT 成分/ST/停牌/涨跌停；`production_ready=false` | +| Tushare local → Qlib | live mirror 只读盘点、scoped release、不可变 provider 与本地研究 | 同上 + `pyarrow==24.0.0`；本地镜像 | selected jobs double verified、仍 latest；独立 lineage 验证 221 jobs×6 fields、mismatch 0、converter current true；v2 verifier 通过：1,111 instruments、1,942 sessions、1,975,455 rows；97 observed/96 effective snapshots 严格 500 成分、次日生效；双次 JSON byte-identical | event-time/保守次日生效 PIT 近似，不是严格 knowledge-time PIT；不同 start 的首观测锚 provider 不可直接拼接；research-only、`gate_credit=[]`；ST/统一 9.5% 仍缺 | | Mock parity exporter | JoinQuant/QMT wrapper 的 L2—L4 合同回归 | Python ≥3.10 | `mock_contract_only`、`real_platform_pass=false`、`gate_credit=[]` | 不计 G9；必须换成真实平台导出 | G1—G10 当前都未通过。JoinQuant TargetPackage run 支持且只支持 diff --git a/docs/README.md b/docs/README.md index 7d52a5d..338f30a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,3 +8,5 @@ - 当前实现架构：[`../docs/ARCHITECTURE.md`](ARCHITECTURE.md) - 平台矩阵：[`PLATFORM_MATRIX.md`](PLATFORM_MATRIX.md) - Tushare 数据：[`TUSHARE_LOCAL_DATA.md`](TUSHARE_LOCAL_DATA.md) +- Tushare → Qlib 本机运行： + [`runbooks/TUSHARE_QLIB_LOCAL_RUN.md`](runbooks/TUSHARE_QLIB_LOCAL_RUN.md) diff --git a/docs/TUSHARE_LOCAL_DATA.md b/docs/TUSHARE_LOCAL_DATA.md index 739db6c..43fbf2a 100644 --- a/docs/TUSHARE_LOCAL_DATA.md +++ b/docs/TUSHARE_LOCAL_DATA.md @@ -1,225 +1,329 @@ -# Tushare 本地数据盘点、Qlib 接入与回测手册 +# Tushare 本地数据盘点、冻结边界与 Qlib 接入 -截至 2026-07-26，Quant OS 已能只读盘点另一个任务生成的 Tushare -Parquet 镜像，校验已完成分区，并转换为不可变的 Qlib 0.9.7 provider。 -这条路径已经完成两次 byte-identical 的本地动量回测。 - -当前证据的边界是： +点时证据：2026-07-31 00:40 CST。原先“93 个 completed 文件、日线只到 +1993-10”的结论已经失效，不得再用于判断当前镜像。当前镜像约 7.7 GB， +SQLite job ledger 在该点时已有 378,707 条 `completed` 版本记录、约 +1.80 亿行；同时仍有与本次中证 500 研究无关的指数任务处于 `running`。 ```text -completed_partitions_only_not_full_mirror +tushare_daily_completed_partitions = 428 +tushare_daily_planned_partitions = 428 +tushare_daily_completed_rows = 18,041,386 +tushare_daily_market_data_from = 1990-12-19 +tushare_daily_market_data_through = 2026-07-28 production_ready = false investment_value_claim = false gate_credit = [] ``` -它证明“本地 Tushare 数据可以进入 Qlib 并可重放”，不证明镜像已经下载完整， -也不证明策略收益有效或 Quant OS 已达到 `BASELINE_60`。 +这意味着数据已经足够做一条真实的 2018—2025 本地 Qlib 研究链，但不能把 +“live raw mirror 很大”直接等同于“已经有可复现、可用于发布结论的数据集”。 +正确边界是三层： -## 1. 2026-07-26 点时盘点 +| 层 | 本机位置/产物 | 用途 | 是否可变 | +| --- | --- | --- | --- | +| live raw mirror | `$TUSHARE_MIRROR_ROOT` | 下载器持续落盘的 Parquet、权限探测和 job ledger | 是；不能直接作为可重放 run 的唯一引用 | +| scoped frozen release | `tools/tushare_snapshot.py` 生成的范围化 manifest | 固定日期、API、指数代码、所选 job/file hash 和选择规则 | manifest 不变；引用的 raw bytes 仍需留存 | +| Qlib provider | `data/qlib/...` + provider manifest | Qlib 0.9.7 可直接读取的 calendar/instruments/features | 否；是 frozen release 的派生物，不是原始数据备份 | -镜像中共有 93 个状态为 `completed` 的 Parquet，均通过 SQLite 记录的文件大小、 -SHA-256 和 Parquet row count 校验；另有 1 个遗留 `running` job。 +`tools/tushare_qlib.py` 和冻结工具都只读镜像，不调用 Tushare，也不读取或保存 +token。snapshot 工具不复制 Parquet；manifest 能在源文件被覆盖时检测不一致， +但若要长期从 raw bytes 重建，还需要后续接入 content-addressed archive。 +原始 Parquet、绝对路径、账号信息和本地运行 artifacts 不进入 Git。 -| 表 | 文件数 | 行数 | 当前范围/说明 | -| --- | ---: | ---: | --- | -| `daily` | 35 | 28,731 | 1990-12-19—1993-10-29，138 个代码、731 个交易日 | -| `trade_cal` | 37 | 13,004 | SSE 1990—2026 日历，8,689 个开市日 | -| `stock_basic` | 4 | 5,869 | 当前上市/退市/暂停上市等基础快照 | -| `stock_company` | 3 | 6,294 | SSE/SZSE/BSE 公司资料 | -| `index_basic` | 7 | 9,643 | CSI/SW/SSE/SZSE 等指数基础资料 | -| `index_classify` | 6 | 870 | 申万 2014/2021 分类 | -| `bse_mapping` | 1 | 248 | 北交所新旧代码映射 | -| **合计** | **93** | **64,659** | **5,734,035 bytes Parquet** | +## 1. 当前真实覆盖 -`daily` 原计划覆盖 1990-12—2026-07，共 428 个月；当前完成 35 个月，约 -8.2%。下载进程已停止，SQLite 仍遗留 -`daily/month=1993-11` 的 `running` 状态。日志显示首先发生 DNS 解析失败， -错误处理阶段又发生 `sqlite3.OperationalError: unable to open database -file`。因此不能把这个状态解释为仍在后台下载，也不要在 Quant OS -转换过程中修改或自动恢复原下载器。 +下面同时区分“自然分区文件”和 SQLite 的“completed 版本记录”。下载器在 +2026-07 月内多次刷新同一个分区，所以 `daily` 和 `adj_factor` 都是 428 个 +自然月文件，却各有 430 条 completed 版本记录。job 行数求和会重复计算旧版 +2026-07，不能当作当前文件行数。 -尚未落盘的生产关键表包括： +| 数据 | 当前自然文件 | 当前行数 | 实际可见范围 | 可直接支持什么 | +| --- | ---: | ---: | --- | --- | +| `daily` | 428 | 18,041,386 | 1990-12-19—2026-07-28 | OHLC、前收、成交量额、收益与本地撮合输入 | +| `adj_factor` | 428 | 18,868,457 | 1990-12—2026-07；最大 `trade_date=20260729` | 区间内累计复权 | +| `trade_cal` | 39 completed 版本 | 13,423 | SSE 1990—2026 | 交易日历与完整性检查 | +| `stock_basic` | 4 | 5,869 | 当前上市、退市、暂停上市等快照 | 代码、上市/退市日期；不是 PIT 成分表 | +| `daily_basic` | 331 | 17,335,584 | 1999-01-04—2026-07-28 | 市值、换手、估值与横截面特征 | +| `stk_limit` | 235 | 17,878,521 | 2007-01-04—2026-07-29 | 逐股逐日真实涨跌停价格 | +| `suspend_d` | 37 | 641,379 | 非空记录 1999-05-04—2026-07-29 | 停复牌与可交易性 | +| `index_daily`，`000905.SH` | 37 年 job | 5,239 | 2004-12-31—2026-07-29 | 真实中证 500 benchmark | +| `index_weight`，`000905.SH` | 37 年 job | 129,000 | 2005-01-31—2026-06-30 | 中证 500 的历史定期成分/权重快照 | +| `index_member_all` | 1 | 3,000 | 申万行业层级及进出日期 | 行业分类；不能冒充中证 500 历史成分 | -- `adj_factor`、`index_daily`； -- `index_member_all`、`index_weight`； -- `stk_limit`、`suspend_d`、`stock_st`； -- `daily_basic`。 +`daily` 原计划覆盖 1990-12—2026-07，共 428 个月；当前完成 +428 个月，约 100.0%。这里的 +“完成”只指自然月文件覆盖，不代表 live mirror 已冻结，也不代表 +production-ready。 -其中历史 `stock_st` 权限探测被拒绝。即使其他下载完成，历史 ST 仍需要新的 -授权数据源或明确的降级门禁。 +另外已有 `income`、`balancesheet`、`cashflow`、`fina_indicator`、 +`disclosure_date`、`moneyflow` 等数据。它们可以用于下一阶段的基本面、质量、 +资金流特征，但财务报表必须按公告/披露可得日做 as-of join，不能直接按报告期 +回填到历史日期。 -## 2. 已确认的数据质量 +### 仍然存在的硬缺口 -- 已完成 Parquet 自然键无重复，核心日线字段无空值； -- 无负成交量或负成交额，日线日期与同期 SSE 开市日完全对应； -- 17 条早期 OHLC 包络异常；默认拒绝，只有显式 - `--ohlc-policy expand-range` 才按原始 `open/high/low/close` 四价的 - min/max 修复，并记录数量与异常键 hash； -- 2 条 `pct_chg` 与保存精度下重算结果的偏差超过 0.02 个百分点； -- 日线代码 `000022.SZ` 不在当前 `stock_basic` 快照； -- `stock_basic` 存在遗留非标准代码 `T600018.SH`，转换器不会把它当成 - A 股代码； -- `index_basic.base_point` 存在分区类型漂移，后续 canonical 合并时必须 - 显式 cast。 +- 2026-07-26 的权限探测明确显示 `stock_st` 为 `denied`。在获得授权历史 + ST 源之前，不能声称完整复现 ST 股票的逐日交易限制； +- live ledger 在本次点时仍有 24 个 `running`：19 个 + `931632CNY04.CSI` 的 `index_daily` 年任务和 5 个 `931632.CSI` 的 + `index_weight` 年任务。它们与 `000905.SH` 的 2018—2025 release 无关， + 但说明全镜像仍是移动目标； +- `daily` 的最新 completed 请求元数据包含 `20260729`，但当前 Parquet + 的最大实际日期是 `20260728`；任何冻结工具都必须验证 Parquet 内容，而不能 + 只相信请求参数或 job 状态； +- 多个 completed 版本可指向同一个最终路径。冻结时必须按明确规则选择当前 + canonical 版本并复核 size/SHA-256/Parquet row count，不能把所有 + `completed` 盲目拼接； +- `stock_basic` 是当前快照，不得用它反向构造历史指数 universe。 -Tushare 的 `daily` 是未复权行情，停牌期间不返回记录；`vol` 单位为手， -`amount` 单位为千元。转换器固定执行： +## 2. 数据如何进入 Quant OS + +第一条可用研究路径是中证 500 横截面研究： + +```text +daily + adj_factor + trade_cal + + index_daily(000905.SH) + + index_weight(000905.SH) + -> scoped frozen release (2018-01-01—2025-12-31) + -> managed Qlib provider + -> momentum / Alpha158 / LightGBM + -> evidence JSON + provider verifier +``` + +第二条是本地事件回测输入： + +```text +raw daily/pre_close + stk_limit + suspend_d + + PIT universe + signal/target + -> Quant OS reference execution + -> ledger / replay / platform parity +``` + +两条路径不能混为一谈。Qlib 负责因子、模型和组合研究；Quant OS 的 +Signal/Target/Order/Broker 合同、涨跌停、停牌、T+1、成交和对账仍由本地 +事件链负责。历史 ST 未补齐时，本地执行链必须保留显式缺口或 fail-closed， +不能静默按普通股票处理后再把结果标为完整市场仿真。 + +### 复权口径 + +新 provider 对每个 symbol 使用 release 区间内第一个有效 `adj_factor` 作为 +锚点： + +```text +factor_t = adj_factor_t / first_valid_adj_factor +adjusted_OHLC_t = raw_OHLC_t * factor_t +adjusted_volume_t = raw_volume_in_shares_t / factor_t +money_t = raw_amount_in_CNY_t +``` + +这是“首个有效因子锚定的区间累计复权”，文档和证据中不要把它简写成 +`qfq`。不用区间末因子做归一化，是为了避免研究区间末端信息参与早期价格的 +缩放。停牌缺口保留为 `NaN`，不会伪造 K 线。 + +固定单位仍为： ```text 600000.SH -> SH600000 000001.SZ -> SZ000001 430047.BJ -> BJ430047 -volume = vol * 100 -money = amount * 1000 -factor = 1 +raw_volume_in_shares = Tushare vol * 100 +raw_amount_in_CNY = Tushare amount * 1000 ``` -未上市、退市后和停牌缺口在 Qlib feature 中保留为 `NaN`。字段语义来源见 -[Tushare 日线接口](https://tushare.pro/document/2?doc_id=27)与 -[复权因子接口](https://tushare.pro/document/2?doc_id=28)。 +### universe 与 benchmark 口径 -## 3. Quant OS 如何使用这些数据 +- benchmark 使用 `index_daily(000905.SH)`，Qlib symbol 为 `SH000905`； +- universe 使用 `index_weight(000905.SH)` 的历史快照，并按“只向未来 + carry-forward、绝不从未来 backfill、月末权重下一交易日生效”生成时变 + 成分区间。这是 event-time / 保守次日生效 PIT 近似；源数据没有单独验证的 + `published_at`，不能声称严格 knowledge-time PIT； +- `index_member_all` 是申万行业数据，不参与中证 500 universe； +- 研究报告必须分别披露 benchmark、universe、价格复权和可交易性过滤来源。 -当前已经实现的链路是研究/引擎验证路径： +## 3. 现有代码基线与 v2 接口 -```text -Tushare mirror - -> 只读取 SQLite completed jobs - -> 文件大小 + SHA-256 + Parquet row count/footer 校验 - -> 新目录原子发布 Qlib binary provider - -> provider tree hash + data_version + manifest verifier - -> Qlib momentum / Alpha158 research runner -``` +### 现有基线 -正式 Quant OS 还需要第二条生产数据路径： - -```text -完整 Tushare/JQData PIT 数据 - -> canonical immutable snapshot - -> Quant OS 本地事件回测 / decision - -> 聚宽、QMT、Qlib 的同输入分层比较 -``` - -Qlib 适合因子、模型和组合研究，不拥有原始数据，也不替代 Quant OS 的 -Signal/Target/Order/Broker 合同。当前 Tushare adapter 只实现第一条链路； -在复权、真实 benchmark、历史成分、停牌、涨跌停和 ST 数据补齐前，不会把它 -接入生产 decision。 - -## 4. 可运行命令 - -从 Quant OS 根目录执行。镜像路径只放在当前 shell 环境变量中，不写入 Git： +仓库当前基线的 `build` 只消费 `daily/trade_cal/stock_basic`，必须显式传 +`--allow-unadjusted`，会生成 synthetic equal-weight benchmark，并按整个 +研究区间的最少观测数筛选股票。它只能用于转换器/运行时技术验证： ```bash -export QUANT_OS_ROOT=/path/to/quant-os -export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror -cd "$QUANT_OS_ROOT" - -python3.12 -m venv .venv-qlib312 -source .venv-qlib312/bin/activate -python -m pip install -r requirements/research-py312.txt -export PYTHONPATH=src:. -``` - -盘点并验证全部 completed 文件： - -```bash -python tools/tushare_qlib.py inventory \ +PYTHONPATH=src:. python tools/tushare_qlib.py build \ --mirror-root "$TUSHARE_MIRROR_ROOT" \ - --output-json artifacts/tushare-inventory.json -``` - -当前未复权数据只能显式构建技术验证 provider；每次必须使用新目录，工具拒绝 -覆盖已有 provider： - -```bash -python tools/tushare_qlib.py build \ - --mirror-root "$TUSHARE_MIRROR_ROOT" \ - --output-dir data/qlib/tushare-early-v1 \ - --start 1990-12-19 \ - --end 1993-10-29 \ + --output-dir data/qlib/tushare-legacy-2018-2025 \ + --start 2018-01-01 \ + --end 2025-12-31 \ --minimum-observations 60 \ --allow-unadjusted \ - --ohlc-policy expand-range \ - --output-json artifacts/tushare-qlib-build.json - -python tools/tushare_qlib.py verify \ - data/qlib/tushare-early-v1 \ - --output-json artifacts/tushare-qlib-verify.json + --ohlc-policy fail \ + --output-json artifacts/tushare/legacy-build.json ``` -运行 Qlib 0.9.7 本地回测： +不要用这个结果回答“中证 500 策略是否有效”，也不要给它任何发布门禁分数。 + +### 已验证的 v2 接口 + +v2 已消费 `adj_factor`、真实指数行情和历史指数权重，并完成 provider +构建、验证和两次本地 run。实际构建命令为： ```bash -PYTHONHASHSEED=0 python -m platforms.qlib_runner \ - --provider-uri data/qlib/tushare-early-v1 \ - --market tushare_a \ - --benchmark SH999999 \ - --start 1992-01-02 \ - --end 1993-10-28 \ - --feature-start 1991-01-02 \ - --lookback 20 \ - --topk 10 \ - --n-drop 2 \ - --rebalance weekly \ - --output-json artifacts/tushare-qlib-momentum.json +PYTHONPATH=src:. python tools/tushare_qlib.py build \ + --mirror-root "$TUSHARE_MIRROR_ROOT" \ + --output-dir data/qlib/tushare-csi500-2018-2025-v2 \ + --start 2018-01-01 \ + --end 2025-12-31 \ + --market-name tushare_csi500 \ + --benchmark-index-code 000905.SH \ + --universe-index-code 000905.SH \ + --minimum-observations 60 \ + --ohlc-policy fail \ + --output-json artifacts/tushare/csi500-2018-2025-build.json ``` -Qlib simulator 会消费结束日后的下一 provider session，所以 backtest `end` -必须早于 provider 最后一个交易日。Quant OS 对受管 provider 会在 Qlib -初始化前检查 market、benchmark、起始边界和这个终点条件。 +这条命令不再传 `--allow-unadjusted`。provider manifest 必须记录复权口径、 +真实 benchmark、时变 universe、所有 source job/file hash、转换器 hash、 +质量统计和可重算 `data_version`。 -`artifacts/` 与 `data/qlib/` 均被 Git 忽略。原始 Parquet、token、绝对镜像 -路径和回测临时产物不得提交到仓库。 +`tools/tushare_snapshot.py` 是独立的 raw scoped freeze 入口。Qlib 核心输入 +可以这样冻结： -## 5. 本次真实本地运行证据 +```bash +python tools/tushare_snapshot.py \ + --mirror-root "$TUSHARE_MIRROR_ROOT" \ + --start 2018-01-01 \ + --end 2025-12-31 \ + --apis daily adj_factor trade_cal stock_basic index_daily index_weight \ + --index-codes 000905.SH \ + --output artifacts/local-tushare-20260731/scoped-source-manifest.json +``` -最终 v4 provider： +默认会验证 size、SHA-256 和 Parquet row count。需要研究 +`daily_basic/stk_limit/suspend_d` 时应再冻结一个明确包含它们的扩展 +release；不要把未列入 manifest 的表暗中加入既有 run。当前 `build` 仍从 +mirror 重新选择 source jobs，并不接受这份 manifest 作为参数，因此 run +验收还必须比较 provider manifest 与 scoped manifest 的 source +job/file hashes。 -| 证据 | 值 | +snapshot 工具在请求 `index_weight` 时会自动包含 `start.year-1` 的 pre-start +PIT anchor。本次 scoped manifest 与 provider 均为 221 个 source jobs， +全字段比对零差异，已经覆盖有效 roster 日期 `2017-12-29`： + +```text +scoped_manifest_sha256 = 744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2 +scoped_selection_sha256 = 66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5 +scoped_source_jobs = 221 +provider_source_jobs = 221 +source_job_full_field_diff = 0 +provider_source_jobs_before_read = 221 +provider_source_jobs_after_read = 221 +provider_selected_jobs_still_latest = true +provider_post_read_file_verification = true +``` + +冻结期间 SQLite 主文件/WAL 仍被无关下载任务更新： +`changed_through_file_verification=true`。这不是 selected scope 未冻结： +SQLite 事务读取期间 `changed_while_reading=false`，221 个 selected jobs +均完成两次 size/SHA-256/Parquet row-count 校验，验证后仍是各自 partition +的 latest，并与 provider 的 221 个 source jobs 全字段零差异。这里的证据边界 +是“scoped jobs 稳定”，不是“整个 live DB/WAL 静止”。 + +scoped manifest 与 provider 的正式 lineage 由独立工具核对： + +```bash +python tools/tushare_lineage.py \ + --source-manifest artifacts/local-tushare-20260731/scoped-source-manifest.json \ + --provider-dir data/qlib/tushare-csi500-2018-2025-v2 \ + --output-json artifacts/tushare/csi500-2018-2025-lineage.json +``` + +结果为 221 个 jobs、6 个逐 job 字段 +`id/path/row_count/byte_count/sha256/request_sha256` 全部一致， +`mismatch_count=0`、`converter_source_matches_current=true`。lineage artifact +同样明确 `investment_value_claim=false`、`gate_credit=[]`。 + +## 4. 2018—2025 provider 与双次回测实证 + +managed provider 已构建在新目录，并由独立 verifier 通过： + +| 证据 | 实测值 | | --- | --- | -| adapter source SHA-256 | `2228c23af176a0f2fcf311e88dca9bdc5d63fd36fa5a3c2d3e42808a405e7243` | -| data version | `3378aa75a5601bde476ad07bea90418966a66a037ca59195dec93d77b41cc3f8` | -| provider tree SHA-256 | `dc85b8daf692a66641afef64398700264c6054f1dac84b1211cb258a06b62948` | -| manifest SHA-256 | `862e7435bb0be7c8a3c66180e49761c110056c1a3cc02c1618e70d9eb566d945` | -| provider 文件/大小 | 975 / 1,085,886 bytes | -| calendar | 1990-12-19—1993-10-29，731 sessions | -| market | 107 个至少有 60 条记录的 A 股代码 | -| selected rows | 27,998 | +| data version | `5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963` | +| provider tree SHA-256 | `f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f` | +| provider manifest SHA-256 | `27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0` | +| provider files / bytes | 10,011 / 71,707,194 | +| source stability | 221/221 jobs 在消费前后稳定；读取后重新校验文件且仍为 latest | +| calendar | 2018-01-02—2025-12-31，1,942 sessions | +| event-time PIT 近似 market | 1,111 个曾进入 `000905.SH` roster 的 instruments | +| snapshot coverage | 97 observed / 96 effective；最大日历间隔 36 天 | +| roster quality | 每个 observed snapshot 严格 500 constituents；weight sum 容差 ±0.5 | +| availability | 月末权重从下一 provider 交易日生效；`same_session_membership_use=false` | +| selected daily rows | 1,975,455 | +| price adjustment | 完整；0 个缺失 daily/adj keys；首个有效因子锚定 | +| OHLC quality | `--ohlc-policy fail`；0 个 envelope anomalies | +| benchmark | 真实 `index_daily(000905.SH)` / `SH000905` | +| release flags | `production_ready=false`、`investment_value_claim=false`、`gate_credit=[]` | -回测参数为 1992-01-02—1993-10-28、20 日动量、周频、Top 10、每期 drop 2， -初始资金 1,000 万，Qlib 0.9.7、Python 3.12.13、`PYTHONHASHSEED=0`。 -相同命令连续执行两次，结果 JSON 逐 byte 相同： +周频 20 日动量参数为 2019-01-02—2025-12-30、Top 50、每期 drop 5、 +初始资金 1,000 万、开仓费 0.03%、平仓费 0.08%、最低费用 5 元、 +`PYTHONHASHSEED=0`。相同命令连续执行两次，两个 JSON 逐 byte 相同： -| 证据/指标 | 值 | +| 证据/指标 | 实测值 | | --- | ---: | -| result JSON SHA-256 | `92ff9b8cea746e9c89ddf62fcfe3feb21248ca9112d9e10d24e0639058a86020` | -| runner source SHA-256 | `fa8118819252c55e99e67de356cc961d8a4a22f82256b4d62579a45426d96167` | -| signal | 4,700 行；hash `cfad4087d68b7f71e33f0f46fc5bb97db985e02ac2515f01857e2ab6813a5909` | -| portfolio report | 466 行；hash `5f8606fb64963d3e2618f66503e6ae94048c24b62c5d8e1806ff44ec57558b04` | -| strategy cumulative return | -52.5251% | -| max drawdown | -87.7604% | -| synthetic benchmark cumulative return | 196,302.0822% | -| total cost / turnover | 0.016013 / 29.567521 | +| run JSON SHA-256 | `803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3` | +| signal rows | 176,615 | +| portfolio report rows | 1,698 | +| strategy cumulative return | 0.253151986388197（25.3152%） | +| benchmark cumulative return | 0.789559383530678（78.9559%） | +| max drawdown | -0.6041641365538415（-60.4164%） | +| total cost / turnover | 0.03579348112198462 / 65.51393782463258 | -这些收益数字没有投资解释。原因包括未复权早期行情、没有真实指数 benchmark、 -没有历史成分/ST/停牌/涨跌停、按全区间至少 60 条观测筛选带来的非 PIT -偏差，以及 1992—1993 特殊市场阶段。极端 synthetic benchmark 恰好说明 -为什么“程序跑完”不能等价为“回测有效”。 +这组数字证明真实 Tushare → event-time/保守次日生效 PIT 近似 universe → +复权 Qlib provider → +momentum runner 的链路可重放，不证明策略可投资。策略显著跑输 benchmark， +且最大回撤超过 60%，不能把正累计收益单独拿出来宣传。 -## 6. 何时可以升级为正式研究数据 +最终身份变化来自 verifier/转换器源码绑定；provider tree 与回测数值未变。 +verifier 已确认 recorded/current converter SHA-256 一致。由于 run evidence +绑定 provider identity，两个 run JSON 的 SHA-256 随之更新。 -至少完成以下步骤后，才能新建 production-eligible provider： +## 5. 验证顺序 -1. 修复下载器错误状态并完成目标日期的 `daily`； -2. 补齐 `adj_factor`，冻结复权基准日和公式； -3. 使用 `index_daily` 替换合成 benchmark； -4. 指数策略补齐 `index_member_all/index_weight`； -5. 补齐 `suspend_d/stk_limit`，为历史 `stock_st` 找到授权来源或保持硬阻断； -6. 冻结下载 job、请求 hash、Parquet hash、转换器 hash 和新 `data_version`； -7. 运行真实长样本 walk-forward/OOS，而不是复用本次早期 smoke； -8. 同一冻结输入进入 Quant OS 本地回测、聚宽和可用 QMT，生成 L1—L4 - 差异报告。 +1. 对 live mirror 做快速元数据盘点，确认目标 API 和 `000905.SH` 范围； +2. 生成 2018—2025 scoped frozen release，并验证所选文件的 size、 + SHA-256、Parquet footer/row count、自然键和日期边界； +3. 在冻结后从同一 source state 构建新 provider，绝不覆盖已有目录，并核对 + provider/scoped manifest 的 source hashes； +4. 运行 `tools/tushare_qlib.py verify`，核对 manifest、tree hash、 + calendar、instrument 和 feature 文件； +5. 运行同一条 Qlib momentum 命令两次，比较 evidence JSON 的关键输入、 + signal/report hash 和重算指标； +6. 再进入 walk-forward、Alpha158/LightGBM、成本/风险扰动和本地事件回测。 -Qlib provider 的目录格式和缺失值/复权约定参见 -[Qlib Data Layer 文档](https://qlib.readthedocs.io/en/latest/component/data.html)。 +完整可复制命令见 +[`runbooks/TUSHARE_QLIB_LOCAL_RUN.md`](runbooks/TUSHARE_QLIB_LOCAL_RUN.md)。 + +## 6. 结果边界 + +2018—2025 的真实行情、复权、指数行情和历史权重会让回测比旧的 +1990—1993 technical smoke 有意义得多，但仍只是一条研究证据： + +- momentum smoke 的收益不能证明投资价值； +- Qlib 的成交模型不能替代 A 股逐股涨跌停、停牌、T+1 和真实容量仿真； +- 本次 Qlib run 使用统一 `limit_threshold=0.095`，只是 9.5% 市场级近似； + 它没有消费已有 `stk_limit` 的逐股逐日价格，也不能表达不同板块、日期和 + ST 股票的真实限制； +- `stock_st` 缺口没有因为已有 `namechange` 或 `stk_limit` 就自动消失； +- `index_weight` 没有单独验证的 `published_at`，因此这里不是严格 + knowledge-time PIT； +- 复权以每个 symbol 在 provider 区间内的首个有效因子为锚。不同 `start` + 构建的 provider 归一化基准不同，不能直接拼接其价格 level；需要统一起点 + 重建或显式重新归一化； +- 必须做时间切分、walk-forward/OOS、参数稳定性、成本压力和跨引擎差异； +- 本文不据此声称 Quant OS 已达到 60 分或 80 分标准。 + +相关字段语义参见 +[Tushare 日线接口](https://tushare.pro/document/2?doc_id=27)、 +[复权因子接口](https://tushare.pro/document/2?doc_id=28)和 +[Qlib Data Layer](https://qlib.readthedocs.io/en/latest/component/data.html)。 diff --git a/docs/getting-started/GETTING_STARTED_80.md b/docs/getting-started/GETTING_STARTED_80.md index cc064bb..317fda0 100644 --- a/docs/getting-started/GETTING_STARTED_80.md +++ b/docs/getting-started/GETTING_STARTED_80.md @@ -19,13 +19,14 @@ Investment value claim: false 独立仓： ```text -~/boat-workspace/Code/quant-os +$QUANT_OS_ROOT ``` 推荐 Python 3.12： ```bash -cd ~/boat-workspace/Code/quant-os +export QUANT_OS_ROOT=/path/to/quant-os +cd "$QUANT_OS_ROOT" python3.12 -m venv .venv-core source .venv-core/bin/activate python -m pip install -e . @@ -36,7 +37,7 @@ python -m pip install -e . `quant-os` 应在 checkout 内运行；从其他目录调用新命令时显式写： ```bash -quant-os --project-root ~/boat-workspace/Code/quant-os doctor +quant-os --project-root "$QUANT_OS_ROOT" doctor ``` wheel 不是独立的数据/runtime 分发包。 @@ -117,16 +118,29 @@ quant-os verify-research-manifest artifacts/research-smoke/manifest.json ## 5. Tushare 数据与 Qlib -状态：`[已实现到技术 smoke｜真实镜像仍不完整]` +状态：`[v2 provider/run 已验证｜research-only]` -先阅读 [`../TUSHARE_LOCAL_DATA.md`](../TUSHARE_LOCAL_DATA.md)。当前盘点表明 -completed Parquet 能校验并转换为 managed Qlib provider，但缺复权、真实 -benchmark、历史成分、ST、停牌和涨跌停，不能进入生产 decision。 +先阅读 [`../TUSHARE_LOCAL_DATA.md`](../TUSHARE_LOCAL_DATA.md) 和 +[`../runbooks/TUSHARE_QLIB_LOCAL_RUN.md`](../runbooks/TUSHARE_QLIB_LOCAL_RUN.md)。 +当前镜像已有复权因子、真实中证 500 benchmark/历史权重、停牌和涨跌停； +raw mirror 仍可变化，历史 `stock_st` 权限被拒，所以必须先冻结 scoped +release，再构建、验证 provider。2018—2025 provider 和双次 momentum run +已经完成；月末指数权重保守地从下一 provider 交易日生效，不做 same-session +使用，97 个 observed snapshots 形成 96 个 effective rosters，均严格 500 +成分、weight sum 在 100±0.5 内、最大间隔 36 天。但统一 9.5% 涨跌停近似和 +ST 缺口仍使其不能进入完整 production decision，`gate_credit=[]`。 +冻结期间全 DB/WAL 可能因无关下载任务变化；验收看的是 221 个 scoped jobs +double verified、验证后仍为 latest；再用 `tools/tushare_lineage.py` 核对 +221 jobs 的 6 个身份字段，要求 mismatch 0 且 converter current true。 +历史权重没有单独验证的 `published_at`，所以 universe 是 event-time/保守 +次日生效 PIT 近似，不是严格 knowledge-time PIT。复权使用首观测锚；不同 +`start` 构建的 provider 不能直接拼接价格 level。 典型流程： ```bash PYTHONPATH=src:. python tools/tushare_qlib.py inventory --help +python tools/tushare_snapshot.py --help PYTHONPATH=src:. python tools/tushare_qlib.py build --help PYTHONPATH=src:. python tools/tushare_qlib.py verify --help PYTHONPATH=src:. python -m platforms.qlib_runner --help diff --git a/docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md b/docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md new file mode 100644 index 0000000..52bbf86 --- /dev/null +++ b/docs/runbooks/TUSHARE_QLIB_LOCAL_RUN.md @@ -0,0 +1,221 @@ +# 本机 Tushare → 冻结 release → Qlib 回测 + +目标：只使用本机已有数据，冻结 `000905.SH` 的 2018—2025 输入，构建 +managed Qlib provider，验证后运行一条可重放的周频动量 smoke。 + +## 1. 固定本机路径与 Python 环境 + +```bash +export QUANT_OS_ROOT=/path/to/quant-os +export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror +cd "$QUANT_OS_ROOT" + +python3.12 -m venv .venv-qlib312 +source .venv-qlib312/bin/activate +python -m pip install -r requirements/research-py312.txt +python -c "import qlib; assert qlib.__version__ == '0.9.7'" +export PYTHONPATH=src:. +``` + +当前正式项目目录内应新建自己的 `.venv-qlib312`。不要复用 +`quant-os-migration-backup-20260730` 下的迁移遗留环境。 + +### 推荐的 Makefile 全链调用 + +```bash +export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror +export TUSHARE_PROVIDER=data/qlib/tushare-csi500-2018-2025-next +export TUSHARE_SNAPSHOT=artifacts/local-tushare/scoped-source-manifest.json + +make tushare-snapshot +make tushare-build +make tushare-verify +make tushare-lineage +make tushare-backtest +make tushare-replay +``` + +每次 build 必须使用一个尚不存在的 `TUSHARE_PROVIDER` 目录，不能覆盖本页记录的 +正式 provider。`make tushare-replay` 依赖 `tushare-backtest`，随后运行第二次 +并用 `cmp` 要求两个 evidence JSON 逐 byte 相同。下面保留展开命令，便于审计 +每一个参数。 + +## 2. 先盘点 live mirror + +全镜像仍可能下载指数数据。先做快速 job-ledger 盘点： + +```bash +python tools/tushare_qlib.py inventory \ + --mirror-root "$TUSHARE_MIRROR_ROOT" \ + --skip-file-verification \ + --output-json artifacts/tushare/live-inventory.json +``` + +对几十万文件做全镜像 SHA 校验成本较高，也不能解决“扫描过程中镜像变化”的 +问题。正式 run 应在下一步冻结范围后，只校验 release 选中的文件。 + +## 3. 冻结 2018—2025 scoped release + +本次 Qlib 核心 release 包含： + +```text +daily,adj_factor,trade_cal,stock_basic,index_daily,index_weight +index code = 000905.SH +date = 2018-01-01 ... 2025-12-31 +``` + +独立入口为 `tools/tushare_snapshot.py`。先确认接口，再生成 manifest： + +```bash +python tools/tushare_snapshot.py --help + +python tools/tushare_snapshot.py \ + --mirror-root "$TUSHARE_MIRROR_ROOT" \ + --start 2018-01-01 \ + --end 2025-12-31 \ + --apis daily adj_factor trade_cal stock_basic index_daily index_weight \ + --index-codes 000905.SH \ + --output artifacts/local-tushare-20260731/scoped-source-manifest.json +``` + +默认开启文件验证，manifest 记录选择规则、目标 API/指数/日期、source job、 +实际 Parquet 路径、size、row count、SHA-256 和 release hash。生成后不要 +继续用 live ledger 代替这份 manifest。若下一条执行仿真需要 +`daily_basic/stk_limit/suspend_d`，另建包含这些 API 的 release，不要静默 +扩充本次 Qlib 输入。该 manifest 是 hash 锁定证据，不复制 raw Parquet； +需要长期 raw 重建时还要保留对应内容寻址副本。 + +## 4. 构建并验证 v2 provider + +先确认 v2 参数： + +```bash +python tools/tushare_qlib.py build --help +``` + +当前已验证命令如下： + +```bash +python tools/tushare_qlib.py build \ + --mirror-root "$TUSHARE_MIRROR_ROOT" \ + --output-dir data/qlib/tushare-csi500-2018-2025-v2 \ + --start 2018-01-01 \ + --end 2025-12-31 \ + --market-name tushare_csi500 \ + --benchmark-index-code 000905.SH \ + --universe-index-code 000905.SH \ + --minimum-observations 60 \ + --ohlc-policy fail \ + --output-json artifacts/tushare/csi500-2018-2025-build.json + +python tools/tushare_qlib.py verify \ + data/qlib/tushare-csi500-2018-2025-v2 \ + --output-json artifacts/tushare/csi500-2018-2025-verify.json + +python tools/tushare_lineage.py \ + --source-manifest artifacts/local-tushare-20260731/scoped-source-manifest.json \ + --provider-dir data/qlib/tushare-csi500-2018-2025-v2 \ + --output-json artifacts/tushare/csi500-2018-2025-lineage.json +``` + +输出目录必须是新目录。失败时保留错误和 inventory/release 证据，不要删除门禁 +或回退到 synthetic benchmark。v2 的复权是每个 symbol 以区间首个有效因子 +为锚：`factor_t=adj_t/first_adj`；OHLC 乘 factor、share volume 除 factor、 +money 不变。不同 `start` 构建的 provider 具有不同归一化锚点，价格 level +不能直接拼接；需要以统一起点重建或显式重新归一化。 + +当前 `build` 会自行从 mirror 选择 source jobs，并把它们写进 provider +manifest；它不会直接读取上一步的 scoped manifest。验收时必须比较两份 +manifest 的 source job/file hashes。snapshot 工具会为 `index_weight` 自动 +包含 `start.year-1` 的 pre-start anchor。本次两边均为 221 个 jobs，全字段 +diff 为 0；scoped manifest SHA-256 为 +`744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2`， +selection SHA-256 为 +`66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5`。 +provider 对 221 个 jobs 在消费前完成校验，消费后重新读取 SQLite 并再次校验 +221 个文件；`selected_jobs_still_latest=true`、 +`post_read_file_verification=true`。 + +live 下载器在整个验证窗口内继续写入无关任务，所以全 DB/WAL 的 +`changed_through_file_verification=true`。scoped 证据仍成立：事务读取期间 +`changed_while_reading=false`，221 个 jobs 均 double verified、验证后仍为 +latest，且与 build 的 221 个 source jobs 全字段零差异。不要把“全库有变化” +误写成“本次 scoped inputs 有变化”。 + +lineage 结果还必须满足：`source_job_count=221`，匹配字段严格为 +`id/path/row_count/byte_count/sha256/request_sha256`， +`mismatch_count=0`，并且 `converter_source_matches_current=true`。 + +## 5. 运行本地 momentum smoke + +provider 以 2025-12-31 结束；Qlib simulator 会访问回测结束日之后的下一个 +provider session，因此回测结束日使用 2025-12-30： + +```bash +PYTHONHASHSEED=0 python -m platforms.qlib_runner \ + --provider-uri data/qlib/tushare-csi500-2018-2025-v2 \ + --market tushare_csi500 \ + --benchmark SH000905 \ + --start 2019-01-02 \ + --end 2025-12-30 \ + --feature-start 2018-01-02 \ + --lookback 20 \ + --topk 50 \ + --n-drop 5 \ + --rebalance weekly \ + --output-json artifacts/tushare/csi500-momentum-run-1.json +``` + +原参数不变再运行一次，只改输出文件名为 +`csi500-momentum-run-2.json`。比较输入/provider identity、signal hash、 +portfolio report hash、表边界和重算指标；`created_at` 等非决定性元数据若 +存在，应由 verifier 的语义比较处理，而不是只看终端是否打印成功。 + +本次两个 run JSON 实际逐 byte 相同，SHA-256 均为 +`803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3`。 + +## 6. 已获得的正式研究证据 + +| 项目 | 实测值 | +| --- | --- | +| data version | `5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963` | +| provider tree SHA-256 | `f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f` | +| provider manifest SHA-256 | `27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0` | +| lineage | 221 jobs × 6 fields；mismatch 0；converter current true | +| provider files / bytes | 10,011 / 71,707,194 | +| source stability | 221/221 jobs before/after stable；post-read file verification 通过 | +| instruments / sessions | 1,111 / 1,942 | +| selected rows | 1,975,455 | +| observed / effective snapshots | 97 / 96 | +| universe quality | 每个 snapshot 严格 500 constituents；weight sum ±0.5；最大间隔 36 天 | +| membership availability | 月末权重下一 provider 交易日生效；same-session use 为 false | +| signal / report rows | 176,615 / 1,698 | +| strategy cumulative return | 0.253151986388197 | +| benchmark cumulative return | 0.789559383530678 | +| max drawdown | -0.6041641365538415 | +| evidence flags | `production_ready=false`、`investment_value_claim=false`、`gate_credit=[]` | + +## 7. 如何读结果 + +先回答四个工程问题： + +1. frozen release 与 provider 是否通过 hash/语义验证； +2. benchmark 是否确为 `SH000905`，universe 是否来自历史权重并从下一交易日 + 生效； +3. 两次 run 的关键表和指标是否可重放； +4. 日期、股票数、缺失率、换手、成本和极端收益是否合理。 + +然后才看年化收益、最大回撤、信息比率等研究指标。该 momentum run 只是数据桥 +和研究运行时 smoke，不是选股结论；它尚未完整表达历史 ST、逐股成交约束、 +冲击成本、容量和券商执行，也不自动获得 60/80 标准中的任何门禁分数。 + +尤其注意，本次 Qlib run 的 `limit_threshold=0.095` 是统一 9.5% 近似，没有 +使用镜像内 `stk_limit` 的逐股逐日价格，无法表达主板/创业板/科创板、规则变更 +和 ST 股票的不同涨跌停限制；历史 `stock_st` 权限仍被拒。因此这些结果严格是 +research-only，不能升级为 production-ready。 + +`index_weight` 源没有单独验证的 `published_at`，所以该 universe 只能称为 +event-time / 保守次日生效 PIT 近似，不能称为严格 knowledge-time PIT。 + +更完整的数据边界与当前覆盖见 +[`../TUSHARE_LOCAL_DATA.md`](../TUSHARE_LOCAL_DATA.md)。 diff --git a/evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json b/evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json new file mode 100644 index 0000000..629c8c2 --- /dev/null +++ b/evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json @@ -0,0 +1,85 @@ +{ + "schema_version": 1, + "record_type": "quant_os_tushare_qlib_research_run", + "run_id": "5bf19d2d", + "audit_as_of": "2026-07-31", + "scope": { + "index_code": "000905.SH", + "start": "2018-01-01", + "end": "2025-12-31", + "daily_market_data_from": "2018-01-02", + "daily_market_data_through": "2025-12-31", + "daily_partitions_completed": 96, + "daily_partitions_planned": 96, + "daily_rows": 8813885 + }, + "snapshot": { + "selected_jobs": 221, + "verified_jobs": 221, + "double_verified_jobs": 221, + "manifest_sha256": "744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2", + "selection_sha256": "66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5" + }, + "provider": { + "verified": true, + "data_version": "5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963", + "tree_sha256": "f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f", + "manifest_sha256": "27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0", + "file_count": 10011, + "byte_count": 71707194, + "calendar_sessions": 1942, + "instrument_count": 1111, + "selected_daily_rows": 1975455, + "observed_membership_snapshots": 97, + "effective_membership_snapshots": 96, + "same_session_membership_use": false, + "price_adjustment_complete": true, + "benchmark": "SH000905", + "converter_source_matches_current": true + }, + "lineage": { + "verified": true, + "source_job_count": 221, + "matched_fields": [ + "id", + "path", + "row_count", + "byte_count", + "sha256", + "request_sha256" + ], + "mismatch_count": 0, + "converter_source_matches_current": true + }, + "source_stability": { + "state_db_changed_through_file_verification": true, + "state_db_change_scope": "unrelated live downloader jobs", + "snapshot_jobs_still_latest": true, + "selected_scope_stable": true, + "post_read_file_verification": true, + "provider_source_jobs": 221, + "snapshot_source_jobs": 221, + "source_job_full_field_diff": 0 + }, + "reproducibility": { + "runs": 2, + "byte_identical": true, + "run_sha256": "803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3", + "signal_rows": 176615, + "portfolio_days": 1698 + }, + "performance": { + "strategy_cumulative_return": 0.2531519864, + "benchmark_cumulative_return": 0.7895593835, + "max_drawdown": -0.6041641366 + }, + "limitations": { + "stock_st": "denied", + "qlib_limit_threshold": 0.095, + "limit_rule_fidelity": "market-level approximation; not per-symbol daily limit prices", + "research_only": true + }, + "production_ready": false, + "investment_value_claim": false, + "gate_credit": [] +} diff --git a/gate_scorecard.json b/gate_scorecard.json index b324274..b9f946c 100644 --- a/gate_scorecard.json +++ b/gate_scorecard.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "system": "Quant OS / quant60 baseline kernel", - "audit_as_of": "2026-07-30", + "audit_as_of": "2026-07-31", "claim_policy": { "baseline_threshold": 60, "all_gates_required": true, @@ -35,7 +35,7 @@ "status": "not_scored", "reason": "Quant OS 已接通本地 synthetic 五层候选链，能冻结 Ridge ModelBundle、逐层 trace 和无股数 TargetPackageV1；一个 synthetic-research TargetPackage 已在真实 JoinQuant hosted 环境完成消费、账户/开盘价绑定、3 笔委托和 3 笔成交，因此 execution 层已有真实平台观察。但真实 provider-data 长样本/OOS 尚未接入，JoinQuant 没有重跑上游 Universe/Alpha/Portfolio/Risk，QMT peer 与逐层真实数据 parity 仍缺失，factor risk、CVXPY 和 guarded TWAP/POV 也尚未进入当前主链。因此它仍是 NOT_BASELINE_60，G9 未通过。", "verified_supporting_facts": [ - "The independent-repository standard-library suite completed 315 tests with OK and 6 optional-runtime skips; successful local tests do not prove real-platform, broker or compliance gates.", + "The independent-repository standard-library suite completed 325 tests with OK and 1 optional-runtime skip; successful local tests do not prove real-platform, broker or compliance gates.", "The local synthetic research entrypoint now executes one causally linked Universe → Ridge Alpha → cost/risk-aware Portfolio → post-risk gate → reference Execution path, freezes a replayable ModelBundle, and publishes a hash-linked TargetPackageV1 and five-layer trace. This is structural synthetic evidence only.", "The connected candidate currently uses 60-session diagonal realized variance, explicit spread/fee/square-root-impact cost forecasts, the deterministic constrained optimizer, a post-target risk gate, and reference lot/T+1 order deltas. factor_risk.py, the optional CVXPY solver mode, and guarded TWAP/POV are not selected by the current main slice.", "JoinQuant and QMT target-package modes verify an exact embedded package tape, match the declared dual clock, bind platform equity/positions/sellable quantities/prices, and reject missing, duplicate or tampered packages without falling back to momentum. Both modes have local harness/contract evidence; JoinQuant execution alone now also has one narrow real hosted smoke.", @@ -44,7 +44,7 @@ "The local snapshot can drive snapshot-backtest and snapshot-decision with manifest verification; it freezes a cross-checked provider trading calendar, and a T-close signal can bind only to a broker observation in the unique next session's Asia/Shanghai [09:00,09:30) window without collapsing the two clocks.", "CPython 3.12 with pyqlib 0.9.7 completed a native momentum fixture smoke with 24 signals and saved audited portfolio-table hashes, bounds and recomputed metrics.", "Qlib Alpha158, LightGBM and Recorder completed a real local fixture smoke with audited portfolio output; the deterministic two-stock synthetic fixture has no investment-value meaning.", - "The local Tushare inventory verified all 93 completed Parquet files against recorded size, SHA-256 and row count. An immutable managed Qlib provider and the same early-history momentum command produced byte-identical evidence JSON twice with PYTHONHASHSEED=0. The mirror is only about 8.2% complete for daily partitions and lacks adjustment, real benchmark, point-in-time membership, ST, suspension and limit data, so production_ready and investment_value_claim remain false and no gate credit is earned.", + "The local Tushare CSI500 2018-2025 scoped release double-verified 221/221 selected jobs and confirmed that all remained latest; the live SQLite ledger changed only because of unrelated downloader jobs. The dedicated lineage verifier matched all 221 converter source jobs on id, path, row_count, byte_count, sha256 and request_sha256 with mismatch_count=0, so converter_source_matches_current=true. The verified Qlib provider has data version 5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963, manifest SHA-256 27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0, 10,011 files, 71,707,194 bytes, 1,942 sessions, 1,111 instruments, adjusted prices, a real SH000905 benchmark and next-session-effective historical membership. Two identical runs produced SHA-256 803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3, 176,615 signals and 1,698 portfolio days. Historical stock_st remains unavailable and the Qlib 9.5% limit threshold is only a market-level approximation, so production_ready and investment_value_claim remain false and gate_credit remains empty.", "A real JoinQuant daily hosted backtest completed for 2024-01-02 through 2024-12-31 with CNY 1,000,000; the visible completed-run log had zero ERROR/Traceback/order-rejection entries. That run used portable_momentum_smoke, not the Ridge/TargetPackage Baseline candidate, and earns no candidate or cross-engine gate credit.", "The QMT shadow CLI is strictly read-only and produces HMAC account-bound broker observation, broker snapshot and target-diff artifacts. QMT_SHADOW_EVIDENCE_V1 authenticates semantic content, deterministic published bytes, the original decision, planner/engine/operator source and fixed policy; semantic replay verifies exact derived state. Independent local QA completed 58 deterministic scenarios, 500 malformed fuzz cases and 300 legal-observation fuzz cases with no P0/P1. This is Quant OS publisher integrity, not broker attestation, and it still has only fake-broker evidence.", "JoinQuant/QMT mock parity is mock_contract_only, real_platform_pass is false and it earns no G9 gate credit." diff --git a/status/status-data.js b/status/status-data.js index c1a3d79..5f58966 100644 --- a/status/status-data.js +++ b/status/status-data.js @@ -1,6 +1,6 @@ // Generated by tools/build_public_status.py; do not edit by hand. window.QUANT_OS_PUBLIC_STATUS = { - "auditAsOf": "2026-07-30", + "auditAsOf": "2026-07-31", "bundles": { "artifacts": [ { @@ -198,6 +198,12 @@ window.QUANT_OS_PUBLIC_STATUS = { "label": "Tushare inventory and Qlib bridge record", "path": "docs/TUSHARE_LOCAL_DATA.md" }, + { + "id": "tushare-csi500-research-run", + "kind": "machine_record", + "label": "Tushare CSI500 2018-2025 reproducible research run", + "path": "evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json" + }, { "id": "platform-deployment", "kind": "runbook", @@ -213,7 +219,7 @@ window.QUANT_OS_PUBLIC_STATUS = { ], "freshness": [ { - "asOf": "2026-07-30", + "asOf": "2026-07-31", "evidenceLinks": [ "scorecard", "audit" @@ -222,7 +228,7 @@ window.QUANT_OS_PUBLIC_STATUS = { "label": "项目证据审计", "semantics": "证据盘点日期；不是部署时间，也不是行情截止日。", "status": "recorded", - "value": "2026-07-30" + "value": "2026-07-31" }, { "evidenceLinks": [ @@ -257,23 +263,58 @@ window.QUANT_OS_PUBLIC_STATUS = { "value": "2024-03-11 — 2024-03-13; 3 orders / 3 fills" }, { - "completedPartitions": 35, - "completedRows": 28731, - "completionLabel": "35/428（约 8.2%）", - "completionRatio": 0.081776, + "benchmarkCumulativeReturn": 0.7895593835, + "calendarSessions": 1942, + "completedPartitions": 96, + "completedRows": 8813885, + "completionLabel": "96/96（约 100%）", + "completionRatio": 1.0, + "converterSourceMatchesCurrent": true, + "effectiveMembershipSnapshots": 96, "evidenceLinks": [ + "tushare-csi500-research-run", "tushare-data", "scorecard" ], + "gateCredit": [], "id": "tushare-daily-market-data", - "label": "Tushare daily 行情覆盖", - "marketDataFrom": "1990-12-19", - "marketDataThrough": "1993-10-29", - "plannedPartitions": 428, + "instrumentCount": 1111, + "investmentValueClaim": false, + "label": "Tushare CSI500 scoped research release", + "lineageMatchedFields": [ + "id", + "path", + "row_count", + "byte_count", + "sha256", + "request_sha256" + ], + "lineageMismatchCount": 0, + "marketDataFrom": "2018-01-02", + "marketDataThrough": "2025-12-31", + "maxDrawdown": -0.6041641366, + "observedMembershipSnapshots": 97, + "plannedPartitions": 96, + "portfolioDays": 1698, "productionReady": false, - "semantics": "marketDataThrough 仅取自 daily 行情分区；绝不使用覆盖更晚的 trade_cal 日期。", - "status": "partial", - "value": "35/428（约 8.2%），行情截至 1993-10-29" + "providerBytes": 71707194, + "providerDataVersion": "5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963", + "providerFiles": 10011, + "providerManifestSha256": "27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0", + "providerTreeSha256": "f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f", + "runSha256": "803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3", + "sameSessionMembershipUse": false, + "selectedDailyRows": 1975455, + "semantics": "96/96 只表示 2018—2025 scoped daily 输入完整且 221/221 source jobs 经双重校验；全局 ledger 因无关下载任务发生变化，但 scoped jobs 仍为 latest；lineage 工具按 id/path/row_count/byte_count/sha256/request_sha256 验证 converter 221/221 零差异。universe 成分次一交易日生效。该 Qlib run 未使用逐股涨跌停，stock_st 不可用，不代表 production-ready 或投资价值。", + "signalRows": 176615, + "snapshotJobs": 221, + "sourceJobCount": 221, + "sourceStable": true, + "stateDbChangedDuringVerification": true, + "status": "research_only", + "strategyCumulativeReturn": 0.2531519864, + "value": "96/96 scoped daily 分区；221/221 source jobs；行情 2018-01-02 — 2025-12-31", + "verifiedSnapshotJobs": 221 }, { "evidenceLinks": [ @@ -525,24 +566,25 @@ window.QUANT_OS_PUBLIC_STATUS = { "status": "not_passed" } ], - "generatedAt": "2026-07-30T00:00:00Z", + "generatedAt": "2026-07-31T00:00:00Z", "generatedAtSource": "audit_as_of", "layers": [ { "capabilities": [ "JQData raw/factor/limits/paused/PIT membership/ST snapshot", - "Tushare completed-Parquet inventory and immutable Qlib provider", + "Tushare scoped source release and verified immutable Qlib provider", "semantic manifest and data-version verification" ], "claim": "不可变快照、PIT 字段合同和 Tushare→Qlib 技术链路已有本地合同/运行证据；这些 provider-data 路径尚未接成当前 Ridge 五层候选。", "evidenceLinks": [ "architecture", "tushare-data", + "tushare-csi500-research-run", "scorecard" ], "id": "data", "limitations": [ - "尚无授权 JQData 真实快照与许可 lineage；Tushare 镜像仅部分完成且缺生产关键字段；当前五层候选仍使用 synthetic 数据。" + "Tushare 2018—2025 中证 500 scoped research 链已验证，但 live raw 仍可变、stock_st 不可用、9.5% 涨跌停为市场级近似，且尚未接入当前 Ridge 五层候选；JQData 授权快照与许可 lineage 仍缺失。" ], "name": "数据与时点", "releaseVerified": false, @@ -1439,15 +1481,16 @@ window.QUANT_OS_PUBLIC_STATUS = { "status": "real_runtime" }, { - "claim": "93 个 completed 文件全部通过 size/SHA/row-count；v4 provider verifier 成功；同一动量命令连续两次 evidence JSON 逐 byte 相同", + "claim": "selected jobs double verified、仍 latest；独立 lineage 验证 221 jobs×6 fields、mismatch 0、converter current true；v2 verifier 通过：1,111 instruments、1,942 sessions、1,975,455 rows；97 observed/96 effective snapshots 严格 500 成分、次日生效；双次 JSON byte-identical", "evidenceLinks": [ "platform-matrix", - "tushare-data" + "tushare-data", + "tushare-csi500-research-run" ], "id": "tushare-local-to-qlib", - "intendedUse": "completed Parquet 的只读盘点、不可变 provider 与本地研究 smoke", + "intendedUse": "live mirror 只读盘点、scoped release、不可变 provider 与本地研究", "limitations": [ - "daily 仅完成约 8.2%，缺复权、真实 benchmark、PIT 成分/ST/停牌/涨跌停；production_ready=false" + "event-time/保守次日生效 PIT 近似，不是严格 knowledge-time PIT；不同 start 的首观测锚 provider 不可直接拼接；research-only、gate_credit=[]；ST/统一 9.5% 仍缺" ], "liveReady": false, "name": "Tushare local → Qlib", @@ -1475,18 +1518,18 @@ window.QUANT_OS_PUBLIC_STATUS = { "provenance": { "baselineConfig": "configs/baseline.json", "bundleManifest": "dist/bundle_manifest.json", - "payloadSha256": "2582133135aa956bb567fd009e73d31c3242ffb4f89b8cf70d2932027cc5d763", + "payloadSha256": "c607f08ef7450714084a562a39afe8d98783550882fbb26b4a844c7161f5bfe1", "payloadSha256Semantics": "SHA-256 of canonical payload before payloadSha256 is added.", "platformMatrix": "docs/PLATFORM_MATRIX.md", "production80Assessment": "profiles/production-80/current-assessment.json", "production80Standard": "profiles/production-80/standard.json", "releaseReachability": "releases/quant60-research-candidate-v1/reachability.json", "scorecard": "gate_scorecard.json", - "sourceSetSha256": "92485e439b1ecfb75166f8750e26b4d392d44ff614755295635e7761fa255c5c", + "sourceSetSha256": "de518af50036a250cf75ea53e0b6804b14043f55abd19125e279e4117c833f87", "sources": [ { "path": "gate_scorecard.json", - "sha256": "2cc29f8d3670cd2e1e90ebe10a7d4d5a1ab9fc6a6b08612df2bbad485cd19655" + "sha256": "8723af0d0a29ea540f91bb9f82fbdd982f31903bc8d0959dfa96d8740564d925" }, { "path": "profiles/baseline-60/profile.json", @@ -1512,17 +1555,21 @@ window.QUANT_OS_PUBLIC_STATUS = { "path": "evidence/joinquant/TP-20240311-795dcfba/2457a7c39a276e09e0fabf99e28978e1/run.json", "sha256": "9956c5e61bdc1c82bc24f20f24c1d65a201bab2f71c2c9e4e23eb5bd40eca6ce" }, + { + "path": "evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json", + "sha256": "6374776487a4a10c6351e2844eba74f7af4ab1db5696dc4d13d5672da9daab76" + }, { "path": "configs/baseline.json", "sha256": "939a7df386b06bc82869940572c35544e9361913b94d9e03027411e51b1006e1" }, { "path": "README.md", - "sha256": "0bf0562ae9247a88a5b0248f66b9b314d7ab0ea435bc8f8e5efa78f7d2f785bb" + "sha256": "e67e9fb71550cedbf3f8436c06ceafc5d1fc50e34ba0feaf66310aee8ec5ea78" }, { "path": "docs/ARCHITECTURE.md", - "sha256": "8aae46f1b371651ac6f72905b7bcb3d72fdfda2a793e5505ab890352921c3219" + "sha256": "a2f9be07b138d587ffa1cb01608e8a469af8a951ef7e6ac371752eb8cdd2ae2a" }, { "path": "docs/architecture/ARCHITECTURE_80.md", @@ -1542,11 +1589,11 @@ window.QUANT_OS_PUBLIC_STATUS = { }, { "path": "docs/getting-started/GETTING_STARTED_80.md", - "sha256": "5c18a558bd2f9e1640144a1729309bf38ada3185188c985476bd609b28c4479e" + "sha256": "14339c6324b142d176813bd274b70cd55b4532d7bfc8f5a39408cc33737df7ba" }, { "path": "docs/PLATFORM_MATRIX.md", - "sha256": "9c7952b019c92efd1055488b58838045ee30979ccc84008cd1218fab2a6d6717" + "sha256": "90c8936b2c51d9e8375e0934f1e9894e9c6fc9c2f29eb2920a8ed226869ba392" }, { "path": "docs/CROSS_ENGINE_CONSISTENCY.md", @@ -1562,11 +1609,11 @@ window.QUANT_OS_PUBLIC_STATUS = { }, { "path": "docs/TUSHARE_LOCAL_DATA.md", - "sha256": "611b120ba59b59b443b5cd9237356c509f2f2699063784a4685506e88a95adea" + "sha256": "560d5e039e19393b64aee1d2b1b213ef32c6d67734688627c261da138a860246" }, { "path": "MIGRATION.md", - "sha256": "2b86f9f4ced687de61c4a3268d57bb38cde4fb1fac75be61ec8cd18c12ccdd26" + "sha256": "5f16fc828879a33a686b6d10c30129938c29b2669b0da2808d889b4542d4165e" }, { "path": "ops/README.md", @@ -1602,7 +1649,7 @@ window.QUANT_OS_PUBLIC_STATUS = { }, { "path": "tools/build_public_status.py", - "sha256": "ec24dd12a6383ab1210a475c6f982b911f6ce45a7c205b9474bb2bb70665bc2a" + "sha256": "17d78b2fbe8e3373ebb283483a4f00f5fae0b0ec2e93a3f9a751c1d69f2004e5" }, { "path": "tools/check_tracked_secrets.py", @@ -1612,7 +1659,8 @@ window.QUANT_OS_PUBLIC_STATUS = { "path": "tools/inventory_local_assets.py", "sha256": "f52a89b2e42d0f3aed7e3c79e0110c37cab779794f550658116e07b9e338160b" } - ] + ], + "tushareResearchRun": "evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json" }, "schemaVersion": 2, "summary": { @@ -1621,17 +1669,17 @@ window.QUANT_OS_PUBLIC_STATUS = { "candidateRealPlatformObserved": true, "gateCoverageLabel": "0/10 passed", "investmentValueClaim": false, - "optionalSkipsRecorded": 6, + "optionalSkipsRecorded": 1, "productionReady": false, "realRuntimeSmokes": [ "JoinQuant TargetPackage execution consumer: one synthetic-research package, 3 orders / 3 fills (not Baseline/G9 evidence)", "JoinQuant portable_momentum_smoke hosted daily backtest (not Baseline/TargetPackage evidence)", "Qlib native momentum fixture", "Qlib Alpha158 + LightGBM fixture", - "Tushare completed-partition Qlib momentum smoke" + "Tushare CSI500 2018—2025 scoped Qlib momentum reproducibility run" ], "testEvidenceQualifier": "Scorecard-recorded run; tests do not prove real-platform or broker gates.", - "testsRecorded": 315 + "testsRecorded": 325 }, "system": "Quant OS / quant60 baseline kernel" }; diff --git a/tests/test_data_tushare.py b/tests/test_data_tushare.py index f728b90..955c054 100644 --- a/tests/test_data_tushare.py +++ b/tests/test_data_tushare.py @@ -13,9 +13,11 @@ sys.path.insert(0, str(ROOT)) from adapters.tushare_local import ( TushareMirrorError, build_qlib_provider, + inventory_mirror, tushare_to_qlib_symbol, verify_qlib_provider, ) +from tools.tushare_qlib import _assert_json_outside_provider class TushareSymbolTest(unittest.TestCase): @@ -26,6 +28,20 @@ class TushareSymbolTest(unittest.TestCase): with self.assertRaises(ValueError): tushare_to_qlib_symbol("T600018.SH") + def test_provider_json_output_cannot_mutate_provider_tree(self): + with self.assertRaisesRegex( + ValueError, + "outside the immutable provider", + ): + _assert_json_outside_provider( + "/tmp/provider/evidence.json", + "/tmp/provider", + ) + _assert_json_outside_provider( + "/tmp/evidence.json", + "/tmp/provider", + ) + @unittest.skipUnless( importlib.util.find_spec("pandas") is not None @@ -33,7 +49,14 @@ class TushareSymbolTest(unittest.TestCase): "pandas and pyarrow are optional outside the Qlib research environment", ) class TushareProviderTest(unittest.TestCase): - def _mirror(self, root: Path, *, bad_ohlc: bool = False) -> Path: + def _mirror( + self, + root: Path, + *, + bad_ohlc: bool = False, + advanced: bool = False, + shadowed_daily: bool = False, + ) -> Path: import pandas mirror = root / "mirror" @@ -43,7 +66,10 @@ class TushareProviderTest(unittest.TestCase): (parquet_root / "stock_basic").mkdir(parents=True) dates = pandas.bdate_range("2024-01-02", periods=8) daily_rows = [] - for symbol, base in (("600000.SH", 10.0), ("000001.SZ", 20.0)): + symbols = [("600000.SH", 10.0), ("000001.SZ", 20.0)] + if advanced: + symbols.append(("000002.SZ", 30.0)) + for symbol, base in symbols: previous = base for index, value in enumerate(dates): close = base * (1.01 ** index) @@ -78,11 +104,121 @@ class TushareProviderTest(unittest.TestCase): ), "stock_basic/list_status=L.parquet": pandas.DataFrame( { - "ts_code": ["600000.SH", "000001.SZ"], - "curr_type": ["CNY", "CNY"], + "ts_code": [value[0] for value in symbols], + "curr_type": ["CNY"] * len(symbols), } ), } + partitions = {} + if advanced: + adj_rows = [] + for symbol, _ in symbols: + for index, value in enumerate(dates): + factor = 2.0 if symbol == "600000.SH" and index >= 4 else 1.0 + adj_rows.append( + { + "ts_code": symbol, + "trade_date": value.strftime("%Y%m%d"), + "adj_factor": factor, + } + ) + index_rows = [] + previous = 1000.0 + for index, value in enumerate(dates): + close = 1000.0 + index * 10.0 + index_rows.append( + { + "ts_code": "000905.SH", + "trade_date": value.strftime("%Y%m%d"), + "open": close - 2.0, + "high": close + 5.0, + "low": close - 5.0, + "close": close, + "pre_close": previous, + "change": close - previous, + "pct_chg": (close / previous - 1.0) * 100.0, + "vol": 2000.0, + "amount": close * 200.0, + } + ) + previous = close + first_snapshot = dates[0].strftime("%Y%m%d") + second_snapshot = dates[4].strftime("%Y%m%d") + anchor_weight_rows = [ + { + "index_code": "000905.SH", + "con_code": "600000.SH", + "trade_date": "20231229", + "weight": 50.0, + }, + { + "index_code": "000905.SH", + "con_code": "000001.SZ", + "trade_date": "20231229", + "weight": 50.0, + }, + ] + weight_rows = [ + { + "index_code": "000905.SH", + "con_code": "600000.SH", + "trade_date": first_snapshot, + "weight": 50.0, + }, + { + "index_code": "000905.SH", + "con_code": "000001.SZ", + "trade_date": first_snapshot, + "weight": 50.0, + }, + { + "index_code": "000905.SH", + "con_code": "000001.SZ", + "trade_date": second_snapshot, + "weight": 55.0, + }, + { + "index_code": "000905.SH", + "con_code": "000002.SZ", + "trade_date": second_snapshot, + "weight": 45.0, + }, + ] + frames.update( + { + "adj_factor/month=2024-01.parquet": pandas.DataFrame( + adj_rows + ), + ( + "index_daily/" + "ts_code=000905_SH_year=2024.parquet" + ): pandas.DataFrame(index_rows), + ( + "index_weight/" + "index_code=000905_SH_year=2023.parquet" + ): pandas.DataFrame(anchor_weight_rows), + ( + "index_weight/" + "index_code=000905_SH_year=2024.parquet" + ): pandas.DataFrame(weight_rows), + } + ) + partitions.update( + { + ( + "index_daily/" + "ts_code=000905_SH_year=2024.parquet" + ): "ts_code=000905_SH/year=2024", + ( + "index_weight/" + "index_code=000905_SH_year=2023.parquet" + ): "index_code=000905_SH/year=2023", + ( + "index_weight/" + "index_code=000905_SH_year=2024.parquet" + ): "index_code=000905_SH/year=2024", + } + ) state = mirror / "data/state.sqlite3" connection = sqlite3.connect(state) connection.execute( @@ -117,13 +253,24 @@ class TushareProviderTest(unittest.TestCase): ( index, api, - path.stem, + partitions.get(relative, path.stem), len(frame), path.stat().st_size, hashlib.sha256(path.read_bytes()).hexdigest(), str(path), ), ) + if shadowed_daily: + daily_path = parquet_root / "daily/month=2024-01.parquet" + connection.execute( + """ + INSERT INTO jobs VALUES + (0, 'daily', 'month=2024-01', 'completed', 1, 1, + 'obsolete-checksum', ?, NULL, + '2024-01-01T00:00:00Z', NULL) + """, + (str(daily_path),), + ) connection.execute( """ INSERT INTO jobs VALUES @@ -150,7 +297,9 @@ class TushareProviderTest(unittest.TestCase): ) self.assertFalse(result["investment_value_claim"]) self.assertEqual(result["market"]["instrument_count"], 2) - self.assertTrue(verify_qlib_provider(output)["ok"]) + verified = verify_qlib_provider(output) + self.assertTrue(verified["ok"]) + self.assertTrue(verified["converter_source_matches_current"]) manifest = json.loads( (output / "quant_os_tushare_manifest.json").read_text() ) @@ -225,6 +374,175 @@ class TushareProviderTest(unittest.TestCase): self.assertAlmostEqual(high[1], 10.0) self.assertAlmostEqual(low[1], 8.0) + def test_inventory_ignores_shadowed_completed_file_version(self): + with tempfile.TemporaryDirectory() as temporary: + mirror = self._mirror( + Path(temporary), + shadowed_daily=True, + ) + result = inventory_mirror(mirror) + self.assertEqual(result["shadowed_completed_versions"], 1) + self.assertEqual( + result["shadowed_completed_versions_by_api"], + {"daily": 1}, + ) + self.assertEqual(result["tables"]["daily"]["files"], 1) + self.assertEqual(result["verified_completed_files"], 3) + + def test_adjusted_real_benchmark_and_pit_universe(self): + from array import array + import math + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + mirror = self._mirror(root, advanced=True) + output = root / "provider" + result = build_qlib_provider( + mirror, + output, + start="2024-01-02", + end="2024-01-11", + minimum_observations=999, + benchmark_index_code="000905.SH", + universe_index_code="000905.SH", + expected_constituent_count=2, + ) + self.assertEqual(result["schema_version"], 2) + self.assertEqual( + result["price_adjustment"]["mode"], + "cumulative_adjusted_first_observation_anchor", + ) + self.assertTrue(result["price_adjustment"]["component_complete"]) + self.assertFalse(result["price_adjustment"]["production_eligible"]) + self.assertEqual( + result["benchmark"]["kind"], + "tushare_index_daily", + ) + self.assertTrue(result["market"]["point_in_time"]) + self.assertFalse(result["market"]["minimum_observations_applied"]) + self.assertFalse(result["market"]["same_session_membership_use"]) + self.assertEqual( + result["market"]["availability_contract"], + "conservative next-session effectiveness because the source " + "does not expose a separately verified published_at timestamp", + ) + self.assertEqual(result["market"]["instrument_count"], 3) + self.assertTrue( + result["source_stability"]["post_read_file_verification"] + ) + self.assertEqual( + {item["api_name"] for item in result["source_jobs"]}, + { + "adj_factor", + "daily", + "index_daily", + "index_weight", + "stock_basic", + "trade_cal", + }, + ) + + factor = array("f") + factor.frombytes( + (output / "features/sh600000/factor.day.bin").read_bytes() + ) + close = array("f") + close.frombytes( + (output / "features/sh600000/close.day.bin").read_bytes() + ) + volume = array("f") + volume.frombytes( + (output / "features/sh600000/volume.day.bin").read_bytes() + ) + change = array("f") + change.frombytes( + (output / "features/sh600000/change.day.bin").read_bytes() + ) + self.assertAlmostEqual(factor[1], 1.0) + self.assertAlmostEqual(factor[5], 2.0) + raw_close = 10.0 * (1.01 ** 4) + self.assertAlmostEqual(close[5] / factor[5], raw_close, places=5) + self.assertAlmostEqual(volume[5], 50_000.0, places=3) + self.assertTrue(math.isnan(change[1])) + self.assertAlmostEqual( + change[5], + (raw_close * 2.0) / (10.0 * (1.01 ** 3)) - 1.0, + places=5, + ) + + index_close = array("f") + index_close.frombytes( + (output / "features/sh000905/close.day.bin").read_bytes() + ) + self.assertAlmostEqual(index_close[1], 1000.0) + market_lines = ( + output / "instruments/tushare_a.txt" + ).read_text(encoding="utf-8").splitlines() + self.assertEqual( + market_lines, + [ + "SZ000001\t2024-01-02\t2024-01-11", + "SZ000002\t2024-01-09\t2024-01-11", + "SH600000\t2024-01-02\t2024-01-08", + ], + ) + self.assertTrue(verify_qlib_provider(output)["ok"]) + + def test_pit_universe_fails_closed_on_missing_anchor_or_bad_roster(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + mirror = self._mirror(root, advanced=True) + state = mirror / "data/state.sqlite3" + connection = sqlite3.connect(state) + connection.execute( + "DELETE FROM jobs WHERE partition_key = ?", + ("index_code=000905_SH/year=2023",), + ) + connection.commit() + connection.close() + with self.assertRaisesRegex( + TushareMirrorError, + "missing required partitions", + ): + build_qlib_provider( + mirror, + root / "provider-missing-anchor", + start="2024-01-02", + end="2024-01-11", + benchmark_index_code="000905.SH", + universe_index_code="000905.SH", + expected_constituent_count=2, + ) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + mirror = self._mirror(root, advanced=True) + with self.assertRaisesRegex( + TushareMirrorError, + "constituents; expected 3", + ): + build_qlib_provider( + mirror, + root / "provider-bad-roster", + start="2024-01-02", + end="2024-01-11", + benchmark_index_code="000905.SH", + universe_index_code="000905.SH", + expected_constituent_count=3, + ) + with self.assertRaisesRegex( + ValueError, + "requires benchmark_index_code", + ): + build_qlib_provider( + mirror, + root / "provider-synthetic-benchmark", + start="2024-01-02", + end="2024-01-11", + universe_index_code="000905.SH", + expected_constituent_count=2, + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_public_status.py b/tests/test_public_status.py index 77c8eb5..13d8c20 100644 --- a/tests/test_public_status.py +++ b/tests/test_public_status.py @@ -76,6 +76,8 @@ class PublicStatusTests(unittest.TestCase): self.assertFalse(payload["marketScope"]["bseTradingEnabled"]) self.assertIn("北交所", payload["marketScope"]["statement"]) self.assertIn("fail closed", payload["marketScope"]["statement"]) + self.assertEqual(payload["summary"]["testsRecorded"], 325) + self.assertEqual(payload["summary"]["optionalSkipsRecorded"], 1) def test_public_payload_is_deliberately_non_secret_and_non_local(self): payload = _read_payload() @@ -99,6 +101,25 @@ class PublicStatusTests(unittest.TestCase): self.assertFalse(Path(evidence["path"]).is_absolute()) self.assertNotIn("..", Path(evidence["path"]).parts) + def test_public_tushare_claims_cannot_regress_to_early_mirror(self): + payload = _read_payload() + encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True) + scorecard = (PROJECT / "gate_scorecard.json").read_text( + encoding="utf-8" + ) + stale_claims = ( + "93 个", + "8.2%", + "1993-10-29", + "CSI500-2018-2025/de9657cd", + "de5b96253ff765fa4e4be9736ddc2fad528f72c7721964313956d44b02969173", + "3c550c0ba986f7eb736b421ac318fdc2a58eacaa65cfbdeaf6b3f920b5288304", + "aed22a7b83e4defd45c4874e568d6aeafdabe43987cea301e0c1d09543dccb7c", + ) + for stale_claim in stale_claims: + self.assertNotIn(stale_claim, encoded) + self.assertNotIn(stale_claim, scorecard) + def test_static_page_has_fail_closed_fallback_and_no_local_paths(self): page = INDEX_PATH.read_text(encoding="utf-8") self.assertIn('src="./status-data.js"', page) @@ -272,6 +293,10 @@ class PublicStatusTests(unittest.TestCase): evidence["joinquant-target-package-evidence"]["path"], "docs/JOINQUANT_TARGET_PACKAGE_EVIDENCE_2026-07-26.md", ) + self.assertEqual( + evidence["tushare-csi500-research-run"]["path"], + "evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json", + ) gaps = payload["deliveryGaps"] self.assertGreaterEqual(len(gaps["internal"]), 3) @@ -374,7 +399,7 @@ class PublicStatusTests(unittest.TestCase): def test_freshness_uses_daily_bars_not_the_later_trade_calendar(self): payload = _read_payload() freshness = {item["id"]: item for item in payload["freshness"]} - self.assertEqual(freshness["project-audit"]["value"], "2026-07-30") + self.assertEqual(freshness["project-audit"]["value"], "2026-07-31") self.assertEqual( freshness["joinquant-hosted-market-window"]["marketDataThrough"], "2024-12-31", @@ -394,17 +419,67 @@ class PublicStatusTests(unittest.TestCase): self.assertIn("G9", target_package["semantics"]) tushare = freshness["tushare-daily-market-data"] - self.assertEqual(tushare["completedPartitions"], 35) - self.assertEqual(tushare["plannedPartitions"], 428) - self.assertAlmostEqual(tushare["completionRatio"], 35 / 428, places=6) - self.assertEqual(tushare["marketDataThrough"], "1993-10-29") + self.assertEqual(tushare["status"], "research_only") + self.assertEqual(tushare["completedPartitions"], 96) + self.assertEqual(tushare["plannedPartitions"], 96) + self.assertEqual(tushare["completionRatio"], 1.0) + self.assertEqual(tushare["completedRows"], 8_813_885) + self.assertEqual(tushare["marketDataFrom"], "2018-01-02") + self.assertEqual(tushare["marketDataThrough"], "2025-12-31") + self.assertEqual( + (tushare["verifiedSnapshotJobs"], tushare["snapshotJobs"]), + (221, 221), + ) + self.assertEqual(tushare["providerFiles"], 10_011) + self.assertEqual(tushare["providerBytes"], 71_707_194) + self.assertEqual(tushare["calendarSessions"], 1_942) + self.assertEqual(tushare["instrumentCount"], 1_111) + self.assertEqual(tushare["selectedDailyRows"], 1_975_455) + self.assertEqual(tushare["observedMembershipSnapshots"], 97) + self.assertEqual(tushare["effectiveMembershipSnapshots"], 96) + self.assertFalse(tushare["sameSessionMembershipUse"]) + self.assertTrue(tushare["sourceStable"]) + self.assertTrue(tushare["stateDbChangedDuringVerification"]) + self.assertTrue(tushare["converterSourceMatchesCurrent"]) + self.assertEqual( + tushare["providerDataVersion"], + "5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963", + ) + self.assertEqual( + tushare["providerManifestSha256"], + "27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0", + ) + self.assertEqual(tushare["sourceJobCount"], 221) + self.assertEqual( + tushare["lineageMatchedFields"], + [ + "id", + "path", + "row_count", + "byte_count", + "sha256", + "request_sha256", + ], + ) + self.assertEqual(tushare["lineageMismatchCount"], 0) + self.assertEqual( + tushare["runSha256"], + "803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3", + ) + self.assertEqual(tushare["signalRows"], 176_615) + self.assertEqual(tushare["portfolioDays"], 1_698) self.assertFalse(tushare["productionReady"]) + self.assertFalse(tushare["investmentValueClaim"]) + self.assertEqual(tushare["gateCredit"], []) source = (PROJECT / "docs" / "TUSHARE_LOCAL_DATA.md").read_text( encoding="utf-8" ) self.assertRegex(source, r"`trade_cal`.*2026") - self.assertNotIn("2026", tushare["marketDataThrough"]) - self.assertIn("daily", tushare["semantics"]) + self.assertIn("221/221", tushare["semantics"]) + self.assertIn( + "tushare-csi500-research-run", + tushare["evidenceLinks"], + ) self.assertEqual(freshness["jqdata-real-snapshot"]["value"], "N/A") self.assertEqual(freshness["jqdata-real-snapshot"]["status"], "blocked") diff --git a/tests/test_tushare_lineage.py b/tests/test_tushare_lineage.py new file mode 100644 index 0000000..e16b65a --- /dev/null +++ b/tests/test_tushare_lineage.py @@ -0,0 +1,105 @@ +import hashlib +import json +from pathlib import Path +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from tools.tushare_lineage import ( + TushareLineageError, + _compare_source_jobs, + _validate_snapshot, +) + + +def _canonical(value): + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +class TushareLineageTest(unittest.TestCase): + def _snapshot(self): + job = { + "api_name": "daily", + "byte_count": 123, + "completed_at": "2026-07-31T00:00:00Z", + "id": 7, + "observed": { + "row_count": 11, + "sha256": "a" * 64, + "size_bytes": 123, + }, + "partition_key": "month=2024-01", + "path": "data/parquet/daily/month=2024-01.parquet", + "request_sha256": "b" * 64, + "row_count": 11, + "sha256": "a" * 64, + "shadowed_completed_count": 0, + "verified": True, + } + selection_identity = { + key: value + for key, value in job.items() + if key not in {"observed", "verified"} + } + snapshot = { + "format": "quant-os.tushare-scoped-snapshot/v2", + "selected_jobs": [job], + "selection_sha256": hashlib.sha256( + _canonical({"selected_jobs": [selection_identity]}) + ).hexdigest(), + "verification": { + "double_verified_jobs": 1, + "selected_jobs_still_latest_after_verification": True, + "verified_jobs": 1, + }, + } + snapshot["manifest_sha256"] = hashlib.sha256( + _canonical(snapshot) + ).hexdigest() + return snapshot + + def test_exact_source_lineage_and_tamper_detection(self): + snapshot = self._snapshot() + jobs = _validate_snapshot(snapshot) + provider_job = { + "api_name": "daily", + "byte_count": 123, + "job_id": 7, + "partition_key": "month=2024-01", + "path": "data/parquet/daily/month=2024-01.parquet", + "request_sha256": "b" * 64, + "row_count": 11, + "sha256": "a" * 64, + } + _compare_source_jobs(jobs, [provider_job]) + provider_job["sha256"] = "c" * 64 + with self.assertRaisesRegex( + TushareLineageError, + "source job fields differ", + ): + _compare_source_jobs(jobs, [provider_job]) + + snapshot["selected_jobs"][0]["path"] = "../escape.parquet" + snapshot_without_hash = dict(snapshot) + snapshot_without_hash.pop("manifest_sha256") + snapshot["manifest_sha256"] = hashlib.sha256( + _canonical(snapshot_without_hash) + ).hexdigest() + with self.assertRaisesRegex( + TushareLineageError, + "stay relative", + ): + _validate_snapshot(snapshot) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tushare_snapshot.py b/tests/test_tushare_snapshot.py new file mode 100644 index 0000000..86c4445 --- /dev/null +++ b/tests/test_tushare_snapshot.py @@ -0,0 +1,379 @@ +import hashlib +import importlib.util +import json +from pathlib import Path +import sqlite3 +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from tools.tushare_snapshot import ( + TushareSnapshotError, + create_snapshot, + write_manifest, +) + + +def _manifest_hash(manifest): + payload = dict(manifest) + payload.pop("manifest_sha256") + encoded = json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +class TushareSnapshotTest(unittest.TestCase): + def _new_mirror(self, root: Path): + mirror = root / "mirror" + (mirror / "data/parquet").mkdir(parents=True) + connection = sqlite3.connect(mirror / "data/state.sqlite3") + connection.execute( + """ + CREATE TABLE jobs ( + id INTEGER PRIMARY KEY, + api_name TEXT NOT NULL, + partition_key TEXT NOT NULL, + status TEXT NOT NULL, + row_count INTEGER, + byte_count INTEGER, + sha256 TEXT, + file_path TEXT, + completed_at TEXT, + params_json TEXT, + fields TEXT + ) + """ + ) + return mirror, connection + + def _completed( + self, + mirror: Path, + connection: sqlite3.Connection, + *, + job_id: int, + api: str, + partition: str, + content: bytes = b"not-a-parquet", + path_suffix: str = "", + row_count: int = 1, + ): + safe_partition = ( + partition.replace("/", "_").replace("=", "-") + path_suffix + ) + path = mirror / "data/parquet" / api / f"{safe_partition}.parquet" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + connection.execute( + """ + INSERT INTO jobs + (id, api_name, partition_key, status, row_count, byte_count, + sha256, file_path, completed_at, params_json, fields) + VALUES (?, ?, ?, 'completed', ?, ?, ?, ?, ?, '[]', NULL) + """, + ( + job_id, + api, + partition, + row_count, + path.stat().st_size, + hashlib.sha256(content).hexdigest(), + str(path), + f"2018-03-{min(job_id, 28):02d}T00:00:00Z", + ), + ) + return path + + def _core_fixture(self, root: Path): + mirror, connection = self._new_mirror(root) + self._completed( + mirror, + connection, + job_id=1, + api="daily", + partition="month=2018-01", + path_suffix="-old", + ) + self._completed( + mirror, + connection, + job_id=2, + api="daily", + partition="month=2018-01", + content=b"newer", + ) + self._completed( + mirror, + connection, + job_id=3, + api="daily", + partition="month=2018-02", + ) + self._completed( + mirror, + connection, + job_id=4, + api="daily", + partition="month=2018-03", + ) + self._completed( + mirror, + connection, + job_id=5, + api="adj_factor", + partition="month=2018-01", + ) + self._completed( + mirror, + connection, + job_id=6, + api="adj_factor", + partition="month=2018-02", + ) + self._completed( + mirror, + connection, + job_id=7, + api="trade_cal", + partition="exchange=SSE/year=2018", + ) + self._completed( + mirror, + connection, + job_id=8, + api="stock_basic", + partition="list_status=L", + ) + self._completed( + mirror, + connection, + job_id=9, + api="index_daily", + partition="ts_code=000905_SH/year=2018", + ) + self._completed( + mirror, + connection, + job_id=10, + api="index_weight", + partition="index_code=000905_SH/year=2018", + ) + self._completed( + mirror, + connection, + job_id=20, + api="index_weight", + partition="index_code=000905_SH/year=2017", + ) + connection.execute( + """ + INSERT INTO jobs + (id, api_name, partition_key, status, row_count, byte_count, + sha256, file_path, completed_at) + VALUES + (11, 'index_daily', 'ts_code=931643_CNI/year=2018', 'running', + NULL, NULL, NULL, NULL, NULL) + """ + ) + connection.commit() + return mirror, connection + + def test_scoped_selection_shadows_old_completed_and_excludes_other_index(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + mirror, connection = self._core_fixture(root) + connection.close() + + manifest = create_snapshot( + mirror, + start="2018-01-15", + end="2018-02-03", + verify=False, + ) + + self.assertEqual(manifest["summary"]["selected_jobs"], 9) + self.assertEqual( + manifest["summary"]["shadowed_completed_jobs"], + 1, + ) + selected_ids = {job["id"] for job in manifest["selected_jobs"]} + self.assertIn(2, selected_ids) + self.assertNotIn(1, selected_ids) + self.assertNotIn(4, selected_ids) + self.assertIn(20, selected_ids) + self.assertEqual( + manifest["scope"]["index_weight_anchor_year"], + 2017, + ) + self.assertEqual( + manifest["excluded_live_jobs"][0]["reason"], + "different_index_code", + ) + self.assertEqual( + manifest["verification"]["mode"], + "not_performed", + ) + self.assertTrue( + manifest["verification"][ + "selected_jobs_still_latest_after_verification" + ] + ) + self.assertRegex( + manifest["selected_jobs"][0]["request_sha256"], + r"^[0-9a-f]{64}$", + ) + self.assertNotIn( + "params_json", + manifest["selected_jobs"][0], + ) + self.assertEqual( + manifest["manifest_sha256"], + _manifest_hash(manifest), + ) + serialized = json.dumps(manifest) + self.assertNotIn(str(root), serialized) + self.assertTrue( + all( + not Path(job["path"]).is_absolute() + for job in manifest["selected_jobs"] + ) + ) + + output = root / "snapshot.json" + self.assertEqual(write_manifest(output, manifest), output.resolve()) + self.assertEqual( + json.loads(output.read_text())["manifest_sha256"], + manifest["manifest_sha256"], + ) + + def test_in_scope_running_job_fails_closed(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + mirror, connection = self._core_fixture(root) + connection.execute( + """ + INSERT INTO jobs + (id, api_name, partition_key, status, row_count, byte_count, + sha256, file_path, completed_at) + VALUES + (12, 'daily', 'month=2018-02', 'running', + NULL, NULL, NULL, NULL, NULL) + """ + ) + connection.commit() + connection.close() + with self.assertRaisesRegex( + TushareSnapshotError, + "contains running/deferred", + ): + create_snapshot( + mirror, + start="2018-01-15", + end="2018-02-03", + verify=False, + ) + + def test_missing_partition_fails_closed(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + mirror, connection = self._new_mirror(root) + self._completed( + mirror, + connection, + job_id=1, + api="daily", + partition="month=2018-01", + ) + connection.commit() + connection.close() + with self.assertRaisesRegex( + TushareSnapshotError, + "missing daily:month=2018-02", + ): + create_snapshot( + mirror, + start="2018-01-01", + end="2018-02-28", + apis=("daily",), + verify=False, + ) + + @unittest.skipUnless( + importlib.util.find_spec("pyarrow") is not None, + "pyarrow is needed for Parquet metadata verification", + ) + def test_default_verification_detects_file_tampering(self): + import pyarrow as pa + import pyarrow.parquet as pq + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + mirror, connection = self._new_mirror(root) + table = pa.table( + { + "ts_code": ["600000.SH", "000001.SZ"], + "trade_date": ["20180102", "20180102"], + } + ) + parquet_path = ( + mirror / "data/parquet/daily/month-2018-01.parquet" + ) + parquet_path.parent.mkdir(parents=True) + pq.write_table(table, parquet_path) + content = parquet_path.read_bytes() + connection.execute( + """ + INSERT INTO jobs + (id, api_name, partition_key, status, row_count, byte_count, + sha256, file_path, completed_at, params_json, fields) + VALUES + (1, 'daily', 'month=2018-01', 'completed', 2, ?, ?, ?, + '2018-02-01T00:00:00Z', '[]', + 'ts_code,trade_date') + """, + ( + len(content), + hashlib.sha256(content).hexdigest(), + str(parquet_path), + ), + ) + connection.commit() + connection.close() + + manifest = create_snapshot( + mirror, + start="2018-01-01", + end="2018-01-31", + apis=("daily",), + ) + self.assertEqual(manifest["verification"]["verified_jobs"], 1) + self.assertEqual( + manifest["verification"]["double_verified_jobs"], + 1, + ) + self.assertEqual( + manifest["selected_jobs"][0]["observed"]["row_count"], + 2, + ) + + parquet_path.write_bytes(content + b"tampered") + with self.assertRaisesRegex(TushareSnapshotError, "size mismatch"): + create_snapshot( + mirror, + start="2018-01-01", + end="2018-01-31", + apis=("daily",), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_public_status.py b/tools/build_public_status.py index 2e8c277..c1bda9f 100644 --- a/tools/build_public_status.py +++ b/tools/build_public_status.py @@ -9,7 +9,7 @@ import json import os import re import sys -from datetime import datetime, timezone +from datetime import date, datetime, timezone from pathlib import Path from typing import Any, Dict, Iterable, List, Mapping, Sequence, Tuple @@ -33,6 +33,9 @@ JOINQUANT_TARGET_PACKAGE_RUN_PATH = ( JOINQUANT_TARGET_PACKAGE_DOC_PATH = ( "docs/JOINQUANT_TARGET_PACKAGE_EVIDENCE_2026-07-26.md" ) +TUSHARE_RESEARCH_RUN_PATH = ( + "evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json" +) JOINQUANT_TARGET_PACKAGE_RUN_ID = "2457a7c39a276e09e0fabf99e28978e1" JOINQUANT_TARGET_PACKAGE_OBSERVED_AT = "2026-07-26T08:50:15Z" JOINQUANT_TARGET_PACKAGE_BUNDLE_SHA256 = ( @@ -47,6 +50,7 @@ SOURCE_PATHS: Tuple[str, ...] = ( "dist/bundle_manifest.json", REACHABILITY_PATH, JOINQUANT_TARGET_PACKAGE_RUN_PATH, + TUSHARE_RESEARCH_RUN_PATH, "configs/baseline.json", "README.md", "docs/ARCHITECTURE.md", @@ -566,11 +570,209 @@ def _validate_joinquant_target_package_run( } +def _validate_tushare_research_run( + project: Path, + run: Mapping[str, Any], +) -> Dict[str, Any]: + expected_scope = { + "index_code": "000905.SH", + "start": "2018-01-01", + "end": "2025-12-31", + "daily_market_data_from": "2018-01-02", + "daily_market_data_through": "2025-12-31", + "daily_partitions_completed": 96, + "daily_partitions_planned": 96, + "daily_rows": 8_813_885, + } + expected_snapshot = { + "selected_jobs": 221, + "verified_jobs": 221, + "double_verified_jobs": 221, + "manifest_sha256": ( + "744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2" + ), + "selection_sha256": ( + "66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5" + ), + } + expected_provider = { + "verified": True, + "data_version": ( + "5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963" + ), + "tree_sha256": ( + "f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f" + ), + "manifest_sha256": ( + "27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0" + ), + "file_count": 10_011, + "byte_count": 71_707_194, + "calendar_sessions": 1_942, + "instrument_count": 1_111, + "selected_daily_rows": 1_975_455, + "observed_membership_snapshots": 97, + "effective_membership_snapshots": 96, + "same_session_membership_use": False, + "price_adjustment_complete": True, + "benchmark": "SH000905", + "converter_source_matches_current": True, + } + expected_lineage = { + "verified": True, + "source_job_count": 221, + "matched_fields": [ + "id", + "path", + "row_count", + "byte_count", + "sha256", + "request_sha256", + ], + "mismatch_count": 0, + "converter_source_matches_current": True, + } + expected_stability = { + "state_db_changed_through_file_verification": True, + "state_db_change_scope": "unrelated live downloader jobs", + "snapshot_jobs_still_latest": True, + "selected_scope_stable": True, + "post_read_file_verification": True, + "provider_source_jobs": 221, + "snapshot_source_jobs": 221, + "source_job_full_field_diff": 0, + } + expected_reproducibility = { + "runs": 2, + "byte_identical": True, + "run_sha256": ( + "803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3" + ), + "signal_rows": 176_615, + "portfolio_days": 1_698, + } + expected_performance = { + "strategy_cumulative_return": 0.2531519864, + "benchmark_cumulative_return": 0.7895593835, + "max_drawdown": -0.6041641366, + } + expected_limitations = { + "stock_st": "denied", + "qlib_limit_threshold": 0.095, + "limit_rule_fidelity": ( + "market-level approximation; not per-symbol daily limit prices" + ), + "research_only": True, + } + if ( + run.get("schema_version") != 1 + or run.get("record_type") + != "quant_os_tushare_qlib_research_run" + or run.get("run_id") != "5bf19d2d" + or run.get("audit_as_of") != "2026-07-31" + or run.get("scope") != expected_scope + or run.get("snapshot") != expected_snapshot + or run.get("provider") != expected_provider + or run.get("lineage") != expected_lineage + or run.get("source_stability") != expected_stability + or run.get("reproducibility") != expected_reproducibility + or run.get("performance") != expected_performance + or run.get("limitations") != expected_limitations + or run.get("production_ready") is not False + or run.get("investment_value_claim") is not False + or run.get("gate_credit") != [] + ): + raise PublicStatusError( + "Tushare CSI500 research machine record drifted" + ) + encoded = json.dumps(run, ensure_ascii=False, sort_keys=True) + if ABSOLUTE_PATH_PATTERN.search(encoded): + raise PublicStatusError( + "Tushare CSI500 research record contains an absolute local path" + ) + path = _safe_repo_path( + TUSHARE_RESEARCH_RUN_PATH, + label="Tushare CSI500 research run", + ) + if not (project / path).is_file(): + raise PublicStatusError( + "tracked Tushare CSI500 research run is missing" + ) + return { + "runId": str(run["run_id"]), + "auditAsOf": str(run["audit_as_of"]), + "indexCode": str(expected_scope["index_code"]), + "scopeStart": str(expected_scope["start"]), + "scopeEnd": str(expected_scope["end"]), + "marketDataFrom": str(expected_scope["daily_market_data_from"]), + "marketDataThrough": str( + expected_scope["daily_market_data_through"] + ), + "completedPartitions": int( + expected_scope["daily_partitions_completed"] + ), + "plannedPartitions": int( + expected_scope["daily_partitions_planned"] + ), + "completedRows": int(expected_scope["daily_rows"]), + "snapshotJobs": int(expected_snapshot["selected_jobs"]), + "verifiedSnapshotJobs": int(expected_snapshot["verified_jobs"]), + "snapshotManifestSha256": str( + expected_snapshot["manifest_sha256"] + ), + "snapshotSelectionSha256": str( + expected_snapshot["selection_sha256"] + ), + "providerDataVersion": str(expected_provider["data_version"]), + "providerTreeSha256": str(expected_provider["tree_sha256"]), + "providerManifestSha256": str( + expected_provider["manifest_sha256"] + ), + "sourceJobCount": int(expected_lineage["source_job_count"]), + "lineageMatchedFields": list(expected_lineage["matched_fields"]), + "lineageMismatchCount": int(expected_lineage["mismatch_count"]), + "providerFiles": int(expected_provider["file_count"]), + "providerBytes": int(expected_provider["byte_count"]), + "calendarSessions": int(expected_provider["calendar_sessions"]), + "instrumentCount": int(expected_provider["instrument_count"]), + "selectedDailyRows": int( + expected_provider["selected_daily_rows"] + ), + "observedMembershipSnapshots": int( + expected_provider["observed_membership_snapshots"] + ), + "effectiveMembershipSnapshots": int( + expected_provider["effective_membership_snapshots"] + ), + "sameSessionMembershipUse": False, + "sourceStable": True, + "stateDbChangedDuringVerification": True, + "converterSourceMatchesCurrent": True, + "runSha256": str(expected_reproducibility["run_sha256"]), + "signalRows": int(expected_reproducibility["signal_rows"]), + "portfolioDays": int(expected_reproducibility["portfolio_days"]), + "strategyCumulativeReturn": float( + expected_performance["strategy_cumulative_return"] + ), + "benchmarkCumulativeReturn": float( + expected_performance["benchmark_cumulative_return"] + ), + "maxDrawdown": float(expected_performance["max_drawdown"]), + "stockStAvailable": False, + "limitThreshold": float( + expected_limitations["qlib_limit_threshold"] + ), + "productionReady": False, + "investmentValueClaim": False, + "gateCredit": [], + } + + def _extract_test_summary(facts: Sequence[Any]) -> Tuple[int | None, int | None]: for fact in facts: text = str(fact) match = re.search( - r"completed\s+(\d+)\s+tests.*?(\d+)\s+optional-runtime skips", + r"completed\s+(\d+)\s+tests.*?(\d+)\s+optional-runtime skips?", text, flags=re.IGNORECASE, ) @@ -618,6 +820,8 @@ def _platform_evidence_links(name: str) -> List[str]: ) if "Qlib" in name or "Tushare" in name: links.append("tushare-data") + if "Tushare" in name: + links.append("tushare-csi500-research-run") if "QMT" in name or "XtTrader" in name: links.append("platform-deployment") return list(dict.fromkeys(links)) @@ -719,6 +923,12 @@ def _build_evidence(project: Path) -> List[Dict[str, Any]]: "docs/TUSHARE_LOCAL_DATA.md", "runtime_evidence", ), + ( + "tushare-csi500-research-run", + "Tushare CSI500 2018-2025 reproducible research run", + TUSHARE_RESEARCH_RUN_PATH, + "machine_record", + ), ( "platform-deployment", "Platform deployment runbook", @@ -788,13 +998,20 @@ def _build_layers( "claim": "不可变快照、PIT 字段合同和 Tushare→Qlib 技术链路已有本地合同/运行证据；这些 provider-data 路径尚未接成当前 Ridge 五层候选。", "capabilities": [ "JQData raw/factor/limits/paused/PIT membership/ST snapshot", - "Tushare completed-Parquet inventory and immutable Qlib provider", + "Tushare scoped source release and verified immutable Qlib provider", "semantic manifest and data-version verification", ], "limitations": [ - "尚无授权 JQData 真实快照与许可 lineage；Tushare 镜像仅部分完成且缺生产关键字段；当前五层候选仍使用 synthetic 数据。" + "Tushare 2018—2025 中证 500 scoped research 链已验证，但 live raw " + "仍可变、stock_st 不可用、9.5% 涨跌停为市场级近似，且尚未接入" + "当前 Ridge 五层候选；JQData 授权快照与许可 lineage 仍缺失。" + ], + "evidenceLinks": [ + "architecture", + "tushare-data", + "tushare-csi500-research-run", + "scorecard", ], - "evidenceLinks": ["architecture", "tushare-data", "scorecard"], "releaseVerified": False, }, { @@ -1046,6 +1263,7 @@ def _build_freshness( scorecard: Mapping[str, Any], joinquant_markdown: str, joinquant_target_package_run: Mapping[str, Any], + tushare_research_run: Mapping[str, Any], tushare_markdown: str, baseline_shadow_days: int, production_shadow_days: int, @@ -1061,49 +1279,100 @@ def _build_freshness( ) joinquant_start, joinquant_end = period_match.groups() - daily_match = re.search( - r"(?m)^\|\s*`daily`\s*\|\s*([\d,]+)\s*\|\s*([\d,]+)\s*\|\s*" - r"(\d{4}-\d{2}-\d{2})[—-](\d{4}-\d{2}-\d{2})", - tushare_markdown, - ) - plan_match = re.search( - r"`daily`\s*原计划覆盖.*?共\s*(\d+)\s*个月；当前完成\s*" - r"(\d+)\s*个月，约\s*([\d.]+)%", - tushare_markdown, - flags=re.DOTALL, - ) + tushare_fields = { + key: re.search( + rf"(?m)^{key}\s*=\s*([^\s]+)\s*$", + tushare_markdown, + ) + for key in ( + "tushare_daily_completed_partitions", + "tushare_daily_planned_partitions", + "tushare_daily_completed_rows", + "tushare_daily_market_data_from", + "tushare_daily_market_data_through", + ) + } ready_match = re.search( r"(?m)^production_ready\s*=\s*(true|false)\s*$", tushare_markdown, ) - if daily_match is None or plan_match is None or ready_match is None: + if any(match is None for match in tushare_fields.values()) or ready_match is None: raise PublicStatusError( "Tushare evidence must declare daily coverage and production readiness" ) - ( - daily_partitions_text, - daily_rows_text, - tushare_start, - tushare_end, - ) = daily_match.groups() - planned_months_text, completed_months_text, percentage_text = plan_match.groups() - completed_partitions = int(daily_partitions_text.replace(",", "")) - completed_months = int(completed_months_text) - planned_months = int(planned_months_text) - if completed_partitions != completed_months: + live_completed_partitions = int( + tushare_fields["tushare_daily_completed_partitions"] + .group(1) + .replace(",", "") + ) + live_planned_months = int( + tushare_fields["tushare_daily_planned_partitions"] + .group(1) + .replace(",", "") + ) + live_daily_rows = int( + tushare_fields["tushare_daily_completed_rows"] + .group(1) + .replace(",", "") + ) + live_tushare_start = tushare_fields[ + "tushare_daily_market_data_from" + ].group(1) + live_tushare_end = tushare_fields[ + "tushare_daily_market_data_through" + ].group(1) + if ( + live_planned_months <= 0 + or live_completed_partitions > live_planned_months + or live_daily_rows <= 0 + ): raise PublicStatusError( - "Tushare daily partition count differs from completed-month statement" + "Tushare daily partition coverage is invalid" ) - calculated_ratio = completed_months / planned_months - recorded_percentage = float(percentage_text) - if abs(calculated_ratio * 100.0 - recorded_percentage) > 0.1: + for label, value in ( + ("market_data_from", live_tushare_start), + ("market_data_through", live_tushare_end), + ): + try: + date.fromisoformat(value) + except ValueError as exc: + raise PublicStatusError( + f"Tushare {label} must be an ISO date" + ) from exc + if live_tushare_start > live_tushare_end: raise PublicStatusError( - "Tushare recorded coverage percentage does not match month counts" + "Tushare daily market-data start must not follow its end" ) + completed_partitions = int( + tushare_research_run["completedPartitions"] + ) + planned_months = int(tushare_research_run["plannedPartitions"]) + daily_rows = int(tushare_research_run["completedRows"]) + tushare_start = str(tushare_research_run["marketDataFrom"]) + tushare_end = str(tushare_research_run["marketDataThrough"]) + if ( + completed_partitions != planned_months + or completed_partitions > live_completed_partitions + or daily_rows > live_daily_rows + or date.fromisoformat(tushare_start) + < date.fromisoformat(live_tushare_start) + or date.fromisoformat(tushare_end) + > date.fromisoformat(live_tushare_end) + ): + raise PublicStatusError( + "Tushare scoped research record is inconsistent with live inventory" + ) + calculated_ratio = completed_partitions / planned_months + recorded_percentage = calculated_ratio * 100.0 production_ready = ready_match.group(1) == "true" - if production_ready: + if ( + production_ready + or tushare_research_run["productionReady"] is not False + or tushare_research_run["investmentValueClaim"] is not False + or tushare_research_run["gateCredit"] != [] + ): raise PublicStatusError( - "partial Tushare source must not be publicized as production ready" + "current Tushare research bridge must not be publicized as production ready" ) audit_as_of = str(scorecard.get("audit_as_of")) @@ -1159,24 +1428,94 @@ def _build_freshness( }, { "id": "tushare-daily-market-data", - "label": "Tushare daily 行情覆盖", - "status": "partial", + "label": "Tushare CSI500 scoped research release", + "status": "research_only", "marketDataFrom": tushare_start, "marketDataThrough": tushare_end, "completedPartitions": completed_partitions, "plannedPartitions": planned_months, - "completedRows": int(daily_rows_text.replace(",", "")), + "completedRows": daily_rows, "completionRatio": round(calculated_ratio, 6), "completionLabel": ( f"{completed_partitions}/{planned_months}（约 {recorded_percentage:g}%）" ), "value": ( - f"{completed_partitions}/{planned_months}（约 {recorded_percentage:g}%），" - f"行情截至 {tushare_end}" + f"{completed_partitions}/{planned_months} scoped daily 分区；" + f"{tushare_research_run['verifiedSnapshotJobs']}/" + f"{tushare_research_run['snapshotJobs']} source jobs；" + f"行情 {tushare_start} — {tushare_end}" ), "productionReady": production_ready, - "semantics": "marketDataThrough 仅取自 daily 行情分区；绝不使用覆盖更晚的 trade_cal 日期。", - "evidenceLinks": ["tushare-data", "scorecard"], + "investmentValueClaim": False, + "gateCredit": [], + "snapshotJobs": int(tushare_research_run["snapshotJobs"]), + "verifiedSnapshotJobs": int( + tushare_research_run["verifiedSnapshotJobs"] + ), + "providerDataVersion": str( + tushare_research_run["providerDataVersion"] + ), + "providerTreeSha256": str( + tushare_research_run["providerTreeSha256"] + ), + "providerManifestSha256": str( + tushare_research_run["providerManifestSha256"] + ), + "sourceJobCount": int( + tushare_research_run["sourceJobCount"] + ), + "lineageMatchedFields": list( + tushare_research_run["lineageMatchedFields"] + ), + "lineageMismatchCount": int( + tushare_research_run["lineageMismatchCount"] + ), + "providerFiles": int(tushare_research_run["providerFiles"]), + "providerBytes": int(tushare_research_run["providerBytes"]), + "calendarSessions": int( + tushare_research_run["calendarSessions"] + ), + "instrumentCount": int( + tushare_research_run["instrumentCount"] + ), + "selectedDailyRows": int( + tushare_research_run["selectedDailyRows"] + ), + "observedMembershipSnapshots": int( + tushare_research_run["observedMembershipSnapshots"] + ), + "effectiveMembershipSnapshots": int( + tushare_research_run["effectiveMembershipSnapshots"] + ), + "sameSessionMembershipUse": False, + "sourceStable": True, + "stateDbChangedDuringVerification": True, + "converterSourceMatchesCurrent": True, + "runSha256": str(tushare_research_run["runSha256"]), + "signalRows": int(tushare_research_run["signalRows"]), + "portfolioDays": int(tushare_research_run["portfolioDays"]), + "strategyCumulativeReturn": float( + tushare_research_run["strategyCumulativeReturn"] + ), + "benchmarkCumulativeReturn": float( + tushare_research_run["benchmarkCumulativeReturn"] + ), + "maxDrawdown": float(tushare_research_run["maxDrawdown"]), + "semantics": ( + "96/96 只表示 2018—2025 scoped daily 输入完整且 221/221 " + "source jobs 经双重校验；全局 ledger 因无关下载任务发生变化，" + "但 scoped jobs 仍为 latest；lineage 工具按 id/path/row_count/" + "byte_count/sha256/request_sha256 验证 converter 221/221 " + "零差异。" + "universe 成分次一交易日生效。该 Qlib " + "run 未使用逐股涨跌停，stock_st 不可用，不代表 production-ready " + "或投资价值。" + ), + "evidenceLinks": [ + "tushare-csi500-research-run", + "tushare-data", + "scorecard", + ], }, { "id": "jqdata-real-snapshot", @@ -1413,6 +1752,9 @@ def build_public_status( joinquant_target_package_run_raw = _load_json( project / JOINQUANT_TARGET_PACKAGE_RUN_PATH ) + tushare_research_run_raw = _load_json( + project / TUSHARE_RESEARCH_RUN_PATH + ) baseline = _load_json(project / "configs" / "baseline.json") platform_markdown = (project / "docs" / "PLATFORM_MATRIX.md").read_text( encoding="utf-8" @@ -1501,6 +1843,10 @@ def build_public_status( project, joinquant_target_package_run_raw, ) + tushare_research_run = _validate_tushare_research_run( + project, + tushare_research_run_raw, + ) maturity80 = _build_maturity80( project, production_80_standard, @@ -1582,7 +1928,7 @@ def build_public_status( "JoinQuant portable_momentum_smoke hosted daily backtest (not Baseline/TargetPackage evidence)", "Qlib native momentum fixture", "Qlib Alpha158 + LightGBM fixture", - "Tushare completed-partition Qlib momentum smoke", + "Tushare CSI500 2018—2025 scoped Qlib momentum reproducibility run", ], "gateCoverageLabel": f"{passed}/{len(gates)} passed", "productionReady": bool(maturity80["qualified"]), @@ -1630,6 +1976,7 @@ def build_public_status( scorecard=scorecard, joinquant_markdown=joinquant_markdown, joinquant_target_package_run=joinquant_target_package_run, + tushare_research_run=tushare_research_run, tushare_markdown=tushare_markdown, baseline_shadow_days=baseline_shadow_days, production_shadow_days=production_shadow_days, @@ -1672,6 +2019,7 @@ def build_public_status( "production80Assessment": PRODUCTION_80_ASSESSMENT_PATH, "bundleManifest": "dist/bundle_manifest.json", "releaseReachability": REACHABILITY_PATH, + "tushareResearchRun": TUSHARE_RESEARCH_RUN_PATH, "platformMatrix": "docs/PLATFORM_MATRIX.md", "baselineConfig": "configs/baseline.json", "sourceSetSha256": source_set_sha, diff --git a/tools/tushare_lineage.py b/tools/tushare_lineage.py new file mode 100644 index 0000000..d0d8092 --- /dev/null +++ b/tools/tushare_lineage.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Verify exact lineage from a Tushare scoped manifest to a Qlib provider.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import sys +import tempfile +from typing import Any, Mapping, Optional, Sequence + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from adapters.tushare_local import MANIFEST_NAME, verify_qlib_provider + + +MATCHED_FIELDS = ( + ("id", "job_id"), + ("path", "path"), + ("row_count", "row_count"), + ("byte_count", "byte_count"), + ("sha256", "sha256"), + ("request_sha256", "request_sha256"), +) + + +class TushareLineageError(RuntimeError): + """Raised when a scoped manifest and provider do not match exactly.""" + + +def _canonical_json(value: Mapping[str, Any]) -> bytes: + return json.dumps( + dict(value), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _load_object(path: Path, *, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise TushareLineageError(f"cannot read {label}: {path}") from exc + if not isinstance(value, dict): + raise TushareLineageError(f"{label} must be a JSON object") + return value + + +def _validate_snapshot( + snapshot: Mapping[str, Any], +) -> Sequence[Mapping[str, Any]]: + if snapshot.get("format") != "quant-os.tushare-scoped-snapshot/v2": + raise TushareLineageError("source snapshot must use scoped format v2") + manifest_sha = str(snapshot.get("manifest_sha256") or "") + without_manifest_sha = dict(snapshot) + without_manifest_sha.pop("manifest_sha256", None) + computed_manifest_sha = hashlib.sha256( + _canonical_json(without_manifest_sha) + ).hexdigest() + if manifest_sha != computed_manifest_sha: + raise TushareLineageError("source snapshot manifest SHA-256 mismatch") + + jobs = snapshot.get("selected_jobs") + if not isinstance(jobs, list) or not jobs: + raise TushareLineageError("source snapshot has no selected_jobs") + selection_identity = [] + seen: set[tuple[str, str]] = set() + for raw_job in jobs: + if not isinstance(raw_job, Mapping): + raise TushareLineageError("source snapshot job must be an object") + key = ( + str(raw_job.get("api_name") or ""), + str(raw_job.get("partition_key") or ""), + ) + if not all(key) or key in seen: + raise TushareLineageError( + "source snapshot has empty or duplicate job keys" + ) + seen.add(key) + path = Path(str(raw_job.get("path") or "")) + if path.is_absolute() or ".." in path.parts: + raise TushareLineageError( + "source snapshot job paths must stay relative to the mirror" + ) + observed = raw_job.get("observed") + if raw_job.get("verified") is not True or not isinstance( + observed, + Mapping, + ): + raise TushareLineageError( + "every source snapshot job must carry verified observations" + ) + if ( + observed.get("row_count") != raw_job.get("row_count") + or observed.get("size_bytes") != raw_job.get("byte_count") + or observed.get("sha256") != raw_job.get("sha256") + ): + raise TushareLineageError( + "source snapshot observed file evidence is inconsistent" + ) + selection_identity.append( + { + field: value + for field, value in raw_job.items() + if field not in {"observed", "verified"} + } + ) + computed_selection_sha = hashlib.sha256( + _canonical_json({"selected_jobs": selection_identity}) + ).hexdigest() + if computed_selection_sha != snapshot.get("selection_sha256"): + raise TushareLineageError("source snapshot selection SHA-256 mismatch") + + verification = snapshot.get("verification") + if ( + not isinstance(verification, Mapping) + or verification.get("verified_jobs") != len(jobs) + or verification.get("double_verified_jobs") != len(jobs) + or verification.get( + "selected_jobs_still_latest_after_verification" + ) + is not True + ): + raise TushareLineageError( + "source snapshot lacks closed double-verification evidence" + ) + return jobs + + +def _compare_source_jobs( + snapshot_jobs: Sequence[Mapping[str, Any]], + provider_jobs: Sequence[Mapping[str, Any]], +) -> None: + def key(job: Mapping[str, Any]) -> tuple[str, str]: + return ( + str(job.get("api_name") or ""), + str(job.get("partition_key") or ""), + ) + + snapshot_by_key = {key(job): job for job in snapshot_jobs} + provider_by_key = {key(job): job for job in provider_jobs} + if len(snapshot_by_key) != len(snapshot_jobs): + raise TushareLineageError("source snapshot has duplicate job keys") + if len(provider_by_key) != len(provider_jobs): + raise TushareLineageError("provider manifest has duplicate job keys") + if set(snapshot_by_key) != set(provider_by_key): + missing = sorted(set(snapshot_by_key).difference(provider_by_key)) + extra = sorted(set(provider_by_key).difference(snapshot_by_key)) + raise TushareLineageError( + "source job sets differ: " + f"missing_from_provider={missing[:5]}, " + f"extra_in_provider={extra[:5]}" + ) + mismatches: list[str] = [] + for job_key in sorted(snapshot_by_key): + source = snapshot_by_key[job_key] + provider = provider_by_key[job_key] + for source_field, provider_field in MATCHED_FIELDS: + if source.get(source_field) != provider.get(provider_field): + mismatches.append( + f"{job_key[0]}:{job_key[1]}:{source_field}" + ) + if mismatches: + raise TushareLineageError( + "source job fields differ: " + ", ".join(mismatches[:12]) + ) + + +def verify_lineage( + source_manifest: str | Path, + provider_dir: str | Path, +) -> dict[str, Any]: + source_path = Path(source_manifest).expanduser().resolve() + provider = Path(provider_dir).expanduser().resolve() + snapshot = _load_object(source_path, label="source snapshot") + snapshot_jobs = _validate_snapshot(snapshot) + provider_verification = verify_qlib_provider(provider) + provider_manifest = _load_object( + provider / MANIFEST_NAME, + label="provider manifest", + ) + provider_jobs = provider_manifest.get("source_jobs") + if not isinstance(provider_jobs, list): + raise TushareLineageError("provider manifest has no source_jobs") + _compare_source_jobs(snapshot_jobs, provider_jobs) + return { + "artifact_type": "quant_os_tushare_source_provider_lineage", + "ok": True, + "snapshot_format": snapshot["format"], + "snapshot_manifest_sha256": snapshot["manifest_sha256"], + "snapshot_selection_sha256": snapshot["selection_sha256"], + "source_job_count": len(snapshot_jobs), + "matched_fields": [ + source_field for source_field, _ in MATCHED_FIELDS + ], + "mismatch_count": 0, + "provider": provider_verification, + "tool_source_sha256": hashlib.sha256( + Path(__file__).resolve().read_bytes() + ).hexdigest(), + "gate_credit": [], + "investment_value_claim": False, + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + destination = path.expanduser().resolve() + if destination.exists() and destination.is_dir(): + raise ValueError("output points to a directory") + destination.parent.mkdir(parents=True, exist_ok=True) + provider_bytes = ( + json.dumps( + dict(payload), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{destination.name}.", + suffix=".tmp", + dir=destination.parent, + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(provider_bytes) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, destination) + finally: + if temporary.exists(): + temporary.unlink() + + +def _main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-manifest", required=True) + parser.add_argument("--provider-dir", required=True) + parser.add_argument("--output-json") + args = parser.parse_args(argv) + try: + result = verify_lineage( + args.source_manifest, + args.provider_dir, + ) + if args.output_json: + output = Path(args.output_json).expanduser().resolve() + provider = Path(args.provider_dir).expanduser().resolve() + if output == provider or provider in output.parents: + raise ValueError( + "output-json must be outside the immutable provider " + "directory" + ) + _write_json(output, result) + except (OSError, ValueError, TushareLineageError) as exc: + parser.error(str(exc)) + print( + json.dumps( + result, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/tools/tushare_qlib.py b/tools/tushare_qlib.py index 51553a3..58f5e10 100644 --- a/tools/tushare_qlib.py +++ b/tools/tushare_qlib.py @@ -8,6 +8,10 @@ from pathlib import Path import sys from typing import Any, Mapping, Optional +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + from adapters.tushare_local import ( TushareMirrorError, build_qlib_provider, @@ -16,6 +20,20 @@ from adapters.tushare_local import ( ) +def _assert_json_outside_provider( + output_json: Optional[str], + provider_dir: str, +) -> None: + if output_json is None: + return + destination = Path(output_json).expanduser().resolve() + provider = Path(provider_dir).expanduser().resolve() + if destination == provider or provider in destination.parents: + raise ValueError( + "output-json must be outside the immutable provider directory" + ) + + def _write_json(path: Optional[str], payload: Mapping[str, Any]) -> None: if path is None: return @@ -52,6 +70,22 @@ def build_parser() -> argparse.ArgumentParser: build.add_argument("--end", required=True) build.add_argument("--market-name", default="tushare_a") build.add_argument("--benchmark-symbol", default="SH999999") + build.add_argument( + "--benchmark-index-code", + help="use real Tushare index_daily data, for example 000905.SH", + ) + build.add_argument( + "--universe-index-code", + help="build point-in-time market intervals from index_weight snapshots", + ) + build.add_argument( + "--expected-constituent-count", + type=int, + help=( + "required for unknown index codes; known indices such as " + "000905.SH use a built-in exact count" + ), + ) build.add_argument("--minimum-observations", type=int, default=60) build.add_argument( "--allow-unadjusted", @@ -81,6 +115,10 @@ def _main(argv: Optional[list[str]] = None) -> int: verify_files=not args.skip_file_verification, ) elif args.command == "build": + _assert_json_outside_provider( + args.output_json, + args.output_dir, + ) result = build_qlib_provider( args.mirror_root, args.output_dir, @@ -91,8 +129,15 @@ def _main(argv: Optional[list[str]] = None) -> int: minimum_observations=args.minimum_observations, allow_unadjusted=args.allow_unadjusted, expand_range=args.ohlc_policy == "expand-range", + benchmark_index_code=args.benchmark_index_code, + universe_index_code=args.universe_index_code, + expected_constituent_count=args.expected_constituent_count, ) else: + _assert_json_outside_provider( + args.output_json, + args.provider_dir, + ) result = verify_qlib_provider(args.provider_dir) _write_json(args.output_json, result) except (OSError, ValueError, TushareMirrorError) as exc: diff --git a/tools/tushare_snapshot.py b/tools/tushare_snapshot.py new file mode 100644 index 0000000..2427c7a --- /dev/null +++ b/tools/tushare_snapshot.py @@ -0,0 +1,977 @@ +"""Create an auditable, scoped manifest from a live local Tushare mirror. + +The command is intentionally read-only with respect to the mirror. It records +the latest completed job for every selected ``(api_name, partition_key)`` in one +SQLite read transaction, then optionally verifies the immutable Parquet files. +It never reads Tushare credentials and never copies source data. +""" + +from __future__ import annotations + +import argparse +from collections import Counter, defaultdict +from datetime import date, datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import sqlite3 +import sys +import tempfile +from typing import Any, Iterable, Mapping, Optional, Sequence + + +TOOL_VERSION = "2" +DEFAULT_APIS = ( + "daily", + "adj_factor", + "trade_cal", + "stock_basic", + "index_daily", + "index_weight", +) +DEFAULT_INDEX_CODES = ("000905.SH",) +BLOCKING_LIVE_STATUSES = frozenset(("running", "deferred")) +_MONTH_RE = re.compile(r"(?:^|[/_])month=(\d{4}-\d{2})(?:$|[/_])") +_YEAR_RE = re.compile(r"(?:^|[/_])year=(\d{4})(?:$|[/_])") +_INDEX_CODE_RE = re.compile( + r"(?:^|/)(?:ts_code|index_code)=([^/]+)", + re.IGNORECASE, +) +_API_RE = re.compile(r"^[A-Za-z0-9_]+$") +_INDEX_INPUT_RE = re.compile(r"^([A-Za-z0-9]+)[._]([A-Za-z0-9]+)$") + + +class TushareSnapshotError(RuntimeError): + """Raised when a requested snapshot cannot be frozen safely.""" + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace( + "+00:00", + "Z", + ) + + +def _canonical_json(payload: Mapping[str, Any]) -> bytes: + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _normalize_index_code(raw: str) -> str: + value = raw.strip().upper() + match = _INDEX_INPUT_RE.fullmatch(value) + if match is None: + raise ValueError( + f"invalid index code {raw!r}; expected a value such as 000905.SH" + ) + return f"{match.group(1)}.{match.group(2)}" + + +def _partition_index_code(partition_key: str) -> Optional[str]: + match = _INDEX_CODE_RE.search(partition_key) + if match is None: + return None + raw = match.group(1) + try: + return _normalize_index_code(raw) + except ValueError: + return None + + +def _partition_month(partition_key: str) -> Optional[str]: + match = _MONTH_RE.search(partition_key) + return match.group(1) if match else None + + +def _partition_year(partition_key: str) -> Optional[int]: + match = _YEAR_RE.search(partition_key) + return int(match.group(1)) if match else None + + +def _month_keys(start: date, end: date) -> list[str]: + year = start.year + month = start.month + result: list[str] = [] + while (year, month) <= (end.year, end.month): + result.append(f"{year:04d}-{month:02d}") + month += 1 + if month == 13: + year += 1 + month = 1 + return result + + +def _parse_values(values: Sequence[str]) -> list[str]: + result: list[str] = [] + for value in values: + result.extend(part.strip() for part in value.split(",") if part.strip()) + return result + + +def _validate_apis(apis: Iterable[str]) -> tuple[str, ...]: + result: list[str] = [] + seen: set[str] = set() + for raw in apis: + api = raw.strip() + if not _API_RE.fullmatch(api): + raise ValueError(f"invalid API name: {raw!r}") + if api not in seen: + seen.add(api) + result.append(api) + if not result: + raise ValueError("at least one API must be requested") + return tuple(result) + + +def _scope_reason( + api_name: str, + partition_key: str, + *, + apis: set[str], + months: set[str], + years: set[int], + index_weight_years: set[int], + index_codes: set[str], +) -> tuple[bool, str]: + if api_name not in apis: + return False, "api_not_requested" + + if api_name in {"daily", "adj_factor"}: + month = _partition_month(partition_key) + if month is None: + return False, "unrecognized_partition" + return (True, "in_scope") if month in months else ( + False, + "outside_date_range", + ) + + if api_name == "trade_cal": + year = _partition_year(partition_key) + if year is None: + return False, "unrecognized_partition" + return (True, "in_scope") if year in years else ( + False, + "outside_date_range", + ) + + if api_name in {"index_daily", "index_weight"}: + code = _partition_index_code(partition_key) + if code not in index_codes: + return False, "different_index_code" + year = _partition_year(partition_key) + if year is None: + return False, "unrecognized_partition" + accepted_years = ( + index_weight_years if api_name == "index_weight" else years + ) + return (True, "in_scope") if year in accepted_years else ( + False, + "outside_date_range", + ) + + if api_name == "stock_basic": + return True, "in_scope" + + month = _partition_month(partition_key) + if month is not None: + return (True, "in_scope") if month in months else ( + False, + "outside_date_range", + ) + year = _partition_year(partition_key) + if year is not None: + return (True, "in_scope") if year in years else ( + False, + "outside_date_range", + ) + return True, "in_scope" + + +def _state_file_identity(path: Path) -> dict[str, Any]: + stat = path.stat() + return { + "device": stat.st_dev, + "inode": stat.st_ino, + "mtime_ns": stat.st_mtime_ns, + "path": "data/state.sqlite3", + "size_bytes": stat.st_size, + } + + +def _wal_identity(state_path: Path) -> Optional[dict[str, int]]: + wal_path = state_path.with_name(state_path.name + "-wal") + if not wal_path.exists(): + return None + stat = wal_path.stat() + return { + "inode": stat.st_ino, + "mtime_ns": stat.st_mtime_ns, + "size_bytes": stat.st_size, + } + + +def _candidate_query( + connection: sqlite3.Connection, + api_name: str, + *, + months: Sequence[str], + years: Sequence[int], + index_codes: Sequence[str], +) -> list[sqlite3.Row]: + columns = ( + "id, api_name, partition_key, status, row_count, byte_count, " + "sha256, file_path, completed_at, params_json, fields" + ) + where = ["api_name = ?", "status = 'completed'"] + parameters: list[Any] = [api_name] + + if api_name in {"daily", "adj_factor"}: + placeholders = ",".join("?" for _ in months) + where.append(f"partition_key IN ({placeholders})") + parameters.extend(f"month={month}" for month in months) + elif api_name == "trade_cal": + clauses = [] + for year in years: + clauses.append("partition_key LIKE ?") + parameters.append(f"%year={year}%") + where.append("(" + " OR ".join(clauses) + ")") + elif api_name in {"index_daily", "index_weight"}: + code_clauses = [] + for code in index_codes: + code_clauses.append("partition_key LIKE ?") + parameters.append(f"%={code.replace('.', '_')}%") + year_clauses = [] + for year in years: + year_clauses.append("partition_key LIKE ?") + parameters.append(f"%year={year}%") + where.append("(" + " OR ".join(code_clauses) + ")") + where.append("(" + " OR ".join(year_clauses) + ")") + + query = f"SELECT {columns} FROM jobs WHERE " + " AND ".join(where) + return list(connection.execute(query, parameters)) + + +def _assert_scope_complete( + selected: Sequence[sqlite3.Row], + *, + apis: Sequence[str], + months: Sequence[str], + years: Sequence[int], + index_weight_years: Sequence[int], + index_codes: Sequence[str], +) -> None: + by_api: dict[str, list[sqlite3.Row]] = defaultdict(list) + for row in selected: + by_api[str(row["api_name"])].append(row) + + missing: list[str] = [] + for api in apis: + rows = by_api.get(api, []) + if api in {"daily", "adj_factor"}: + found = { + value + for value in ( + _partition_month(str(row["partition_key"])) for row in rows + ) + if value is not None + } + missing.extend( + f"{api}:month={month}" for month in months if month not in found + ) + elif api == "trade_cal": + found = { + value + for value in ( + _partition_year(str(row["partition_key"])) for row in rows + ) + if value is not None + } + missing.extend( + f"{api}:year={year}" for year in years if year not in found + ) + elif api in {"index_daily", "index_weight"}: + found = { + ( + _partition_index_code(str(row["partition_key"])), + _partition_year(str(row["partition_key"])), + ) + for row in rows + } + required_years = ( + index_weight_years if api == "index_weight" else years + ) + missing.extend( + f"{api}:{code}/year={year}" + for code in index_codes + for year in required_years + if (code, year) not in found + ) + elif not rows: + missing.append(f"{api}:no_completed_partition") + + if missing: + preview = ", ".join(missing[:12]) + suffix = "" if len(missing) <= 12 else f" (+{len(missing) - 12} more)" + raise TushareSnapshotError( + f"requested scope is incomplete; missing {preview}{suffix}" + ) + + +def _resolve_source_file(mirror_root: Path, raw_path: str) -> tuple[Path, str]: + path = Path(raw_path) + if not path.is_absolute(): + path = mirror_root / path + resolved = path.expanduser().resolve(strict=True) + root = mirror_root.resolve(strict=True) + try: + relative = resolved.relative_to(root) + except ValueError as exc: + raise TushareSnapshotError( + "source file escapes the declared mirror root" + ) from exc + return resolved, relative.as_posix() + + +def _parquet_row_count(path: Path) -> int: + try: + import pyarrow.parquet as parquet + except ImportError as exc: + raise TushareSnapshotError( + "Parquet row verification requires pyarrow; install the Qlib " + "research requirements or use --no-verify explicitly" + ) from exc + try: + metadata = parquet.ParquetFile(path).metadata + except Exception as exc: + raise TushareSnapshotError( + f"cannot read Parquet metadata for {path.name}: {exc}" + ) from exc + return int(metadata.num_rows) + + +def _verify_job_file( + mirror_root: Path, + row: sqlite3.Row, +) -> tuple[str, dict[str, Any]]: + raw_path = row["file_path"] + if not isinstance(raw_path, str) or not raw_path: + raise TushareSnapshotError( + f"completed job {row['id']} has no source file path" + ) + path, relative = _resolve_source_file(mirror_root, raw_path) + stat = path.stat() + observed_size = int(stat.st_size) + expected_size = row["byte_count"] + if expected_size is None or observed_size != int(expected_size): + raise TushareSnapshotError( + f"size mismatch for job {row['id']} ({relative}): " + f"expected {expected_size}, observed {observed_size}" + ) + + expected_sha = str(row["sha256"] or "").lower() + if not re.fullmatch(r"[0-9a-f]{64}", expected_sha): + raise TushareSnapshotError( + f"completed job {row['id']} has no valid SHA-256" + ) + observed_sha = _sha256_file(path) + if observed_sha != expected_sha: + raise TushareSnapshotError( + f"SHA-256 mismatch for job {row['id']} ({relative})" + ) + + expected_rows = row["row_count"] + observed_rows = _parquet_row_count(path) + if expected_rows is None or observed_rows != int(expected_rows): + raise TushareSnapshotError( + f"row-count mismatch for job {row['id']} ({relative}): " + f"expected {expected_rows}, observed {observed_rows}" + ) + return relative, { + "row_count": observed_rows, + "sha256": observed_sha, + "size_bytes": observed_size, + } + + +def _relative_job_path(mirror_root: Path, row: sqlite3.Row) -> str: + raw_path = row["file_path"] + if not isinstance(raw_path, str) or not raw_path: + raise TushareSnapshotError( + f"completed job {row['id']} has no source file path" + ) + _, relative = _resolve_source_file(mirror_root, raw_path) + return relative + + +def _expected_job_metadata(row: sqlite3.Row) -> tuple[int, int, str]: + try: + row_count = int(row["row_count"]) + byte_count = int(row["byte_count"]) + except (TypeError, ValueError) as exc: + raise TushareSnapshotError( + f"completed job {row['id']} has incomplete count metadata" + ) from exc + if row_count < 0 or byte_count < 0: + raise TushareSnapshotError( + f"completed job {row['id']} has negative count metadata" + ) + source_sha = str(row["sha256"] or "").lower() + if not re.fullmatch(r"[0-9a-f]{64}", source_sha): + raise TushareSnapshotError( + f"completed job {row['id']} has no valid SHA-256" + ) + return row_count, byte_count, source_sha + + +def _build_table_summaries( + jobs: Sequence[Mapping[str, Any]], +) -> dict[str, dict[str, Any]]: + grouped: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for job in jobs: + grouped[str(job["api_name"])].append(job) + summaries: dict[str, dict[str, Any]] = {} + for api_name in sorted(grouped): + rows = grouped[api_name] + partitions = sorted(str(row["partition_key"]) for row in rows) + summaries[api_name] = { + "bytes": sum(int(row["byte_count"]) for row in rows), + "first_partition": partitions[0], + "last_partition": partitions[-1], + "partitions": len(rows), + "rows": sum(int(row["row_count"]) for row in rows), + "shadowed_completed_jobs": sum( + int(row["shadowed_completed_count"]) for row in rows + ), + } + return summaries + + +def _request_sha256(row: sqlite3.Row) -> str: + raw_params = row["params_json"] + if not isinstance(raw_params, str): + raise TushareSnapshotError( + f"completed job {row['id']} has no params_json provenance" + ) + try: + params = json.loads(raw_params) + except json.JSONDecodeError as exc: + raise TushareSnapshotError( + f"completed job {row['id']} has invalid params_json provenance" + ) from exc + request = { + "api_name": str(row["api_name"]), + "partition_key": str(row["partition_key"]), + "fields": str(row["fields"] or ""), + "params": params, + } + return hashlib.sha256(_canonical_json(request)).hexdigest() + + +def _latest_completed_ids( + state_path: Path, +) -> dict[tuple[str, str], int]: + connection = sqlite3.connect( + state_path.as_uri() + "?mode=ro", + uri=True, + timeout=30, + ) + try: + rows = connection.execute( + """ + SELECT api_name, partition_key, MAX(id) + FROM jobs + WHERE status = 'completed' + GROUP BY api_name, partition_key + """ + ) + return { + (str(api_name), str(partition_key)): int(job_id) + for api_name, partition_key, job_id in rows + } + finally: + connection.close() + + +def create_snapshot( + mirror_root: str | Path, + *, + start: str | date, + end: str | date, + apis: Sequence[str] = DEFAULT_APIS, + index_codes: Sequence[str] = DEFAULT_INDEX_CODES, + verify: bool = True, +) -> dict[str, Any]: + """Read and verify one deterministic Tushare mirror selection.""" + + root = Path(mirror_root).expanduser().resolve(strict=True) + state_path = root / "data" / "state.sqlite3" + if not state_path.is_file(): + raise TushareSnapshotError( + "mirror is missing its data/state.sqlite3 catalog" + ) + + start_date = ( + start if isinstance(start, date) else date.fromisoformat(str(start)) + ) + end_date = end if isinstance(end, date) else date.fromisoformat(str(end)) + if start_date > end_date: + raise ValueError("start must be on or before end") + + requested_apis = _validate_apis(apis) + requested_codes = tuple( + dict.fromkeys(_normalize_index_code(code) for code in index_codes) + ) + if ( + {"index_daily", "index_weight"} & set(requested_apis) + and not requested_codes + ): + raise ValueError("index APIs require at least one --index-codes value") + + months = _month_keys(start_date, end_date) + years = list(range(start_date.year, end_date.year + 1)) + index_weight_years = ( + [start_date.year - 1, *years] + if "index_weight" in requested_apis + else years + ) + month_set = set(months) + year_set = set(years) + index_weight_year_set = set(index_weight_years) + code_set = set(requested_codes) + api_set = set(requested_apis) + + identity_before = _state_file_identity(state_path) + wal_before = _wal_identity(state_path) + read_started_at = _utc_now() + connection = sqlite3.connect( + state_path.as_uri() + "?mode=ro", + uri=True, + timeout=30, + ) + connection.row_factory = sqlite3.Row + try: + connection.execute("BEGIN") + required_columns = { + "id", + "api_name", + "partition_key", + "status", + "row_count", + "byte_count", + "sha256", + "file_path", + "completed_at", + "fields", + "params_json", + } + available_columns = { + str(row["name"]) + for row in connection.execute("PRAGMA table_info(jobs)") + } + missing_columns = required_columns - available_columns + if missing_columns: + raise TushareSnapshotError( + "jobs table is missing required columns: " + + ", ".join(sorted(missing_columns)) + ) + + sqlite_identity = { + "data_version": int( + connection.execute("PRAGMA data_version").fetchone()[0] + ), + "journal_mode": str( + connection.execute("PRAGMA journal_mode").fetchone()[0] + ), + "page_count": int( + connection.execute("PRAGMA page_count").fetchone()[0] + ), + "page_size": int( + connection.execute("PRAGMA page_size").fetchone()[0] + ), + "schema_version": int( + connection.execute("PRAGMA schema_version").fetchone()[0] + ), + "user_version": int( + connection.execute("PRAGMA user_version").fetchone()[0] + ), + } + + scoped_completed: list[sqlite3.Row] = [] + for api_name in requested_apis: + api_years = ( + index_weight_years + if api_name == "index_weight" + else years + ) + rows = _candidate_query( + connection, + api_name, + months=months, + years=api_years, + index_codes=requested_codes, + ) + scoped_completed.extend( + row + for row in rows + if _scope_reason( + str(row["api_name"]), + str(row["partition_key"]), + apis=api_set, + months=month_set, + years=year_set, + index_weight_years=index_weight_year_set, + index_codes=code_set, + )[0] + ) + + live_rows = list( + connection.execute( + """ + SELECT id, api_name, partition_key, status + FROM jobs + WHERE status IN ('running', 'deferred') + ORDER BY id + """ + ) + ) + connection.commit() + finally: + connection.close() + read_finished_at = _utc_now() + identity_after = _state_file_identity(state_path) + wal_after = _wal_identity(state_path) + + excluded_live_jobs: list[dict[str, Any]] = [] + blocking_live_jobs: list[sqlite3.Row] = [] + for row in live_rows: + in_scope, reason = _scope_reason( + str(row["api_name"]), + str(row["partition_key"]), + apis=api_set, + months=month_set, + years=year_set, + index_weight_years=index_weight_year_set, + index_codes=code_set, + ) + if in_scope and str(row["status"]) in BLOCKING_LIVE_STATUSES: + blocking_live_jobs.append(row) + else: + excluded_live_jobs.append( + { + "api_name": str(row["api_name"]), + "id": int(row["id"]), + "partition_key": str(row["partition_key"]), + "reason": reason, + "status": str(row["status"]), + } + ) + if blocking_live_jobs: + details = ", ".join( + f"{row['api_name']}:{row['partition_key']}#{row['id']}" + f"[{row['status']}]" + for row in blocking_live_jobs[:12] + ) + raise TushareSnapshotError( + "requested scope contains running/deferred jobs; retry after they " + f"reach a terminal state: {details}" + ) + + grouped: dict[tuple[str, str], list[sqlite3.Row]] = defaultdict(list) + for row in scoped_completed: + grouped[(str(row["api_name"]), str(row["partition_key"]))].append(row) + latest_rows: list[sqlite3.Row] = [] + shadow_counts: dict[tuple[str, str], int] = {} + for key, rows in grouped.items(): + ordered = sorted(rows, key=lambda row: int(row["id"])) + latest_rows.append(ordered[-1]) + shadow_counts[key] = len(ordered) - 1 + latest_rows.sort( + key=lambda row: ( + str(row["api_name"]), + str(row["partition_key"]), + int(row["id"]), + ) + ) + + _assert_scope_complete( + latest_rows, + apis=requested_apis, + months=months, + years=years, + index_weight_years=index_weight_years, + index_codes=requested_codes, + ) + + selected_jobs: list[dict[str, Any]] = [] + for row in latest_rows: + key = (str(row["api_name"]), str(row["partition_key"])) + row_count, byte_count, expected_sha = _expected_job_metadata(row) + if verify: + relative_path, observed = _verify_job_file(root, row) + else: + relative_path = _relative_job_path(root, row) + observed = None + job: dict[str, Any] = { + "api_name": key[0], + "byte_count": byte_count, + "completed_at": row["completed_at"], + "id": int(row["id"]), + "partition_key": key[1], + "path": relative_path, + "row_count": row_count, + "request_sha256": _request_sha256(row), + "sha256": expected_sha, + "shadowed_completed_count": shadow_counts[key], + "verified": verify, + } + if observed is not None: + job["observed"] = observed + selected_jobs.append(job) + + latest_ids_after_verification = _latest_completed_ids(state_path) + stale_jobs = [ + { + "api_name": job["api_name"], + "partition_key": job["partition_key"], + "selected_job_id": job["id"], + "latest_job_id": latest_ids_after_verification.get( + (job["api_name"], job["partition_key"]) + ), + } + for job in selected_jobs + if latest_ids_after_verification.get( + (job["api_name"], job["partition_key"]) + ) + != job["id"] + ] + if stale_jobs: + raise TushareSnapshotError( + "selected jobs changed during file verification: " + f"{stale_jobs[:12]}" + ) + if verify: + for row, job in zip(latest_rows, selected_jobs): + relative_path, observed = _verify_job_file(root, row) + if ( + relative_path != job["path"] + or observed != job.get("observed") + ): + raise TushareSnapshotError( + "selected source bytes changed during double verification" + ) + identity_after_verification = _state_file_identity(state_path) + wal_after_verification = _wal_identity(state_path) + + selection_identity = [ + { + key: value + for key, value in job.items() + if key not in {"observed", "verified"} + } + for job in selected_jobs + ] + selection_sha = hashlib.sha256( + _canonical_json({"selected_jobs": selection_identity}) + ).hexdigest() + changed_while_reading = ( + identity_before != identity_after or wal_before != wal_after + ) + status_counts = Counter(str(row["status"]) for row in live_rows) + manifest: dict[str, Any] = { + "as_of": read_started_at, + "excluded_live_jobs": excluded_live_jobs, + "format": "quant-os.tushare-scoped-snapshot/v2", + "hash_algorithms": { + "canonical_json": ( + "UTF-8 JSON; keys sorted; separators ',' and ':'; " + "ensure_ascii=false; allow_nan=false" + ), + "manifest_sha256": ( + "SHA-256 of canonical_json(the complete manifest excluding " + "the manifest_sha256 member)" + ), + "request_sha256": ( + "SHA-256 of canonical_json({api_name, partition_key, fields, " + "params}), with params_json parsed as JSON and without " + "embedding it in this manifest" + ), + "selection_sha256": ( + "SHA-256 of canonical_json({selected_jobs: [...]}), excluding " + "each job's observed and verified members so verify/no-verify " + "runs over the same SQLite selection have the same identity" + ), + }, + "mirror": { + "label": root.name, + "state_db": { + "changed_while_reading": changed_while_reading, + "changed_through_file_verification": ( + identity_before != identity_after_verification + or wal_before != wal_after_verification + ), + "file_after": identity_after, + "file_after_verification": identity_after_verification, + "file_before": identity_before, + "read_finished_at": read_finished_at, + "read_started_at": read_started_at, + "sqlite_snapshot": sqlite_identity, + "wal_after": wal_after, + "wal_after_verification": wal_after_verification, + "wal_before": wal_before, + }, + }, + "scope": { + "apis": list(requested_apis), + "end": end_date.isoformat(), + "index_codes": list(requested_codes), + "partition_policy": ( + "calendar partitions overlapping [start,end]; timeless " + "stock_basic partitions; exact requested index codes; " + "index_weight also includes start.year-1 as the pre-start " + "point-in-time membership anchor" + ), + "index_weight_anchor_year": ( + start_date.year - 1 + if "index_weight" in requested_apis + else None + ), + "start": start_date.isoformat(), + }, + "selected_jobs": selected_jobs, + "selection_sha256": selection_sha, + "summary": { + "excluded_live_job_status_counts": dict(sorted(status_counts.items())), + "excluded_live_jobs": len(excluded_live_jobs), + "selected_bytes": sum(job["byte_count"] for job in selected_jobs), + "selected_jobs": len(selected_jobs), + "selected_rows": sum(job["row_count"] for job in selected_jobs), + "shadowed_completed_jobs": sum( + job["shadowed_completed_count"] for job in selected_jobs + ), + "tables": _build_table_summaries(selected_jobs), + }, + "tool": { + "name": "tools/tushare_snapshot.py", + "version": TOOL_VERSION, + }, + "verification": { + "mode": "size+sha256+parquet_row_count" if verify else "not_performed", + "verified_jobs": len(selected_jobs) if verify else 0, + "double_verified_jobs": len(selected_jobs) if verify else 0, + "selected_jobs_still_latest_after_verification": True, + }, + } + manifest["manifest_sha256"] = hashlib.sha256( + _canonical_json(manifest) + ).hexdigest() + return manifest + + +def write_manifest(path: str | Path, manifest: Mapping[str, Any]) -> Path: + destination = Path(path).expanduser().resolve() + if destination.exists() and destination.is_dir(): + raise ValueError(f"output points to a directory: {destination}") + destination.parent.mkdir(parents=True, exist_ok=True) + payload = ( + json.dumps( + manifest, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n" + ) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{destination.name}.", + suffix=".tmp", + dir=destination.parent, + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, destination) + finally: + if temporary.exists(): + temporary.unlink() + return destination + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mirror-root", required=True) + parser.add_argument("--start", required=True) + parser.add_argument("--end", required=True) + parser.add_argument( + "--apis", + nargs="+", + default=list(DEFAULT_APIS), + help="space- or comma-separated API names", + ) + parser.add_argument( + "--index-codes", + nargs="+", + default=list(DEFAULT_INDEX_CODES), + help="space- or comma-separated Tushare index codes", + ) + parser.add_argument( + "--verify", + action=argparse.BooleanOptionalAction, + default=True, + help="verify file size, SHA-256 and Parquet row count (default: true)", + ) + parser.add_argument("--output", "--output-json", dest="output", required=True) + return parser + + +def _main(argv: Optional[Sequence[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + manifest = create_snapshot( + args.mirror_root, + start=args.start, + end=args.end, + apis=_parse_values(args.apis), + index_codes=_parse_values(args.index_codes), + verify=args.verify, + ) + output = write_manifest(args.output, manifest) + except (OSError, ValueError, sqlite3.Error, TushareSnapshotError) as exc: + parser.error(str(exc)) + print( + json.dumps( + { + "manifest_sha256": manifest["manifest_sha256"], + "output": str(output), + "selected_jobs": manifest["summary"]["selected_jobs"], + "verified_jobs": manifest["verification"]["verified_jobs"], + }, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(_main())