boat/quant-os
Public Access
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: operationalize local Tushare Qlib research

Browse Source
This commit is contained in:
zebzhang
2026-07-31 01:26:49 +08:00
parent f81c116bfe
commit 6275370afc
21 changed files with 4438 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/build_public_status.py
+391 -43
View File
@@ -9,7 +9,7 @@ import json
import os
import re
import sys
from datetime import datetime, timezone
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Mapping, Sequence, Tuple
@@ -33,6 +33,9 @@ JOINQUANT_TARGET_PACKAGE_RUN_PATH = (
JOINQUANT_TARGET_PACKAGE_DOC_PATH = (
"docs/JOINQUANT_TARGET_PACKAGE_EVIDENCE_2026-07-26.md"
)
TUSHARE_RESEARCH_RUN_PATH = (
"evidence/tushare/CSI500-2018-2025/5bf19d2d/run.json"
)
JOINQUANT_TARGET_PACKAGE_RUN_ID = "2457a7c39a276e09e0fabf99e28978e1"
JOINQUANT_TARGET_PACKAGE_OBSERVED_AT = "2026-07-26T08:50:15Z"
JOINQUANT_TARGET_PACKAGE_BUNDLE_SHA256 = (
@@ -47,6 +50,7 @@ SOURCE_PATHS: Tuple[str, ...] = (
"dist/bundle_manifest.json",
REACHABILITY_PATH,
JOINQUANT_TARGET_PACKAGE_RUN_PATH,
TUSHARE_RESEARCH_RUN_PATH,
"configs/baseline.json",
"README.md",
"docs/ARCHITECTURE.md",
@@ -566,11 +570,209 @@ def _validate_joinquant_target_package_run(
}
def _validate_tushare_research_run(
project: Path,
run: Mapping[str, Any],
) -> Dict[str, Any]:
expected_scope = {
"index_code": "000905.SH",
"start": "2018-01-01",
"end": "2025-12-31",
"daily_market_data_from": "2018-01-02",
"daily_market_data_through": "2025-12-31",
"daily_partitions_completed": 96,
"daily_partitions_planned": 96,
"daily_rows": 8_813_885,
}
expected_snapshot = {
"selected_jobs": 221,
"verified_jobs": 221,
"double_verified_jobs": 221,
"manifest_sha256": (
"744a69828f527594e9f2787280095368c50deffa2f269c359ae65e2e0ac2d5f2"
),
"selection_sha256": (
"66e91fffecba5fa042922c49e339f24212c3febf5b874279866fdca660f54fb5"
),
}
expected_provider = {
"verified": True,
"data_version": (
"5bf19d2da064357ad1802bca4bfa0c0505ed63fe2b24785ec6c56ecff1724963"
),
"tree_sha256": (
"f173b8095fd9a62a63807324fa48bad83f0c282eea3abba871d0b0efd30b199f"
),
"manifest_sha256": (
"27fb6fedb6a4b114eae2aba44505fb6c4d32c7a9ae81e58c724dfadb8dd10ed0"
),
"file_count": 10_011,
"byte_count": 71_707_194,
"calendar_sessions": 1_942,
"instrument_count": 1_111,
"selected_daily_rows": 1_975_455,
"observed_membership_snapshots": 97,
"effective_membership_snapshots": 96,
"same_session_membership_use": False,
"price_adjustment_complete": True,
"benchmark": "SH000905",
"converter_source_matches_current": True,
}
expected_lineage = {
"verified": True,
"source_job_count": 221,
"matched_fields": [
"id",
"path",
"row_count",
"byte_count",
"sha256",
"request_sha256",
],
"mismatch_count": 0,
"converter_source_matches_current": True,
}
expected_stability = {
"state_db_changed_through_file_verification": True,
"state_db_change_scope": "unrelated live downloader jobs",
"snapshot_jobs_still_latest": True,
"selected_scope_stable": True,
"post_read_file_verification": True,
"provider_source_jobs": 221,
"snapshot_source_jobs": 221,
"source_job_full_field_diff": 0,
}
expected_reproducibility = {
"runs": 2,
"byte_identical": True,
"run_sha256": (
"803035a95ee9cf6f90af20cdf7e7024149b97e0113b585209a80f80048be87d3"
),
"signal_rows": 176_615,
"portfolio_days": 1_698,
}
expected_performance = {
"strategy_cumulative_return": 0.2531519864,
"benchmark_cumulative_return": 0.7895593835,
"max_drawdown": -0.6041641366,
}
expected_limitations = {
"stock_st": "denied",
"qlib_limit_threshold": 0.095,
"limit_rule_fidelity": (
"market-level approximation; not per-symbol daily limit prices"
),
"research_only": True,
}
if (
run.get("schema_version") != 1
or run.get("record_type")
!= "quant_os_tushare_qlib_research_run"
or run.get("run_id") != "5bf19d2d"
or run.get("audit_as_of") != "2026-07-31"
or run.get("scope") != expected_scope
or run.get("snapshot") != expected_snapshot
or run.get("provider") != expected_provider
or run.get("lineage") != expected_lineage
or run.get("source_stability") != expected_stability
or run.get("reproducibility") != expected_reproducibility
or run.get("performance") != expected_performance
or run.get("limitations") != expected_limitations
or run.get("production_ready") is not False
or run.get("investment_value_claim") is not False
or run.get("gate_credit") != []
):
raise PublicStatusError(
"Tushare CSI500 research machine record drifted"
)
encoded = json.dumps(run, ensure_ascii=False, sort_keys=True)
if ABSOLUTE_PATH_PATTERN.search(encoded):
raise PublicStatusError(
"Tushare CSI500 research record contains an absolute local path"
)
path = _safe_repo_path(
TUSHARE_RESEARCH_RUN_PATH,
label="Tushare CSI500 research run",
)
if not (project / path).is_file():
raise PublicStatusError(
"tracked Tushare CSI500 research run is missing"
)
return {
"runId": str(run["run_id"]),
"auditAsOf": str(run["audit_as_of"]),
"indexCode": str(expected_scope["index_code"]),
"scopeStart": str(expected_scope["start"]),
"scopeEnd": str(expected_scope["end"]),
"marketDataFrom": str(expected_scope["daily_market_data_from"]),
"marketDataThrough": str(
expected_scope["daily_market_data_through"]
),
"completedPartitions": int(
expected_scope["daily_partitions_completed"]
),
"plannedPartitions": int(
expected_scope["daily_partitions_planned"]
),
"completedRows": int(expected_scope["daily_rows"]),
"snapshotJobs": int(expected_snapshot["selected_jobs"]),
"verifiedSnapshotJobs": int(expected_snapshot["verified_jobs"]),
"snapshotManifestSha256": str(
expected_snapshot["manifest_sha256"]
),
"snapshotSelectionSha256": str(
expected_snapshot["selection_sha256"]
),
"providerDataVersion": str(expected_provider["data_version"]),
"providerTreeSha256": str(expected_provider["tree_sha256"]),
"providerManifestSha256": str(
expected_provider["manifest_sha256"]
),
"sourceJobCount": int(expected_lineage["source_job_count"]),
"lineageMatchedFields": list(expected_lineage["matched_fields"]),
"lineageMismatchCount": int(expected_lineage["mismatch_count"]),
"providerFiles": int(expected_provider["file_count"]),
"providerBytes": int(expected_provider["byte_count"]),
"calendarSessions": int(expected_provider["calendar_sessions"]),
"instrumentCount": int(expected_provider["instrument_count"]),
"selectedDailyRows": int(
expected_provider["selected_daily_rows"]
),
"observedMembershipSnapshots": int(
expected_provider["observed_membership_snapshots"]
),
"effectiveMembershipSnapshots": int(
expected_provider["effective_membership_snapshots"]
),
"sameSessionMembershipUse": False,
"sourceStable": True,
"stateDbChangedDuringVerification": True,
"converterSourceMatchesCurrent": True,
"runSha256": str(expected_reproducibility["run_sha256"]),
"signalRows": int(expected_reproducibility["signal_rows"]),
"portfolioDays": int(expected_reproducibility["portfolio_days"]),
"strategyCumulativeReturn": float(
expected_performance["strategy_cumulative_return"]
),
"benchmarkCumulativeReturn": float(
expected_performance["benchmark_cumulative_return"]
),
"maxDrawdown": float(expected_performance["max_drawdown"]),
"stockStAvailable": False,
"limitThreshold": float(
expected_limitations["qlib_limit_threshold"]
),
"productionReady": False,
"investmentValueClaim": False,
"gateCredit": [],
}
def _extract_test_summary(facts: Sequence[Any]) -> Tuple[int | None, int | None]:
for fact in facts:
text = str(fact)
match = re.search(
r"completed\s+(\d+)\s+tests.*?(\d+)\s+optional-runtime skips",
r"completed\s+(\d+)\s+tests.*?(\d+)\s+optional-runtime skips?",
text,
flags=re.IGNORECASE,
)
@@ -618,6 +820,8 @@ def _platform_evidence_links(name: str) -> List[str]:
)
if "Qlib" in name or "Tushare" in name:
links.append("tushare-data")
if "Tushare" in name:
links.append("tushare-csi500-research-run")
if "QMT" in name or "XtTrader" in name:
links.append("platform-deployment")
return list(dict.fromkeys(links))
@@ -719,6 +923,12 @@ def _build_evidence(project: Path) -> List[Dict[str, Any]]:
"docs/TUSHARE_LOCAL_DATA.md",
"runtime_evidence",
),
(
"tushare-csi500-research-run",
"Tushare CSI500 2018-2025 reproducible research run",
TUSHARE_RESEARCH_RUN_PATH,
"machine_record",
),
(
"platform-deployment",
"Platform deployment runbook",
@@ -788,13 +998,20 @@ def _build_layers(
"claim": "不可变快照、PIT 字段合同和 Tushare→Qlib 技术链路已有本地合同/运行证据；这些 provider-data 路径尚未接成当前 Ridge 五层候选。",
"capabilities": [
"JQData raw/factor/limits/paused/PIT membership/ST snapshot",
"Tushare completed-Parquet inventory and immutable Qlib provider",
"Tushare scoped source release and verified immutable Qlib provider",
"semantic manifest and data-version verification",
],
"limitations": [
"尚无授权 JQData 真实快照与许可 lineage；Tushare 镜像仅部分完成且缺生产关键字段；当前五层候选仍使用 synthetic 数据。"
"Tushare 2018—2025 中证 500 scoped research 链已验证，但 live raw "
"仍可变、stock_st 不可用、9.5% 涨跌停为市场级近似，且尚未接入"
"当前 Ridge 五层候选；JQData 授权快照与许可 lineage 仍缺失。"
],
"evidenceLinks": [
"architecture",
"tushare-data",
"tushare-csi500-research-run",
"scorecard",
],
"evidenceLinks": ["architecture", "tushare-data", "scorecard"],
"releaseVerified": False,
},
{
@@ -1046,6 +1263,7 @@ def _build_freshness(
scorecard: Mapping[str, Any],
joinquant_markdown: str,
joinquant_target_package_run: Mapping[str, Any],
tushare_research_run: Mapping[str, Any],
tushare_markdown: str,
baseline_shadow_days: int,
production_shadow_days: int,
@@ -1061,49 +1279,100 @@ def _build_freshness(
)
joinquant_start, joinquant_end = period_match.groups()
daily_match = re.search(
r"(?m)^\|\s*`daily`\s*\|\s*([\d,]+)\s*\|\s*([\d,]+)\s*\|\s*"
r"(\d{4}-\d{2}-\d{2})[—-](\d{4}-\d{2}-\d{2})",
tushare_markdown,
)
plan_match = re.search(
r"`daily`\s*原计划覆盖.*?共\s*(\d+)\s*个月；当前完成\s*"
r"(\d+)\s*个月，约\s*([\d.]+)%",
tushare_markdown,
flags=re.DOTALL,
)
tushare_fields = {
key: re.search(
rf"(?m)^{key}\s*=\s*([^\s]+)\s*$",
tushare_markdown,
)
for key in (
"tushare_daily_completed_partitions",
"tushare_daily_planned_partitions",
"tushare_daily_completed_rows",
"tushare_daily_market_data_from",
"tushare_daily_market_data_through",
)
}
ready_match = re.search(
r"(?m)^production_ready\s*=\s*(true|false)\s*$",
tushare_markdown,
)
if daily_match is None or plan_match is None or ready_match is None:
if any(match is None for match in tushare_fields.values()) or ready_match is None:
raise PublicStatusError(
"Tushare evidence must declare daily coverage and production readiness"
)
(
daily_partitions_text,
daily_rows_text,
tushare_start,
tushare_end,
) = daily_match.groups()
planned_months_text, completed_months_text, percentage_text = plan_match.groups()
completed_partitions = int(daily_partitions_text.replace(",", ""))
completed_months = int(completed_months_text)
planned_months = int(planned_months_text)
if completed_partitions != completed_months:
live_completed_partitions = int(
tushare_fields["tushare_daily_completed_partitions"]
.group(1)
.replace(",", "")
)
live_planned_months = int(
tushare_fields["tushare_daily_planned_partitions"]
.group(1)
.replace(",", "")
)
live_daily_rows = int(
tushare_fields["tushare_daily_completed_rows"]
.group(1)
.replace(",", "")
)
live_tushare_start = tushare_fields[
"tushare_daily_market_data_from"
].group(1)
live_tushare_end = tushare_fields[
"tushare_daily_market_data_through"
].group(1)
if (
live_planned_months <= 0
or live_completed_partitions > live_planned_months
or live_daily_rows <= 0
):
raise PublicStatusError(
"Tushare daily partition count differs from completed-month statement"
"Tushare daily partition coverage is invalid"
)
calculated_ratio = completed_months / planned_months
recorded_percentage = float(percentage_text)
if abs(calculated_ratio * 100.0 - recorded_percentage) > 0.1:
for label, value in (
("market_data_from", live_tushare_start),
("market_data_through", live_tushare_end),
):
try:
date.fromisoformat(value)
except ValueError as exc:
raise PublicStatusError(
f"Tushare {label} must be an ISO date"
) from exc
if live_tushare_start > live_tushare_end:
raise PublicStatusError(
"Tushare recorded coverage percentage does not match month counts"
"Tushare daily market-data start must not follow its end"
)
completed_partitions = int(
tushare_research_run["completedPartitions"]
)
planned_months = int(tushare_research_run["plannedPartitions"])
daily_rows = int(tushare_research_run["completedRows"])
tushare_start = str(tushare_research_run["marketDataFrom"])
tushare_end = str(tushare_research_run["marketDataThrough"])
if (
completed_partitions != planned_months
or completed_partitions > live_completed_partitions
or daily_rows > live_daily_rows
or date.fromisoformat(tushare_start)
< date.fromisoformat(live_tushare_start)
or date.fromisoformat(tushare_end)
> date.fromisoformat(live_tushare_end)
):
raise PublicStatusError(
"Tushare scoped research record is inconsistent with live inventory"
)
calculated_ratio = completed_partitions / planned_months
recorded_percentage = calculated_ratio * 100.0
production_ready = ready_match.group(1) == "true"
if production_ready:
if (
production_ready
or tushare_research_run["productionReady"] is not False
or tushare_research_run["investmentValueClaim"] is not False
or tushare_research_run["gateCredit"] != []
):
raise PublicStatusError(
"partial Tushare source must not be publicized as production ready"
"current Tushare research bridge must not be publicized as production ready"
)
audit_as_of = str(scorecard.get("audit_as_of"))
@@ -1159,24 +1428,94 @@ def _build_freshness(
},
{
"id": "tushare-daily-market-data",
"label": "Tushare daily 行情覆盖",
"status": "partial",
"label": "Tushare CSI500 scoped research release",
"status": "research_only",
"marketDataFrom": tushare_start,
"marketDataThrough": tushare_end,
"completedPartitions": completed_partitions,
"plannedPartitions": planned_months,
"completedRows": int(daily_rows_text.replace(",", "")),
"completedRows": daily_rows,
"completionRatio": round(calculated_ratio, 6),
"completionLabel": (
f"{completed_partitions}/{planned_months}（约 {recorded_percentage:g}%）"
),
"value": (
f"{completed_partitions}/{planned_months}（约 {recorded_percentage:g}%），"
f"行情截至 {tushare_end}"
f"{completed_partitions}/{planned_months} scoped daily 分区；"
f"{tushare_research_run['verifiedSnapshotJobs']}/"
f"{tushare_research_run['snapshotJobs']} source jobs；"
f"行情 {tushare_start} — {tushare_end}"
),
"productionReady": production_ready,
"semantics": "marketDataThrough 仅取自 daily 行情分区；绝不使用覆盖更晚的 trade_cal 日期。",
"evidenceLinks": ["tushare-data", "scorecard"],
"investmentValueClaim": False,
"gateCredit": [],
"snapshotJobs": int(tushare_research_run["snapshotJobs"]),
"verifiedSnapshotJobs": int(
tushare_research_run["verifiedSnapshotJobs"]
),
"providerDataVersion": str(
tushare_research_run["providerDataVersion"]
),
"providerTreeSha256": str(
tushare_research_run["providerTreeSha256"]
),
"providerManifestSha256": str(
tushare_research_run["providerManifestSha256"]
),
"sourceJobCount": int(
tushare_research_run["sourceJobCount"]
),
"lineageMatchedFields": list(
tushare_research_run["lineageMatchedFields"]
),
"lineageMismatchCount": int(
tushare_research_run["lineageMismatchCount"]
),
"providerFiles": int(tushare_research_run["providerFiles"]),
"providerBytes": int(tushare_research_run["providerBytes"]),
"calendarSessions": int(
tushare_research_run["calendarSessions"]
),
"instrumentCount": int(
tushare_research_run["instrumentCount"]
),
"selectedDailyRows": int(
tushare_research_run["selectedDailyRows"]
),
"observedMembershipSnapshots": int(
tushare_research_run["observedMembershipSnapshots"]
),
"effectiveMembershipSnapshots": int(
tushare_research_run["effectiveMembershipSnapshots"]
),
"sameSessionMembershipUse": False,
"sourceStable": True,
"stateDbChangedDuringVerification": True,
"converterSourceMatchesCurrent": True,
"runSha256": str(tushare_research_run["runSha256"]),
"signalRows": int(tushare_research_run["signalRows"]),
"portfolioDays": int(tushare_research_run["portfolioDays"]),
"strategyCumulativeReturn": float(
tushare_research_run["strategyCumulativeReturn"]
),
"benchmarkCumulativeReturn": float(
tushare_research_run["benchmarkCumulativeReturn"]
),
"maxDrawdown": float(tushare_research_run["maxDrawdown"]),
"semantics": (
"96/96 只表示 2018—2025 scoped daily 输入完整且 221/221 "
"source jobs 经双重校验；全局 ledger 因无关下载任务发生变化，"
"但 scoped jobs 仍为 latest；lineage 工具按 id/path/row_count/"
"byte_count/sha256/request_sha256 验证 converter 221/221 "
"零差异。"
"universe 成分次一交易日生效。该 Qlib "
"run 未使用逐股涨跌停，stock_st 不可用，不代表 production-ready "
"或投资价值。"
),
"evidenceLinks": [
"tushare-csi500-research-run",
"tushare-data",
"scorecard",
],
},
{
"id": "jqdata-real-snapshot",
@@ -1413,6 +1752,9 @@ def build_public_status(
joinquant_target_package_run_raw = _load_json(
project / JOINQUANT_TARGET_PACKAGE_RUN_PATH
)
tushare_research_run_raw = _load_json(
project / TUSHARE_RESEARCH_RUN_PATH
)
baseline = _load_json(project / "configs" / "baseline.json")
platform_markdown = (project / "docs" / "PLATFORM_MATRIX.md").read_text(
encoding="utf-8"
@@ -1501,6 +1843,10 @@ def build_public_status(
project,
joinquant_target_package_run_raw,
)
tushare_research_run = _validate_tushare_research_run(
project,
tushare_research_run_raw,
)
maturity80 = _build_maturity80(
project,
production_80_standard,
@@ -1582,7 +1928,7 @@ def build_public_status(
"JoinQuant portable_momentum_smoke hosted daily backtest (not Baseline/TargetPackage evidence)",
"Qlib native momentum fixture",
"Qlib Alpha158 + LightGBM fixture",
"Tushare completed-partition Qlib momentum smoke",
"Tushare CSI500 2018—2025 scoped Qlib momentum reproducibility run",
],
"gateCoverageLabel": f"{passed}/{len(gates)} passed",
"productionReady": bool(maturity80["qualified"]),
@@ -1630,6 +1976,7 @@ def build_public_status(
scorecard=scorecard,
joinquant_markdown=joinquant_markdown,
joinquant_target_package_run=joinquant_target_package_run,
tushare_research_run=tushare_research_run,
tushare_markdown=tushare_markdown,
baseline_shadow_days=baseline_shadow_days,
production_shadow_days=production_shadow_days,
@@ -1672,6 +2019,7 @@ def build_public_status(
"production80Assessment": PRODUCTION_80_ASSESSMENT_PATH,
"bundleManifest": "dist/bundle_manifest.json",
"releaseReachability": REACHABILITY_PATH,
"tushareResearchRun": TUSHARE_RESEARCH_RUN_PATH,
"platformMatrix": "docs/PLATFORM_MATRIX.md",
"baselineConfig": "configs/baseline.json",
"sourceSetSha256": source_set_sha,