feat: add Tushare Qlib data bridge

This commit is contained in:
2026-07-26 13:41:24 +08:00
parent 48c5f64bbd
commit e5911ca454
15 changed files with 1804 additions and 8 deletions
+1 -1
View File
@@ -6,7 +6,7 @@ Quant OS 是项目名;`quant60` 是第一版 A 股 Baseline 内核、CLI names
回测/影子验证,并且所有 G1—G10 都由版本绑定的真实证据支持。
当前结论仍为 `NOT_BASELINE_60`。最近一次标准库全量测试记录为
`Ran 211 tests``OK (skipped=3)`个 skip 是可选依赖/运行时路径。
`Ran 216 tests``OK (skipped=6)`个 skip 是可选依赖/运行时路径。
该结果证明代码合同,不证明真实平台、券商或合规 Gate。
## Stage 1: 独立仓库与确定性执行
+35
View File
@@ -26,6 +26,7 @@ Quant OS 是项目名;`quant60` 是第一版 A 股 Baseline 的 Python
| 聚宽 hosted | 可上传的 Python 单文件,直接回测 portable baseline | 2024 全年、100 万资金的真实 hosted run 已完成且可见日志无 ERROR;另有 bundle/fake/mock 合约 | 仍需完整导出、相同输入本地 peer 与逐层差异报告 |
| QMT built-in/qmttools | 可上传/驱动的硬 backtest-only portable baseline;固定中证 500 benchmark、10% 参与率、PIT 成分/ST | Python 3.6 语法、fake/qmttools 合约 | 需要已授权 QMT 客户端、历史 ST 数据权限/正例探针与实跑导出 |
| Qlib 0.9.7 | native momentum backtestAlpha158/LightGBM CLI/Recorder 工作流 | Python 3.12 两条 fixture 已真实跑通,并保存表级 hash、行列/时间边界和重算指标 | synthetic 两股票没有投资价值;无真实 OOS、L3/L4 parity |
| Tushare → Qlib | 只读盘点 completed Parquet、校验 checksum、构建/验证不可变 provider、运行本地回测 | 93 个 completed 文件已验证;早期未复权 provider 与两次 byte-identical momentum run | 下载仅到 1993-10;缺复权、真实 benchmark、历史成分/ST/停牌/涨跌停,不能进入生产 decision |
| XtTrader shadow | 只读账户查询、HMAC 账户绑定、broker observation/snapshot、零下单 target-diff 计划与完整 verifier | fake broker/QMT 合约;所有 broker mutation 均不可达 | 需要授权 QMT 环境的账户合同探针、20 日影子、恢复/对账和券商确认 |
跨四个引擎当前共同的、可冻结生产基线是
@@ -375,6 +376,40 @@ python -m pip install -r requirements/research-py312.txt
python -c "import qlib; assert qlib.__version__ == '0.9.7'"
```
### 使用本地 Tushare 镜像
`tools/tushare_qlib.py` 不调用 Tushare、不读取 token,也不修改镜像。它只接受
SQLite 中状态为 `completed` 且文件大小、SHA-256、Parquet 行数一致的分区;
生成新 Qlib provider 时保存 source job、build parameter、converter hash、
tree hash 与可重算 `data_version`,并拒绝覆盖已有目录。
```bash
export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror
PYTHONPATH=src:. python tools/tushare_qlib.py inventory \
--mirror-root "$TUSHARE_MIRROR_ROOT" \
--output-json artifacts/tushare-inventory.json
PYTHONPATH=src:. python tools/tushare_qlib.py build \
--mirror-root "$TUSHARE_MIRROR_ROOT" \
--output-dir data/qlib/tushare-early-v1 \
--start 1990-12-19 --end 1993-10-29 \
--minimum-observations 60 \
--allow-unadjusted \
--ohlc-policy expand-range \
--output-json artifacts/tushare-qlib-build.json
PYTHONPATH=src:. python tools/tushare_qlib.py verify \
data/qlib/tushare-early-v1
```
当前镜像只有 1990-12—1993-10 日线,且没有 `adj_factor`、真实指数行情、
历史成分、ST、停牌和涨跌停数据。因此 `--allow-unadjusted` 是显式技术
smoke 降级,provider 固定 `production_eligible=false`
`investment_value_claim=false``gate_credit=[]`。盘点、字段口径、完整命令、
v4 hash 与两次本地回测结果见
[`docs/TUSHARE_LOCAL_DATA.md`](docs/TUSHARE_LOCAL_DATA.md)。
native momentum smoke
```bash
+943
View File
@@ -0,0 +1,943 @@
"""Read-only Tushare mirror inventory and immutable Qlib provider builder.
The adapter never calls Tushare and never reads a token. It trusts only jobs
marked ``completed`` in the mirror SQLite database, verifies their recorded
size/SHA-256/Parquet row count, and publishes a new Qlib directory without
modifying the mirror.
"""
from __future__ import annotations
from array import array
import datetime as dt
import hashlib
import json
import math
import os
from pathlib import Path
import re
import shutil
import sqlite3
import sys
import tempfile
from typing import Any, Dict, Iterable, Mapping, Optional, Sequence
MANIFEST_NAME = "quant_os_tushare_manifest.json"
REQUIRED_DAILY_FIELDS = {
"ts_code",
"trade_date",
"open",
"high",
"low",
"close",
"pre_close",
"pct_chg",
"vol",
"amount",
}
QLIB_FIELDS = (
"open",
"high",
"low",
"close",
"vwap",
"volume",
"money",
"factor",
"change",
)
MISSING_PRODUCTION_TABLES = (
"adj_factor",
"daily_basic",
"index_daily",
"index_member_all",
"index_weight",
"stk_limit",
"stock_st",
"suspend_d",
)
REQUIRED_PRODUCTION_TABLES = (
"daily",
"trade_cal",
"stock_basic",
*MISSING_PRODUCTION_TABLES,
)
class TushareMirrorError(RuntimeError):
"""Raised when mirror or provider evidence fails closed."""
def _canonical_bytes(value: Mapping[str, Any]) -> bytes:
return (
json.dumps(
dict(value),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
+ "\n"
).encode("utf-8")
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def _iso_date(value: str, *, name: str) -> str:
normalized = str(value or "").strip()
try:
parsed = dt.date.fromisoformat(normalized)
except ValueError as exc:
raise ValueError(f"{name} must use YYYY-MM-DD") from exc
if parsed.isoformat() != normalized:
raise ValueError(f"{name} must use YYYY-MM-DD")
return normalized
def _yyyymmdd(value: str) -> str:
return value.replace("-", "")
def tushare_to_qlib_symbol(value: str) -> str:
"""Convert ``600000.SH`` into Qlib's ``SH600000`` spelling."""
normalized = str(value or "").strip().upper()
match = re.fullmatch(r"(\d{6})\.(SH|SZ|BJ)", normalized)
if match is None:
raise ValueError(f"unsupported Tushare A-share symbol: {value!r}")
code, exchange = match.groups()
return f"{exchange}{code}"
def _looks_like_a_share(value: str) -> bool:
normalized = str(value or "").strip().upper()
match = re.fullmatch(r"(\d{6})\.(SH|SZ|BJ)", normalized)
if match is None:
return False
code, exchange = match.groups()
if exchange == "SH":
return code.startswith(("600", "601", "603", "605", "688", "689"))
if exchange == "SZ":
return code.startswith(("000", "001", "002", "003", "300", "301"))
return code[0] in {"4", "8", "9"}
def _market_name(value: str) -> str:
normalized = str(value or "").strip().lower()
if re.fullmatch(r"[a-z][a-z0-9_]{0,63}", normalized) is None:
raise ValueError("market_name must match [a-z][a-z0-9_]{0,63}")
return normalized
def _mirror_root(value: str | Path) -> Path:
root = Path(value).expanduser().resolve()
if not root.is_dir():
raise FileNotFoundError(f"Tushare mirror root does not exist: {root}")
state = root / "data/state.sqlite3"
parquet = root / "data/parquet"
if not state.is_file() or not parquet.is_dir():
raise TushareMirrorError(
"mirror root must contain data/state.sqlite3 and data/parquet"
)
return root
def _read_jobs(root: Path) -> list[Dict[str, Any]]:
state = root / "data/state.sqlite3"
connection = sqlite3.connect(f"file:{state}?mode=ro", uri=True)
connection.row_factory = sqlite3.Row
try:
connection.execute("BEGIN")
rows = connection.execute(
"""
SELECT id, api_name, partition_key, status, row_count, byte_count,
sha256, file_path, started_at, completed_at, error_code
FROM jobs
ORDER BY id
"""
).fetchall()
finally:
connection.close()
return [dict(row) for row in rows]
def _job_path(root: Path, job: Mapping[str, Any]) -> Path:
raw = str(job.get("file_path") or "").strip()
if not raw:
raise TushareMirrorError(
f"completed job has no file_path: {job.get('api_name')}/"
f"{job.get('partition_key')}"
)
path = Path(raw).expanduser()
if not path.is_absolute():
path = root / path
resolved = path.resolve()
data_root = (root / "data").resolve()
try:
resolved.relative_to(data_root)
except ValueError as exc:
raise TushareMirrorError(
f"job file escapes mirror data root: {resolved}"
) from exc
return resolved
def _relative_job_path(root: Path, path: Path) -> str:
return path.relative_to(root.resolve()).as_posix()
def _load_parquet_module() -> Any:
try:
import pyarrow.parquet as parquet
except ImportError as exc:
raise TushareMirrorError(
"pyarrow is required; install requirements/research-py312.txt"
) from exc
return parquet
def _load_pandas() -> Any:
try:
import pandas
except ImportError as exc:
raise TushareMirrorError(
"pandas is required; install requirements/research-py312.txt"
) from exc
return pandas
def _verify_completed_job(
root: Path,
job: Mapping[str, Any],
*,
parquet: Any,
) -> Dict[str, Any]:
if job.get("status") != "completed":
raise TushareMirrorError("only completed mirror jobs may be verified")
path = _job_path(root, job)
if not path.is_file():
raise TushareMirrorError(f"completed job file is missing: {path}")
expected_bytes = int(job.get("byte_count") or -1)
actual_bytes = path.stat().st_size
if actual_bytes != expected_bytes:
raise TushareMirrorError(
f"byte_count mismatch for {path.name}: "
f"{actual_bytes} != {expected_bytes}"
)
expected_sha = str(job.get("sha256") or "")
actual_sha = _sha256_file(path)
if actual_sha != expected_sha:
raise TushareMirrorError(f"SHA-256 mismatch for {path.name}")
parquet_file = parquet.ParquetFile(path)
actual_rows = int(parquet_file.metadata.num_rows)
expected_rows = int(job.get("row_count") or 0)
if actual_rows != expected_rows:
raise TushareMirrorError(
f"row_count mismatch for {path.name}: "
f"{actual_rows} != {expected_rows}"
)
footer = {}
for raw_key, raw_value in (parquet_file.metadata.metadata or {}).items():
try:
key = raw_key.decode("utf-8")
value = raw_value.decode("utf-8")
except (AttributeError, UnicodeDecodeError):
continue
if key != "ARROW:schema":
footer[key] = value
footer_api = footer.get("tushare_api")
footer_partition = footer.get("partition_key")
if footer_api and footer_api != str(job["api_name"]):
raise TushareMirrorError(
f"Parquet tushare_api mismatch for {path.name}"
)
if footer_partition and footer_partition != str(job["partition_key"]):
raise TushareMirrorError(
f"Parquet partition_key mismatch for {path.name}"
)
query_sha256 = footer.get("query_sha256")
if query_sha256 and re.fullmatch(r"[0-9a-f]{64}", query_sha256) is None:
raise TushareMirrorError(
f"invalid Parquet query_sha256 for {path.name}"
)
return {
"api_name": str(job["api_name"]),
"partition_key": str(job["partition_key"]),
"path": _relative_job_path(root, path),
"row_count": actual_rows,
"byte_count": actual_bytes,
"sha256": actual_sha,
"query_sha256": query_sha256,
"downloaded_at": footer.get("downloaded_at"),
"completed_at": job.get("completed_at"),
}
def inventory_mirror(
mirror_root: str | Path,
*,
verify_files: bool = True,
) -> Dict[str, Any]:
"""Return a token-free point-in-time inventory of the mirror."""
root = _mirror_root(mirror_root)
jobs = _read_jobs(root)
statuses: Dict[str, int] = {}
tables: Dict[str, Dict[str, Any]] = {}
for job in jobs:
status = str(job["status"])
statuses[status] = statuses.get(status, 0) + 1
if status != "completed":
continue
api = str(job["api_name"])
table = tables.setdefault(
api,
{
"files": 0,
"rows": 0,
"bytes": 0,
"first_partition": None,
"last_partition": None,
},
)
partition = str(job["partition_key"])
table["files"] += 1
table["rows"] += int(job.get("row_count") or 0)
table["bytes"] += int(job.get("byte_count") or 0)
values = [
item
for item in (table["first_partition"], table["last_partition"])
if item is not None
] + [partition]
table["first_partition"] = min(values)
table["last_partition"] = max(values)
verified = 0
if verify_files:
parquet = _load_parquet_module()
for job in jobs:
if job["status"] == "completed":
_verify_completed_job(root, job, parquet=parquet)
verified += 1
running = [
{
"api_name": str(job["api_name"]),
"partition_key": str(job["partition_key"]),
"status": str(job["status"]),
"started_at": job.get("started_at"),
}
for job in jobs
if job["status"] != "completed"
]
completed_apis = set(tables)
missing_production_tables = [
value
for value in REQUIRED_PRODUCTION_TABLES
if value not in completed_apis
]
return {
"schema_version": 1,
"artifact_type": "quant_os_tushare_mirror_inventory",
"mirror_root": str(root),
"observed_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"job_status_counts": dict(sorted(statuses.items())),
"tables": dict(sorted(tables.items())),
"verified_completed_files": verified,
"noncompleted_jobs": running,
"missing_production_tables": missing_production_tables,
"coverage_claim": "completed_partitions_only_not_full_mirror",
"production_ready": False,
"production_readiness_reason": (
"job inventory cannot prove planned partition/date coverage, "
"point-in-time semantics, adjustment completeness or entitlement"
),
"token_read": False,
"investment_value_claim": False,
}
def _write_lines(path: Path, lines: Iterable[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _write_feature(
path: Path,
start_index: int,
values: Sequence[float],
) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
payload = array("f", [float(start_index), *(float(value) for value in values)])
if sys.byteorder != "little":
payload.byteswap()
path.write_bytes(payload.tobytes())
def _tree_evidence(root: Path) -> Dict[str, Any]:
records = []
for path in sorted(item for item in root.rglob("*") if item.is_file()):
if path.name == MANIFEST_NAME:
continue
relative = path.relative_to(root).as_posix()
records.append(
{
"path": relative,
"byte_count": path.stat().st_size,
"sha256": _sha256_file(path),
}
)
digest = hashlib.sha256(_canonical_bytes({"files": records})).hexdigest()
return {
"provider_tree_sha256": digest,
"provider_file_count": len(records),
"provider_byte_count": sum(item["byte_count"] for item in records),
}
def _month_overlaps(partition: str, start: str, end: str) -> bool:
match = re.search(r"month=(\d{4})-(\d{2})", partition)
if match is None:
return False
month = "".join(match.groups())
return _yyyymmdd(start)[:6] <= month <= _yyyymmdd(end)[:6]
def _year_overlaps(partition: str, start: str, end: str) -> bool:
match = re.search(r"year=(\d{4})", partition)
if match is None:
return False
year = match.group(1)
return start[:4] <= year <= end[:4]
def _data_version_payload(
*,
source_jobs: Sequence[Mapping[str, Any]],
build_parameters: Mapping[str, Any],
converter_sha256: str,
) -> Dict[str, Any]:
return {
"source_jobs": [dict(value) for value in source_jobs],
"range": [
build_parameters["start"],
build_parameters["end"],
],
"market": build_parameters["market_name"],
"benchmark": build_parameters["benchmark_symbol"],
"minimum_observations": build_parameters["minimum_observations"],
"allow_unadjusted": build_parameters["allow_unadjusted"],
"price_adjustment": "raw_unadjusted_factor_1",
"ohlc_policy": build_parameters["ohlc_policy"],
"converter_sha256": converter_sha256,
}
def _eligible_a_share(
ts_code: str,
metadata: Mapping[str, Mapping[str, Any]],
) -> bool:
if not _looks_like_a_share(ts_code):
return False
row = metadata.get(ts_code)
if row is None:
return True
currency = str(row.get("curr_type") or "").strip().upper()
return not currency or currency == "CNY"
def build_qlib_provider(
mirror_root: str | Path,
output_dir: str | Path,
*,
start: str,
end: str,
market_name: str = "tushare_a",
benchmark_symbol: str = "SH999999",
minimum_observations: int = 60,
allow_unadjusted: bool = False,
expand_range: bool = False,
) -> Dict[str, Any]:
"""Freeze completed daily partitions into a Qlib 0.9.7 binary provider."""
if not allow_unadjusted:
raise TushareMirrorError(
"adj_factor is unavailable; pass allow_unadjusted=True only for "
"an explicitly non-investment technical run"
)
start_iso = _iso_date(start, name="start")
end_iso = _iso_date(end, name="end")
if start_iso > end_iso:
raise ValueError("start must be on or before end")
market = _market_name(market_name)
benchmark = str(benchmark_symbol).strip().upper()
if re.fullmatch(r"(SH|SZ|BJ)\d{6}", benchmark) is None:
raise ValueError("benchmark_symbol must use Qlib SH999999 form")
if int(minimum_observations) < 2:
raise ValueError("minimum_observations must be at least 2")
root = _mirror_root(mirror_root)
output = Path(output_dir).expanduser().resolve()
if output.exists():
raise FileExistsError(f"output_dir already exists: {output}")
output.parent.mkdir(parents=True, exist_ok=True)
jobs = _read_jobs(root)
completed = [job for job in jobs if job["status"] == "completed"]
daily_jobs = [
job
for job in completed
if job["api_name"] == "daily"
and _month_overlaps(str(job["partition_key"]), start_iso, end_iso)
]
calendar_jobs = [
job
for job in completed
if job["api_name"] == "trade_cal"
and _year_overlaps(str(job["partition_key"]), start_iso, end_iso)
]
stock_jobs = [
job for job in completed if job["api_name"] == "stock_basic"
]
if not daily_jobs or not calendar_jobs or not stock_jobs:
raise TushareMirrorError(
"completed daily, trade_cal and stock_basic jobs are required"
)
parquet = _load_parquet_module()
selected_jobs = daily_jobs + calendar_jobs + stock_jobs
source_jobs = [
_verify_completed_job(root, job, parquet=parquet)
for job in selected_jobs
]
pandas = _load_pandas()
daily = pandas.concat(
[pandas.read_parquet(_job_path(root, job)) for job in daily_jobs],
ignore_index=True,
)
missing = REQUIRED_DAILY_FIELDS.difference(daily.columns)
if missing:
raise TushareMirrorError(
f"daily partitions miss required fields: {sorted(missing)}"
)
daily["trade_date"] = daily["trade_date"].astype(str)
start_raw, end_raw = _yyyymmdd(start_iso), _yyyymmdd(end_iso)
daily = daily[
(daily["trade_date"] >= start_raw)
& (daily["trade_date"] <= end_raw)
].copy()
if daily.empty:
raise TushareMirrorError("requested range contains no completed daily rows")
if daily.duplicated(["ts_code", "trade_date"]).any():
raise TushareMirrorError("daily contains duplicate ts_code/trade_date keys")
numeric_fields = (
"open",
"high",
"low",
"close",
"pre_close",
"pct_chg",
"vol",
"amount",
)
for field in numeric_fields:
daily[field] = pandas.to_numeric(daily[field], errors="coerce")
if daily[list(numeric_fields)].isna().any().any():
raise TushareMirrorError("daily contains null/non-numeric core values")
if (daily[["open", "high", "low", "close", "pre_close"]] <= 0).any().any():
raise TushareMirrorError("daily contains non-positive price values")
if (daily[["vol", "amount"]] < 0).any().any():
raise TushareMirrorError("daily contains negative volume or amount")
high_low_bad = daily["high"] < daily["low"]
envelope_bad = (
high_low_bad
| (daily["open"] > daily["high"])
| (daily["open"] < daily["low"])
| (daily["close"] > daily["high"])
| (daily["close"] < daily["low"])
)
envelope_bad_count = int(envelope_bad.sum())
anomaly_keys = (
daily.loc[envelope_bad, ["ts_code", "trade_date"]]
.sort_values(["trade_date", "ts_code"])
.astype(str)
.to_dict(orient="records")
)
anomaly_keys_sha256 = hashlib.sha256(
_canonical_bytes({"keys": anomaly_keys})
).hexdigest()
if envelope_bad_count and not expand_range:
raise TushareMirrorError(
f"{envelope_bad_count} OHLC envelope anomalies found; "
"use expand_range=True only for an explicitly recorded repair"
)
if envelope_bad_count:
original_ohlc = daily[["open", "high", "low", "close"]].copy()
daily["high"] = original_ohlc.max(axis=1)
daily["low"] = original_ohlc.min(axis=1)
calendar_frame = pandas.concat(
[pandas.read_parquet(_job_path(root, job)) for job in calendar_jobs],
ignore_index=True,
)
calendar_frame["cal_date"] = calendar_frame["cal_date"].astype(str)
open_mask = calendar_frame["is_open"].astype(str).isin({"1", "1.0", "True"})
exchange_mask = calendar_frame["exchange"].astype(str).eq("SSE")
calendar_raw = sorted(
set(
calendar_frame.loc[
open_mask
& exchange_mask
& (calendar_frame["cal_date"] >= start_raw)
& (calendar_frame["cal_date"] <= end_raw),
"cal_date",
].tolist()
)
)
if not calendar_raw:
raise TushareMirrorError("trade_cal has no open dates in requested range")
observed_dates = set(daily["trade_date"].tolist())
missing_market_dates = sorted(set(calendar_raw).difference(observed_dates))
unexpected_dates = sorted(observed_dates.difference(calendar_raw))
if missing_market_dates or unexpected_dates:
raise TushareMirrorError(
"daily/trade_cal mismatch: "
f"missing={missing_market_dates[:5]}, "
f"unexpected={unexpected_dates[:5]}"
)
stock = pandas.concat(
[pandas.read_parquet(_job_path(root, job)) for job in stock_jobs],
ignore_index=True,
)
if stock.duplicated(["ts_code"]).any():
raise TushareMirrorError("stock_basic contains duplicate ts_code keys")
metadata = {
str(row["ts_code"]): row.to_dict()
for _, row in stock.iterrows()
}
daily = daily[
daily["ts_code"].map(
lambda value: _eligible_a_share(str(value), metadata)
)
].copy()
counts = daily.groupby("ts_code").size()
selected_codes = sorted(
str(value)
for value in counts[counts >= int(minimum_observations)].index
)
if len(selected_codes) < 2:
raise TushareMirrorError(
"fewer than two eligible instruments meet minimum_observations"
)
daily = daily[daily["ts_code"].isin(selected_codes)].copy()
daily["qlib_symbol"] = daily["ts_code"].map(tushare_to_qlib_symbol)
symbol_pairs = (
daily[["ts_code", "qlib_symbol"]].drop_duplicates().sort_values("ts_code")
)
if symbol_pairs["qlib_symbol"].duplicated().any():
raise TushareMirrorError("Tushare symbols collide after Qlib conversion")
if benchmark in set(symbol_pairs["qlib_symbol"]):
raise TushareMirrorError("benchmark_symbol collides with a stock")
pct_recomputed = (
(daily["close"] - daily["pre_close"]) / daily["pre_close"] * 100.0
)
pct_discrepancy_count = int(
(pct_recomputed - daily["pct_chg"]).abs().gt(0.02).sum()
)
missing_metadata = sorted(set(selected_codes).difference(metadata))
calendar = [
dt.datetime.strptime(value, "%Y%m%d").date().isoformat()
for value in calendar_raw
]
calendar_index = {value: index for index, value in enumerate(calendar_raw)}
temporary = Path(
tempfile.mkdtemp(prefix=f".{output.name}.", dir=str(output.parent))
)
try:
_write_lines(temporary / "calendars/day.txt", calendar)
instrument_lines = []
stock_symbols = []
for qlib_symbol, group in daily.groupby("qlib_symbol", sort=True):
ordered = group.sort_values("trade_date")
first_raw = str(ordered["trade_date"].iloc[0])
last_raw = str(ordered["trade_date"].iloc[-1])
first_index = calendar_index[first_raw]
last_index = calendar_index[last_raw]
length = last_index - first_index + 1
fields = {
name: [float("nan")] * length for name in QLIB_FIELDS
}
for _, row in ordered.iterrows():
offset = calendar_index[str(row["trade_date"])] - first_index
volume_shares = float(row["vol"]) * 100.0
money_cny = float(row["amount"]) * 1000.0
vwap = (
money_cny / volume_shares
if volume_shares > 0.0
else float(row["close"])
)
change = float(row["close"]) / float(row["pre_close"]) - 1.0
values = {
"open": float(row["open"]),
"high": float(row["high"]),
"low": float(row["low"]),
"close": float(row["close"]),
"vwap": vwap,
"volume": volume_shares,
"money": money_cny,
"factor": 1.0,
"change": change,
}
for field, value in values.items():
fields[field][offset] = value
for field, values in fields.items():
_write_feature(
temporary
/ "features"
/ str(qlib_symbol).lower()
/ f"{field}.day.bin",
first_index,
values,
)
first_iso = dt.datetime.strptime(first_raw, "%Y%m%d").date().isoformat()
last_iso = dt.datetime.strptime(last_raw, "%Y%m%d").date().isoformat()
instrument_lines.append(f"{qlib_symbol}\t{first_iso}\t{last_iso}")
stock_symbols.append(str(qlib_symbol))
returns = (
daily.assign(
_return=daily["close"] / daily["pre_close"] - 1.0
)
.groupby("trade_date")["_return"]
.mean()
.to_dict()
)
benchmark_fields = {name: [] for name in QLIB_FIELDS}
level = 100.0
for raw_date in calendar_raw:
change = float(returns.get(raw_date, 0.0))
if not math.isfinite(change) or change <= -1.0:
raise TushareMirrorError(
f"invalid derived benchmark return on {raw_date}"
)
level *= 1.0 + change
benchmark_fields["open"].append(level)
benchmark_fields["high"].append(level)
benchmark_fields["low"].append(level)
benchmark_fields["close"].append(level)
benchmark_fields["vwap"].append(level)
benchmark_fields["volume"].append(100_000_000.0)
benchmark_fields["money"].append(level * 100_000_000.0)
benchmark_fields["factor"].append(1.0)
benchmark_fields["change"].append(change)
for field, values in benchmark_fields.items():
_write_feature(
temporary
/ "features"
/ benchmark.lower()
/ f"{field}.day.bin",
0,
values,
)
benchmark_line = (
f"{benchmark}\t{calendar[0]}\t{calendar[-1]}"
)
_write_lines(
temporary / f"instruments/{market}.txt",
instrument_lines,
)
_write_lines(
temporary / "instruments/all.txt",
[*instrument_lines, benchmark_line],
)
tree = _tree_evidence(temporary)
completed_apis = {str(job["api_name"]) for job in completed}
converter_sha256 = _sha256_file(Path(__file__).resolve())
build_parameters = {
"start": start_iso,
"end": end_iso,
"market_name": market,
"benchmark_symbol": benchmark,
"minimum_observations": int(minimum_observations),
"allow_unadjusted": True,
"ohlc_policy": "expand_range" if expand_range else "fail",
}
version_payload = _data_version_payload(
source_jobs=source_jobs,
build_parameters=build_parameters,
converter_sha256=converter_sha256,
)
data_version = hashlib.sha256(
_canonical_bytes(version_payload)
).hexdigest()
status_counts: Dict[str, int] = {}
for job in jobs:
key = str(job["status"])
status_counts[key] = status_counts.get(key, 0) + 1
manifest: Dict[str, Any] = {
"schema_version": 1,
"artifact_type": "quant_os_tushare_qlib_provider",
"source": "tushare_local_mirror",
"created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"data_version": data_version,
**tree,
"converter": {
"source_sha256": converter_sha256,
"output_format": "qlib_0_9_7_day_binary",
},
"build_parameters": build_parameters,
"calendar": {
"frequency": "day",
"start": calendar[0],
"end": calendar[-1],
"sessions": len(calendar),
"source": "trade_cal/SSE/is_open=1",
},
"market": {
"name": market,
"instrument_count": len(stock_symbols),
"minimum_observations": int(minimum_observations),
"benchmark_excluded": True,
},
"benchmark": {
"symbol": benchmark,
"kind": "synthetic_equal_weight",
"method": (
"daily arithmetic mean of eligible stock "
"close/pre_close-1 observations"
),
"real_index_claim": False,
},
"fields": list(QLIB_FIELDS),
"units": {
"volume": "shares; Tushare vol(hand) multiplied by 100",
"money": "CNY; Tushare amount(thousand CNY) multiplied by 1000",
},
"price_adjustment": {
"mode": "raw_unadjusted",
"factor": 1.0,
"production_eligible": False,
},
"quality": {
"duplicate_keys": 0,
"whole_market_calendar_gaps": 0,
"ohlc_envelope_anomalies": envelope_bad_count,
"ohlc_policy": "expand_range" if expand_range else "fail",
"ohlc_anomaly_keys_sha256": anomaly_keys_sha256,
"pct_chg_discrepancies_gt_0_02_percentage_points": (
pct_discrepancy_count
),
"selected_rows": int(len(daily)),
"selected_symbols_missing_stock_basic": missing_metadata,
"universe_selection": (
"eligible A-share symbols with at least "
f"{int(minimum_observations)} observations over the full "
"requested range; not point-in-time eligibility"
),
},
"source_jobs": source_jobs,
"mirror_job_status_counts_at_build": dict(sorted(status_counts.items())),
"missing_production_tables": [
value
for value in MISSING_PRODUCTION_TABLES
if value not in completed_apis
],
"verification_boundary": (
"completed checksummed partitions and generated provider "
"integrity only; not point-in-time production market evidence"
),
"gate_credit": [],
"investment_value_claim": False,
}
(temporary / MANIFEST_NAME).write_bytes(_canonical_bytes(manifest))
os.replace(temporary, output)
except BaseException:
if temporary.exists():
shutil.rmtree(temporary)
raise
return {
"ok": True,
"output_dir": str(output),
"manifest": str(output / MANIFEST_NAME),
**manifest,
}
def verify_qlib_provider(provider_dir: str | Path) -> Dict[str, Any]:
"""Verify a provider tree against its Quant OS manifest."""
root = Path(provider_dir).expanduser().resolve()
if not root.is_dir():
raise FileNotFoundError(f"provider_dir does not exist: {root}")
manifest_path = root / MANIFEST_NAME
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise TushareMirrorError(
f"cannot read provider manifest: {manifest_path}"
) from exc
if not isinstance(manifest, dict):
raise TushareMirrorError("provider manifest must be a JSON object")
if manifest.get("artifact_type") != "quant_os_tushare_qlib_provider":
raise TushareMirrorError("unexpected provider artifact_type")
build_parameters = manifest.get("build_parameters")
converter = manifest.get("converter")
source_jobs = manifest.get("source_jobs")
if (
not isinstance(build_parameters, Mapping)
or not isinstance(converter, Mapping)
or not isinstance(source_jobs, list)
):
raise TushareMirrorError(
"provider manifest misses build/data-version evidence"
)
try:
version_payload = _data_version_payload(
source_jobs=source_jobs,
build_parameters=build_parameters,
converter_sha256=str(converter["source_sha256"]),
)
computed_data_version = hashlib.sha256(
_canonical_bytes(version_payload)
).hexdigest()
except (KeyError, TypeError, ValueError) as exc:
raise TushareMirrorError(
"provider build/data-version evidence is invalid"
) from exc
if computed_data_version != manifest.get("data_version"):
raise TushareMirrorError(
"provider data_version does not match source/build evidence"
)
tree = _tree_evidence(root)
for name in (
"provider_tree_sha256",
"provider_file_count",
"provider_byte_count",
):
if tree[name] != manifest.get(name):
raise TushareMirrorError(
f"provider tree mismatch for {name}: "
f"{tree[name]!r} != {manifest.get(name)!r}"
)
for required in ("calendars/day.txt", "instruments/all.txt"):
if not (root / required).is_file():
raise TushareMirrorError(f"provider misses {required}")
return {
"ok": True,
"data_version": manifest.get("data_version"),
**tree,
"manifest_sha256": _sha256_file(manifest_path),
"investment_value_claim": False,
}
+12 -1
View File
@@ -32,6 +32,7 @@ synthetic PIT/features -> purged walk-forward -> Ridge challenger
(not deployment-frozen)
Qlib 0.9.7:
Tushare completed/checksummed Parquet -> immutable managed provider
native portable-momentum signal bridge
Alpha158 + LightGBM + Recorder research workflow
(no L3 target / L4 order-intent parity)
@@ -62,6 +63,14 @@ XtTrader:
## Data plane and point-in-time semantics
本地 Tushare 镜像是独立只读输入边界。adapter 只信任 SQLite 中 completed
job,并重新核对文件大小、SHA-256、Parquet 行数/footerQlib provider
发布到新目录,manifest 绑定 source job、build parameter、converter source、
完整 provider tree 和可重算 `data_version`。当前镜像未完成且缺复权、
真实 benchmark、历史成分/ST/停牌/涨跌停,因此只能进入 Qlib 技术 smoke
不能直接进入 canonical production decision。完整合同与实跑证据见
[`TUSHARE_LOCAL_DATA.md`](TUSHARE_LOCAL_DATA.md)。
JQData adapter 的输入合同是逐交易日历史指数成员和 `get_bars` 日线字段:
- 未复权 `open/high/low/close``volume``money`
@@ -173,6 +182,8 @@ Qlib 路径固定为 `0.9.7`。CPython 3.12 实际 fixture smoke 已得到:
重算的有限值指标,明确 `investment_value_claim=false`
- local MLflow file store 使用时,runner 以
`os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true")` 显式确认。
- 本地 Tushare completed 分区已生成 managed provider;早期未复权动量
smoke 在 `PYTHONHASHSEED=0` 下连续两次得到逐 byte 相同的 evidence JSON。
该 fixture 只有两只股票和一个 benchmark,性能没有投资意义。Qlib 的单一
`limit_threshold` 也不能表达逐日板块/ST 规则,所以它只参与 L1/L2 和诊断
@@ -217,7 +228,7 @@ saved/current source aggregate、saved/current schema aggregate 和 payload
schema。`verify-snapshot-decision` 与 data snapshot semantic verifier 承担
各自边界的同类职责。
这些合同已有自动测试;最近一次标准库套件为 211 tests、OK、3 个可选路径
这些合同已有自动测试;最近一次标准库套件为 216 tests、OK、6 个可选路径
skip。测试结果不能替代 schema 迁移演练、真实平台导出或券商回调认证。
## Clocks and reconciliation
+3 -2
View File
@@ -25,7 +25,7 @@ A 股 Baseline 内核。它的新增得分必须在授权数据、真实平台
截至 2026-07-26 的实现事实快照:
- 截至 2026-07-26 的最新标准库全量测试为 `Ran 211 tests``OK (skipped=3)`
- 截至 2026-07-26 的最新标准库全量测试为 `Ran 216 tests``OK (skipped=6)`
- JQData raw/factor/pre-close/limits/paused + 每日 PIT `is_st`/指数成员
已连接到 semantic snapshot verifier、`snapshot-backtest`
`snapshot-decision`,并拒绝把抓取当日未到 24:00 的日线标成完整收盘,
@@ -152,7 +152,8 @@ A 股 Baseline 内核。它的新增得分必须在授权数据、真实平台
T 日 Signal 与 T+1 broker fact 绑定、完整 artifact verifier
- Colab 默认无账号流程,以及显式可选的 JQData/Qlib cell。
最近一次 211-test suite两条 Qlib fixture smoke 证明上述代码路径能执行,
最近一次 216-test suite两条 Qlib fixture smoke 和 Tushare
managed-provider 技术 smoke 证明上述代码路径能执行,
但仍明确不证明:
- synthetic 数据或两股票 Qlib fixture 的收益具有投资价值;
+2 -1
View File
@@ -5,7 +5,7 @@
| Path | Intended use | Runtime / prerequisite | Current verified fact | Missing release evidence |
| --- | --- | --- | --- | --- |
| Local synthetic execution | 确定性事件回测、ledger/replay | Python ≥3.10,无账号 | 标准库套件最近记录为 211 tests、OK、3 skipsmoke + exact manifest verifier 可运行 | synthetic 不代表真实市场/收益 |
| Local synthetic execution | 确定性事件回测、ledger/replay | Python ≥3.10,无账号 | 标准库套件最近记录为 216 tests、OK、6 skipsmoke + exact manifest verifier 可运行 | synthetic 不代表真实市场/收益 |
| Local synthetic research | PIT/features/walk-forward/Ridge/risk/cost/target | Python ≥3.10,无账号 | research smoke + manifest verifier 可运行 | Ridge 未冻结部署,未做授权长样本 OOS |
| JQData ingestion | 授权数据到 canonical immutable snapshot | `jqdatasdk==1.9.8` + 授权账号 | fake-provider 测试覆盖 raw/factor/pre-close/limits/paused/PIT `is_st`/membership、24:00 完整收盘约束和语义 verifier | 尚未登录真实账号保存快照、许可与 lineage |
| Snapshot local backtest | PIT membership 的 provider-data 本地回测 | semantic verifier 通过的 snapshot | `snapshot-backtest` 复用 production decision path,输出普通 run manifest | 尚无授权真实 snapshot;仍是本地 fill model |
@@ -19,6 +19,7 @@
| XtTrader read-only shadow | 账户绑定 target-diff 与 broker observation | MiniQMT/QMT、账户查询权限、既有本地 HMAC key | HMAC 账户绑定 + authenticated evidence envelope、decision/observation semantic replay、资产/持仓/委托/成交恒等式、fail-closed exact verifier 与 fake broker 合同 | HMAC 不是 broker attestation;仍缺官方 query 失败/空结果合同、callback/restart、20 日真实 shadownot live-ready |
| Qlib native momentum | signal/model research | CPython 3.12 + exactly `pyqlib==0.9.7` | 真实本地 fixture 成功,24 signal;保存 signal/report hash、表边界和重算 portfolio 指标 | synthetic 两股票;无真实数据/OOS,无 L3/L4 parity |
| Qlib Alpha158/LightGBM | model fit + Recorder workflow | 同上 + LightGBM/MLflow stack | 真实 fixture 完成 model/Recorder;读取 portfolio report,保存表 hash、边界、重算指标与 artifact path | synthetic 两股票性能无意义;真实 provider/OOS 和冻结模型缺失 |
| Tushare local → Qlib | completed Parquet 的只读盘点、不可变 provider 与本地研究 smoke | 同上 + `pyarrow==24.0.0`;本地镜像 | 93 个 completed 文件全部通过 size/SHA/row-countv4 provider verifier 成功;同一动量命令连续两次 evidence JSON 逐 byte 相同 | `daily` 仅完成约 8.2%,缺复权、真实 benchmark、PIT 成分/ST/停牌/涨跌停;`production_ready=false` |
| Mock parity exporter | JoinQuant/QMT wrapper 的 L2—L4 合同回归 | Python ≥3.10 | `mock_contract_only``real_platform_pass=false``gate_credit=[]` | 不计 G9;必须换成真实平台导出 |
G1—G10 当前都未通过。JoinQuant hosted smoke 与 Qlib fixture smoke 是真实
+225
View File
@@ -0,0 +1,225 @@
# Tushare 本地数据盘点、Qlib 接入与回测手册
截至 2026-07-26Quant OS 已能只读盘点另一个任务生成的 Tushare
Parquet 镜像,校验已完成分区,并转换为不可变的 Qlib 0.9.7 provider。
这条路径已经完成两次 byte-identical 的本地动量回测。
当前证据的边界是:
```text
completed_partitions_only_not_full_mirror
production_ready = false
investment_value_claim = false
gate_credit = []
```
它证明“本地 Tushare 数据可以进入 Qlib 并可重放”,不证明镜像已经下载完整,
也不证明策略收益有效或 Quant OS 已达到 `BASELINE_60`
## 1. 2026-07-26 点时盘点
镜像中共有 93 个状态为 `completed` 的 Parquet,均通过 SQLite 记录的文件大小、
SHA-256 和 Parquet row count 校验;另有 1 个遗留 `running` job。
| 表 | 文件数 | 行数 | 当前范围/说明 |
| --- | ---: | ---: | --- |
| `daily` | 35 | 28,731 | 1990-12-19—1993-10-29138 个代码、731 个交易日 |
| `trade_cal` | 37 | 13,004 | SSE 1990—2026 日历,8,689 个开市日 |
| `stock_basic` | 4 | 5,869 | 当前上市/退市/暂停上市等基础快照 |
| `stock_company` | 3 | 6,294 | SSE/SZSE/BSE 公司资料 |
| `index_basic` | 7 | 9,643 | CSI/SW/SSE/SZSE 等指数基础资料 |
| `index_classify` | 6 | 870 | 申万 2014/2021 分类 |
| `bse_mapping` | 1 | 248 | 北交所新旧代码映射 |
| **合计** | **93** | **64,659** | **5,734,035 bytes Parquet** |
`daily` 原计划覆盖 1990-12—2026-07,共 428 个月;当前完成 35 个月,约
8.2%。下载进程已停止,SQLite 仍遗留
`daily/month=1993-11``running` 状态。日志显示首先发生 DNS 解析失败,
错误处理阶段又发生 `sqlite3.OperationalError: unable to open database
file`。因此不能把这个状态解释为仍在后台下载,也不要在 Quant OS
转换过程中修改或自动恢复原下载器。
尚未落盘的生产关键表包括:
- `adj_factor``index_daily`
- `index_member_all``index_weight`
- `stk_limit``suspend_d``stock_st`
- `daily_basic`
其中历史 `stock_st` 权限探测被拒绝。即使其他下载完成,历史 ST 仍需要新的
授权数据源或明确的降级门禁。
## 2. 已确认的数据质量
- 已完成 Parquet 自然键无重复,核心日线字段无空值;
- 无负成交量或负成交额,日线日期与同期 SSE 开市日完全对应;
- 17 条早期 OHLC 包络异常;默认拒绝,只有显式
`--ohlc-policy expand-range` 才按原始 `open/high/low/close` 四价的
min/max 修复,并记录数量与异常键 hash;
- 2 条 `pct_chg` 与保存精度下重算结果的偏差超过 0.02 个百分点;
- 日线代码 `000022.SZ` 不在当前 `stock_basic` 快照;
- `stock_basic` 存在遗留非标准代码 `T600018.SH`,转换器不会把它当成
A 股代码;
- `index_basic.base_point` 存在分区类型漂移,后续 canonical 合并时必须
显式 cast。
Tushare 的 `daily` 是未复权行情,停牌期间不返回记录;`vol` 单位为手,
`amount` 单位为千元。转换器固定执行:
```text
600000.SH -> SH600000
000001.SZ -> SZ000001
430047.BJ -> BJ430047
volume = vol * 100
money = amount * 1000
factor = 1
```
未上市、退市后和停牌缺口在 Qlib feature 中保留为 `NaN`。字段语义来源见
[Tushare 日线接口](https://tushare.pro/document/2?doc_id=27)与
[复权因子接口](https://tushare.pro/document/2?doc_id=28)。
## 3. Quant OS 如何使用这些数据
当前已经实现的链路是研究/引擎验证路径:
```text
Tushare mirror
-> 只读取 SQLite completed jobs
-> 文件大小 + SHA-256 + Parquet row count/footer 校验
-> 新目录原子发布 Qlib binary provider
-> provider tree hash + data_version + manifest verifier
-> Qlib momentum / Alpha158 research runner
```
正式 Quant OS 还需要第二条生产数据路径:
```text
完整 Tushare/JQData PIT 数据
-> canonical immutable snapshot
-> Quant OS 本地事件回测 / decision
-> 聚宽、QMT、Qlib 的同输入分层比较
```
Qlib 适合因子、模型和组合研究,不拥有原始数据,也不替代 Quant OS 的
Signal/Target/Order/Broker 合同。当前 Tushare adapter 只实现第一条链路;
在复权、真实 benchmark、历史成分、停牌、涨跌停和 ST 数据补齐前,不会把它
接入生产 decision。
## 4. 可运行命令
从 Quant OS 根目录执行。镜像路径只放在当前 shell 环境变量中,不写入 Git:
```bash
export QUANT_OS_ROOT=/path/to/quants-strategies/quant-os
export TUSHARE_MIRROR_ROOT=/path/to/tushare-mirror
cd "$QUANT_OS_ROOT"
python3.12 -m venv .venv-qlib312
source .venv-qlib312/bin/activate
python -m pip install -r requirements/research-py312.txt
export PYTHONPATH=src:.
```
盘点并验证全部 completed 文件:
```bash
python tools/tushare_qlib.py inventory \
--mirror-root "$TUSHARE_MIRROR_ROOT" \
--output-json artifacts/tushare-inventory.json
```
当前未复权数据只能显式构建技术验证 provider;每次必须使用新目录,工具拒绝
覆盖已有 provider
```bash
python tools/tushare_qlib.py build \
--mirror-root "$TUSHARE_MIRROR_ROOT" \
--output-dir data/qlib/tushare-early-v1 \
--start 1990-12-19 \
--end 1993-10-29 \
--minimum-observations 60 \
--allow-unadjusted \
--ohlc-policy expand-range \
--output-json artifacts/tushare-qlib-build.json
python tools/tushare_qlib.py verify \
data/qlib/tushare-early-v1 \
--output-json artifacts/tushare-qlib-verify.json
```
运行 Qlib 0.9.7 本地回测:
```bash
PYTHONHASHSEED=0 python -m platforms.qlib_runner \
--provider-uri data/qlib/tushare-early-v1 \
--market tushare_a \
--benchmark SH999999 \
--start 1992-01-02 \
--end 1993-10-28 \
--feature-start 1991-01-02 \
--lookback 20 \
--topk 10 \
--n-drop 2 \
--rebalance weekly \
--output-json artifacts/tushare-qlib-momentum.json
```
Qlib simulator 会消费结束日后的下一 provider session,所以 backtest `end`
必须早于 provider 最后一个交易日。Quant OS 对受管 provider 会在 Qlib
初始化前检查 market、benchmark、起始边界和这个终点条件。
`artifacts/``data/qlib/` 均被 Git 忽略。原始 Parquet、token、绝对镜像
路径和回测临时产物不得提交到仓库。
## 5. 本次真实本地运行证据
最终 v4 provider
| 证据 | 值 |
| --- | --- |
| adapter source SHA-256 | `2228c23af176a0f2fcf311e88dca9bdc5d63fd36fa5a3c2d3e42808a405e7243` |
| data version | `3378aa75a5601bde476ad07bea90418966a66a037ca59195dec93d77b41cc3f8` |
| provider tree SHA-256 | `dc85b8daf692a66641afef64398700264c6054f1dac84b1211cb258a06b62948` |
| manifest SHA-256 | `862e7435bb0be7c8a3c66180e49761c110056c1a3cc02c1618e70d9eb566d945` |
| provider 文件/大小 | 975 / 1,085,886 bytes |
| calendar | 1990-12-19—1993-10-29731 sessions |
| market | 107 个至少有 60 条记录的 A 股代码 |
| selected rows | 27,998 |
回测参数为 1992-01-02—1993-10-28、20 日动量、周频、Top 10、每期 drop 2
初始资金 1,000 万,Qlib 0.9.7、Python 3.12.13、`PYTHONHASHSEED=0`
相同命令连续执行两次,结果 JSON 逐 byte 相同:
| 证据/指标 | 值 |
| --- | ---: |
| result JSON SHA-256 | `92ff9b8cea746e9c89ddf62fcfe3feb21248ca9112d9e10d24e0639058a86020` |
| runner source SHA-256 | `fa8118819252c55e99e67de356cc961d8a4a22f82256b4d62579a45426d96167` |
| signal | 4,700 行;hash `cfad4087d68b7f71e33f0f46fc5bb97db985e02ac2515f01857e2ab6813a5909` |
| portfolio report | 466 行;hash `5f8606fb64963d3e2618f66503e6ae94048c24b62c5d8e1806ff44ec57558b04` |
| strategy cumulative return | -52.5251% |
| max drawdown | -87.7604% |
| synthetic benchmark cumulative return | 196,302.0822% |
| total cost / turnover | 0.016013 / 29.567521 |
这些收益数字没有投资解释。原因包括未复权早期行情、没有真实指数 benchmark、
没有历史成分/ST/停牌/涨跌停、按全区间至少 60 条观测筛选带来的非 PIT
偏差,以及 1992—1993 特殊市场阶段。极端 synthetic benchmark 恰好说明
为什么“程序跑完”不能等价为“回测有效”。
## 6. 何时可以升级为正式研究数据
至少完成以下步骤后,才能新建 production-eligible provider
1. 修复下载器错误状态并完成目标日期的 `daily`
2. 补齐 `adj_factor`,冻结复权基准日和公式;
3. 使用 `index_daily` 替换合成 benchmark
4. 指数策略补齐 `index_member_all/index_weight`
5. 补齐 `suspend_d/stk_limit`,为历史 `stock_st` 找到授权来源或保持硬阻断;
6. 冻结下载 job、请求 hash、Parquet hash、转换器 hash 和新 `data_version`
7. 运行真实长样本 walk-forward/OOS,而不是复用本次早期 smoke;
8. 同一冻结输入进入 Quant OS 本地回测、聚宽和可用 QMT,生成 L1—L4
差异报告。
Qlib provider 的目录格式和缺失值/复权约定参见
[Qlib Data Layer 文档](https://qlib.readthedocs.io/en/latest/component/data.html)。
+3 -2
View File
@@ -33,13 +33,14 @@
"current_delivery": {
"score": null,
"status": "not_scored",
"reason": "Quant OS now has executable local pipelines, real Qlib fixture smokes and one completed real JoinQuant hosted smoke, but it has not produced the same-input JoinQuant comparison, real QMT, broker-shadow, continuous reconciliation or compliance evidence required to score Baseline 60.",
"reason": "Quant OS now has executable local pipelines, real Qlib fixture smokes, a checksummed incomplete Tushare-to-Qlib technical smoke and one completed real JoinQuant hosted smoke, but it has not produced complete adjusted PIT data, the same-input JoinQuant comparison, real QMT, broker-shadow, continuous reconciliation or compliance evidence required to score Baseline 60.",
"verified_supporting_facts": [
"The latest standard-library suite completed 211 tests with OK and 3 optional-runtime skips; the CPython 3.12 pyqlib environment completed the same 211 tests with no skips.",
"The latest standard-library suite completed 216 tests with OK and 6 optional-runtime skips; the CPython 3.12 pyqlib environment completed the same 216 tests with the native Qlib smoke enabled and no skips.",
"The JQData adapter maps raw bars, adjustment factor, previous close, daily limits, pause state, PIT is_st and daily PIT index membership into a semantically revalidated snapshot; it rejects an end date that has not reached the provider's complete daily-bar boundary, and only fake-provider tests have been run so far.",
"The local snapshot can drive snapshot-backtest and snapshot-decision with manifest verification; it freezes a cross-checked provider trading calendar, and a T-close signal can bind only to a broker observation in the unique next session's Asia/Shanghai [09:00,09:30) window without collapsing the two clocks.",
"CPython 3.12 with pyqlib 0.9.7 completed a native momentum fixture smoke with 24 signals and saved audited portfolio-table hashes, bounds and recomputed metrics.",
"Qlib Alpha158, LightGBM and Recorder completed a real local fixture smoke with audited portfolio output; the deterministic two-stock synthetic fixture has no investment-value meaning.",
"The local Tushare inventory verified all 93 completed Parquet files against recorded size, SHA-256 and row count. An immutable managed Qlib provider and the same early-history momentum command produced byte-identical evidence JSON twice with PYTHONHASHSEED=0. The mirror is only about 8.2% complete for daily partitions and lacks adjustment, real benchmark, point-in-time membership, ST, suspension and limit data, so production_ready and investment_value_claim remain false and no gate credit is earned.",
"A real JoinQuant daily hosted backtest completed for 2024-01-02 through 2024-12-31 with CNY 1,000,000; the visible completed-run log had zero ERROR/Traceback/order-rejection entries. This proves hosted runtime execution only: exact input/plan/order/fill export and a local same-input peer are still missing.",
"The QMT shadow CLI is strictly read-only and produces HMAC account-bound broker observation, broker snapshot and target-diff artifacts. QMT_SHADOW_EVIDENCE_V1 authenticates semantic content, deterministic published bytes, the original decision, planner/engine/operator source and fixed policy; semantic replay verifies exact derived state. Independent local QA completed 58 deterministic scenarios, 500 malformed fuzz cases and 300 legal-observation fuzz cases with no P0/P1. This is Quant OS publisher integrity, not broker attestation, and it still has only fake-broker evidence.",
"JoinQuant/QMT mock parity is mock_contract_only, real_platform_pass is false and it earns no G9 gate credit."
+180 -1
View File
@@ -16,6 +16,7 @@ import json
import math
import os
from pathlib import Path
import sys
import tempfile
from typing import Any, Dict, Iterable, Mapping, Optional, Sequence
@@ -26,6 +27,19 @@ class QlibUnavailableError(RuntimeError):
"""Raised when Qlib or its configured provider cannot be used."""
def _runner_evidence() -> Dict[str, Any]:
source = Path(__file__).resolve().read_bytes()
return {
"runner_source_sha256": hashlib.sha256(source).hexdigest(),
"python_version": (
f"{sys.version_info.major}."
f"{sys.version_info.minor}."
f"{sys.version_info.micro}"
),
"python_hash_seed": os.environ.get("PYTHONHASHSEED"),
}
def _table_audit(table: Any) -> Optional[Dict[str, Any]]:
"""Return compact deterministic evidence for a pandas Series/DataFrame."""
@@ -154,6 +168,121 @@ def _validate_provider(provider_uri: str) -> str:
return value
def _managed_provider_evidence(provider_uri: str) -> Dict[str, Any]:
"""Verify and summarize a Quant OS-managed local provider when present."""
if "://" in provider_uri:
return {
"managed": False,
"manifest_name": None,
"verification": "remote provider URI; no local manifest",
}
root = Path(provider_uri)
manifest_path = root / "quant_os_tushare_manifest.json"
if not manifest_path.is_file():
return {
"managed": False,
"manifest_name": None,
"verification": "no Quant OS provider manifest",
}
try:
from adapters.tushare_local import verify_qlib_provider
verification = verify_qlib_provider(root)
manifest_bytes = manifest_path.read_bytes()
manifest = json.loads(manifest_bytes)
except Exception as exc:
raise QlibUnavailableError(
f"managed Qlib provider verification failed: {exc}"
) from exc
if not isinstance(manifest, Mapping):
raise QlibUnavailableError(
"managed Qlib provider manifest must be a JSON object"
)
evidence: Dict[str, Any] = {
"managed": True,
"manifest_name": manifest_path.name,
"manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(),
"data_version": manifest.get("data_version"),
"provider_tree_sha256": manifest.get("provider_tree_sha256"),
"source": manifest.get("source", "tushare_local_mirror"),
"converter": manifest.get("converter"),
"build_parameters": manifest.get("build_parameters"),
"calendar": manifest.get("calendar"),
"market": manifest.get("market"),
"benchmark": manifest.get("benchmark"),
"price_adjustment": manifest.get("price_adjustment"),
"investment_value_claim": False,
}
if isinstance(verification, Mapping):
evidence["verified_file_count"] = verification.get(
"provider_file_count",
verification.get("file_count"),
)
return evidence
def _validate_managed_provider_request(
provider_evidence: Mapping[str, Any],
*,
market: str,
benchmark: str,
start_time: str,
end_time: str,
) -> None:
"""Fail early on managed-provider mismatches and terminal-calendar use."""
if not provider_evidence.get("managed"):
return
provider_market = provider_evidence.get("market")
if isinstance(provider_market, Mapping):
expected_market = str(provider_market.get("name") or "").strip()
if expected_market and market != expected_market:
raise ValueError(
f"managed provider market is {expected_market!r}, "
f"not {market!r}"
)
provider_benchmark = provider_evidence.get("benchmark")
if isinstance(provider_benchmark, Mapping):
expected_benchmark = str(
provider_benchmark.get("symbol") or ""
).strip()
if expected_benchmark and benchmark != expected_benchmark:
raise ValueError(
f"managed provider benchmark is {expected_benchmark!r}, "
f"not {benchmark!r}"
)
calendar = provider_evidence.get("calendar")
if not isinstance(calendar, Mapping):
raise QlibUnavailableError(
"managed provider manifest has no calendar evidence"
)
try:
calendar_start = dt.date.fromisoformat(str(calendar["start"]))
calendar_end = dt.date.fromisoformat(str(calendar["end"]))
requested_start = dt.date.fromisoformat(str(start_time)[:10])
requested_end = dt.date.fromisoformat(str(end_time)[:10])
except (KeyError, TypeError, ValueError) as exc:
raise QlibUnavailableError(
"managed provider calendar/request dates must use YYYY-MM-DD"
) from exc
if requested_start < calendar_start:
raise ValueError(
f"start_time {requested_start} precedes managed provider "
f"calendar start {calendar_start}"
)
if requested_end >= calendar_end:
raise ValueError(
f"end_time must be before managed provider terminal session "
f"{calendar_end}; Qlib's simulator consumes the next provider "
"session. Rebuild with a later calendar end or choose the "
"previous open session."
)
def _infer_columns(frame: Any) -> tuple[str, str, str]:
columns = [str(column) for column in frame.columns]
instrument = next(
@@ -315,6 +444,21 @@ def run_native_momentum_backtest(
) -> Dict[str, Any]:
"""Run the official SimulatorExecutor + TopkDropoutStrategy stack."""
provider = _validate_provider(provider_uri)
provider_evidence = _managed_provider_evidence(provider)
_validate_managed_provider_request(
provider_evidence,
market=market,
benchmark=benchmark,
start_time=start_time,
end_time=end_time,
)
resolved_feature_start = feature_start_time
if resolved_feature_start is None:
start = dt.datetime.fromisoformat(str(start_time)[:10])
padding_days = max(14, 3 * (int(lookback) + int(skip) + 2))
resolved_feature_start = (
start - dt.timedelta(days=padding_days)
).date().isoformat()
api = _load_qlib()
constant = importlib.import_module("qlib.constant")
api["qlib"].init(provider_uri=provider, region=constant.REG_CN)
@@ -323,7 +467,7 @@ def run_native_momentum_backtest(
market,
start_time=start_time,
end_time=end_time,
feature_start_time=feature_start_time,
feature_start_time=resolved_feature_start,
lookback=lookback,
skip=skip,
rebalance=rebalance,
@@ -374,6 +518,26 @@ def run_native_momentum_backtest(
"uniform_limit_threshold_approximation_not_point_in_time_"
"board_or_ST_rules"
),
"run_parameters": {
"market": market,
"benchmark": benchmark,
"start_time": start_time,
"end_time": end_time,
"feature_start_time": resolved_feature_start,
"lookback": int(lookback),
"skip": int(skip),
"rebalance": rebalance,
"topk": int(topk),
"n_drop": int(n_drop),
"account": float(account),
"deal_price": deal_price,
"open_cost": float(open_cost),
"close_cost": float(close_cost),
"min_cost": float(min_cost),
"limit_threshold": float(limit_threshold),
},
"runner_evidence": _runner_evidence(),
"provider_evidence": provider_evidence,
}
@@ -482,6 +646,14 @@ def run_alpha158_lightgbm_workflow(
) -> Dict[str, Any]:
"""Fit, record signals and run PortAnaRecord with official Qlib APIs."""
provider = _validate_provider(provider_uri)
provider_evidence = _managed_provider_evidence(provider)
_validate_managed_provider_request(
provider_evidence,
market=market,
benchmark=benchmark,
start_time=train[0],
end_time=test[1],
)
# Qlib 0.9.7 defaults to a local MLflow file store. MLflow 3.14 requires
# an explicit acknowledgement before it will open that backend. Keep the
# acknowledgement local to this process; a caller-provided tracking
@@ -556,6 +728,8 @@ def run_alpha158_lightgbm_workflow(
"recorded_metrics": recorded_metrics,
"portfolio_artifact_paths": artifact_paths,
},
"runner_evidence": _runner_evidence(),
"provider_evidence": provider_evidence,
}
@@ -682,6 +856,9 @@ def _momentum_cli_summary(result: Mapping[str, Any]) -> Dict[str, Any]:
"clock_contract": result.get("clock_contract"),
"strategy_fidelity": result.get("strategy_fidelity"),
"a_share_rule_fidelity": result.get("a_share_rule_fidelity"),
"run_parameters": result.get("run_parameters"),
"runner_evidence": result.get("runner_evidence"),
"provider_evidence": result.get("provider_evidence"),
"investment_value_claim": False,
}
@@ -712,6 +889,8 @@ def _alpha158_cli_summary(
"test": list(test),
},
"recorder_evidence": result.get("recorder_evidence"),
"runner_evidence": result.get("runner_evidence"),
"provider_evidence": result.get("provider_evidence"),
"investment_value_claim": False,
}
+1
View File
@@ -7,6 +7,7 @@
numpy==2.5.1
pandas==2.3.3
pyarrow==24.0.0
scipy==1.18.0
lightgbm==4.7.0
cvxpy==1.9.2
+3
View File
@@ -119,11 +119,14 @@ class CapabilityProbeTest(unittest.TestCase):
self.assertTrue(report["artifacts"]["portable_core"])
self.assertTrue(report["artifacts"]["colab_notebook"])
self.assertTrue(report["artifacts"]["jqdata_snapshot_adapter"])
self.assertTrue(report["artifacts"]["tushare_qlib_adapter"])
self.assertTrue(report["artifacts"]["tushare_qlib_cli"])
self.assertTrue(report["artifacts"]["qmt_shadow_planner"])
self.assertTrue(report["artifacts"]["qmt_shadow_cli"])
self.assertIn("no external platform", report["verification_boundary"])
self.assertTrue(report["matrix"]["xttrader"]["safe_default"])
self.assertIn("rejects BJ", report["matrix"]["xttrader"]["market_scope"])
self.assertTrue(report["matrix"]["tushare_local_qlib"]["safe_default"])
for probe in report["optional_runtimes"].values():
self.assertIs(probe["import_ok"], None)
+35
View File
@@ -17,6 +17,7 @@ sys.path.insert(0, str(ROOT / "src"))
from platforms.qlib_runner import (
QlibUnavailableError,
_main,
_validate_managed_provider_request,
build_alpha158_lightgbm_task,
generate_portable_momentum_signal,
run_native_momentum_backtest,
@@ -91,6 +92,35 @@ class QlibFixtureTest(unittest.TestCase):
end_time="2024-02-01",
)
def test_managed_provider_request_fails_before_terminal_session(self):
evidence = {
"managed": True,
"calendar": {
"start": "2024-01-02",
"end": "2024-01-11",
},
"market": {"name": "tushare_a"},
"benchmark": {"symbol": "SH999999"},
}
with self.assertRaisesRegex(
ValueError,
"simulator consumes the next provider session",
):
_validate_managed_provider_request(
evidence,
market="tushare_a",
benchmark="SH999999",
start_time="2024-01-02",
end_time="2024-01-11",
)
_validate_managed_provider_request(
evidence,
market="tushare_a",
benchmark="SH999999",
start_time="2024-01-02",
end_time="2024-01-10",
)
@unittest.skipUnless(
importlib.util.find_spec("pandas") is not None,
"pandas is optional outside the Qlib environment",
@@ -147,6 +177,7 @@ class QlibCliTest(unittest.TestCase):
"clock_contract": "test-clock",
"strategy_fidelity": "signal-only",
"a_share_rule_fidelity": "approximation",
"run_parameters": {"start_time": "2024-01-02"},
}
with tempfile.TemporaryDirectory() as temp:
output = Path(temp) / "nested" / "summary.json"
@@ -194,6 +225,10 @@ class QlibCliTest(unittest.TestCase):
self.assertEqual(payload["workflow"], "momentum")
self.assertEqual(payload["signal_rows"], 2)
self.assertEqual(payload["portfolio_frequencies"], ["1day"])
self.assertEqual(
payload["run_parameters"]["start_time"],
"2024-01-02",
)
self.assertIn('"workflow": "momentum"', printer.call_args.args[0])
def test_alpha158_cli_dispatches_with_validated_segments(self):
+230
View File
@@ -0,0 +1,230 @@
import importlib.util
import json
from pathlib import Path
import sqlite3
import sys
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from adapters.tushare_local import (
TushareMirrorError,
build_qlib_provider,
tushare_to_qlib_symbol,
verify_qlib_provider,
)
class TushareSymbolTest(unittest.TestCase):
def test_symbol_conversion(self):
self.assertEqual(tushare_to_qlib_symbol("600000.SH"), "SH600000")
self.assertEqual(tushare_to_qlib_symbol("000001.SZ"), "SZ000001")
self.assertEqual(tushare_to_qlib_symbol("430047.BJ"), "BJ430047")
with self.assertRaises(ValueError):
tushare_to_qlib_symbol("T600018.SH")
@unittest.skipUnless(
importlib.util.find_spec("pandas") is not None
and importlib.util.find_spec("pyarrow") is not None,
"pandas and pyarrow are optional outside the Qlib research environment",
)
class TushareProviderTest(unittest.TestCase):
def _mirror(self, root: Path, *, bad_ohlc: bool = False) -> Path:
import pandas
mirror = root / "mirror"
parquet_root = mirror / "data/parquet"
(parquet_root / "daily").mkdir(parents=True)
(parquet_root / "trade_cal").mkdir(parents=True)
(parquet_root / "stock_basic").mkdir(parents=True)
dates = pandas.bdate_range("2024-01-02", periods=8)
daily_rows = []
for symbol, base in (("600000.SH", 10.0), ("000001.SZ", 20.0)):
previous = base
for index, value in enumerate(dates):
close = base * (1.01 ** index)
daily_rows.append(
{
"ts_code": symbol,
"trade_date": value.strftime("%Y%m%d"),
"open": close * 0.99,
"high": close * 1.01,
"low": close * 0.98,
"close": close,
"pre_close": previous,
"change": close - previous,
"pct_chg": (close / previous - 1.0) * 100.0,
"vol": 1000.0,
"amount": close * 100.0,
}
)
previous = close
if bad_ohlc:
daily_rows[0]["high"] = 8.0
daily_rows[0]["low"] = 9.0
frames = {
"daily/month=2024-01.parquet": pandas.DataFrame(daily_rows),
"trade_cal/exchange=SSE_year=2024.parquet": pandas.DataFrame(
{
"exchange": ["SSE"] * len(dates),
"cal_date": [value.strftime("%Y%m%d") for value in dates],
"is_open": [1] * len(dates),
"pretrade_date": [None] * len(dates),
}
),
"stock_basic/list_status=L.parquet": pandas.DataFrame(
{
"ts_code": ["600000.SH", "000001.SZ"],
"curr_type": ["CNY", "CNY"],
}
),
}
state = mirror / "data/state.sqlite3"
connection = sqlite3.connect(state)
connection.execute(
"""
CREATE TABLE jobs (
id INTEGER PRIMARY KEY,
api_name TEXT NOT NULL,
partition_key TEXT NOT NULL,
status TEXT NOT NULL,
row_count INTEGER,
byte_count INTEGER,
sha256 TEXT,
file_path TEXT,
started_at TEXT,
completed_at TEXT,
error_code INTEGER
)
"""
)
import hashlib
for index, (relative, frame) in enumerate(frames.items(), start=1):
path = parquet_root / relative
path.parent.mkdir(parents=True, exist_ok=True)
frame.to_parquet(path, index=False)
api = relative.split("/", 1)[0]
connection.execute(
"""
INSERT INTO jobs VALUES (?, ?, ?, 'completed', ?, ?, ?, ?,
NULL, '2024-01-31T00:00:00Z', NULL)
""",
(
index,
api,
path.stem,
len(frame),
path.stat().st_size,
hashlib.sha256(path.read_bytes()).hexdigest(),
str(path),
),
)
connection.execute(
"""
INSERT INTO jobs VALUES
(99, 'daily', 'month=2024-02', 'running', NULL, NULL, NULL, NULL,
'2024-02-01T00:00:00Z', NULL, NULL)
"""
)
connection.commit()
connection.close()
return mirror
def test_build_and_verify_ignores_running_job(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
mirror = self._mirror(root)
output = root / "provider"
result = build_qlib_provider(
mirror,
output,
start="2024-01-02",
end="2024-01-11",
minimum_observations=2,
allow_unadjusted=True,
)
self.assertFalse(result["investment_value_claim"])
self.assertEqual(result["market"]["instrument_count"], 2)
self.assertTrue(verify_qlib_provider(output)["ok"])
manifest = json.loads(
(output / "quant_os_tushare_manifest.json").read_text()
)
self.assertRegex(
manifest["converter"]["source_sha256"],
r"^[0-9a-f]{64}$",
)
self.assertEqual(
manifest["mirror_job_status_counts_at_build"]["running"],
1,
)
manifest["build_parameters"]["start"] = "2024-01-03"
(output / "quant_os_tushare_manifest.json").write_text(
json.dumps(manifest),
encoding="utf-8",
)
with self.assertRaisesRegex(
TushareMirrorError,
"data_version",
):
verify_qlib_provider(output)
def test_unadjusted_and_ohlc_gates_fail_closed(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
mirror = self._mirror(root)
with self.assertRaises(TushareMirrorError):
build_qlib_provider(
mirror,
root / "provider",
start="2024-01-02",
end="2024-01-11",
minimum_observations=2,
)
def test_expand_range_uses_the_original_ohlc_envelope(self):
from array import array
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
mirror = self._mirror(root, bad_ohlc=True)
with self.assertRaisesRegex(
TushareMirrorError,
"OHLC envelope anomalies",
):
build_qlib_provider(
mirror,
root / "provider-fail",
start="2024-01-02",
end="2024-01-11",
minimum_observations=2,
allow_unadjusted=True,
)
output = root / "provider-expanded"
build_qlib_provider(
mirror,
output,
start="2024-01-02",
end="2024-01-11",
minimum_observations=2,
allow_unadjusted=True,
expand_range=True,
)
high = array("f")
high.frombytes(
(output / "features/sh600000/high.day.bin").read_bytes()
)
low = array("f")
low.frombytes(
(output / "features/sh600000/low.day.bin").read_bytes()
)
self.assertAlmostEqual(high[1], 10.0)
self.assertAlmostEqual(low[1], 8.0)
if __name__ == "__main__":
unittest.main()
+18
View File
@@ -79,6 +79,17 @@ CAPABILITY_MATRIX: Dict[str, Dict[str, Any]] = {
"not target/order parity"
),
},
"tushare_local_qlib": {
"backtest": True,
"hosted": False,
"live_orders": False,
"safe_default": True,
"market_scope": (
"completed checksummed local Tushare partitions converted into "
"an immutable Qlib provider; source completeness and adjustment "
"gates remain explicit"
),
},
}
@@ -162,6 +173,12 @@ def probe_capabilities(
"qlib_momentum_and_alpha158_cli": (
root / "platforms/qlib_runner.py"
).is_file(),
"tushare_qlib_adapter": (
root / "adapters/tushare_local.py"
).is_file(),
"tushare_qlib_cli": (
root / "tools/tushare_qlib.py"
).is_file(),
"qmt_shadow_planner": (
root / "src/quant60/qmt_shadow.py"
).is_file(),
@@ -172,6 +189,7 @@ def probe_capabilities(
"optional_runtimes": {
"jqdatasdk": _module_probe("jqdatasdk", deep),
"cvxpy": _module_probe("cvxpy", deep),
"pyarrow": _module_probe("pyarrow", deep),
"qlib": _module_probe("qlib", deep),
"xtquant": _module_probe("xtquant", deep),
"xtquant.qmttools": _module_probe("xtquant.qmttools", deep),
+113
View File
@@ -0,0 +1,113 @@
"""Inventory a local Tushare mirror and build/verify a frozen Qlib provider."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import sys
from typing import Any, Mapping, Optional
from adapters.tushare_local import (
TushareMirrorError,
build_qlib_provider,
inventory_mirror,
verify_qlib_provider,
)
def _write_json(path: Optional[str], payload: Mapping[str, Any]) -> None:
if path is None:
return
destination = Path(path).expanduser().resolve()
if destination.exists() and destination.is_dir():
raise ValueError(f"output-json points to a directory: {destination}")
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(
json.dumps(
dict(payload),
ensure_ascii=False,
indent=2,
sort_keys=True,
allow_nan=False,
)
+ "\n",
encoding="utf-8",
)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
inventory = subparsers.add_parser("inventory")
inventory.add_argument("--mirror-root", required=True)
inventory.add_argument("--skip-file-verification", action="store_true")
inventory.add_argument("--output-json")
build = subparsers.add_parser("build")
build.add_argument("--mirror-root", required=True)
build.add_argument("--output-dir", required=True)
build.add_argument("--start", required=True)
build.add_argument("--end", required=True)
build.add_argument("--market-name", default="tushare_a")
build.add_argument("--benchmark-symbol", default="SH999999")
build.add_argument("--minimum-observations", type=int, default=60)
build.add_argument(
"--allow-unadjusted",
action="store_true",
help="required while adj_factor is absent; never grants investment credit",
)
build.add_argument(
"--ohlc-policy",
choices=("fail", "expand-range"),
default="fail",
)
build.add_argument("--output-json")
verify = subparsers.add_parser("verify")
verify.add_argument("provider_dir")
verify.add_argument("--output-json")
return parser
def _main(argv: Optional[list[str]] = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
try:
if args.command == "inventory":
result = inventory_mirror(
args.mirror_root,
verify_files=not args.skip_file_verification,
)
elif args.command == "build":
result = build_qlib_provider(
args.mirror_root,
args.output_dir,
start=args.start,
end=args.end,
market_name=args.market_name,
benchmark_symbol=args.benchmark_symbol,
minimum_observations=args.minimum_observations,
allow_unadjusted=args.allow_unadjusted,
expand_range=args.ohlc_policy == "expand-range",
)
else:
result = verify_qlib_provider(args.provider_dir)
_write_json(args.output_json, result)
except (OSError, ValueError, TushareMirrorError) as exc:
parser.error(str(exc))
print(
json.dumps(
result,
ensure_ascii=False,
indent=2,
sort_keys=True,
allow_nan=False,
)
)
return 0
if __name__ == "__main__":
sys.exit(_main())