1865 lines
67 KiB
Python
1865 lines
67 KiB
Python
"""Read-only Tushare mirror inventory and immutable Qlib provider builder.
|
|
|
|
The adapter never calls Tushare and never reads a token. It trusts only jobs
|
|
marked ``completed`` in the mirror SQLite database, verifies their recorded
|
|
size/SHA-256/Parquet row count, and publishes a new Qlib directory without
|
|
modifying the mirror.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from array import array
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
from typing import Any, Dict, Iterable, Mapping, Optional, Sequence
|
|
|
|
|
|
MANIFEST_NAME = "quant_os_tushare_manifest.json"
|
|
REQUIRED_DAILY_FIELDS = {
|
|
"ts_code",
|
|
"trade_date",
|
|
"open",
|
|
"high",
|
|
"low",
|
|
"close",
|
|
"pre_close",
|
|
"pct_chg",
|
|
"vol",
|
|
"amount",
|
|
}
|
|
QLIB_FIELDS = (
|
|
"open",
|
|
"high",
|
|
"low",
|
|
"close",
|
|
"vwap",
|
|
"volume",
|
|
"money",
|
|
"factor",
|
|
"change",
|
|
)
|
|
MISSING_PRODUCTION_TABLES = (
|
|
"adj_factor",
|
|
"daily_basic",
|
|
"index_daily",
|
|
"index_member_all",
|
|
"index_weight",
|
|
"stk_limit",
|
|
"stock_st",
|
|
"suspend_d",
|
|
)
|
|
KNOWN_INDEX_CONSTITUENT_COUNTS = {
|
|
"000016.SH": 50,
|
|
"000300.SH": 300,
|
|
"000905.SH": 500,
|
|
"000852.SH": 1000,
|
|
}
|
|
REQUIRED_PRODUCTION_TABLES = (
|
|
"daily",
|
|
"trade_cal",
|
|
"stock_basic",
|
|
*MISSING_PRODUCTION_TABLES,
|
|
)
|
|
|
|
|
|
class TushareMirrorError(RuntimeError):
|
|
"""Raised when mirror or provider evidence fails closed."""
|
|
|
|
|
|
def _canonical_json_bytes(value: Mapping[str, Any]) -> bytes:
|
|
return json.dumps(
|
|
dict(value),
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
).encode("utf-8")
|
|
|
|
|
|
def _canonical_bytes(value: Mapping[str, Any]) -> bytes:
|
|
return _canonical_json_bytes(value) + b"\n"
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _iso_date(value: str, *, name: str) -> str:
|
|
normalized = str(value or "").strip()
|
|
try:
|
|
parsed = dt.date.fromisoformat(normalized)
|
|
except ValueError as exc:
|
|
raise ValueError(f"{name} must use YYYY-MM-DD") from exc
|
|
if parsed.isoformat() != normalized:
|
|
raise ValueError(f"{name} must use YYYY-MM-DD")
|
|
return normalized
|
|
|
|
|
|
def _yyyymmdd(value: str) -> str:
|
|
return value.replace("-", "")
|
|
|
|
|
|
def _month_keys(start_iso: str, end_iso: str) -> list[str]:
|
|
current = dt.date.fromisoformat(start_iso).replace(day=1)
|
|
end_month = dt.date.fromisoformat(end_iso).replace(day=1)
|
|
result: list[str] = []
|
|
while current <= end_month:
|
|
result.append(current.strftime("%Y-%m"))
|
|
if current.month == 12:
|
|
current = current.replace(
|
|
year=current.year + 1,
|
|
month=1,
|
|
)
|
|
else:
|
|
current = current.replace(month=current.month + 1)
|
|
return result
|
|
|
|
|
|
def _partition_year(partition_key: str) -> Optional[int]:
|
|
match = re.search(
|
|
r"(?:^|[/_])year=(\d{4})(?:$|[/_])",
|
|
partition_key,
|
|
)
|
|
return int(match.group(1)) if match else None
|
|
|
|
|
|
def _partition_month(partition_key: str) -> Optional[str]:
|
|
match = re.search(
|
|
r"(?:^|[/_])month=(\d{4}-\d{2})(?:$|[/_])",
|
|
partition_key,
|
|
)
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def _require_partition_values(
|
|
jobs: Sequence[Mapping[str, Any]],
|
|
*,
|
|
api_name: str,
|
|
expected: Sequence[Any],
|
|
extractor: Any,
|
|
) -> None:
|
|
observed = {
|
|
value
|
|
for value in (
|
|
extractor(str(job["partition_key"])) for job in jobs
|
|
)
|
|
if value is not None
|
|
}
|
|
missing = [value for value in expected if value not in observed]
|
|
if missing:
|
|
raise TushareMirrorError(
|
|
f"{api_name} is missing required partitions: {missing[:12]}"
|
|
)
|
|
|
|
|
|
def tushare_to_qlib_symbol(value: str) -> str:
|
|
"""Convert ``600000.SH`` into Qlib's ``SH600000`` spelling."""
|
|
|
|
normalized = str(value or "").strip().upper()
|
|
match = re.fullmatch(r"(\d{6})\.(SH|SZ|BJ)", normalized)
|
|
if match is None:
|
|
raise ValueError(f"unsupported Tushare A-share symbol: {value!r}")
|
|
code, exchange = match.groups()
|
|
return f"{exchange}{code}"
|
|
|
|
|
|
def _looks_like_a_share(value: str) -> bool:
|
|
normalized = str(value or "").strip().upper()
|
|
match = re.fullmatch(r"(\d{6})\.(SH|SZ|BJ)", normalized)
|
|
if match is None:
|
|
return False
|
|
code, exchange = match.groups()
|
|
if exchange == "SH":
|
|
return code.startswith(("600", "601", "603", "605", "688", "689"))
|
|
if exchange == "SZ":
|
|
return code.startswith(
|
|
("000", "001", "002", "003", "300", "301", "302")
|
|
)
|
|
return code[0] in {"4", "8", "9"}
|
|
|
|
|
|
def _market_name(value: str) -> str:
|
|
normalized = str(value or "").strip().lower()
|
|
if re.fullmatch(r"[a-z][a-z0-9_]{0,63}", normalized) is None:
|
|
raise ValueError("market_name must match [a-z][a-z0-9_]{0,63}")
|
|
return normalized
|
|
|
|
|
|
def _mirror_root(value: str | Path) -> Path:
|
|
root = Path(value).expanduser().resolve()
|
|
if not root.is_dir():
|
|
raise FileNotFoundError(f"Tushare mirror root does not exist: {root}")
|
|
state = root / "data/state.sqlite3"
|
|
parquet = root / "data/parquet"
|
|
if not state.is_file() or not parquet.is_dir():
|
|
raise TushareMirrorError(
|
|
"mirror root must contain data/state.sqlite3 and data/parquet"
|
|
)
|
|
return root
|
|
|
|
|
|
def _read_jobs(root: Path) -> list[Dict[str, Any]]:
|
|
state = root / "data/state.sqlite3"
|
|
connection = sqlite3.connect(f"file:{state}?mode=ro", uri=True)
|
|
connection.row_factory = sqlite3.Row
|
|
try:
|
|
connection.execute("BEGIN")
|
|
columns = {
|
|
str(row["name"])
|
|
for row in connection.execute("PRAGMA table_info(jobs)").fetchall()
|
|
}
|
|
optional_columns = ", ".join(
|
|
(
|
|
name
|
|
if name in columns
|
|
else f"NULL AS {name}"
|
|
)
|
|
for name in ("params_json", "fields")
|
|
)
|
|
rows = connection.execute(
|
|
f"""
|
|
SELECT id, api_name, partition_key, status, row_count, byte_count,
|
|
sha256, file_path, started_at, completed_at, error_code,
|
|
{optional_columns}
|
|
FROM jobs
|
|
ORDER BY id
|
|
"""
|
|
).fetchall()
|
|
finally:
|
|
connection.close()
|
|
return [dict(row) for row in rows]
|
|
|
|
|
|
def _latest_completed_jobs(
|
|
jobs: Sequence[Mapping[str, Any]],
|
|
) -> tuple[list[Dict[str, Any]], list[Dict[str, Any]]]:
|
|
"""Select the newest completed version of every logical partition.
|
|
|
|
The mirror replaces a partition file in place when it is refreshed. Older
|
|
completed rows can therefore carry a perfectly valid historical checksum
|
|
for bytes which are no longer present at ``file_path``. Treating every
|
|
completed database row as a live file produces false corruption failures.
|
|
"""
|
|
|
|
newest: Dict[tuple[str, str], Mapping[str, Any]] = {}
|
|
completed = [
|
|
job for job in jobs if str(job.get("status")) == "completed"
|
|
]
|
|
for job in completed:
|
|
key = (str(job["api_name"]), str(job["partition_key"]))
|
|
incumbent = newest.get(key)
|
|
if incumbent is None or int(job["id"]) > int(incumbent["id"]):
|
|
newest[key] = job
|
|
selected_ids = {int(job["id"]) for job in newest.values()}
|
|
selected = [
|
|
dict(job)
|
|
for job in completed
|
|
if int(job["id"]) in selected_ids
|
|
]
|
|
shadowed = [
|
|
dict(job)
|
|
for job in completed
|
|
if int(job["id"]) not in selected_ids
|
|
]
|
|
return selected, shadowed
|
|
|
|
|
|
def _job_path(root: Path, job: Mapping[str, Any]) -> Path:
|
|
raw = str(job.get("file_path") or "").strip()
|
|
if not raw:
|
|
raise TushareMirrorError(
|
|
f"completed job has no file_path: {job.get('api_name')}/"
|
|
f"{job.get('partition_key')}"
|
|
)
|
|
path = Path(raw).expanduser()
|
|
if not path.is_absolute():
|
|
path = root / path
|
|
resolved = path.resolve()
|
|
data_root = (root / "data").resolve()
|
|
try:
|
|
resolved.relative_to(data_root)
|
|
except ValueError as exc:
|
|
raise TushareMirrorError(
|
|
f"job file escapes mirror data root: {resolved}"
|
|
) from exc
|
|
return resolved
|
|
|
|
|
|
def _relative_job_path(root: Path, path: Path) -> str:
|
|
return path.relative_to(root.resolve()).as_posix()
|
|
|
|
|
|
def _load_parquet_module() -> Any:
|
|
try:
|
|
import pyarrow.parquet as parquet
|
|
except ImportError as exc:
|
|
raise TushareMirrorError(
|
|
"pyarrow is required; install requirements/research-py312.txt"
|
|
) from exc
|
|
return parquet
|
|
|
|
|
|
def _load_pandas() -> Any:
|
|
try:
|
|
import pandas
|
|
except ImportError as exc:
|
|
raise TushareMirrorError(
|
|
"pandas is required; install requirements/research-py312.txt"
|
|
) from exc
|
|
return pandas
|
|
|
|
|
|
def _verify_completed_job(
|
|
root: Path,
|
|
job: Mapping[str, Any],
|
|
*,
|
|
parquet: Any,
|
|
) -> Dict[str, Any]:
|
|
if job.get("status") != "completed":
|
|
raise TushareMirrorError("only completed mirror jobs may be verified")
|
|
path = _job_path(root, job)
|
|
if not path.is_file():
|
|
raise TushareMirrorError(f"completed job file is missing: {path}")
|
|
expected_bytes = int(job.get("byte_count") or -1)
|
|
actual_bytes = path.stat().st_size
|
|
if actual_bytes != expected_bytes:
|
|
raise TushareMirrorError(
|
|
f"byte_count mismatch for {path.name}: "
|
|
f"{actual_bytes} != {expected_bytes}"
|
|
)
|
|
expected_sha = str(job.get("sha256") or "")
|
|
actual_sha = _sha256_file(path)
|
|
if actual_sha != expected_sha:
|
|
raise TushareMirrorError(f"SHA-256 mismatch for {path.name}")
|
|
parquet_file = parquet.ParquetFile(path)
|
|
actual_rows = int(parquet_file.metadata.num_rows)
|
|
expected_rows = int(job.get("row_count") or 0)
|
|
if actual_rows != expected_rows:
|
|
raise TushareMirrorError(
|
|
f"row_count mismatch for {path.name}: "
|
|
f"{actual_rows} != {expected_rows}"
|
|
)
|
|
footer = {}
|
|
for raw_key, raw_value in (parquet_file.metadata.metadata or {}).items():
|
|
try:
|
|
key = raw_key.decode("utf-8")
|
|
value = raw_value.decode("utf-8")
|
|
except (AttributeError, UnicodeDecodeError):
|
|
continue
|
|
if key != "ARROW:schema":
|
|
footer[key] = value
|
|
footer_api = footer.get("tushare_api")
|
|
footer_partition = footer.get("partition_key")
|
|
if footer_api and footer_api != str(job["api_name"]):
|
|
raise TushareMirrorError(
|
|
f"Parquet tushare_api mismatch for {path.name}"
|
|
)
|
|
if footer_partition and footer_partition != str(job["partition_key"]):
|
|
raise TushareMirrorError(
|
|
f"Parquet partition_key mismatch for {path.name}"
|
|
)
|
|
query_sha256 = footer.get("query_sha256")
|
|
if query_sha256 and re.fullmatch(r"[0-9a-f]{64}", query_sha256) is None:
|
|
raise TushareMirrorError(
|
|
f"invalid Parquet query_sha256 for {path.name}"
|
|
)
|
|
raw_params = job.get("params_json")
|
|
try:
|
|
params = json.loads(str(raw_params)) if raw_params is not None else None
|
|
except json.JSONDecodeError as exc:
|
|
raise TushareMirrorError(
|
|
f"invalid params_json for completed job {job.get('id')}"
|
|
) from exc
|
|
request_sha256 = hashlib.sha256(
|
|
_canonical_json_bytes(
|
|
{
|
|
"api_name": str(job["api_name"]),
|
|
"partition_key": str(job["partition_key"]),
|
|
"fields": str(job.get("fields") or ""),
|
|
"params": params,
|
|
}
|
|
)
|
|
).hexdigest()
|
|
return {
|
|
"job_id": int(job["id"]),
|
|
"api_name": str(job["api_name"]),
|
|
"partition_key": str(job["partition_key"]),
|
|
"path": _relative_job_path(root, path),
|
|
"row_count": actual_rows,
|
|
"byte_count": actual_bytes,
|
|
"sha256": actual_sha,
|
|
"query_sha256": query_sha256,
|
|
"request_sha256": request_sha256,
|
|
"downloaded_at": footer.get("downloaded_at"),
|
|
"completed_at": job.get("completed_at"),
|
|
}
|
|
|
|
|
|
def inventory_mirror(
|
|
mirror_root: str | Path,
|
|
*,
|
|
verify_files: bool = True,
|
|
) -> Dict[str, Any]:
|
|
"""Return a token-free point-in-time inventory of the mirror."""
|
|
|
|
root = _mirror_root(mirror_root)
|
|
jobs = _read_jobs(root)
|
|
completed, shadowed = _latest_completed_jobs(jobs)
|
|
statuses: Dict[str, int] = {}
|
|
tables: Dict[str, Dict[str, Any]] = {}
|
|
for job in jobs:
|
|
status = str(job["status"])
|
|
statuses[status] = statuses.get(status, 0) + 1
|
|
for job in completed:
|
|
api = str(job["api_name"])
|
|
table = tables.setdefault(
|
|
api,
|
|
{
|
|
"files": 0,
|
|
"rows": 0,
|
|
"bytes": 0,
|
|
"first_partition": None,
|
|
"last_partition": None,
|
|
},
|
|
)
|
|
partition = str(job["partition_key"])
|
|
table["files"] += 1
|
|
table["rows"] += int(job.get("row_count") or 0)
|
|
table["bytes"] += int(job.get("byte_count") or 0)
|
|
values = [
|
|
item
|
|
for item in (table["first_partition"], table["last_partition"])
|
|
if item is not None
|
|
] + [partition]
|
|
table["first_partition"] = min(values)
|
|
table["last_partition"] = max(values)
|
|
|
|
verified = 0
|
|
if verify_files:
|
|
parquet = _load_parquet_module()
|
|
for job in completed:
|
|
_verify_completed_job(root, job, parquet=parquet)
|
|
verified += 1
|
|
running = [
|
|
{
|
|
"api_name": str(job["api_name"]),
|
|
"partition_key": str(job["partition_key"]),
|
|
"status": str(job["status"]),
|
|
"started_at": job.get("started_at"),
|
|
}
|
|
for job in jobs
|
|
if job["status"] != "completed"
|
|
]
|
|
completed_apis = set(tables)
|
|
missing_production_tables = [
|
|
value
|
|
for value in REQUIRED_PRODUCTION_TABLES
|
|
if value not in completed_apis
|
|
]
|
|
return {
|
|
"schema_version": 1,
|
|
"artifact_type": "quant_os_tushare_mirror_inventory",
|
|
"mirror_root": str(root),
|
|
"observed_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"job_status_counts": dict(sorted(statuses.items())),
|
|
"tables": dict(sorted(tables.items())),
|
|
"verified_completed_files": verified,
|
|
"selected_completed_partitions": len(completed),
|
|
"shadowed_completed_versions": len(shadowed),
|
|
"shadowed_completed_versions_by_api": dict(
|
|
sorted(
|
|
{
|
|
api: sum(
|
|
1 for job in shadowed if str(job["api_name"]) == api
|
|
)
|
|
for api in {str(job["api_name"]) for job in shadowed}
|
|
}.items()
|
|
)
|
|
),
|
|
"noncompleted_jobs": running,
|
|
"missing_production_tables": missing_production_tables,
|
|
"coverage_claim": "completed_partitions_only_not_full_mirror",
|
|
"production_ready": False,
|
|
"production_readiness_reason": (
|
|
"job inventory cannot prove planned partition/date coverage, "
|
|
"point-in-time semantics, adjustment completeness or entitlement"
|
|
),
|
|
"token_read": False,
|
|
"investment_value_claim": False,
|
|
}
|
|
|
|
|
|
def _write_lines(path: Path, lines: Iterable[str]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def _write_feature(
|
|
path: Path,
|
|
start_index: int,
|
|
values: Sequence[float],
|
|
) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
payload = array("f", [float(start_index), *(float(value) for value in values)])
|
|
if sys.byteorder != "little":
|
|
payload.byteswap()
|
|
path.write_bytes(payload.tobytes())
|
|
|
|
|
|
def _tree_evidence(root: Path) -> Dict[str, Any]:
|
|
records = []
|
|
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
|
if path.name == MANIFEST_NAME:
|
|
continue
|
|
relative = path.relative_to(root).as_posix()
|
|
records.append(
|
|
{
|
|
"path": relative,
|
|
"byte_count": path.stat().st_size,
|
|
"sha256": _sha256_file(path),
|
|
}
|
|
)
|
|
digest = hashlib.sha256(_canonical_bytes({"files": records})).hexdigest()
|
|
return {
|
|
"provider_tree_sha256": digest,
|
|
"provider_file_count": len(records),
|
|
"provider_byte_count": sum(item["byte_count"] for item in records),
|
|
}
|
|
|
|
|
|
def _month_overlaps(partition: str, start: str, end: str) -> bool:
|
|
match = re.search(r"month=(\d{4})-(\d{2})", partition)
|
|
if match is None:
|
|
return False
|
|
month = "".join(match.groups())
|
|
return _yyyymmdd(start)[:6] <= month <= _yyyymmdd(end)[:6]
|
|
|
|
|
|
def _year_overlaps(partition: str, start: str, end: str) -> bool:
|
|
match = re.search(r"year=(\d{4})", partition)
|
|
if match is None:
|
|
return False
|
|
year = match.group(1)
|
|
return start[:4] <= year <= end[:4]
|
|
|
|
|
|
def _partition_code_matches(
|
|
partition: str,
|
|
*,
|
|
key: str,
|
|
code: str,
|
|
) -> bool:
|
|
encoded = re.sub(r"[^A-Za-z0-9-]+", "_", code.strip().upper())
|
|
return partition.startswith(f"{key}={encoded}/")
|
|
|
|
|
|
def _normalize_index_code(value: str, *, name: str) -> str:
|
|
normalized = str(value or "").strip().upper()
|
|
if re.fullmatch(r"\d{6}\.(SH|SZ|BJ)", normalized) is None:
|
|
raise ValueError(f"{name} must use Tushare 000905.SH form")
|
|
return normalized
|
|
|
|
|
|
def _source_job_sort_key(job: Mapping[str, Any]) -> tuple[str, str, int]:
|
|
return (
|
|
str(job["api_name"]),
|
|
str(job["partition_key"]),
|
|
int(job["id"]),
|
|
)
|
|
|
|
|
|
def _data_version_payload(
|
|
*,
|
|
source_jobs: Sequence[Mapping[str, Any]],
|
|
build_parameters: Mapping[str, Any],
|
|
converter_sha256: str,
|
|
schema_version: int,
|
|
) -> Dict[str, Any]:
|
|
if schema_version == 1:
|
|
return {
|
|
"source_jobs": [dict(value) for value in source_jobs],
|
|
"range": [
|
|
build_parameters["start"],
|
|
build_parameters["end"],
|
|
],
|
|
"market": build_parameters["market_name"],
|
|
"benchmark": build_parameters["benchmark_symbol"],
|
|
"minimum_observations": build_parameters["minimum_observations"],
|
|
"allow_unadjusted": build_parameters["allow_unadjusted"],
|
|
"price_adjustment": "raw_unadjusted_factor_1",
|
|
"ohlc_policy": build_parameters["ohlc_policy"],
|
|
"converter_sha256": converter_sha256,
|
|
}
|
|
if schema_version != 2:
|
|
raise ValueError(
|
|
f"unsupported provider manifest schema_version: {schema_version}"
|
|
)
|
|
return {
|
|
"source_jobs": [dict(value) for value in source_jobs],
|
|
"build_parameters": dict(build_parameters),
|
|
"converter_sha256": converter_sha256,
|
|
}
|
|
|
|
|
|
def _eligible_a_share(
|
|
ts_code: str,
|
|
metadata: Mapping[str, Mapping[str, Any]],
|
|
) -> bool:
|
|
if not _looks_like_a_share(ts_code):
|
|
return False
|
|
row = metadata.get(ts_code)
|
|
if row is None:
|
|
return True
|
|
currency = str(row.get("curr_type") or "").strip().upper()
|
|
return not currency or currency == "CNY"
|
|
|
|
|
|
def build_qlib_provider(
|
|
mirror_root: str | Path,
|
|
output_dir: str | Path,
|
|
*,
|
|
start: str,
|
|
end: str,
|
|
market_name: str = "tushare_a",
|
|
benchmark_symbol: str = "SH999999",
|
|
minimum_observations: int = 60,
|
|
allow_unadjusted: bool = False,
|
|
expand_range: bool = False,
|
|
benchmark_index_code: Optional[str] = None,
|
|
universe_index_code: Optional[str] = None,
|
|
expected_constituent_count: Optional[int] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Freeze completed daily partitions into a Qlib 0.9.7 binary provider."""
|
|
|
|
start_iso = _iso_date(start, name="start")
|
|
end_iso = _iso_date(end, name="end")
|
|
if start_iso > end_iso:
|
|
raise ValueError("start must be on or before end")
|
|
market = _market_name(market_name)
|
|
benchmark_code = (
|
|
_normalize_index_code(
|
|
benchmark_index_code,
|
|
name="benchmark_index_code",
|
|
)
|
|
if benchmark_index_code is not None
|
|
else None
|
|
)
|
|
universe_code = (
|
|
_normalize_index_code(
|
|
universe_index_code,
|
|
name="universe_index_code",
|
|
)
|
|
if universe_index_code is not None
|
|
else None
|
|
)
|
|
if universe_code is not None and benchmark_code is None:
|
|
raise ValueError(
|
|
"universe_index_code requires benchmark_index_code; a PIT "
|
|
"universe must not use the full-range synthetic benchmark"
|
|
)
|
|
if expected_constituent_count is None and universe_code is not None:
|
|
resolved_constituent_count = KNOWN_INDEX_CONSTITUENT_COUNTS.get(
|
|
universe_code
|
|
)
|
|
elif expected_constituent_count is None:
|
|
resolved_constituent_count = None
|
|
else:
|
|
resolved_constituent_count = int(expected_constituent_count)
|
|
if resolved_constituent_count < 2:
|
|
raise ValueError("expected_constituent_count must be at least 2")
|
|
if universe_code is not None and resolved_constituent_count is None:
|
|
raise ValueError(
|
|
"unknown universe index requires expected_constituent_count"
|
|
)
|
|
if benchmark_code is None:
|
|
benchmark = str(benchmark_symbol).strip().upper()
|
|
if re.fullmatch(r"(SH|SZ|BJ)\d{6}", benchmark) is None:
|
|
raise ValueError("benchmark_symbol must use Qlib SH999999 form")
|
|
else:
|
|
benchmark = tushare_to_qlib_symbol(benchmark_code)
|
|
if int(minimum_observations) < 2:
|
|
raise ValueError("minimum_observations must be at least 2")
|
|
|
|
root = _mirror_root(mirror_root)
|
|
output = Path(output_dir).expanduser().resolve()
|
|
if output.exists():
|
|
raise FileExistsError(f"output_dir already exists: {output}")
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
jobs = _read_jobs(root)
|
|
completed, shadowed = _latest_completed_jobs(jobs)
|
|
daily_jobs = [
|
|
job
|
|
for job in completed
|
|
if job["api_name"] == "daily"
|
|
and _month_overlaps(str(job["partition_key"]), start_iso, end_iso)
|
|
]
|
|
calendar_jobs = [
|
|
job
|
|
for job in completed
|
|
if job["api_name"] == "trade_cal"
|
|
and _year_overlaps(str(job["partition_key"]), start_iso, end_iso)
|
|
]
|
|
stock_jobs = [
|
|
job for job in completed if job["api_name"] == "stock_basic"
|
|
]
|
|
adjustment_jobs = [
|
|
job
|
|
for job in completed
|
|
if job["api_name"] == "adj_factor"
|
|
and _month_overlaps(str(job["partition_key"]), start_iso, end_iso)
|
|
]
|
|
benchmark_jobs = (
|
|
[
|
|
job
|
|
for job in completed
|
|
if job["api_name"] == "index_daily"
|
|
and _partition_code_matches(
|
|
str(job["partition_key"]),
|
|
key="ts_code",
|
|
code=benchmark_code,
|
|
)
|
|
and _year_overlaps(str(job["partition_key"]), start_iso, end_iso)
|
|
]
|
|
if benchmark_code is not None
|
|
else []
|
|
)
|
|
universe_jobs = (
|
|
[
|
|
job
|
|
for job in completed
|
|
if job["api_name"] == "index_weight"
|
|
and _partition_code_matches(
|
|
str(job["partition_key"]),
|
|
key="index_code",
|
|
code=universe_code,
|
|
)
|
|
and (
|
|
(match := re.search(
|
|
r"year=(\d{4})",
|
|
str(job["partition_key"]),
|
|
))
|
|
is not None
|
|
and str(int(start_iso[:4]) - 1)
|
|
<= match.group(1)
|
|
<= end_iso[:4]
|
|
)
|
|
]
|
|
if universe_code is not None
|
|
else []
|
|
)
|
|
if not daily_jobs or not calendar_jobs or not stock_jobs:
|
|
raise TushareMirrorError(
|
|
"completed daily, trade_cal and stock_basic jobs are required"
|
|
)
|
|
if benchmark_code is not None and not benchmark_jobs:
|
|
raise TushareMirrorError(
|
|
f"no completed index_daily partitions for {benchmark_code}"
|
|
)
|
|
if universe_code is not None and not universe_jobs:
|
|
raise TushareMirrorError(
|
|
f"no completed index_weight partitions for {universe_code}"
|
|
)
|
|
expected_months = _month_keys(start_iso, end_iso)
|
|
expected_years = list(
|
|
range(int(start_iso[:4]), int(end_iso[:4]) + 1)
|
|
)
|
|
_require_partition_values(
|
|
daily_jobs,
|
|
api_name="daily",
|
|
expected=expected_months,
|
|
extractor=_partition_month,
|
|
)
|
|
_require_partition_values(
|
|
calendar_jobs,
|
|
api_name="trade_cal",
|
|
expected=expected_years,
|
|
extractor=_partition_year,
|
|
)
|
|
if not allow_unadjusted:
|
|
_require_partition_values(
|
|
adjustment_jobs,
|
|
api_name="adj_factor",
|
|
expected=expected_months,
|
|
extractor=_partition_month,
|
|
)
|
|
if benchmark_code is not None:
|
|
_require_partition_values(
|
|
benchmark_jobs,
|
|
api_name="index_daily",
|
|
expected=expected_years,
|
|
extractor=_partition_year,
|
|
)
|
|
if universe_code is not None:
|
|
_require_partition_values(
|
|
universe_jobs,
|
|
api_name="index_weight",
|
|
expected=[expected_years[0] - 1, *expected_years],
|
|
extractor=_partition_year,
|
|
)
|
|
|
|
parquet = _load_parquet_module()
|
|
selected_jobs_by_id = {
|
|
int(job["id"]): job
|
|
for job in (
|
|
daily_jobs
|
|
+ calendar_jobs
|
|
+ stock_jobs
|
|
+ adjustment_jobs
|
|
+ benchmark_jobs
|
|
+ universe_jobs
|
|
)
|
|
}
|
|
selected_jobs = sorted(
|
|
selected_jobs_by_id.values(),
|
|
key=_source_job_sort_key,
|
|
)
|
|
source_jobs = [
|
|
_verify_completed_job(root, job, parquet=parquet)
|
|
for job in selected_jobs
|
|
]
|
|
pandas = _load_pandas()
|
|
universe_prefilter_codes: Optional[set[str]] = None
|
|
if universe_code is not None:
|
|
prefilter_frames = [
|
|
pandas.read_parquet(
|
|
_job_path(root, job),
|
|
columns=["index_code", "con_code", "trade_date"],
|
|
)
|
|
for job in universe_jobs
|
|
]
|
|
prefilter_weights = pandas.concat(
|
|
prefilter_frames,
|
|
ignore_index=True,
|
|
)
|
|
prefilter_weights["index_code"] = (
|
|
prefilter_weights["index_code"].astype(str).str.upper()
|
|
)
|
|
prefilter_weights["con_code"] = (
|
|
prefilter_weights["con_code"].astype(str).str.upper()
|
|
)
|
|
prefilter_weights["trade_date"] = (
|
|
prefilter_weights["trade_date"].astype(str)
|
|
)
|
|
universe_prefilter_codes = set(
|
|
prefilter_weights.loc[
|
|
(prefilter_weights["index_code"] == universe_code)
|
|
& (prefilter_weights["trade_date"] <= _yyyymmdd(end_iso)),
|
|
"con_code",
|
|
].tolist()
|
|
)
|
|
if not universe_prefilter_codes:
|
|
raise TushareMirrorError(
|
|
f"index_weight has no constituents through {end_iso} "
|
|
f"for {universe_code}"
|
|
)
|
|
daily_frames = []
|
|
for job in daily_jobs:
|
|
frame = pandas.read_parquet(_job_path(root, job))
|
|
if universe_prefilter_codes is not None:
|
|
if "ts_code" not in frame.columns:
|
|
raise TushareMirrorError(
|
|
"daily partitions miss required field: ts_code"
|
|
)
|
|
frame = frame[
|
|
frame["ts_code"].astype(str).isin(universe_prefilter_codes)
|
|
]
|
|
daily_frames.append(frame)
|
|
daily = pandas.concat(
|
|
daily_frames,
|
|
ignore_index=True,
|
|
)
|
|
missing = REQUIRED_DAILY_FIELDS.difference(daily.columns)
|
|
if missing:
|
|
raise TushareMirrorError(
|
|
f"daily partitions miss required fields: {sorted(missing)}"
|
|
)
|
|
daily["trade_date"] = daily["trade_date"].astype(str)
|
|
start_raw, end_raw = _yyyymmdd(start_iso), _yyyymmdd(end_iso)
|
|
daily = daily[
|
|
(daily["trade_date"] >= start_raw)
|
|
& (daily["trade_date"] <= end_raw)
|
|
].copy()
|
|
if daily.empty:
|
|
raise TushareMirrorError("requested range contains no completed daily rows")
|
|
if daily.duplicated(["ts_code", "trade_date"]).any():
|
|
raise TushareMirrorError("daily contains duplicate ts_code/trade_date keys")
|
|
|
|
numeric_fields = (
|
|
"open",
|
|
"high",
|
|
"low",
|
|
"close",
|
|
"pre_close",
|
|
"pct_chg",
|
|
"vol",
|
|
"amount",
|
|
)
|
|
for field in numeric_fields:
|
|
daily[field] = pandas.to_numeric(daily[field], errors="coerce")
|
|
if daily[list(numeric_fields)].isna().any().any():
|
|
raise TushareMirrorError("daily contains null/non-numeric core values")
|
|
if (daily[["open", "high", "low", "close", "pre_close"]] <= 0).any().any():
|
|
raise TushareMirrorError("daily contains non-positive price values")
|
|
if (daily[["vol", "amount"]] < 0).any().any():
|
|
raise TushareMirrorError("daily contains negative volume or amount")
|
|
|
|
high_low_bad = daily["high"] < daily["low"]
|
|
envelope_bad = (
|
|
high_low_bad
|
|
| (daily["open"] > daily["high"])
|
|
| (daily["open"] < daily["low"])
|
|
| (daily["close"] > daily["high"])
|
|
| (daily["close"] < daily["low"])
|
|
)
|
|
envelope_bad_count = int(envelope_bad.sum())
|
|
anomaly_keys = (
|
|
daily.loc[envelope_bad, ["ts_code", "trade_date"]]
|
|
.sort_values(["trade_date", "ts_code"])
|
|
.astype(str)
|
|
.to_dict(orient="records")
|
|
)
|
|
anomaly_keys_sha256 = hashlib.sha256(
|
|
_canonical_bytes({"keys": anomaly_keys})
|
|
).hexdigest()
|
|
if envelope_bad_count and not expand_range:
|
|
raise TushareMirrorError(
|
|
f"{envelope_bad_count} OHLC envelope anomalies found; "
|
|
"use expand_range=True only for an explicitly recorded repair"
|
|
)
|
|
if envelope_bad_count:
|
|
original_ohlc = daily[["open", "high", "low", "close"]].copy()
|
|
daily["high"] = original_ohlc.max(axis=1)
|
|
daily["low"] = original_ohlc.min(axis=1)
|
|
|
|
calendar_frame = pandas.concat(
|
|
[pandas.read_parquet(_job_path(root, job)) for job in calendar_jobs],
|
|
ignore_index=True,
|
|
)
|
|
calendar_frame["cal_date"] = calendar_frame["cal_date"].astype(str)
|
|
open_mask = calendar_frame["is_open"].astype(str).isin({"1", "1.0", "True"})
|
|
exchange_mask = calendar_frame["exchange"].astype(str).eq("SSE")
|
|
calendar_raw = sorted(
|
|
set(
|
|
calendar_frame.loc[
|
|
open_mask
|
|
& exchange_mask
|
|
& (calendar_frame["cal_date"] >= start_raw)
|
|
& (calendar_frame["cal_date"] <= end_raw),
|
|
"cal_date",
|
|
].tolist()
|
|
)
|
|
)
|
|
if not calendar_raw:
|
|
raise TushareMirrorError("trade_cal has no open dates in requested range")
|
|
observed_dates = set(daily["trade_date"].tolist())
|
|
missing_market_dates = sorted(set(calendar_raw).difference(observed_dates))
|
|
unexpected_dates = sorted(observed_dates.difference(calendar_raw))
|
|
if missing_market_dates or unexpected_dates:
|
|
raise TushareMirrorError(
|
|
"daily/trade_cal mismatch: "
|
|
f"missing={missing_market_dates[:5]}, "
|
|
f"unexpected={unexpected_dates[:5]}"
|
|
)
|
|
|
|
stock = pandas.concat(
|
|
[pandas.read_parquet(_job_path(root, job)) for job in stock_jobs],
|
|
ignore_index=True,
|
|
)
|
|
if stock.duplicated(["ts_code"]).any():
|
|
raise TushareMirrorError("stock_basic contains duplicate ts_code keys")
|
|
metadata = {
|
|
str(row["ts_code"]): row.to_dict()
|
|
for _, row in stock.iterrows()
|
|
}
|
|
daily = daily[
|
|
daily["ts_code"].map(
|
|
lambda value: _eligible_a_share(str(value), metadata)
|
|
)
|
|
].copy()
|
|
calendar = [
|
|
dt.datetime.strptime(value, "%Y%m%d").date().isoformat()
|
|
for value in calendar_raw
|
|
]
|
|
calendar_index = {value: index for index, value in enumerate(calendar_raw)}
|
|
|
|
membership_lines: list[str] = []
|
|
universe_summary: Dict[str, Any]
|
|
if universe_code is None:
|
|
counts = daily.groupby("ts_code").size()
|
|
selected_codes = sorted(
|
|
str(value)
|
|
for value in counts[counts >= int(minimum_observations)].index
|
|
)
|
|
universe_summary = {
|
|
"kind": "full_range_minimum_observations",
|
|
"point_in_time": False,
|
|
"index_code": None,
|
|
"snapshot_count": 0,
|
|
"selection_uses_future_range_count": True,
|
|
}
|
|
else:
|
|
weights = pandas.concat(
|
|
[
|
|
pandas.read_parquet(_job_path(root, job))
|
|
for job in universe_jobs
|
|
],
|
|
ignore_index=True,
|
|
)
|
|
required_weight_fields = {
|
|
"index_code",
|
|
"con_code",
|
|
"trade_date",
|
|
"weight",
|
|
}
|
|
missing_weight_fields = required_weight_fields.difference(
|
|
weights.columns
|
|
)
|
|
if missing_weight_fields:
|
|
raise TushareMirrorError(
|
|
"index_weight partitions miss required fields: "
|
|
f"{sorted(missing_weight_fields)}"
|
|
)
|
|
weights["index_code"] = weights["index_code"].astype(str).str.upper()
|
|
weights["con_code"] = weights["con_code"].astype(str).str.upper()
|
|
weights["trade_date"] = weights["trade_date"].astype(str)
|
|
weights = weights[
|
|
(weights["index_code"] == universe_code)
|
|
& (weights["trade_date"] <= end_raw)
|
|
].copy()
|
|
if weights.empty:
|
|
raise TushareMirrorError(
|
|
f"index_weight has no snapshots through {end_iso} "
|
|
f"for {universe_code}"
|
|
)
|
|
weights["weight"] = pandas.to_numeric(
|
|
weights["weight"],
|
|
errors="coerce",
|
|
)
|
|
if (
|
|
weights["weight"].isna().any()
|
|
or (~weights["weight"].map(math.isfinite)).any()
|
|
or (weights["weight"] <= 0).any()
|
|
):
|
|
raise TushareMirrorError(
|
|
"index_weight contains null/non-finite/non-positive weights"
|
|
)
|
|
if weights.duplicated(["trade_date", "con_code"]).any():
|
|
raise TushareMirrorError(
|
|
"index_weight contains duplicate trade_date/con_code keys"
|
|
)
|
|
snapshot_dates_all = sorted(set(weights["trade_date"].tolist()))
|
|
anchor_candidates = [
|
|
value for value in snapshot_dates_all if value < start_raw
|
|
]
|
|
in_range_snapshot_dates = [
|
|
value
|
|
for value in snapshot_dates_all
|
|
if start_raw <= value <= end_raw
|
|
]
|
|
if not anchor_candidates:
|
|
raise TushareMirrorError(
|
|
"index_weight has no pre-start snapshot anchor; same-session "
|
|
"or future backfill is forbidden"
|
|
)
|
|
unknown_snapshot_dates = sorted(
|
|
set(in_range_snapshot_dates).difference(calendar_raw)
|
|
)
|
|
if unknown_snapshot_dates:
|
|
raise TushareMirrorError(
|
|
"index_weight snapshots are outside the SSE calendar: "
|
|
f"{unknown_snapshot_dates[:5]}"
|
|
)
|
|
observed_snapshot_dates = [
|
|
max(anchor_candidates),
|
|
*in_range_snapshot_dates,
|
|
]
|
|
parsed_snapshot_dates = [
|
|
dt.datetime.strptime(value, "%Y%m%d").date()
|
|
for value in observed_snapshot_dates
|
|
]
|
|
snapshot_gaps = [
|
|
(right - left).days
|
|
for left, right in zip(
|
|
parsed_snapshot_dates,
|
|
parsed_snapshot_dates[1:],
|
|
)
|
|
]
|
|
tail_gap = (
|
|
dt.date.fromisoformat(end_iso) - parsed_snapshot_dates[-1]
|
|
).days
|
|
max_snapshot_gap = max([*snapshot_gaps, tail_gap], default=tail_gap)
|
|
if max_snapshot_gap > 45:
|
|
raise TushareMirrorError(
|
|
"index_weight snapshot cadence has a gap greater than "
|
|
f"45 calendar days: {max_snapshot_gap}"
|
|
)
|
|
snapshot_members: Dict[str, list[str]] = {}
|
|
snapshot_stats = []
|
|
for snapshot_date in observed_snapshot_dates:
|
|
frame = weights[weights["trade_date"] == snapshot_date]
|
|
members = sorted(set(frame["con_code"].tolist()))
|
|
if not members:
|
|
raise TushareMirrorError(
|
|
f"index_weight snapshot {snapshot_date} is empty"
|
|
)
|
|
if len(members) != resolved_constituent_count:
|
|
raise TushareMirrorError(
|
|
f"index_weight snapshot {snapshot_date} has "
|
|
f"{len(members)} constituents; expected "
|
|
f"{resolved_constituent_count}"
|
|
)
|
|
weight_sum = float(frame["weight"].sum())
|
|
if abs(weight_sum - 100.0) > 0.5:
|
|
raise TushareMirrorError(
|
|
f"index_weight snapshot {snapshot_date} weight sum "
|
|
f"is {weight_sum}; expected 100 +/- 0.5"
|
|
)
|
|
for code in members:
|
|
try:
|
|
tushare_to_qlib_symbol(code)
|
|
except ValueError as exc:
|
|
raise TushareMirrorError(
|
|
f"invalid index constituent code {code!r}"
|
|
) from exc
|
|
snapshot_members[snapshot_date] = members
|
|
if snapshot_date < start_raw:
|
|
effective_from = calendar[0]
|
|
else:
|
|
snapshot_index = calendar_index[snapshot_date]
|
|
effective_from = (
|
|
calendar[snapshot_index + 1]
|
|
if snapshot_index + 1 < len(calendar)
|
|
else None
|
|
)
|
|
snapshot_stats.append(
|
|
{
|
|
"trade_date": snapshot_date,
|
|
"constituent_count": len(members),
|
|
"weight_sum": weight_sum,
|
|
"effective_from": effective_from,
|
|
}
|
|
)
|
|
effective_snapshots: list[tuple[str, int]] = [
|
|
(observed_snapshot_dates[0], 0)
|
|
]
|
|
for snapshot_date in in_range_snapshot_dates:
|
|
snapshot_index = calendar_index[snapshot_date]
|
|
first_index = snapshot_index + 1
|
|
if first_index < len(calendar):
|
|
effective_snapshots.append((snapshot_date, first_index))
|
|
if not effective_snapshots:
|
|
raise TushareMirrorError(
|
|
"index_weight has no roster effective in the requested range"
|
|
)
|
|
selected_codes = sorted(
|
|
{
|
|
code
|
|
for snapshot_date, _ in effective_snapshots
|
|
for code in snapshot_members[snapshot_date]
|
|
}
|
|
)
|
|
daily_codes = set(daily["ts_code"].astype(str))
|
|
missing_daily_codes = sorted(set(selected_codes).difference(daily_codes))
|
|
if missing_daily_codes:
|
|
raise TushareMirrorError(
|
|
"index_weight constituents have no daily rows in requested "
|
|
f"range: {missing_daily_codes[:10]}"
|
|
)
|
|
intervals: Dict[str, list[tuple[int, int]]] = {}
|
|
for position, (snapshot_date, first_index) in enumerate(
|
|
effective_snapshots
|
|
):
|
|
if position + 1 < len(effective_snapshots):
|
|
last_index = effective_snapshots[position + 1][1] - 1
|
|
else:
|
|
last_index = len(calendar_raw) - 1
|
|
if last_index < first_index:
|
|
continue
|
|
for code in snapshot_members[snapshot_date]:
|
|
bucket = intervals.setdefault(code, [])
|
|
if bucket and first_index <= bucket[-1][1] + 1:
|
|
bucket[-1] = (bucket[-1][0], last_index)
|
|
else:
|
|
bucket.append((first_index, last_index))
|
|
for code in sorted(intervals):
|
|
qlib_symbol = tushare_to_qlib_symbol(code)
|
|
for first_index, last_index in intervals[code]:
|
|
membership_lines.append(
|
|
f"{qlib_symbol}\t"
|
|
f"{calendar[first_index]}\t{calendar[last_index]}"
|
|
)
|
|
universe_summary = {
|
|
"kind": "tushare_index_weight_snapshots",
|
|
"point_in_time": True,
|
|
"index_code": universe_code,
|
|
"snapshot_count": len(observed_snapshot_dates),
|
|
"effective_snapshot_count": len(effective_snapshots),
|
|
"snapshot_constituent_counts": snapshot_stats,
|
|
"expected_constituent_count": resolved_constituent_count,
|
|
"snapshot_max_gap_calendar_days": max_snapshot_gap,
|
|
"weight_sum_tolerance": 0.5,
|
|
"selected_code_semantics": "union_of_effective_snapshots",
|
|
"membership_interval_semantics": (
|
|
"latest snapshot strictly before requested start is anchored "
|
|
"at requested start; every in-range snapshot becomes "
|
|
"effective on the next provider trading session and remains "
|
|
"effective through the session before the next roster"
|
|
),
|
|
"availability_contract": (
|
|
"conservative next-session effectiveness because the source "
|
|
"does not expose a separately verified published_at timestamp"
|
|
),
|
|
"same_session_membership_use": False,
|
|
"coverage_start": calendar[0],
|
|
"selection_uses_future_range_count": False,
|
|
}
|
|
if len(selected_codes) < 2:
|
|
raise TushareMirrorError(
|
|
"fewer than two eligible instruments are available"
|
|
)
|
|
daily = daily[daily["ts_code"].isin(selected_codes)].copy()
|
|
|
|
adjustment_complete = False
|
|
missing_adjustment_keys = int(len(daily))
|
|
if adjustment_jobs:
|
|
adjustment_frames = []
|
|
for job in adjustment_jobs:
|
|
frame = pandas.read_parquet(_job_path(root, job))
|
|
if {"ts_code", "trade_date"}.issubset(frame.columns):
|
|
frame = frame[
|
|
frame["ts_code"].astype(str).isin(selected_codes)
|
|
& (frame["trade_date"].astype(str) >= start_raw)
|
|
& (frame["trade_date"].astype(str) <= end_raw)
|
|
]
|
|
adjustment_frames.append(frame)
|
|
adjustments = pandas.concat(
|
|
adjustment_frames,
|
|
ignore_index=True,
|
|
)
|
|
required_adjustment_fields = {"ts_code", "trade_date", "adj_factor"}
|
|
missing_adjustment_fields = required_adjustment_fields.difference(
|
|
adjustments.columns
|
|
)
|
|
if missing_adjustment_fields:
|
|
raise TushareMirrorError(
|
|
"adj_factor partitions miss required fields: "
|
|
f"{sorted(missing_adjustment_fields)}"
|
|
)
|
|
adjustments["ts_code"] = adjustments["ts_code"].astype(str)
|
|
adjustments["trade_date"] = adjustments["trade_date"].astype(str)
|
|
adjustments = adjustments[
|
|
(adjustments["trade_date"] >= start_raw)
|
|
& (adjustments["trade_date"] <= end_raw)
|
|
& (adjustments["ts_code"].isin(selected_codes))
|
|
].copy()
|
|
if adjustments.duplicated(["ts_code", "trade_date"]).any():
|
|
raise TushareMirrorError(
|
|
"adj_factor contains duplicate ts_code/trade_date keys"
|
|
)
|
|
adjustments["adj_factor"] = pandas.to_numeric(
|
|
adjustments["adj_factor"],
|
|
errors="coerce",
|
|
)
|
|
if (
|
|
adjustments["adj_factor"].isna().any()
|
|
or (~adjustments["adj_factor"].map(math.isfinite)).any()
|
|
or (adjustments["adj_factor"] <= 0).any()
|
|
):
|
|
raise TushareMirrorError(
|
|
"adj_factor contains null/non-finite/non-positive values"
|
|
)
|
|
adjustment_keys = set(
|
|
zip(
|
|
adjustments["ts_code"],
|
|
adjustments["trade_date"],
|
|
)
|
|
)
|
|
daily_keys = set(zip(daily["ts_code"], daily["trade_date"]))
|
|
missing_adjustment_keys = len(daily_keys.difference(adjustment_keys))
|
|
adjustment_complete = missing_adjustment_keys == 0
|
|
if adjustment_complete:
|
|
daily = daily.merge(
|
|
adjustments[["ts_code", "trade_date", "adj_factor"]],
|
|
on=["ts_code", "trade_date"],
|
|
how="left",
|
|
validate="one_to_one",
|
|
)
|
|
if not adjustment_complete:
|
|
if not allow_unadjusted:
|
|
raise TushareMirrorError(
|
|
"complete adj_factor coverage is unavailable "
|
|
f"({missing_adjustment_keys} daily keys missing); pass "
|
|
"allow_unadjusted=True only for an explicitly non-investment "
|
|
"technical run"
|
|
)
|
|
daily["multiplier"] = 1.0
|
|
price_adjustment_mode = "raw_unadjusted"
|
|
else:
|
|
daily["adj_factor"] = pandas.to_numeric(
|
|
daily["adj_factor"],
|
|
errors="raise",
|
|
)
|
|
daily = daily.sort_values(["ts_code", "trade_date"]).copy()
|
|
anchor_factor = daily.groupby("ts_code")["adj_factor"].transform(
|
|
"first"
|
|
)
|
|
daily["multiplier"] = daily["adj_factor"] / anchor_factor
|
|
if (
|
|
daily["multiplier"].isna().any()
|
|
or (~daily["multiplier"].map(math.isfinite)).any()
|
|
or (daily["multiplier"] <= 0).any()
|
|
):
|
|
raise TushareMirrorError(
|
|
"computed first-observation-anchored adjustment multipliers "
|
|
"are invalid"
|
|
)
|
|
price_adjustment_mode = (
|
|
"cumulative_adjusted_first_observation_anchor"
|
|
)
|
|
|
|
daily = daily.sort_values(["ts_code", "trade_date"]).copy()
|
|
daily["_adjusted_close"] = daily["close"] * daily["multiplier"]
|
|
daily["_adjusted_change"] = daily.groupby("ts_code")[
|
|
"_adjusted_close"
|
|
].pct_change(fill_method=None)
|
|
daily["qlib_symbol"] = daily["ts_code"].map(tushare_to_qlib_symbol)
|
|
symbol_pairs = (
|
|
daily[["ts_code", "qlib_symbol"]].drop_duplicates().sort_values("ts_code")
|
|
)
|
|
if symbol_pairs["qlib_symbol"].duplicated().any():
|
|
raise TushareMirrorError("Tushare symbols collide after Qlib conversion")
|
|
if benchmark in set(symbol_pairs["qlib_symbol"]):
|
|
raise TushareMirrorError("benchmark_symbol collides with a stock")
|
|
|
|
pct_recomputed = (
|
|
(daily["close"] - daily["pre_close"]) / daily["pre_close"] * 100.0
|
|
)
|
|
pct_discrepancy_count = int(
|
|
(pct_recomputed - daily["pct_chg"]).abs().gt(0.02).sum()
|
|
)
|
|
missing_metadata = sorted(set(selected_codes).difference(metadata))
|
|
|
|
temporary = Path(
|
|
tempfile.mkdtemp(prefix=f".{output.name}.", dir=str(output.parent))
|
|
)
|
|
try:
|
|
_write_lines(temporary / "calendars/day.txt", calendar)
|
|
instrument_lines = []
|
|
stock_symbols = []
|
|
for qlib_symbol, group in daily.groupby("qlib_symbol", sort=True):
|
|
ordered = group.sort_values("trade_date")
|
|
first_raw = str(ordered["trade_date"].iloc[0])
|
|
last_raw = str(ordered["trade_date"].iloc[-1])
|
|
first_index = calendar_index[first_raw]
|
|
last_index = calendar_index[last_raw]
|
|
length = last_index - first_index + 1
|
|
fields = {
|
|
name: [float("nan")] * length for name in QLIB_FIELDS
|
|
}
|
|
for _, row in ordered.iterrows():
|
|
offset = calendar_index[str(row["trade_date"])] - first_index
|
|
volume_shares = float(row["vol"]) * 100.0
|
|
money_cny = float(row["amount"]) * 1000.0
|
|
raw_vwap = (
|
|
money_cny / volume_shares
|
|
if volume_shares > 0.0
|
|
else float(row["close"])
|
|
)
|
|
multiplier = float(row["multiplier"])
|
|
values = {
|
|
"open": float(row["open"]) * multiplier,
|
|
"high": float(row["high"]) * multiplier,
|
|
"low": float(row["low"]) * multiplier,
|
|
"close": float(row["close"]) * multiplier,
|
|
"vwap": raw_vwap * multiplier,
|
|
"volume": volume_shares / multiplier,
|
|
"money": money_cny,
|
|
"factor": multiplier,
|
|
"change": float(row["_adjusted_change"]),
|
|
}
|
|
for field, value in values.items():
|
|
fields[field][offset] = value
|
|
for field, values in fields.items():
|
|
_write_feature(
|
|
temporary
|
|
/ "features"
|
|
/ str(qlib_symbol).lower()
|
|
/ f"{field}.day.bin",
|
|
first_index,
|
|
values,
|
|
)
|
|
first_iso = dt.datetime.strptime(first_raw, "%Y%m%d").date().isoformat()
|
|
last_iso = dt.datetime.strptime(last_raw, "%Y%m%d").date().isoformat()
|
|
instrument_lines.append(f"{qlib_symbol}\t{first_iso}\t{last_iso}")
|
|
stock_symbols.append(str(qlib_symbol))
|
|
|
|
benchmark_fields = {name: [] for name in QLIB_FIELDS}
|
|
if benchmark_code is None:
|
|
returns = (
|
|
daily.groupby("trade_date")["_adjusted_change"]
|
|
.mean()
|
|
.to_dict()
|
|
)
|
|
level = 100.0
|
|
for raw_date in calendar_raw:
|
|
change = float(returns.get(raw_date, 0.0))
|
|
if not math.isfinite(change):
|
|
change = 0.0
|
|
if change <= -1.0:
|
|
raise TushareMirrorError(
|
|
f"invalid derived benchmark return on {raw_date}"
|
|
)
|
|
level *= 1.0 + change
|
|
benchmark_fields["open"].append(level)
|
|
benchmark_fields["high"].append(level)
|
|
benchmark_fields["low"].append(level)
|
|
benchmark_fields["close"].append(level)
|
|
benchmark_fields["vwap"].append(level)
|
|
benchmark_fields["volume"].append(100_000_000.0)
|
|
benchmark_fields["money"].append(level * 100_000_000.0)
|
|
benchmark_fields["factor"].append(1.0)
|
|
benchmark_fields["change"].append(change)
|
|
benchmark_manifest = {
|
|
"symbol": benchmark,
|
|
"kind": "synthetic_equal_weight",
|
|
"method": (
|
|
"daily arithmetic mean of eligible stock cumulative-"
|
|
"adjusted "
|
|
"close pct_change observations"
|
|
),
|
|
"index_code": None,
|
|
"real_index_claim": False,
|
|
}
|
|
else:
|
|
index_daily = pandas.concat(
|
|
[
|
|
pandas.read_parquet(_job_path(root, job))
|
|
for job in benchmark_jobs
|
|
],
|
|
ignore_index=True,
|
|
)
|
|
required_index_fields = {
|
|
"ts_code",
|
|
"trade_date",
|
|
"open",
|
|
"high",
|
|
"low",
|
|
"close",
|
|
"pre_close",
|
|
"vol",
|
|
"amount",
|
|
}
|
|
missing_index_fields = required_index_fields.difference(
|
|
index_daily.columns
|
|
)
|
|
if missing_index_fields:
|
|
raise TushareMirrorError(
|
|
"index_daily partitions miss required fields: "
|
|
f"{sorted(missing_index_fields)}"
|
|
)
|
|
index_daily["ts_code"] = (
|
|
index_daily["ts_code"].astype(str).str.upper()
|
|
)
|
|
index_daily["trade_date"] = index_daily["trade_date"].astype(str)
|
|
index_daily = index_daily[
|
|
(index_daily["ts_code"] == benchmark_code)
|
|
& (index_daily["trade_date"] >= start_raw)
|
|
& (index_daily["trade_date"] <= end_raw)
|
|
].copy()
|
|
if index_daily.duplicated(["trade_date"]).any():
|
|
raise TushareMirrorError(
|
|
"index_daily contains duplicate benchmark trade_date keys"
|
|
)
|
|
actual_index_dates = set(index_daily["trade_date"].tolist())
|
|
missing_index_dates = sorted(
|
|
set(calendar_raw).difference(actual_index_dates)
|
|
)
|
|
unexpected_index_dates = sorted(
|
|
actual_index_dates.difference(calendar_raw)
|
|
)
|
|
if missing_index_dates or unexpected_index_dates:
|
|
raise TushareMirrorError(
|
|
"index_daily/trade_cal mismatch: "
|
|
f"missing={missing_index_dates[:5]}, "
|
|
f"unexpected={unexpected_index_dates[:5]}"
|
|
)
|
|
index_numeric_fields = (
|
|
"open",
|
|
"high",
|
|
"low",
|
|
"close",
|
|
"pre_close",
|
|
"vol",
|
|
"amount",
|
|
)
|
|
for field in index_numeric_fields:
|
|
index_daily[field] = pandas.to_numeric(
|
|
index_daily[field],
|
|
errors="coerce",
|
|
)
|
|
if index_daily[list(index_numeric_fields)].isna().any().any():
|
|
raise TushareMirrorError(
|
|
"index_daily contains null/non-numeric core values"
|
|
)
|
|
if (
|
|
index_daily[["open", "high", "low", "close", "pre_close"]]
|
|
<= 0
|
|
).any().any():
|
|
raise TushareMirrorError(
|
|
"index_daily contains non-positive price values"
|
|
)
|
|
if (index_daily[["vol", "amount"]] < 0).any().any():
|
|
raise TushareMirrorError(
|
|
"index_daily contains negative volume or amount"
|
|
)
|
|
index_daily = index_daily.set_index("trade_date").loc[calendar_raw]
|
|
index_change = index_daily["close"].pct_change(fill_method=None)
|
|
for position, raw_date in enumerate(calendar_raw):
|
|
row = index_daily.loc[raw_date]
|
|
volume_shares = float(row["vol"]) * 100.0
|
|
money_cny = float(row["amount"]) * 1000.0
|
|
vwap = (
|
|
money_cny / volume_shares
|
|
if volume_shares > 0.0
|
|
else float(row["close"])
|
|
)
|
|
benchmark_fields["open"].append(float(row["open"]))
|
|
benchmark_fields["high"].append(float(row["high"]))
|
|
benchmark_fields["low"].append(float(row["low"]))
|
|
benchmark_fields["close"].append(float(row["close"]))
|
|
benchmark_fields["vwap"].append(vwap)
|
|
benchmark_fields["volume"].append(volume_shares)
|
|
benchmark_fields["money"].append(money_cny)
|
|
benchmark_fields["factor"].append(1.0)
|
|
change = index_change.iloc[position]
|
|
benchmark_fields["change"].append(float(change))
|
|
benchmark_manifest = {
|
|
"symbol": benchmark,
|
|
"kind": "tushare_index_daily",
|
|
"method": "raw index OHLCV from completed index_daily jobs",
|
|
"index_code": benchmark_code,
|
|
"real_index_claim": True,
|
|
}
|
|
for field, values in benchmark_fields.items():
|
|
_write_feature(
|
|
temporary
|
|
/ "features"
|
|
/ benchmark.lower()
|
|
/ f"{field}.day.bin",
|
|
0,
|
|
values,
|
|
)
|
|
benchmark_line = (
|
|
f"{benchmark}\t{calendar[0]}\t{calendar[-1]}"
|
|
)
|
|
market_instrument_lines = (
|
|
membership_lines if universe_code is not None else instrument_lines
|
|
)
|
|
_write_lines(
|
|
temporary / f"instruments/{market}.txt",
|
|
market_instrument_lines,
|
|
)
|
|
_write_lines(
|
|
temporary / "instruments/all.txt",
|
|
[*instrument_lines, benchmark_line],
|
|
)
|
|
|
|
jobs_after_read = _read_jobs(root)
|
|
completed_after_read, _ = _latest_completed_jobs(jobs_after_read)
|
|
latest_after_by_key = {
|
|
(str(job["api_name"]), str(job["partition_key"])): int(job["id"])
|
|
for job in completed_after_read
|
|
}
|
|
changed_source_jobs = [
|
|
{
|
|
"api_name": str(job["api_name"]),
|
|
"partition_key": str(job["partition_key"]),
|
|
"selected_job_id": int(job["id"]),
|
|
"latest_job_id": latest_after_by_key.get(
|
|
(
|
|
str(job["api_name"]),
|
|
str(job["partition_key"]),
|
|
)
|
|
),
|
|
}
|
|
for job in selected_jobs
|
|
if latest_after_by_key.get(
|
|
(
|
|
str(job["api_name"]),
|
|
str(job["partition_key"]),
|
|
)
|
|
)
|
|
!= int(job["id"])
|
|
]
|
|
if changed_source_jobs:
|
|
raise TushareMirrorError(
|
|
"selected source jobs changed while the provider was being "
|
|
f"built: {changed_source_jobs[:5]}"
|
|
)
|
|
source_jobs_after_read = [
|
|
_verify_completed_job(root, job, parquet=parquet)
|
|
for job in selected_jobs
|
|
]
|
|
if source_jobs_after_read != source_jobs:
|
|
raise TushareMirrorError(
|
|
"source bytes or provenance changed while the provider was "
|
|
"being built"
|
|
)
|
|
|
|
tree = _tree_evidence(temporary)
|
|
completed_apis = {str(job["api_name"]) for job in completed}
|
|
converter_sha256 = _sha256_file(Path(__file__).resolve())
|
|
build_parameters = {
|
|
"start": start_iso,
|
|
"end": end_iso,
|
|
"market_name": market,
|
|
"benchmark_symbol": benchmark,
|
|
"benchmark_index_code": benchmark_code,
|
|
"benchmark_mode": (
|
|
"tushare_index_daily"
|
|
if benchmark_code is not None
|
|
else "synthetic_equal_weight"
|
|
),
|
|
"universe_index_code": universe_code,
|
|
"universe_mode": (
|
|
"tushare_index_weight_pit"
|
|
if universe_code is not None
|
|
else "full_range_minimum_observations"
|
|
),
|
|
"expected_constituent_count": resolved_constituent_count,
|
|
"membership_availability": (
|
|
"next_provider_session"
|
|
if universe_code is not None
|
|
else None
|
|
),
|
|
"minimum_observations": int(minimum_observations),
|
|
"minimum_observations_applied": universe_code is None,
|
|
"allow_unadjusted": bool(allow_unadjusted),
|
|
"price_adjustment_mode": price_adjustment_mode,
|
|
"ohlc_policy": "expand_range" if expand_range else "fail",
|
|
}
|
|
version_payload = _data_version_payload(
|
|
source_jobs=source_jobs,
|
|
build_parameters=build_parameters,
|
|
converter_sha256=converter_sha256,
|
|
schema_version=2,
|
|
)
|
|
data_version = hashlib.sha256(
|
|
_canonical_bytes(version_payload)
|
|
).hexdigest()
|
|
status_counts: Dict[str, int] = {}
|
|
for job in jobs:
|
|
key = str(job["status"])
|
|
status_counts[key] = status_counts.get(key, 0) + 1
|
|
manifest: Dict[str, Any] = {
|
|
"schema_version": 2,
|
|
"artifact_type": "quant_os_tushare_qlib_provider",
|
|
"source": "tushare_local_mirror",
|
|
"created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"data_version": data_version,
|
|
**tree,
|
|
"converter": {
|
|
"source_sha256": converter_sha256,
|
|
"output_format": "qlib_0_9_7_day_binary",
|
|
},
|
|
"build_parameters": build_parameters,
|
|
"calendar": {
|
|
"frequency": "day",
|
|
"start": calendar[0],
|
|
"end": calendar[-1],
|
|
"sessions": len(calendar),
|
|
"source": "trade_cal/SSE/is_open=1",
|
|
},
|
|
"market": {
|
|
"name": market,
|
|
"instrument_count": len(stock_symbols),
|
|
"minimum_observations": int(minimum_observations),
|
|
"minimum_observations_applied": universe_code is None,
|
|
"benchmark_excluded": True,
|
|
**universe_summary,
|
|
},
|
|
"benchmark": benchmark_manifest,
|
|
"fields": list(QLIB_FIELDS),
|
|
"units": {
|
|
"volume": "shares; Tushare vol(hand) multiplied by 100",
|
|
"money": "CNY; Tushare amount(thousand CNY) multiplied by 1000",
|
|
},
|
|
"price_adjustment": {
|
|
"mode": price_adjustment_mode,
|
|
"formula": (
|
|
"multiplier=adj_factor/"
|
|
"first_adj_factor_for_symbol_in_requested_interval"
|
|
if adjustment_complete
|
|
else "factor=1"
|
|
),
|
|
"anchor_semantics": (
|
|
"per-symbol first observed adjustment factor is a "
|
|
"constant anchor; no future corporate action is used to "
|
|
"rescale earlier observations"
|
|
if adjustment_complete
|
|
else "not_applicable"
|
|
),
|
|
"coverage_complete": adjustment_complete,
|
|
"missing_daily_keys": missing_adjustment_keys,
|
|
"component_complete": adjustment_complete,
|
|
"production_eligible": False,
|
|
},
|
|
"quality": {
|
|
"duplicate_keys": 0,
|
|
"whole_market_calendar_gaps": 0,
|
|
"ohlc_envelope_anomalies": envelope_bad_count,
|
|
"ohlc_policy": "expand_range" if expand_range else "fail",
|
|
"ohlc_anomaly_keys_sha256": anomaly_keys_sha256,
|
|
"pct_chg_discrepancies_gt_0_02_percentage_points": (
|
|
pct_discrepancy_count
|
|
),
|
|
"selected_rows": int(len(daily)),
|
|
"selected_symbols_missing_stock_basic": missing_metadata,
|
|
"universe_selection": (
|
|
universe_summary["membership_interval_semantics"]
|
|
if universe_code is not None
|
|
else (
|
|
"eligible A-share symbols with at least "
|
|
f"{int(minimum_observations)} observations over the "
|
|
"full requested range; not point-in-time eligibility"
|
|
)
|
|
),
|
|
},
|
|
"source_jobs": source_jobs,
|
|
"source_stability": {
|
|
"post_read_file_verification": True,
|
|
"selected_jobs_still_latest": True,
|
|
"policy": (
|
|
"verify selected bytes before consumption, then re-read "
|
|
"the SQLite ledger and re-verify every selected file "
|
|
"after all provider inputs have been consumed"
|
|
),
|
|
},
|
|
"mirror_job_status_counts_at_build": dict(sorted(status_counts.items())),
|
|
"shadowed_completed_versions_at_build": len(shadowed),
|
|
"missing_production_tables": [
|
|
value
|
|
for value in MISSING_PRODUCTION_TABLES
|
|
if value not in completed_apis
|
|
],
|
|
"verification_boundary": (
|
|
"completed checksummed partitions and generated provider "
|
|
"integrity only; not point-in-time production market evidence"
|
|
),
|
|
"gate_credit": [],
|
|
"investment_value_claim": False,
|
|
}
|
|
(temporary / MANIFEST_NAME).write_bytes(_canonical_bytes(manifest))
|
|
os.replace(temporary, output)
|
|
except BaseException:
|
|
if temporary.exists():
|
|
shutil.rmtree(temporary)
|
|
raise
|
|
return {
|
|
"ok": True,
|
|
"output_dir": str(output),
|
|
"manifest": str(output / MANIFEST_NAME),
|
|
**manifest,
|
|
}
|
|
|
|
|
|
def verify_qlib_provider(provider_dir: str | Path) -> Dict[str, Any]:
|
|
"""Verify a provider tree against its Quant OS manifest."""
|
|
|
|
root = Path(provider_dir).expanduser().resolve()
|
|
if not root.is_dir():
|
|
raise FileNotFoundError(f"provider_dir does not exist: {root}")
|
|
manifest_path = root / MANIFEST_NAME
|
|
try:
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise TushareMirrorError(
|
|
f"cannot read provider manifest: {manifest_path}"
|
|
) from exc
|
|
if not isinstance(manifest, dict):
|
|
raise TushareMirrorError("provider manifest must be a JSON object")
|
|
if manifest.get("artifact_type") != "quant_os_tushare_qlib_provider":
|
|
raise TushareMirrorError("unexpected provider artifact_type")
|
|
build_parameters = manifest.get("build_parameters")
|
|
converter = manifest.get("converter")
|
|
source_jobs = manifest.get("source_jobs")
|
|
if (
|
|
not isinstance(build_parameters, Mapping)
|
|
or not isinstance(converter, Mapping)
|
|
or not isinstance(source_jobs, list)
|
|
):
|
|
raise TushareMirrorError(
|
|
"provider manifest misses build/data-version evidence"
|
|
)
|
|
try:
|
|
version_payload = _data_version_payload(
|
|
source_jobs=source_jobs,
|
|
build_parameters=build_parameters,
|
|
converter_sha256=str(converter["source_sha256"]),
|
|
schema_version=int(manifest.get("schema_version", 1)),
|
|
)
|
|
computed_data_version = hashlib.sha256(
|
|
_canonical_bytes(version_payload)
|
|
).hexdigest()
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise TushareMirrorError(
|
|
"provider build/data-version evidence is invalid"
|
|
) from exc
|
|
if computed_data_version != manifest.get("data_version"):
|
|
raise TushareMirrorError(
|
|
"provider data_version does not match source/build evidence"
|
|
)
|
|
schema_version = int(manifest.get("schema_version", 1))
|
|
if schema_version >= 2:
|
|
source_stability = manifest.get("source_stability")
|
|
if (
|
|
not isinstance(source_stability, Mapping)
|
|
or source_stability.get("post_read_file_verification") is not True
|
|
or source_stability.get("selected_jobs_still_latest") is not True
|
|
):
|
|
raise TushareMirrorError(
|
|
"schema-v2 provider lacks closed source-stability evidence"
|
|
)
|
|
tree = _tree_evidence(root)
|
|
for name in (
|
|
"provider_tree_sha256",
|
|
"provider_file_count",
|
|
"provider_byte_count",
|
|
):
|
|
if tree[name] != manifest.get(name):
|
|
raise TushareMirrorError(
|
|
f"provider tree mismatch for {name}: "
|
|
f"{tree[name]!r} != {manifest.get(name)!r}"
|
|
)
|
|
for required in ("calendars/day.txt", "instruments/all.txt"):
|
|
if not (root / required).is_file():
|
|
raise TushareMirrorError(f"provider misses {required}")
|
|
current_converter_sha256 = _sha256_file(Path(__file__).resolve())
|
|
recorded_converter_sha256 = str(converter["source_sha256"])
|
|
return {
|
|
"ok": True,
|
|
"data_version": manifest.get("data_version"),
|
|
**tree,
|
|
"manifest_sha256": _sha256_file(manifest_path),
|
|
"recorded_converter_sha256": recorded_converter_sha256,
|
|
"current_converter_sha256": current_converter_sha256,
|
|
"converter_source_matches_current": (
|
|
current_converter_sha256 == recorded_converter_sha256
|
|
),
|
|
"investment_value_claim": False,
|
|
}
|