"""Read-only Tushare mirror inventory and immutable Qlib provider builder. The adapter never calls Tushare and never reads a token. It trusts only jobs marked ``completed`` in the mirror SQLite database, verifies their recorded size/SHA-256/Parquet row count, and publishes a new Qlib directory without modifying the mirror. """ from __future__ import annotations from array import array import datetime as dt import hashlib import json import math import os from pathlib import Path import re import shutil import sqlite3 import sys import tempfile from typing import Any, Dict, Iterable, Mapping, Optional, Sequence MANIFEST_NAME = "quant_os_tushare_manifest.json" REQUIRED_DAILY_FIELDS = { "ts_code", "trade_date", "open", "high", "low", "close", "pre_close", "pct_chg", "vol", "amount", } QLIB_FIELDS = ( "open", "high", "low", "close", "vwap", "volume", "money", "factor", "change", ) MISSING_PRODUCTION_TABLES = ( "adj_factor", "daily_basic", "index_daily", "index_member_all", "index_weight", "stk_limit", "stock_st", "suspend_d", ) KNOWN_INDEX_CONSTITUENT_COUNTS = { "000016.SH": 50, "000300.SH": 300, "000905.SH": 500, "000852.SH": 1000, } REQUIRED_PRODUCTION_TABLES = ( "daily", "trade_cal", "stock_basic", *MISSING_PRODUCTION_TABLES, ) class TushareMirrorError(RuntimeError): """Raised when mirror or provider evidence fails closed.""" def _canonical_json_bytes(value: Mapping[str, Any]) -> bytes: return json.dumps( dict(value), ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False, ).encode("utf-8") def _canonical_bytes(value: Mapping[str, Any]) -> bytes: return _canonical_json_bytes(value) + b"\n" def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def _iso_date(value: str, *, name: str) -> str: normalized = str(value or "").strip() try: parsed = dt.date.fromisoformat(normalized) except ValueError as exc: raise ValueError(f"{name} must use YYYY-MM-DD") from exc if parsed.isoformat() != normalized: raise ValueError(f"{name} must use YYYY-MM-DD") return normalized def _yyyymmdd(value: str) -> str: return value.replace("-", "") def _month_keys(start_iso: str, end_iso: str) -> list[str]: current = dt.date.fromisoformat(start_iso).replace(day=1) end_month = dt.date.fromisoformat(end_iso).replace(day=1) result: list[str] = [] while current <= end_month: result.append(current.strftime("%Y-%m")) if current.month == 12: current = current.replace( year=current.year + 1, month=1, ) else: current = current.replace(month=current.month + 1) return result def _partition_year(partition_key: str) -> Optional[int]: match = re.search( r"(?:^|[/_])year=(\d{4})(?:$|[/_])", partition_key, ) return int(match.group(1)) if match else None def _partition_month(partition_key: str) -> Optional[str]: match = re.search( r"(?:^|[/_])month=(\d{4}-\d{2})(?:$|[/_])", partition_key, ) return match.group(1) if match else None def _require_partition_values( jobs: Sequence[Mapping[str, Any]], *, api_name: str, expected: Sequence[Any], extractor: Any, ) -> None: observed = { value for value in ( extractor(str(job["partition_key"])) for job in jobs ) if value is not None } missing = [value for value in expected if value not in observed] if missing: raise TushareMirrorError( f"{api_name} is missing required partitions: {missing[:12]}" ) def tushare_to_qlib_symbol(value: str) -> str: """Convert ``600000.SH`` into Qlib's ``SH600000`` spelling.""" normalized = str(value or "").strip().upper() match = re.fullmatch(r"(\d{6})\.(SH|SZ|BJ)", normalized) if match is None: raise ValueError(f"unsupported Tushare A-share symbol: {value!r}") code, exchange = match.groups() return f"{exchange}{code}" def _looks_like_a_share(value: str) -> bool: normalized = str(value or "").strip().upper() match = re.fullmatch(r"(\d{6})\.(SH|SZ|BJ)", normalized) if match is None: return False code, exchange = match.groups() if exchange == "SH": return code.startswith(("600", "601", "603", "605", "688", "689")) if exchange == "SZ": return code.startswith( ("000", "001", "002", "003", "300", "301", "302") ) return code[0] in {"4", "8", "9"} def _market_name(value: str) -> str: normalized = str(value or "").strip().lower() if re.fullmatch(r"[a-z][a-z0-9_]{0,63}", normalized) is None: raise ValueError("market_name must match [a-z][a-z0-9_]{0,63}") return normalized def _mirror_root(value: str | Path) -> Path: root = Path(value).expanduser().resolve() if not root.is_dir(): raise FileNotFoundError(f"Tushare mirror root does not exist: {root}") state = root / "data/state.sqlite3" parquet = root / "data/parquet" if not state.is_file() or not parquet.is_dir(): raise TushareMirrorError( "mirror root must contain data/state.sqlite3 and data/parquet" ) return root def _read_jobs(root: Path) -> list[Dict[str, Any]]: state = root / "data/state.sqlite3" connection = sqlite3.connect(f"file:{state}?mode=ro", uri=True) connection.row_factory = sqlite3.Row try: connection.execute("BEGIN") columns = { str(row["name"]) for row in connection.execute("PRAGMA table_info(jobs)").fetchall() } optional_columns = ", ".join( ( name if name in columns else f"NULL AS {name}" ) for name in ("params_json", "fields") ) rows = connection.execute( f""" SELECT id, api_name, partition_key, status, row_count, byte_count, sha256, file_path, started_at, completed_at, error_code, {optional_columns} FROM jobs ORDER BY id """ ).fetchall() finally: connection.close() return [dict(row) for row in rows] def _latest_completed_jobs( jobs: Sequence[Mapping[str, Any]], ) -> tuple[list[Dict[str, Any]], list[Dict[str, Any]]]: """Select the newest completed version of every logical partition. The mirror replaces a partition file in place when it is refreshed. Older completed rows can therefore carry a perfectly valid historical checksum for bytes which are no longer present at ``file_path``. Treating every completed database row as a live file produces false corruption failures. """ newest: Dict[tuple[str, str], Mapping[str, Any]] = {} completed = [ job for job in jobs if str(job.get("status")) == "completed" ] for job in completed: key = (str(job["api_name"]), str(job["partition_key"])) incumbent = newest.get(key) if incumbent is None or int(job["id"]) > int(incumbent["id"]): newest[key] = job selected_ids = {int(job["id"]) for job in newest.values()} selected = [ dict(job) for job in completed if int(job["id"]) in selected_ids ] shadowed = [ dict(job) for job in completed if int(job["id"]) not in selected_ids ] return selected, shadowed def _job_path(root: Path, job: Mapping[str, Any]) -> Path: raw = str(job.get("file_path") or "").strip() if not raw: raise TushareMirrorError( f"completed job has no file_path: {job.get('api_name')}/" f"{job.get('partition_key')}" ) path = Path(raw).expanduser() if not path.is_absolute(): path = root / path resolved = path.resolve() data_root = (root / "data").resolve() try: resolved.relative_to(data_root) except ValueError as exc: raise TushareMirrorError( f"job file escapes mirror data root: {resolved}" ) from exc return resolved def _relative_job_path(root: Path, path: Path) -> str: return path.relative_to(root.resolve()).as_posix() def _load_parquet_module() -> Any: try: import pyarrow.parquet as parquet except ImportError as exc: raise TushareMirrorError( "pyarrow is required; install requirements/research-py312.txt" ) from exc return parquet def _load_pandas() -> Any: try: import pandas except ImportError as exc: raise TushareMirrorError( "pandas is required; install requirements/research-py312.txt" ) from exc return pandas def _verify_completed_job( root: Path, job: Mapping[str, Any], *, parquet: Any, ) -> Dict[str, Any]: if job.get("status") != "completed": raise TushareMirrorError("only completed mirror jobs may be verified") path = _job_path(root, job) if not path.is_file(): raise TushareMirrorError(f"completed job file is missing: {path}") expected_bytes = int(job.get("byte_count") or -1) actual_bytes = path.stat().st_size if actual_bytes != expected_bytes: raise TushareMirrorError( f"byte_count mismatch for {path.name}: " f"{actual_bytes} != {expected_bytes}" ) expected_sha = str(job.get("sha256") or "") actual_sha = _sha256_file(path) if actual_sha != expected_sha: raise TushareMirrorError(f"SHA-256 mismatch for {path.name}") parquet_file = parquet.ParquetFile(path) actual_rows = int(parquet_file.metadata.num_rows) expected_rows = int(job.get("row_count") or 0) if actual_rows != expected_rows: raise TushareMirrorError( f"row_count mismatch for {path.name}: " f"{actual_rows} != {expected_rows}" ) footer = {} for raw_key, raw_value in (parquet_file.metadata.metadata or {}).items(): try: key = raw_key.decode("utf-8") value = raw_value.decode("utf-8") except (AttributeError, UnicodeDecodeError): continue if key != "ARROW:schema": footer[key] = value footer_api = footer.get("tushare_api") footer_partition = footer.get("partition_key") if footer_api and footer_api != str(job["api_name"]): raise TushareMirrorError( f"Parquet tushare_api mismatch for {path.name}" ) if footer_partition and footer_partition != str(job["partition_key"]): raise TushareMirrorError( f"Parquet partition_key mismatch for {path.name}" ) query_sha256 = footer.get("query_sha256") if query_sha256 and re.fullmatch(r"[0-9a-f]{64}", query_sha256) is None: raise TushareMirrorError( f"invalid Parquet query_sha256 for {path.name}" ) raw_params = job.get("params_json") try: params = json.loads(str(raw_params)) if raw_params is not None else None except json.JSONDecodeError as exc: raise TushareMirrorError( f"invalid params_json for completed job {job.get('id')}" ) from exc request_sha256 = hashlib.sha256( _canonical_json_bytes( { "api_name": str(job["api_name"]), "partition_key": str(job["partition_key"]), "fields": str(job.get("fields") or ""), "params": params, } ) ).hexdigest() return { "job_id": int(job["id"]), "api_name": str(job["api_name"]), "partition_key": str(job["partition_key"]), "path": _relative_job_path(root, path), "row_count": actual_rows, "byte_count": actual_bytes, "sha256": actual_sha, "query_sha256": query_sha256, "request_sha256": request_sha256, "downloaded_at": footer.get("downloaded_at"), "completed_at": job.get("completed_at"), } def inventory_mirror( mirror_root: str | Path, *, verify_files: bool = True, ) -> Dict[str, Any]: """Return a token-free point-in-time inventory of the mirror.""" root = _mirror_root(mirror_root) jobs = _read_jobs(root) completed, shadowed = _latest_completed_jobs(jobs) statuses: Dict[str, int] = {} tables: Dict[str, Dict[str, Any]] = {} for job in jobs: status = str(job["status"]) statuses[status] = statuses.get(status, 0) + 1 for job in completed: api = str(job["api_name"]) table = tables.setdefault( api, { "files": 0, "rows": 0, "bytes": 0, "first_partition": None, "last_partition": None, }, ) partition = str(job["partition_key"]) table["files"] += 1 table["rows"] += int(job.get("row_count") or 0) table["bytes"] += int(job.get("byte_count") or 0) values = [ item for item in (table["first_partition"], table["last_partition"]) if item is not None ] + [partition] table["first_partition"] = min(values) table["last_partition"] = max(values) verified = 0 if verify_files: parquet = _load_parquet_module() for job in completed: _verify_completed_job(root, job, parquet=parquet) verified += 1 running = [ { "api_name": str(job["api_name"]), "partition_key": str(job["partition_key"]), "status": str(job["status"]), "started_at": job.get("started_at"), } for job in jobs if job["status"] != "completed" ] completed_apis = set(tables) missing_production_tables = [ value for value in REQUIRED_PRODUCTION_TABLES if value not in completed_apis ] return { "schema_version": 1, "artifact_type": "quant_os_tushare_mirror_inventory", "mirror_root": str(root), "observed_at": dt.datetime.now(dt.timezone.utc).isoformat(), "job_status_counts": dict(sorted(statuses.items())), "tables": dict(sorted(tables.items())), "verified_completed_files": verified, "selected_completed_partitions": len(completed), "shadowed_completed_versions": len(shadowed), "shadowed_completed_versions_by_api": dict( sorted( { api: sum( 1 for job in shadowed if str(job["api_name"]) == api ) for api in {str(job["api_name"]) for job in shadowed} }.items() ) ), "noncompleted_jobs": running, "missing_production_tables": missing_production_tables, "coverage_claim": "completed_partitions_only_not_full_mirror", "production_ready": False, "production_readiness_reason": ( "job inventory cannot prove planned partition/date coverage, " "point-in-time semantics, adjustment completeness or entitlement" ), "token_read": False, "investment_value_claim": False, } def _write_lines(path: Path, lines: Iterable[str]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _write_feature( path: Path, start_index: int, values: Sequence[float], ) -> None: path.parent.mkdir(parents=True, exist_ok=True) payload = array("f", [float(start_index), *(float(value) for value in values)]) if sys.byteorder != "little": payload.byteswap() path.write_bytes(payload.tobytes()) def _tree_evidence(root: Path) -> Dict[str, Any]: records = [] for path in sorted(item for item in root.rglob("*") if item.is_file()): if path.name == MANIFEST_NAME: continue relative = path.relative_to(root).as_posix() records.append( { "path": relative, "byte_count": path.stat().st_size, "sha256": _sha256_file(path), } ) digest = hashlib.sha256(_canonical_bytes({"files": records})).hexdigest() return { "provider_tree_sha256": digest, "provider_file_count": len(records), "provider_byte_count": sum(item["byte_count"] for item in records), } def _month_overlaps(partition: str, start: str, end: str) -> bool: match = re.search(r"month=(\d{4})-(\d{2})", partition) if match is None: return False month = "".join(match.groups()) return _yyyymmdd(start)[:6] <= month <= _yyyymmdd(end)[:6] def _year_overlaps(partition: str, start: str, end: str) -> bool: match = re.search(r"year=(\d{4})", partition) if match is None: return False year = match.group(1) return start[:4] <= year <= end[:4] def _partition_code_matches( partition: str, *, key: str, code: str, ) -> bool: encoded = re.sub(r"[^A-Za-z0-9-]+", "_", code.strip().upper()) return partition.startswith(f"{key}={encoded}/") def _normalize_index_code(value: str, *, name: str) -> str: normalized = str(value or "").strip().upper() if re.fullmatch(r"\d{6}\.(SH|SZ|BJ)", normalized) is None: raise ValueError(f"{name} must use Tushare 000905.SH form") return normalized def _source_job_sort_key(job: Mapping[str, Any]) -> tuple[str, str, int]: return ( str(job["api_name"]), str(job["partition_key"]), int(job["id"]), ) def _data_version_payload( *, source_jobs: Sequence[Mapping[str, Any]], build_parameters: Mapping[str, Any], converter_sha256: str, schema_version: int, ) -> Dict[str, Any]: if schema_version == 1: return { "source_jobs": [dict(value) for value in source_jobs], "range": [ build_parameters["start"], build_parameters["end"], ], "market": build_parameters["market_name"], "benchmark": build_parameters["benchmark_symbol"], "minimum_observations": build_parameters["minimum_observations"], "allow_unadjusted": build_parameters["allow_unadjusted"], "price_adjustment": "raw_unadjusted_factor_1", "ohlc_policy": build_parameters["ohlc_policy"], "converter_sha256": converter_sha256, } if schema_version != 2: raise ValueError( f"unsupported provider manifest schema_version: {schema_version}" ) return { "source_jobs": [dict(value) for value in source_jobs], "build_parameters": dict(build_parameters), "converter_sha256": converter_sha256, } def _eligible_a_share( ts_code: str, metadata: Mapping[str, Mapping[str, Any]], ) -> bool: if not _looks_like_a_share(ts_code): return False row = metadata.get(ts_code) if row is None: return True currency = str(row.get("curr_type") or "").strip().upper() return not currency or currency == "CNY" def build_qlib_provider( mirror_root: str | Path, output_dir: str | Path, *, start: str, end: str, market_name: str = "tushare_a", benchmark_symbol: str = "SH999999", minimum_observations: int = 60, allow_unadjusted: bool = False, expand_range: bool = False, benchmark_index_code: Optional[str] = None, universe_index_code: Optional[str] = None, expected_constituent_count: Optional[int] = None, ) -> Dict[str, Any]: """Freeze completed daily partitions into a Qlib 0.9.7 binary provider.""" start_iso = _iso_date(start, name="start") end_iso = _iso_date(end, name="end") if start_iso > end_iso: raise ValueError("start must be on or before end") market = _market_name(market_name) benchmark_code = ( _normalize_index_code( benchmark_index_code, name="benchmark_index_code", ) if benchmark_index_code is not None else None ) universe_code = ( _normalize_index_code( universe_index_code, name="universe_index_code", ) if universe_index_code is not None else None ) if universe_code is not None and benchmark_code is None: raise ValueError( "universe_index_code requires benchmark_index_code; a PIT " "universe must not use the full-range synthetic benchmark" ) if expected_constituent_count is None and universe_code is not None: resolved_constituent_count = KNOWN_INDEX_CONSTITUENT_COUNTS.get( universe_code ) elif expected_constituent_count is None: resolved_constituent_count = None else: resolved_constituent_count = int(expected_constituent_count) if resolved_constituent_count < 2: raise ValueError("expected_constituent_count must be at least 2") if universe_code is not None and resolved_constituent_count is None: raise ValueError( "unknown universe index requires expected_constituent_count" ) if benchmark_code is None: benchmark = str(benchmark_symbol).strip().upper() if re.fullmatch(r"(SH|SZ|BJ)\d{6}", benchmark) is None: raise ValueError("benchmark_symbol must use Qlib SH999999 form") else: benchmark = tushare_to_qlib_symbol(benchmark_code) if int(minimum_observations) < 2: raise ValueError("minimum_observations must be at least 2") root = _mirror_root(mirror_root) output = Path(output_dir).expanduser().resolve() if output.exists(): raise FileExistsError(f"output_dir already exists: {output}") output.parent.mkdir(parents=True, exist_ok=True) jobs = _read_jobs(root) completed, shadowed = _latest_completed_jobs(jobs) daily_jobs = [ job for job in completed if job["api_name"] == "daily" and _month_overlaps(str(job["partition_key"]), start_iso, end_iso) ] calendar_jobs = [ job for job in completed if job["api_name"] == "trade_cal" and _year_overlaps(str(job["partition_key"]), start_iso, end_iso) ] stock_jobs = [ job for job in completed if job["api_name"] == "stock_basic" ] adjustment_jobs = [ job for job in completed if job["api_name"] == "adj_factor" and _month_overlaps(str(job["partition_key"]), start_iso, end_iso) ] benchmark_jobs = ( [ job for job in completed if job["api_name"] == "index_daily" and _partition_code_matches( str(job["partition_key"]), key="ts_code", code=benchmark_code, ) and _year_overlaps(str(job["partition_key"]), start_iso, end_iso) ] if benchmark_code is not None else [] ) universe_jobs = ( [ job for job in completed if job["api_name"] == "index_weight" and _partition_code_matches( str(job["partition_key"]), key="index_code", code=universe_code, ) and ( (match := re.search( r"year=(\d{4})", str(job["partition_key"]), )) is not None and str(int(start_iso[:4]) - 1) <= match.group(1) <= end_iso[:4] ) ] if universe_code is not None else [] ) if not daily_jobs or not calendar_jobs or not stock_jobs: raise TushareMirrorError( "completed daily, trade_cal and stock_basic jobs are required" ) if benchmark_code is not None and not benchmark_jobs: raise TushareMirrorError( f"no completed index_daily partitions for {benchmark_code}" ) if universe_code is not None and not universe_jobs: raise TushareMirrorError( f"no completed index_weight partitions for {universe_code}" ) expected_months = _month_keys(start_iso, end_iso) expected_years = list( range(int(start_iso[:4]), int(end_iso[:4]) + 1) ) _require_partition_values( daily_jobs, api_name="daily", expected=expected_months, extractor=_partition_month, ) _require_partition_values( calendar_jobs, api_name="trade_cal", expected=expected_years, extractor=_partition_year, ) if not allow_unadjusted: _require_partition_values( adjustment_jobs, api_name="adj_factor", expected=expected_months, extractor=_partition_month, ) if benchmark_code is not None: _require_partition_values( benchmark_jobs, api_name="index_daily", expected=expected_years, extractor=_partition_year, ) if universe_code is not None: _require_partition_values( universe_jobs, api_name="index_weight", expected=[expected_years[0] - 1, *expected_years], extractor=_partition_year, ) parquet = _load_parquet_module() selected_jobs_by_id = { int(job["id"]): job for job in ( daily_jobs + calendar_jobs + stock_jobs + adjustment_jobs + benchmark_jobs + universe_jobs ) } selected_jobs = sorted( selected_jobs_by_id.values(), key=_source_job_sort_key, ) source_jobs = [ _verify_completed_job(root, job, parquet=parquet) for job in selected_jobs ] pandas = _load_pandas() universe_prefilter_codes: Optional[set[str]] = None if universe_code is not None: prefilter_frames = [ pandas.read_parquet( _job_path(root, job), columns=["index_code", "con_code", "trade_date"], ) for job in universe_jobs ] prefilter_weights = pandas.concat( prefilter_frames, ignore_index=True, ) prefilter_weights["index_code"] = ( prefilter_weights["index_code"].astype(str).str.upper() ) prefilter_weights["con_code"] = ( prefilter_weights["con_code"].astype(str).str.upper() ) prefilter_weights["trade_date"] = ( prefilter_weights["trade_date"].astype(str) ) universe_prefilter_codes = set( prefilter_weights.loc[ (prefilter_weights["index_code"] == universe_code) & (prefilter_weights["trade_date"] <= _yyyymmdd(end_iso)), "con_code", ].tolist() ) if not universe_prefilter_codes: raise TushareMirrorError( f"index_weight has no constituents through {end_iso} " f"for {universe_code}" ) daily_frames = [] for job in daily_jobs: frame = pandas.read_parquet(_job_path(root, job)) if universe_prefilter_codes is not None: if "ts_code" not in frame.columns: raise TushareMirrorError( "daily partitions miss required field: ts_code" ) frame = frame[ frame["ts_code"].astype(str).isin(universe_prefilter_codes) ] daily_frames.append(frame) daily = pandas.concat( daily_frames, ignore_index=True, ) missing = REQUIRED_DAILY_FIELDS.difference(daily.columns) if missing: raise TushareMirrorError( f"daily partitions miss required fields: {sorted(missing)}" ) daily["trade_date"] = daily["trade_date"].astype(str) start_raw, end_raw = _yyyymmdd(start_iso), _yyyymmdd(end_iso) daily = daily[ (daily["trade_date"] >= start_raw) & (daily["trade_date"] <= end_raw) ].copy() if daily.empty: raise TushareMirrorError("requested range contains no completed daily rows") if daily.duplicated(["ts_code", "trade_date"]).any(): raise TushareMirrorError("daily contains duplicate ts_code/trade_date keys") numeric_fields = ( "open", "high", "low", "close", "pre_close", "pct_chg", "vol", "amount", ) for field in numeric_fields: daily[field] = pandas.to_numeric(daily[field], errors="coerce") if daily[list(numeric_fields)].isna().any().any(): raise TushareMirrorError("daily contains null/non-numeric core values") if (daily[["open", "high", "low", "close", "pre_close"]] <= 0).any().any(): raise TushareMirrorError("daily contains non-positive price values") if (daily[["vol", "amount"]] < 0).any().any(): raise TushareMirrorError("daily contains negative volume or amount") high_low_bad = daily["high"] < daily["low"] envelope_bad = ( high_low_bad | (daily["open"] > daily["high"]) | (daily["open"] < daily["low"]) | (daily["close"] > daily["high"]) | (daily["close"] < daily["low"]) ) envelope_bad_count = int(envelope_bad.sum()) anomaly_keys = ( daily.loc[envelope_bad, ["ts_code", "trade_date"]] .sort_values(["trade_date", "ts_code"]) .astype(str) .to_dict(orient="records") ) anomaly_keys_sha256 = hashlib.sha256( _canonical_bytes({"keys": anomaly_keys}) ).hexdigest() if envelope_bad_count and not expand_range: raise TushareMirrorError( f"{envelope_bad_count} OHLC envelope anomalies found; " "use expand_range=True only for an explicitly recorded repair" ) if envelope_bad_count: original_ohlc = daily[["open", "high", "low", "close"]].copy() daily["high"] = original_ohlc.max(axis=1) daily["low"] = original_ohlc.min(axis=1) calendar_frame = pandas.concat( [pandas.read_parquet(_job_path(root, job)) for job in calendar_jobs], ignore_index=True, ) calendar_frame["cal_date"] = calendar_frame["cal_date"].astype(str) open_mask = calendar_frame["is_open"].astype(str).isin({"1", "1.0", "True"}) exchange_mask = calendar_frame["exchange"].astype(str).eq("SSE") calendar_raw = sorted( set( calendar_frame.loc[ open_mask & exchange_mask & (calendar_frame["cal_date"] >= start_raw) & (calendar_frame["cal_date"] <= end_raw), "cal_date", ].tolist() ) ) if not calendar_raw: raise TushareMirrorError("trade_cal has no open dates in requested range") observed_dates = set(daily["trade_date"].tolist()) missing_market_dates = sorted(set(calendar_raw).difference(observed_dates)) unexpected_dates = sorted(observed_dates.difference(calendar_raw)) if missing_market_dates or unexpected_dates: raise TushareMirrorError( "daily/trade_cal mismatch: " f"missing={missing_market_dates[:5]}, " f"unexpected={unexpected_dates[:5]}" ) stock = pandas.concat( [pandas.read_parquet(_job_path(root, job)) for job in stock_jobs], ignore_index=True, ) if stock.duplicated(["ts_code"]).any(): raise TushareMirrorError("stock_basic contains duplicate ts_code keys") metadata = { str(row["ts_code"]): row.to_dict() for _, row in stock.iterrows() } daily = daily[ daily["ts_code"].map( lambda value: _eligible_a_share(str(value), metadata) ) ].copy() calendar = [ dt.datetime.strptime(value, "%Y%m%d").date().isoformat() for value in calendar_raw ] calendar_index = {value: index for index, value in enumerate(calendar_raw)} membership_lines: list[str] = [] universe_summary: Dict[str, Any] if universe_code is None: counts = daily.groupby("ts_code").size() selected_codes = sorted( str(value) for value in counts[counts >= int(minimum_observations)].index ) universe_summary = { "kind": "full_range_minimum_observations", "point_in_time": False, "index_code": None, "snapshot_count": 0, "selection_uses_future_range_count": True, } else: weights = pandas.concat( [ pandas.read_parquet(_job_path(root, job)) for job in universe_jobs ], ignore_index=True, ) required_weight_fields = { "index_code", "con_code", "trade_date", "weight", } missing_weight_fields = required_weight_fields.difference( weights.columns ) if missing_weight_fields: raise TushareMirrorError( "index_weight partitions miss required fields: " f"{sorted(missing_weight_fields)}" ) weights["index_code"] = weights["index_code"].astype(str).str.upper() weights["con_code"] = weights["con_code"].astype(str).str.upper() weights["trade_date"] = weights["trade_date"].astype(str) weights = weights[ (weights["index_code"] == universe_code) & (weights["trade_date"] <= end_raw) ].copy() if weights.empty: raise TushareMirrorError( f"index_weight has no snapshots through {end_iso} " f"for {universe_code}" ) weights["weight"] = pandas.to_numeric( weights["weight"], errors="coerce", ) if ( weights["weight"].isna().any() or (~weights["weight"].map(math.isfinite)).any() or (weights["weight"] <= 0).any() ): raise TushareMirrorError( "index_weight contains null/non-finite/non-positive weights" ) if weights.duplicated(["trade_date", "con_code"]).any(): raise TushareMirrorError( "index_weight contains duplicate trade_date/con_code keys" ) snapshot_dates_all = sorted(set(weights["trade_date"].tolist())) anchor_candidates = [ value for value in snapshot_dates_all if value < start_raw ] in_range_snapshot_dates = [ value for value in snapshot_dates_all if start_raw <= value <= end_raw ] if not anchor_candidates: raise TushareMirrorError( "index_weight has no pre-start snapshot anchor; same-session " "or future backfill is forbidden" ) unknown_snapshot_dates = sorted( set(in_range_snapshot_dates).difference(calendar_raw) ) if unknown_snapshot_dates: raise TushareMirrorError( "index_weight snapshots are outside the SSE calendar: " f"{unknown_snapshot_dates[:5]}" ) observed_snapshot_dates = [ max(anchor_candidates), *in_range_snapshot_dates, ] parsed_snapshot_dates = [ dt.datetime.strptime(value, "%Y%m%d").date() for value in observed_snapshot_dates ] snapshot_gaps = [ (right - left).days for left, right in zip( parsed_snapshot_dates, parsed_snapshot_dates[1:], ) ] tail_gap = ( dt.date.fromisoformat(end_iso) - parsed_snapshot_dates[-1] ).days max_snapshot_gap = max([*snapshot_gaps, tail_gap], default=tail_gap) if max_snapshot_gap > 45: raise TushareMirrorError( "index_weight snapshot cadence has a gap greater than " f"45 calendar days: {max_snapshot_gap}" ) snapshot_members: Dict[str, list[str]] = {} snapshot_stats = [] for snapshot_date in observed_snapshot_dates: frame = weights[weights["trade_date"] == snapshot_date] members = sorted(set(frame["con_code"].tolist())) if not members: raise TushareMirrorError( f"index_weight snapshot {snapshot_date} is empty" ) if len(members) != resolved_constituent_count: raise TushareMirrorError( f"index_weight snapshot {snapshot_date} has " f"{len(members)} constituents; expected " f"{resolved_constituent_count}" ) weight_sum = float(frame["weight"].sum()) if abs(weight_sum - 100.0) > 0.5: raise TushareMirrorError( f"index_weight snapshot {snapshot_date} weight sum " f"is {weight_sum}; expected 100 +/- 0.5" ) for code in members: try: tushare_to_qlib_symbol(code) except ValueError as exc: raise TushareMirrorError( f"invalid index constituent code {code!r}" ) from exc snapshot_members[snapshot_date] = members if snapshot_date < start_raw: effective_from = calendar[0] else: snapshot_index = calendar_index[snapshot_date] effective_from = ( calendar[snapshot_index + 1] if snapshot_index + 1 < len(calendar) else None ) snapshot_stats.append( { "trade_date": snapshot_date, "constituent_count": len(members), "weight_sum": weight_sum, "effective_from": effective_from, } ) effective_snapshots: list[tuple[str, int]] = [ (observed_snapshot_dates[0], 0) ] for snapshot_date in in_range_snapshot_dates: snapshot_index = calendar_index[snapshot_date] first_index = snapshot_index + 1 if first_index < len(calendar): effective_snapshots.append((snapshot_date, first_index)) if not effective_snapshots: raise TushareMirrorError( "index_weight has no roster effective in the requested range" ) selected_codes = sorted( { code for snapshot_date, _ in effective_snapshots for code in snapshot_members[snapshot_date] } ) daily_codes = set(daily["ts_code"].astype(str)) missing_daily_codes = sorted(set(selected_codes).difference(daily_codes)) if missing_daily_codes: raise TushareMirrorError( "index_weight constituents have no daily rows in requested " f"range: {missing_daily_codes[:10]}" ) intervals: Dict[str, list[tuple[int, int]]] = {} for position, (snapshot_date, first_index) in enumerate( effective_snapshots ): if position + 1 < len(effective_snapshots): last_index = effective_snapshots[position + 1][1] - 1 else: last_index = len(calendar_raw) - 1 if last_index < first_index: continue for code in snapshot_members[snapshot_date]: bucket = intervals.setdefault(code, []) if bucket and first_index <= bucket[-1][1] + 1: bucket[-1] = (bucket[-1][0], last_index) else: bucket.append((first_index, last_index)) for code in sorted(intervals): qlib_symbol = tushare_to_qlib_symbol(code) for first_index, last_index in intervals[code]: membership_lines.append( f"{qlib_symbol}\t" f"{calendar[first_index]}\t{calendar[last_index]}" ) universe_summary = { "kind": "tushare_index_weight_snapshots", "point_in_time": True, "index_code": universe_code, "snapshot_count": len(observed_snapshot_dates), "effective_snapshot_count": len(effective_snapshots), "snapshot_constituent_counts": snapshot_stats, "expected_constituent_count": resolved_constituent_count, "snapshot_max_gap_calendar_days": max_snapshot_gap, "weight_sum_tolerance": 0.5, "selected_code_semantics": "union_of_effective_snapshots", "membership_interval_semantics": ( "latest snapshot strictly before requested start is anchored " "at requested start; every in-range snapshot becomes " "effective on the next provider trading session and remains " "effective through the session before the next roster" ), "availability_contract": ( "conservative next-session effectiveness because the source " "does not expose a separately verified published_at timestamp" ), "same_session_membership_use": False, "coverage_start": calendar[0], "selection_uses_future_range_count": False, } if len(selected_codes) < 2: raise TushareMirrorError( "fewer than two eligible instruments are available" ) daily = daily[daily["ts_code"].isin(selected_codes)].copy() adjustment_complete = False missing_adjustment_keys = int(len(daily)) if adjustment_jobs: adjustment_frames = [] for job in adjustment_jobs: frame = pandas.read_parquet(_job_path(root, job)) if {"ts_code", "trade_date"}.issubset(frame.columns): frame = frame[ frame["ts_code"].astype(str).isin(selected_codes) & (frame["trade_date"].astype(str) >= start_raw) & (frame["trade_date"].astype(str) <= end_raw) ] adjustment_frames.append(frame) adjustments = pandas.concat( adjustment_frames, ignore_index=True, ) required_adjustment_fields = {"ts_code", "trade_date", "adj_factor"} missing_adjustment_fields = required_adjustment_fields.difference( adjustments.columns ) if missing_adjustment_fields: raise TushareMirrorError( "adj_factor partitions miss required fields: " f"{sorted(missing_adjustment_fields)}" ) adjustments["ts_code"] = adjustments["ts_code"].astype(str) adjustments["trade_date"] = adjustments["trade_date"].astype(str) adjustments = adjustments[ (adjustments["trade_date"] >= start_raw) & (adjustments["trade_date"] <= end_raw) & (adjustments["ts_code"].isin(selected_codes)) ].copy() if adjustments.duplicated(["ts_code", "trade_date"]).any(): raise TushareMirrorError( "adj_factor contains duplicate ts_code/trade_date keys" ) adjustments["adj_factor"] = pandas.to_numeric( adjustments["adj_factor"], errors="coerce", ) if ( adjustments["adj_factor"].isna().any() or (~adjustments["adj_factor"].map(math.isfinite)).any() or (adjustments["adj_factor"] <= 0).any() ): raise TushareMirrorError( "adj_factor contains null/non-finite/non-positive values" ) adjustment_keys = set( zip( adjustments["ts_code"], adjustments["trade_date"], ) ) daily_keys = set(zip(daily["ts_code"], daily["trade_date"])) missing_adjustment_keys = len(daily_keys.difference(adjustment_keys)) adjustment_complete = missing_adjustment_keys == 0 if adjustment_complete: daily = daily.merge( adjustments[["ts_code", "trade_date", "adj_factor"]], on=["ts_code", "trade_date"], how="left", validate="one_to_one", ) if not adjustment_complete: if not allow_unadjusted: raise TushareMirrorError( "complete adj_factor coverage is unavailable " f"({missing_adjustment_keys} daily keys missing); pass " "allow_unadjusted=True only for an explicitly non-investment " "technical run" ) daily["multiplier"] = 1.0 price_adjustment_mode = "raw_unadjusted" else: daily["adj_factor"] = pandas.to_numeric( daily["adj_factor"], errors="raise", ) daily = daily.sort_values(["ts_code", "trade_date"]).copy() anchor_factor = daily.groupby("ts_code")["adj_factor"].transform( "first" ) daily["multiplier"] = daily["adj_factor"] / anchor_factor if ( daily["multiplier"].isna().any() or (~daily["multiplier"].map(math.isfinite)).any() or (daily["multiplier"] <= 0).any() ): raise TushareMirrorError( "computed first-observation-anchored adjustment multipliers " "are invalid" ) price_adjustment_mode = ( "cumulative_adjusted_first_observation_anchor" ) daily = daily.sort_values(["ts_code", "trade_date"]).copy() daily["_adjusted_close"] = daily["close"] * daily["multiplier"] daily["_adjusted_change"] = daily.groupby("ts_code")[ "_adjusted_close" ].pct_change(fill_method=None) daily["qlib_symbol"] = daily["ts_code"].map(tushare_to_qlib_symbol) symbol_pairs = ( daily[["ts_code", "qlib_symbol"]].drop_duplicates().sort_values("ts_code") ) if symbol_pairs["qlib_symbol"].duplicated().any(): raise TushareMirrorError("Tushare symbols collide after Qlib conversion") if benchmark in set(symbol_pairs["qlib_symbol"]): raise TushareMirrorError("benchmark_symbol collides with a stock") pct_recomputed = ( (daily["close"] - daily["pre_close"]) / daily["pre_close"] * 100.0 ) pct_discrepancy_count = int( (pct_recomputed - daily["pct_chg"]).abs().gt(0.02).sum() ) missing_metadata = sorted(set(selected_codes).difference(metadata)) temporary = Path( tempfile.mkdtemp(prefix=f".{output.name}.", dir=str(output.parent)) ) try: _write_lines(temporary / "calendars/day.txt", calendar) instrument_lines = [] stock_symbols = [] for qlib_symbol, group in daily.groupby("qlib_symbol", sort=True): ordered = group.sort_values("trade_date") first_raw = str(ordered["trade_date"].iloc[0]) last_raw = str(ordered["trade_date"].iloc[-1]) first_index = calendar_index[first_raw] last_index = calendar_index[last_raw] length = last_index - first_index + 1 fields = { name: [float("nan")] * length for name in QLIB_FIELDS } for _, row in ordered.iterrows(): offset = calendar_index[str(row["trade_date"])] - first_index volume_shares = float(row["vol"]) * 100.0 money_cny = float(row["amount"]) * 1000.0 raw_vwap = ( money_cny / volume_shares if volume_shares > 0.0 else float(row["close"]) ) multiplier = float(row["multiplier"]) values = { "open": float(row["open"]) * multiplier, "high": float(row["high"]) * multiplier, "low": float(row["low"]) * multiplier, "close": float(row["close"]) * multiplier, "vwap": raw_vwap * multiplier, "volume": volume_shares / multiplier, "money": money_cny, "factor": multiplier, "change": float(row["_adjusted_change"]), } for field, value in values.items(): fields[field][offset] = value for field, values in fields.items(): _write_feature( temporary / "features" / str(qlib_symbol).lower() / f"{field}.day.bin", first_index, values, ) first_iso = dt.datetime.strptime(first_raw, "%Y%m%d").date().isoformat() last_iso = dt.datetime.strptime(last_raw, "%Y%m%d").date().isoformat() instrument_lines.append(f"{qlib_symbol}\t{first_iso}\t{last_iso}") stock_symbols.append(str(qlib_symbol)) benchmark_fields = {name: [] for name in QLIB_FIELDS} if benchmark_code is None: returns = ( daily.groupby("trade_date")["_adjusted_change"] .mean() .to_dict() ) level = 100.0 for raw_date in calendar_raw: change = float(returns.get(raw_date, 0.0)) if not math.isfinite(change): change = 0.0 if change <= -1.0: raise TushareMirrorError( f"invalid derived benchmark return on {raw_date}" ) level *= 1.0 + change benchmark_fields["open"].append(level) benchmark_fields["high"].append(level) benchmark_fields["low"].append(level) benchmark_fields["close"].append(level) benchmark_fields["vwap"].append(level) benchmark_fields["volume"].append(100_000_000.0) benchmark_fields["money"].append(level * 100_000_000.0) benchmark_fields["factor"].append(1.0) benchmark_fields["change"].append(change) benchmark_manifest = { "symbol": benchmark, "kind": "synthetic_equal_weight", "method": ( "daily arithmetic mean of eligible stock cumulative-" "adjusted " "close pct_change observations" ), "index_code": None, "real_index_claim": False, } else: index_daily = pandas.concat( [ pandas.read_parquet(_job_path(root, job)) for job in benchmark_jobs ], ignore_index=True, ) required_index_fields = { "ts_code", "trade_date", "open", "high", "low", "close", "pre_close", "vol", "amount", } missing_index_fields = required_index_fields.difference( index_daily.columns ) if missing_index_fields: raise TushareMirrorError( "index_daily partitions miss required fields: " f"{sorted(missing_index_fields)}" ) index_daily["ts_code"] = ( index_daily["ts_code"].astype(str).str.upper() ) index_daily["trade_date"] = index_daily["trade_date"].astype(str) index_daily = index_daily[ (index_daily["ts_code"] == benchmark_code) & (index_daily["trade_date"] >= start_raw) & (index_daily["trade_date"] <= end_raw) ].copy() if index_daily.duplicated(["trade_date"]).any(): raise TushareMirrorError( "index_daily contains duplicate benchmark trade_date keys" ) actual_index_dates = set(index_daily["trade_date"].tolist()) missing_index_dates = sorted( set(calendar_raw).difference(actual_index_dates) ) unexpected_index_dates = sorted( actual_index_dates.difference(calendar_raw) ) if missing_index_dates or unexpected_index_dates: raise TushareMirrorError( "index_daily/trade_cal mismatch: " f"missing={missing_index_dates[:5]}, " f"unexpected={unexpected_index_dates[:5]}" ) index_numeric_fields = ( "open", "high", "low", "close", "pre_close", "vol", "amount", ) for field in index_numeric_fields: index_daily[field] = pandas.to_numeric( index_daily[field], errors="coerce", ) if index_daily[list(index_numeric_fields)].isna().any().any(): raise TushareMirrorError( "index_daily contains null/non-numeric core values" ) if ( index_daily[["open", "high", "low", "close", "pre_close"]] <= 0 ).any().any(): raise TushareMirrorError( "index_daily contains non-positive price values" ) if (index_daily[["vol", "amount"]] < 0).any().any(): raise TushareMirrorError( "index_daily contains negative volume or amount" ) index_daily = index_daily.set_index("trade_date").loc[calendar_raw] index_change = index_daily["close"].pct_change(fill_method=None) for position, raw_date in enumerate(calendar_raw): row = index_daily.loc[raw_date] volume_shares = float(row["vol"]) * 100.0 money_cny = float(row["amount"]) * 1000.0 vwap = ( money_cny / volume_shares if volume_shares > 0.0 else float(row["close"]) ) benchmark_fields["open"].append(float(row["open"])) benchmark_fields["high"].append(float(row["high"])) benchmark_fields["low"].append(float(row["low"])) benchmark_fields["close"].append(float(row["close"])) benchmark_fields["vwap"].append(vwap) benchmark_fields["volume"].append(volume_shares) benchmark_fields["money"].append(money_cny) benchmark_fields["factor"].append(1.0) change = index_change.iloc[position] benchmark_fields["change"].append(float(change)) benchmark_manifest = { "symbol": benchmark, "kind": "tushare_index_daily", "method": "raw index OHLCV from completed index_daily jobs", "index_code": benchmark_code, "real_index_claim": True, } for field, values in benchmark_fields.items(): _write_feature( temporary / "features" / benchmark.lower() / f"{field}.day.bin", 0, values, ) benchmark_line = ( f"{benchmark}\t{calendar[0]}\t{calendar[-1]}" ) market_instrument_lines = ( membership_lines if universe_code is not None else instrument_lines ) _write_lines( temporary / f"instruments/{market}.txt", market_instrument_lines, ) _write_lines( temporary / "instruments/all.txt", [*instrument_lines, benchmark_line], ) jobs_after_read = _read_jobs(root) completed_after_read, _ = _latest_completed_jobs(jobs_after_read) latest_after_by_key = { (str(job["api_name"]), str(job["partition_key"])): int(job["id"]) for job in completed_after_read } changed_source_jobs = [ { "api_name": str(job["api_name"]), "partition_key": str(job["partition_key"]), "selected_job_id": int(job["id"]), "latest_job_id": latest_after_by_key.get( ( str(job["api_name"]), str(job["partition_key"]), ) ), } for job in selected_jobs if latest_after_by_key.get( ( str(job["api_name"]), str(job["partition_key"]), ) ) != int(job["id"]) ] if changed_source_jobs: raise TushareMirrorError( "selected source jobs changed while the provider was being " f"built: {changed_source_jobs[:5]}" ) source_jobs_after_read = [ _verify_completed_job(root, job, parquet=parquet) for job in selected_jobs ] if source_jobs_after_read != source_jobs: raise TushareMirrorError( "source bytes or provenance changed while the provider was " "being built" ) tree = _tree_evidence(temporary) completed_apis = {str(job["api_name"]) for job in completed} converter_sha256 = _sha256_file(Path(__file__).resolve()) build_parameters = { "start": start_iso, "end": end_iso, "market_name": market, "benchmark_symbol": benchmark, "benchmark_index_code": benchmark_code, "benchmark_mode": ( "tushare_index_daily" if benchmark_code is not None else "synthetic_equal_weight" ), "universe_index_code": universe_code, "universe_mode": ( "tushare_index_weight_pit" if universe_code is not None else "full_range_minimum_observations" ), "expected_constituent_count": resolved_constituent_count, "membership_availability": ( "next_provider_session" if universe_code is not None else None ), "minimum_observations": int(minimum_observations), "minimum_observations_applied": universe_code is None, "allow_unadjusted": bool(allow_unadjusted), "price_adjustment_mode": price_adjustment_mode, "ohlc_policy": "expand_range" if expand_range else "fail", } version_payload = _data_version_payload( source_jobs=source_jobs, build_parameters=build_parameters, converter_sha256=converter_sha256, schema_version=2, ) data_version = hashlib.sha256( _canonical_bytes(version_payload) ).hexdigest() status_counts: Dict[str, int] = {} for job in jobs: key = str(job["status"]) status_counts[key] = status_counts.get(key, 0) + 1 manifest: Dict[str, Any] = { "schema_version": 2, "artifact_type": "quant_os_tushare_qlib_provider", "source": "tushare_local_mirror", "created_at": dt.datetime.now(dt.timezone.utc).isoformat(), "data_version": data_version, **tree, "converter": { "source_sha256": converter_sha256, "output_format": "qlib_0_9_7_day_binary", }, "build_parameters": build_parameters, "calendar": { "frequency": "day", "start": calendar[0], "end": calendar[-1], "sessions": len(calendar), "source": "trade_cal/SSE/is_open=1", }, "market": { "name": market, "instrument_count": len(stock_symbols), "minimum_observations": int(minimum_observations), "minimum_observations_applied": universe_code is None, "benchmark_excluded": True, **universe_summary, }, "benchmark": benchmark_manifest, "fields": list(QLIB_FIELDS), "units": { "volume": "shares; Tushare vol(hand) multiplied by 100", "money": "CNY; Tushare amount(thousand CNY) multiplied by 1000", }, "price_adjustment": { "mode": price_adjustment_mode, "formula": ( "multiplier=adj_factor/" "first_adj_factor_for_symbol_in_requested_interval" if adjustment_complete else "factor=1" ), "anchor_semantics": ( "per-symbol first observed adjustment factor is a " "constant anchor; no future corporate action is used to " "rescale earlier observations" if adjustment_complete else "not_applicable" ), "coverage_complete": adjustment_complete, "missing_daily_keys": missing_adjustment_keys, "component_complete": adjustment_complete, "production_eligible": False, }, "quality": { "duplicate_keys": 0, "whole_market_calendar_gaps": 0, "ohlc_envelope_anomalies": envelope_bad_count, "ohlc_policy": "expand_range" if expand_range else "fail", "ohlc_anomaly_keys_sha256": anomaly_keys_sha256, "pct_chg_discrepancies_gt_0_02_percentage_points": ( pct_discrepancy_count ), "selected_rows": int(len(daily)), "selected_symbols_missing_stock_basic": missing_metadata, "universe_selection": ( universe_summary["membership_interval_semantics"] if universe_code is not None else ( "eligible A-share symbols with at least " f"{int(minimum_observations)} observations over the " "full requested range; not point-in-time eligibility" ) ), }, "source_jobs": source_jobs, "source_stability": { "post_read_file_verification": True, "selected_jobs_still_latest": True, "policy": ( "verify selected bytes before consumption, then re-read " "the SQLite ledger and re-verify every selected file " "after all provider inputs have been consumed" ), }, "mirror_job_status_counts_at_build": dict(sorted(status_counts.items())), "shadowed_completed_versions_at_build": len(shadowed), "missing_production_tables": [ value for value in MISSING_PRODUCTION_TABLES if value not in completed_apis ], "verification_boundary": ( "completed checksummed partitions and generated provider " "integrity only; not point-in-time production market evidence" ), "gate_credit": [], "investment_value_claim": False, } (temporary / MANIFEST_NAME).write_bytes(_canonical_bytes(manifest)) os.replace(temporary, output) except BaseException: if temporary.exists(): shutil.rmtree(temporary) raise return { "ok": True, "output_dir": str(output), "manifest": str(output / MANIFEST_NAME), **manifest, } def verify_qlib_provider(provider_dir: str | Path) -> Dict[str, Any]: """Verify a provider tree against its Quant OS manifest.""" root = Path(provider_dir).expanduser().resolve() if not root.is_dir(): raise FileNotFoundError(f"provider_dir does not exist: {root}") manifest_path = root / MANIFEST_NAME try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise TushareMirrorError( f"cannot read provider manifest: {manifest_path}" ) from exc if not isinstance(manifest, dict): raise TushareMirrorError("provider manifest must be a JSON object") if manifest.get("artifact_type") != "quant_os_tushare_qlib_provider": raise TushareMirrorError("unexpected provider artifact_type") build_parameters = manifest.get("build_parameters") converter = manifest.get("converter") source_jobs = manifest.get("source_jobs") if ( not isinstance(build_parameters, Mapping) or not isinstance(converter, Mapping) or not isinstance(source_jobs, list) ): raise TushareMirrorError( "provider manifest misses build/data-version evidence" ) try: version_payload = _data_version_payload( source_jobs=source_jobs, build_parameters=build_parameters, converter_sha256=str(converter["source_sha256"]), schema_version=int(manifest.get("schema_version", 1)), ) computed_data_version = hashlib.sha256( _canonical_bytes(version_payload) ).hexdigest() except (KeyError, TypeError, ValueError) as exc: raise TushareMirrorError( "provider build/data-version evidence is invalid" ) from exc if computed_data_version != manifest.get("data_version"): raise TushareMirrorError( "provider data_version does not match source/build evidence" ) schema_version = int(manifest.get("schema_version", 1)) if schema_version >= 2: source_stability = manifest.get("source_stability") if ( not isinstance(source_stability, Mapping) or source_stability.get("post_read_file_verification") is not True or source_stability.get("selected_jobs_still_latest") is not True ): raise TushareMirrorError( "schema-v2 provider lacks closed source-stability evidence" ) tree = _tree_evidence(root) for name in ( "provider_tree_sha256", "provider_file_count", "provider_byte_count", ): if tree[name] != manifest.get(name): raise TushareMirrorError( f"provider tree mismatch for {name}: " f"{tree[name]!r} != {manifest.get(name)!r}" ) for required in ("calendars/day.txt", "instruments/all.txt"): if not (root / required).is_file(): raise TushareMirrorError(f"provider misses {required}") current_converter_sha256 = _sha256_file(Path(__file__).resolve()) recorded_converter_sha256 = str(converter["source_sha256"]) return { "ok": True, "data_version": manifest.get("data_version"), **tree, "manifest_sha256": _sha256_file(manifest_path), "recorded_converter_sha256": recorded_converter_sha256, "current_converter_sha256": current_converter_sha256, "converter_source_matches_current": ( current_converter_sha256 == recorded_converter_sha256 ), "investment_value_claim": False, }