feat: operationalize local Tushare Qlib research

2026-07-31 01:26:49 +08:00
parent f81c116bfe
commit 6275370afc
21 changed files with 4438 additions and 401 deletions
@@ -0,0 +1,105 @@
+import hashlib
+import json
+from pathlib import Path
+import sys
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from tools.tushare_lineage import (
+    TushareLineageError,
+    _compare_source_jobs,
+    _validate_snapshot,
+)
+
+
+def _canonical(value):
+    return json.dumps(
+        value,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+        allow_nan=False,
+    ).encode()
+
+
+class TushareLineageTest(unittest.TestCase):
+    def _snapshot(self):
+        job = {
+            "api_name": "daily",
+            "byte_count": 123,
+            "completed_at": "2026-07-31T00:00:00Z",
+            "id": 7,
+            "observed": {
+                "row_count": 11,
+                "sha256": "a" * 64,
+                "size_bytes": 123,
+            },
+            "partition_key": "month=2024-01",
+            "path": "data/parquet/daily/month=2024-01.parquet",
+            "request_sha256": "b" * 64,
+            "row_count": 11,
+            "sha256": "a" * 64,
+            "shadowed_completed_count": 0,
+            "verified": True,
+        }
+        selection_identity = {
+            key: value
+            for key, value in job.items()
+            if key not in {"observed", "verified"}
+        }
+        snapshot = {
+            "format": "quant-os.tushare-scoped-snapshot/v2",
+            "selected_jobs": [job],
+            "selection_sha256": hashlib.sha256(
+                _canonical({"selected_jobs": [selection_identity]})
+            ).hexdigest(),
+            "verification": {
+                "double_verified_jobs": 1,
+                "selected_jobs_still_latest_after_verification": True,
+                "verified_jobs": 1,
+            },
+        }
+        snapshot["manifest_sha256"] = hashlib.sha256(
+            _canonical(snapshot)
+        ).hexdigest()
+        return snapshot
+
+    def test_exact_source_lineage_and_tamper_detection(self):
+        snapshot = self._snapshot()
+        jobs = _validate_snapshot(snapshot)
+        provider_job = {
+            "api_name": "daily",
+            "byte_count": 123,
+            "job_id": 7,
+            "partition_key": "month=2024-01",
+            "path": "data/parquet/daily/month=2024-01.parquet",
+            "request_sha256": "b" * 64,
+            "row_count": 11,
+            "sha256": "a" * 64,
+        }
+        _compare_source_jobs(jobs, [provider_job])
+        provider_job["sha256"] = "c" * 64
+        with self.assertRaisesRegex(
+            TushareLineageError,
+            "source job fields differ",
+        ):
+            _compare_source_jobs(jobs, [provider_job])
+
+        snapshot["selected_jobs"][0]["path"] = "../escape.parquet"
+        snapshot_without_hash = dict(snapshot)
+        snapshot_without_hash.pop("manifest_sha256")
+        snapshot["manifest_sha256"] = hashlib.sha256(
+            _canonical(snapshot_without_hash)
+        ).hexdigest()
+        with self.assertRaisesRegex(
+            TushareLineageError,
+            "stay relative",
+        ):
+            _validate_snapshot(snapshot)
+
+
+if __name__ == "__main__":
+    unittest.main()