From 9de34a52319e9c5432dfb3988e79a7d420304405 Mon Sep 17 00:00:00 2001
From: Cai <cai@nbcai.cc>
Date: Tue, 01 Sep 2026 21:12:06 +0800
Subject: [PATCH] bili: land reviewed half-hour pipeline and EOF isolation
---
dev/project-dev/bili_article_image_native_host.py | 117
dev/project-dev/bili_article_image_collector.py | 1025 ++
dev/project-dev/bili_authenticated_extension/config.example.json | 22
dev/project-dev/bili_dynamic_refresh_native_host/native_host.py | 96
dev/project-dev/bili_dynamic_refresh_extension/source-artifact-manifest.json | 39
dev/project-dev/bili_authenticated_extension/background.js | 1918 ++++
dev/project-dev/bili_dynamic_refresh_native_host/source-artifact-manifest.json | 49
dev/project-dev/bili_authenticated_extension/__init__.py | 10
dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/codex_stdio_race_repro.mjs | 53
dev/project-dev/bili_dynamic_refresh_extension/page_extract.js | 88
dev/project-dev/bili_dynamic_refresh_extension/runtime.js | 519 +
dev/project-dev/bili_half_hour_pipeline.config.json | 46
dev/project-dev/bili_dynamic_refresh_extension/service_worker.js | 65
dev/project-dev/bili_article_image_source_manifest.json | 149
dev/project-dev/bili_authenticated_extension/queue_producer.py | 1416 ++++
dev/project-dev/bili_dynamic_refresh_extension/protocol.js | 101
dev/project-dev/bili_dynamic_refresh_native_host/identity.py | 219
dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/test_v009_contract.py | 265
dev/project-dev/bili_dynamic_refresh_extension/package.json | 3
dev/project-dev/bili_dynamic_refresh_native_host/durable.py | 133
dev/project-dev/bili_half_hour_pipeline_source_manifest.json | 50
dev/project-dev/bili_article_image_collector.example.json | 44
dev/project-dev/bili_dynamic_collector.py | 30
dev/project-dev/bili_dynamic_refresh_native_host/__init__.py | 5
dev/project-dev/bili_authenticated_extension/sidepanel.js | 74
dev/project-dev/bili_authenticated_extension/constants.py | 398 +
dev/project-dev/bili_dynamic_refresh_native_host/strict_json.py | 30
dev/project-dev/bili_dynamic_refresh_native_host/protocol.py | 175
dev/project-dev/test/test_bili_article_image_collector.py | 407 +
dev/project-dev/bili_dynamic_refresh_extension/manifest.json | 21
dev/project-dev/bili_authenticated_extension/native_host.py | 1487 +++
dev/project-dev/bili_authenticated_extension/sidepanel.html | 14
dev/project-dev/test/test_bili_article_image_capture.mjs | 102
dev/project-dev/bili_authenticated_extension/build_host.ps1 | 63
dev/project-dev/bili_authenticated_extension/worker.py | 1582 ++++
dev/project-dev/bili_article_image_capture.js | 207
dev/project-dev/bili_authenticated_extension/formal_legacy_identity_manifest.py | 55
dev/project-dev/bili_authenticated_extension/source-artifact-manifest.json | 127
dev/project-dev/bili_authenticated_extension_unpacked_validator.py | 605 +
dev/project-dev/test/bili_authenticated_extension/test_unpacked_projection.py | 290
dev/project-dev/bili_article_image_source_validator.py | 98
dev/project-dev/bili_dynamic_refresh_native_host/native-host-manifest.template.json | 9
dev/project-dev/bili_dynamic_refresh.py | 150
dev/project-dev/bili_authenticated_extension/protocol.py | 425
dev/project-dev/test/test_bili_dynamic_refresh.py | 405
dev/project-dev/bili_half_hour_pipeline.py | 3016 ++++++++
dev/project-dev/bili_authenticated_extension/queue_state.py | 1298 +++
dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/runtime_lifecycle.mjs | 439 +
dev/project-dev/bili_authenticated_extension/queue-producer.example.json | 14
dev/project-dev/test/test_bili_half_hour_pipeline.py | 1555 ++++
dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/js_contract.mjs | 57
dev/project-dev/bili_authenticated_extension/manifest.json | 21
dev/project-dev/bili_dynamic_refresh_native_host/constants.py | 13
dev/project-dev/bili_authenticated_extension/install_native_host.ps1 | 552 +
dev/project-dev/test/bili_authenticated_extension/test_successor_trust_gate.py | 915 ++
55 files changed, 19,778 insertions(+), 1,288 deletions(-)
diff --git a/dev/project-dev/bili_article_image_capture.js b/dev/project-dev/bili_article_image_capture.js
new file mode 100644
index 0000000..c2e09a1
--- /dev/null
+++ b/dev/project-dev/bili_article_image_capture.js
@@ -0,0 +1,207 @@
+(() => {
+ "use strict";
+
+ const PENDING = new Set(["VIDEO_ABSENT", "METADATA_NOT_READY", "OWNER_PENDING", "DIMENSIONS_PENDING"]);
+ const ACCESS_MARKERS = [
+ ["HTTP_412", /412|请求被拦截/i],
+ ["CAPTCHA", /验证码|安全验证|captcha/i],
+ ["LOGIN_REQUIRED", /登录后|请先登录/i],
+ ["PAYWALL", /付费后|购买后|充电专属|会员专享/i],
+ ];
+ const UID = /^[1-9][0-9]{0,19}$/;
+ const ITEM_ID = /^[A-Za-z0-9_-]{1,128}$/;
+ const CONTROL = /[\u0000-\u001f\u007f]/;
+ const IMAGE_HOSTS = new Set(["i0.hdslb.com", "i1.hdslb.com", "i2.hdslb.com"]);
+
+ function fail(code) {
+ const error = new Error(code);
+ error.code = code;
+ throw error;
+ }
+
+ function exactKeys(value, keys, code) {
+ if (!value || typeof value !== "object" || Array.isArray(value)) fail(code);
+ const actual = Object.keys(value).sort();
+ const expected = [...keys].sort();
+ if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) fail(code);
+ return value;
+ }
+
+ function canonicalSpaceUrl(raw, uid, dynamic) {
+ const url = new URL(raw);
+ const path = dynamic ? `/${uid}/dynamic` : `/${uid}`;
+ if (url.protocol !== "https:" || url.hostname !== "space.bilibili.com" || url.search || url.hash || url.pathname.replace(/\/$/, "") !== path) {
+ fail("E_PAGE_IDENTITY");
+ }
+ return `https://space.bilibili.com${path}`;
+ }
+
+ function validatePublicConfig(raw) {
+ const value = exactKeys(raw, ["creator_uid", "creator_name", "dynamic_url", "profile_url", "include_types", "deadline_ms", "observation_interval_ms", "stable_observations"], "E_CONFIG_SCHEMA");
+ if (typeof value.creator_uid !== "string" || !UID.test(value.creator_uid)) fail("E_CONFIG_IDENTITY");
+ if (typeof value.creator_name !== "string" || !value.creator_name.trim() || value.creator_name.length > 80 || CONTROL.test(value.creator_name)) fail("E_CONFIG_IDENTITY");
+ if (!Array.isArray(value.include_types) || !value.include_types.length || new Set(value.include_types).size !== value.include_types.length || value.include_types.some((item) => !["article", "text", "image"].includes(item))) fail("E_CONFIG_SCHEMA");
+ if (!Number.isInteger(value.deadline_ms) || value.deadline_ms < 5000 || value.deadline_ms > 600000) fail("E_CONFIG_SCHEMA");
+ if (!Number.isInteger(value.observation_interval_ms) || value.observation_interval_ms < 100 || value.observation_interval_ms > 10000) fail("E_CONFIG_SCHEMA");
+ if (!Number.isInteger(value.stable_observations) || value.stable_observations < 2 || value.stable_observations > 5) fail("E_CONFIG_SCHEMA");
+ return Object.freeze({
+ ...value,
+ creator_name: value.creator_name.trim(),
+ dynamic_url: canonicalSpaceUrl(value.dynamic_url, value.creator_uid, true),
+ profile_url: canonicalSpaceUrl(value.profile_url, value.creator_uid, false),
+ include_types: Object.freeze([...value.include_types]),
+ });
+ }
+
+ function visibleText(document) {
+ const body = document && document.body;
+ return body && typeof body.innerText === "string" ? body.innerText.slice(0, 20000) : "";
+ }
+
+ function accessState(document) {
+ const text = visibleText(document);
+ for (const [state, pattern] of ACCESS_MARKERS) {
+ if (pattern.test(text)) return state;
+ }
+ return null;
+ }
+
+ function canonicalOpusUrl(raw, stableId) {
+ const url = new URL(raw, "https://www.bilibili.com/");
+ if (url.protocol !== "https:" || url.hostname !== "www.bilibili.com" || url.search || url.hash || url.pathname.replace(/\/$/, "") !== `/opus/${stableId}`) fail("E_ITEM_IDENTITY");
+ return `https://www.bilibili.com/opus/${stableId}`;
+ }
+
+ function imageCandidate(raw) {
+ const url = new URL(raw, "https://www.bilibili.com/");
+ if (url.protocol !== "https:" || !IMAGE_HOSTS.has(url.hostname) || !url.pathname.startsWith("/bfs/") || url.search || url.hash) fail("E_IMAGE_IDENTITY");
+ return `${url.protocol}//${url.hostname}${url.pathname}`;
+ }
+
+ function textFrom(root, selectors) {
+ for (const selector of selectors) {
+ const node = root.querySelector(selector);
+ if (node && typeof node.innerText === "string" && node.innerText.trim()) return node.innerText.replace(/\r\n?/g, "\n").trimEnd();
+ }
+ return "";
+ }
+
+ function canonicalJson(value) {
+ if (value === null || typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
+ if (typeof value === "number" && Number.isSafeInteger(value)) return JSON.stringify(value);
+ if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
+ if (value && typeof value === "object") {
+ return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
+ }
+ fail("E_READINESS_SCHEMA");
+ }
+
+ function canonicalAcceptedItem(raw) {
+ const item = exactKeys(
+ raw,
+ ["stable_id", "item_type", "title", "source_url", "published_at", "body_text", "body_complete", "original_image_candidates"],
+ "E_READINESS_SCHEMA",
+ );
+ if (typeof item.stable_id !== "string" || !ITEM_ID.test(item.stable_id)) fail("E_ITEM_IDENTITY");
+ if (!["article", "text", "image"].includes(item.item_type)) fail("E_READINESS_SCHEMA");
+ if (typeof item.title !== "string" || !item.title.trim() || CONTROL.test(item.title)) fail("E_READINESS_SCHEMA");
+ if (typeof item.body_text !== "string" || !item.body_text || item.body_complete !== true) fail("E_READINESS_SCHEMA");
+ if (!Array.isArray(item.original_image_candidates)) fail("E_READINESS_SCHEMA");
+ const images = item.original_image_candidates.map((candidate) => imageCandidate(candidate));
+ if (new Set(images).size !== images.length) fail("E_IMAGE_IDENTITY");
+ const publishedAtEpochMs = Date.parse(item.published_at);
+ if (!Number.isSafeInteger(publishedAtEpochMs)) fail("E_READINESS_SCHEMA");
+ return {
+ body_complete: true,
+ body_text: item.body_text.replace(/\r\n?/gu, "\n").replace(/\n+$/gu, ""),
+ image_count: images.length,
+ item_type: item.item_type,
+ published_at_epoch_ms: publishedAtEpochMs,
+ source_url: canonicalOpusUrl(item.source_url, item.stable_id),
+ stable_id: item.stable_id,
+ title: item.title.trim(),
+ };
+ }
+
+ async function acceptedSnapshotFingerprint(items) {
+ if (!Array.isArray(items)) fail("E_READINESS_SCHEMA");
+ const canonicalItems = items.map((item) => canonicalAcceptedItem(item)).sort((left, right) => left.stable_id < right.stable_id ? -1 : (left.stable_id > right.stable_id ? 1 : 0));
+ if (new Set(canonicalItems.map((item) => item.stable_id)).size !== canonicalItems.length) fail("E_ITEM_IDENTITY");
+ const bytes = new TextEncoder().encode(canonicalJson({items: canonicalItems, schema_version: 1}));
+ const digest = await crypto.subtle.digest("SHA-256", bytes);
+ return [...new Uint8Array(digest)].map((part) => part.toString(16).padStart(2, "0")).join("").toUpperCase();
+ }
+
+ function currentOpusSnapshot(document, locationLike, rawConfig) {
+ const config = validatePublicConfig(rawConfig);
+ const access = accessState(document);
+ if (access) return { state: access, reason: access, item: null };
+ const current = new URL(locationLike.href);
+ const match = current.protocol === "https:" && current.hostname === "www.bilibili.com" ? current.pathname.match(/^\/opus\/([A-Za-z0-9_-]{1,128})\/?$/) : null;
+ if (!match) return { state: "METADATA_NOT_READY", reason: "METADATA_NOT_READY", item: null };
+ const stableId = match[1];
+ if (!ITEM_ID.test(stableId)) fail("E_ITEM_IDENTITY");
+ const ownerProof = document.querySelector(`[data-mid="${CSS.escape(config.creator_uid)}"], [data-user-id="${CSS.escape(config.creator_uid)}"]`);
+ if (!ownerProof) return { state: "OWNER_PENDING", reason: "OWNER_PENDING", item: null };
+ const bodyText = textFrom(document, [".opus-module-content", ".article-content", ".bili-rich-text__content", "[data-content=\"opus\"]"]);
+ if (!bodyText) return { state: "METADATA_NOT_READY", reason: "METADATA_NOT_READY", item: null };
+ const title = textFrom(document, ["h1", ".opus-module-title", ".article-title"]) || bodyText.split("\n", 1)[0].slice(0, 80);
+ const published = document.querySelector("time[datetime], [data-published-at]");
+ const publishedAt = published ? (published.getAttribute("datetime") || published.getAttribute("data-published-at")) : "";
+ if (!publishedAt || Number.isNaN(Date.parse(publishedAt))) return { state: "METADATA_NOT_READY", reason: "METADATA_NOT_READY", item: null };
+ const imageNodes = [...document.querySelectorAll(".opus-module-content img[src], .article-content img[src], .bili-rich-text__content img[src]")];
+ if (imageNodes.some((node) => !node.naturalWidth || !node.naturalHeight)) return { state: "DIMENSIONS_PENDING", reason: "DIMENSIONS_PENDING", item: null };
+ const images = [...new Set(imageNodes.map((node) => imageCandidate(node.currentSrc || node.src)))];
+ const itemType = document.querySelector(".article-content, .opus-module-title") ? "article" : (images.length ? "image" : "text");
+ if (!config.include_types.includes(itemType)) return { state: "READY", reason: "READY", item: null };
+ return {
+ state: "READY",
+ reason: "READY",
+ item: {
+ stable_id: stableId,
+ item_type: itemType,
+ title,
+ source_url: canonicalOpusUrl(current.href, stableId),
+ published_at: new Date(publishedAt).toISOString(),
+ body_text: bodyText,
+ body_complete: true,
+ original_image_candidates: images,
+ },
+ };
+ }
+
+ async function observeUntilStable(rawConfig, sample, sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds))) {
+ const config = validatePublicConfig(rawConfig);
+ if (typeof sample !== "function") fail("E_CONFIG_SCHEMA");
+ const started = performance.now();
+ let prior = null;
+ let streak = 0;
+ const observations = [];
+ while (true) {
+ const elapsed = Math.floor(performance.now() - started);
+ if (elapsed > config.deadline_ms) fail("E_READINESS_TIMEOUT");
+ const snapshot = await sample();
+ if (!snapshot || typeof snapshot !== "object" || typeof snapshot.state !== "string" || typeof snapshot.reason !== "string") fail("E_READINESS_SCHEMA");
+ if (PENDING.has(snapshot.state)) {
+ if (snapshot.reason !== snapshot.state) fail("E_READINESS_SCHEMA");
+ streak = 0;
+ prior = null;
+ observations.push({ elapsed_ms: elapsed, state: snapshot.state, reason: snapshot.reason, snapshot_sha256: null });
+ } else if (snapshot.state === "READY" && snapshot.reason === "READY") {
+ const acceptedItems = Array.isArray(snapshot.items) ? snapshot.items : (snapshot.item === null ? [] : [snapshot.item]);
+ const digest = await acceptedSnapshotFingerprint(acceptedItems);
+ streak = digest === prior ? streak + 1 : 1;
+ prior = digest;
+ observations.push({ elapsed_ms: elapsed, state: "READY", reason: "READY", snapshot_sha256: digest });
+ if (streak >= config.stable_observations) return { observations, item: snapshot.item, items: acceptedItems };
+ } else {
+ fail(snapshot.state.startsWith("E_") ? snapshot.state : "E_ACCESS_CONTROL");
+ }
+ await sleep(config.observation_interval_ms);
+ }
+ }
+
+ const api = Object.freeze({ validatePublicConfig, currentOpusSnapshot, acceptedSnapshotFingerprint, observeUntilStable });
+ if (typeof module === "object" && module.exports) module.exports = api;
+ globalThis.BiliArticleImageCapture = api;
+})();
diff --git a/dev/project-dev/bili_article_image_collector.example.json b/dev/project-dev/bili_article_image_collector.example.json
new file mode 100644
index 0000000..3bdceb0
--- /dev/null
+++ b/dev/project-dev/bili_article_image_collector.example.json
@@ -0,0 +1,44 @@
+{
+ "schema_version": 1,
+ "creator": {
+ "uid": "10001",
+ "name": "示例创作者"
+ },
+ "page": {
+ "dynamic_url": "https://space.bilibili.com/10001/dynamic",
+ "profile_url": "https://space.bilibili.com/10001"
+ },
+ "output": {
+ "root": null,
+ "intake_root": "../../dev/tmp/bili-article-image-intake",
+ "manifest_name": "manifest.jsonl"
+ },
+ "selection": {
+ "date_start": null,
+ "date_end": null,
+ "window_days": 30,
+ "timezone": "Asia/Shanghai",
+ "include_types": [
+ "article",
+ "text",
+ "image"
+ ]
+ },
+ "readiness": {
+ "deadline_seconds": 120,
+ "observation_interval_ms": 500,
+ "stable_observations": 3
+ },
+ "rerun": {
+ "policy": "verify_only"
+ },
+ "verification": {
+ "summary_path": null
+ },
+ "limits": {
+ "max_items": 500,
+ "max_body_bytes": 8388608,
+ "max_images_per_item": 20,
+ "max_image_bytes": 52428800
+ }
+}
diff --git a/dev/project-dev/bili_article_image_collector.py b/dev/project-dev/bili_article_image_collector.py
new file mode 100644
index 0000000..08f0b66
--- /dev/null
+++ b/dev/project-dev/bili_article_image_collector.py
@@ -0,0 +1,1025 @@
+#!/usr/bin/env python3
+"""Configuration-driven Bilibili article/image collector and verifier.
+
+The browser side may supply only sanitized visible-page capture bundles. This
+module never reads browser storage, credentials, headers, or profiles and never
+performs network requests. It validates, publishes with CreateNew semantics,
+or verifies an existing append-only corpus.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import stat
+import sys
+import uuid
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta, timezone
+from pathlib import Path, PurePosixPath
+from typing import Any, Iterable, Mapping, Sequence
+from urllib.parse import urlsplit, urlunsplit
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+
+CONFIG_SCHEMA = 1
+CAPTURE_SCHEMA = 1
+MANIFEST_SCHEMA = 1
+ALLOWED_TYPES = frozenset({"article", "text", "image"})
+PENDING_STATES = frozenset({"VIDEO_ABSENT", "METADATA_NOT_READY", "OWNER_PENDING", "DIMENSIONS_PENDING"})
+READY_STATE = "READY"
+ACCESS_STATES = frozenset({"ACCESS_BLOCKED", "LOGIN_REQUIRED", "CAPTCHA", "HTTP_412", "PAYWALL"})
+SECRET_KEY = re.compile(
+ r"(?:password|passwd|cookie|token|secret|authorization|captcha|session|localstorage|signed[_-]?url|"
+ r"口令|密码|令牌|验证码|会话)",
+ re.IGNORECASE,
+)
+CONTROL = re.compile(r"[\x00-\x1f\x7f]")
+SHA256 = re.compile(r"[0-9A-F]{64}")
+UID = re.compile(r"[1-9][0-9]{0,19}")
+ITEM_ID = re.compile(r"[A-Za-z0-9_-]{1,128}")
+LEGACY_IMAGE_ID = re.compile(r"([A-Za-z0-9_-]{1,96}):image:([1-9][0-9]{0,3})")
+WINDOWS_BAD = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
+WINDOWS_RESERVED = {"CON", "PRN", "AUX", "NUL", *(f"COM{i}" for i in range(1, 10)), *(f"LPT{i}" for i in range(1, 10))}
+FILE_ATTRIBUTE_REPARSE_POINT = 0x400
+OWNED_PENDING = re.compile(r"\.bili-article-image\.pending\.[0-9a-f]{32}\.json")
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+
+
+class CollectorError(RuntimeError):
+ def __init__(self, code: str, message: str, *, safety: bool = False) -> None:
+ super().__init__(message)
+ self.code = code
+ self.message = message
+ self.safety = safety
+
+
+@dataclass(frozen=True)
+class CollectorConfig:
+ path: Path
+ raw_bytes: bytes
+ sha256: str
+ creator_uid: str
+ creator_name: str
+ dynamic_url: str
+ profile_url: str
+ output_root: Path
+ manifest_path: Path
+ intake_root: Path
+ summary_path: Path | None
+ timezone_name: str
+ date_start: datetime
+ date_end: datetime
+ include_types: frozenset[str]
+ deadline_seconds: int
+ observation_interval_ms: int
+ stable_observations: int
+ rerun_policy: str
+ max_items: int
+ max_body_bytes: int
+ max_images_per_item: int
+ max_image_bytes: int
+
+ @property
+ def tz(self) -> ZoneInfo:
+ return ZoneInfo(self.timezone_name)
+
+
+def _exact_keys(value: Any, expected: Iterable[str], field: str) -> Mapping[str, Any]:
+ expected_set = set(expected)
+ if not isinstance(value, Mapping) or set(value) != expected_set:
+ raise CollectorError("E_CONFIG_SCHEMA", f"{field} keys differ from the strict schema.", safety=True)
+ return value
+
+
+def _reject_secrets(value: Any, path: str = "$") -> None:
+ if isinstance(value, Mapping):
+ for key, child in value.items():
+ if not isinstance(key, str) or SECRET_KEY.search(key):
+ raise CollectorError("E_SECRET_FIELD", f"Secret-bearing field is forbidden at {path}.", safety=True)
+ _reject_secrets(child, f"{path}.{key}")
+ elif isinstance(value, list):
+ for index, child in enumerate(value):
+ _reject_secrets(child, f"{path}[{index}]")
+
+
+def _strict_json(path: Path, description: str) -> tuple[Any, bytes]:
+ try:
+ raw = path.read_bytes()
+ except OSError as exc:
+ raise CollectorError("E_INPUT", f"{description} is unreadable.", safety=True) from exc
+ if raw.startswith(b"\xef\xbb\xbf") or b"\r" in raw or not raw.endswith(b"\n") or raw.endswith(b"\n\n"):
+ raise CollectorError("E_INPUT_ENCODING", f"{description} must be strict UTF-8 LF with one final LF.", safety=True)
+ try:
+ text = raw[:-1].decode("utf-8", errors="strict")
+ value = json.loads(text)
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+ raise CollectorError("E_INPUT_SCHEMA", f"{description} is not strict JSON.", safety=True) from exc
+ _reject_secrets(value)
+ return value, raw
+
+
+def _canonical_bytes(value: Any) -> bytes:
+ return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
+
+
+def _canonical_payload(value: Any) -> bytes:
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+
+
+def _sha256(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest().upper()
+
+
+def _is_reparse(path: Path) -> bool:
+ try:
+ return bool(path.lstat().st_file_attributes & FILE_ATTRIBUTE_REPARSE_POINT)
+ except AttributeError:
+ return path.is_symlink()
+
+
+def _ordinary_file(path: Path) -> None:
+ if not path.exists() or not path.is_file() or _is_reparse(path):
+ raise CollectorError("E_PATH", "Required file is missing, non-ordinary, or reparse-backed.", safety=True)
+
+
+def _safe_existing_chain(path: Path, *, allow_missing_leaf: bool = False) -> None:
+ candidate = path.resolve(strict=False)
+ current = Path(candidate.anchor)
+ parts = candidate.parts[1:]
+ for index, part in enumerate(parts):
+ current = current / part
+ if not current.exists():
+ if allow_missing_leaf and index == len(parts) - 1:
+ return
+ continue
+ if _is_reparse(current):
+ raise CollectorError("E_PATH_REPARSE", "Path chain contains a reparse point.", safety=True)
+ if index < len(parts) - 1 and not current.is_dir():
+ raise CollectorError("E_PATH", "Path chain contains a non-directory component.", safety=True)
+
+
+def _within(child: Path, parent: Path) -> bool:
+ try:
+ child.resolve(strict=False).relative_to(parent.resolve(strict=False))
+ return True
+ except ValueError:
+ return False
+
+
+def _safe_component(value: str, *, max_length: int = 80) -> str:
+ cleaned = WINDOWS_BAD.sub("_", value).strip(" .")
+ cleaned = re.sub(r"\s+", "", cleaned)
+ if not cleaned or cleaned.upper() in WINDOWS_RESERVED:
+ raise CollectorError("E_CONFIG", "creator.name cannot form a safe output component.", safety=True)
+ return cleaned[:max_length].rstrip(" .")
+
+
+def _parse_datetime(value: Any, field: str) -> datetime:
+ if not isinstance(value, str) or not value.strip():
+ raise CollectorError("E_CONFIG", f"{field} must be an offset-aware ISO-8601 string.")
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise CollectorError("E_CONFIG", f"{field} is invalid.") from exc
+ if parsed.tzinfo is None:
+ raise CollectorError("E_CONFIG", f"{field} must include an offset.")
+ return parsed.astimezone(timezone.utc)
+
+
+def _validate_page_url(value: Any, uid: str, *, dynamic: bool) -> str:
+ if not isinstance(value, str) or CONTROL.search(value):
+ raise CollectorError("E_CONFIG", "Configured page URL is invalid.", safety=True)
+ parsed = urlsplit(value)
+ expected_path = f"/{uid}/dynamic" if dynamic else f"/{uid}"
+ if parsed.scheme != "https" or parsed.hostname != "space.bilibili.com" or parsed.query or parsed.fragment:
+ raise CollectorError("E_CONFIG", "Configured page URL must be a query-free Bilibili space URL.", safety=True)
+ if parsed.path.rstrip("/") != expected_path:
+ raise CollectorError("E_CONFIG", "Configured page URL does not bind the creator UID.", safety=True)
+ return urlunsplit(("https", "space.bilibili.com", expected_path, "", ""))
+
+
+def _bounded_int(value: Any, field: str, lower: int, upper: int) -> int:
+ if not isinstance(value, int) or isinstance(value, bool) or not lower <= value <= upper:
+ raise CollectorError("E_CONFIG", f"{field} must be {lower}..{upper}.")
+ return value
+
+
+def load_config(path: Path, *, now: datetime | None = None) -> CollectorConfig:
+ path = path.resolve(strict=True)
+ if not _within(path, PROJECT_ROOT):
+ raise CollectorError("E_PATH_ESCAPE", "config path must remain inside the project root.", safety=True)
+ _ordinary_file(path)
+ value, raw = _strict_json(path, "config")
+ root = _exact_keys(value, {"schema_version", "creator", "page", "output", "selection", "readiness", "rerun", "verification", "limits"}, "config")
+ if root["schema_version"] != CONFIG_SCHEMA:
+ raise CollectorError("E_CONFIG_SCHEMA", "config.schema_version differs.")
+ creator = _exact_keys(root["creator"], {"uid", "name"}, "creator")
+ page = _exact_keys(root["page"], {"dynamic_url", "profile_url"}, "page")
+ output = _exact_keys(root["output"], {"root", "intake_root", "manifest_name"}, "output")
+ selection = _exact_keys(root["selection"], {"date_start", "date_end", "window_days", "timezone", "include_types"}, "selection")
+ readiness = _exact_keys(root["readiness"], {"deadline_seconds", "observation_interval_ms", "stable_observations"}, "readiness")
+ rerun = _exact_keys(root["rerun"], {"policy"}, "rerun")
+ verification = _exact_keys(root["verification"], {"summary_path"}, "verification")
+ limits = _exact_keys(root["limits"], {"max_items", "max_body_bytes", "max_images_per_item", "max_image_bytes"}, "limits")
+
+ uid = creator["uid"]
+ name = creator["name"]
+ if not isinstance(uid, str) or UID.fullmatch(uid) is None:
+ raise CollectorError("E_CONFIG", "creator.uid must be a positive decimal string.")
+ if not isinstance(name, str) or not name.strip() or len(name.strip()) > 80 or CONTROL.search(name):
+ raise CollectorError("E_CONFIG", "creator.name is invalid.")
+ name = name.strip()
+ dynamic_url = _validate_page_url(page["dynamic_url"], uid, dynamic=True)
+ profile_url = _validate_page_url(page["profile_url"], uid, dynamic=False)
+
+ project_root = PROJECT_ROOT
+ root_value = output["root"]
+ if root_value is None:
+ output_root = project_root / "ana-data" / f"news-{_safe_component(name)}"
+ elif isinstance(root_value, str) and root_value.strip():
+ raw_root = Path(root_value)
+ output_root = raw_root if raw_root.is_absolute() else path.parent / raw_root
+ else:
+ raise CollectorError("E_CONFIG", "output.root must be null or a non-empty path.")
+ output_root = output_root.resolve(strict=False)
+ if not _within(output_root, project_root):
+ raise CollectorError("E_PATH_ESCAPE", "output.root must remain inside the project root.", safety=True)
+ _safe_existing_chain(output_root)
+ intake_value = output["intake_root"]
+ if not isinstance(intake_value, str) or not intake_value.strip():
+ raise CollectorError("E_CONFIG", "output.intake_root must be a non-empty path.")
+ intake_root = Path(intake_value)
+ intake_root = (intake_root if intake_root.is_absolute() else path.parent / intake_root).resolve(strict=False)
+ if not _within(intake_root, project_root):
+ raise CollectorError("E_PATH_ESCAPE", "output.intake_root must remain inside the project root.", safety=True)
+ _safe_existing_chain(intake_root)
+ manifest_name = output["manifest_name"]
+ if manifest_name != "manifest.jsonl":
+ raise CollectorError("E_CONFIG", "output.manifest_name must equal manifest.jsonl.")
+ summary_value = verification["summary_path"]
+ if summary_value is None:
+ summary_path = None
+ elif isinstance(summary_value, str) and summary_value.strip():
+ summary_relative = PurePosixPath(summary_value)
+ if summary_relative.is_absolute() or ".." in summary_relative.parts or not summary_relative.parts:
+ raise CollectorError("E_CONFIG", "verification.summary_path must be output-root relative.", safety=True)
+ summary_path = (output_root / Path(*summary_relative.parts)).resolve(strict=False)
+ if not _within(summary_path, output_root):
+ raise CollectorError("E_PATH_ESCAPE", "verification.summary_path escapes output.root.", safety=True)
+ _ordinary_file(summary_path)
+ else:
+ raise CollectorError("E_CONFIG", "verification.summary_path must be null or a relative path.")
+
+ timezone_name = selection["timezone"]
+ if not isinstance(timezone_name, str):
+ raise CollectorError("E_CONFIG", "selection.timezone must be a zoneinfo name.")
+ try:
+ tz = ZoneInfo(timezone_name)
+ except ZoneInfoNotFoundError as exc:
+ raise CollectorError("E_CONFIG", "selection.timezone is unknown.") from exc
+ explicit = selection["date_start"] is not None or selection["date_end"] is not None
+ window_days = selection["window_days"]
+ if explicit:
+ if selection["date_start"] is None or selection["date_end"] is None or window_days is not None:
+ raise CollectorError("E_CONFIG", "Use either date_start/date_end or window_days.")
+ start = _parse_datetime(selection["date_start"], "selection.date_start")
+ end = _parse_datetime(selection["date_end"], "selection.date_end")
+ else:
+ days = _bounded_int(window_days, "selection.window_days", 1, 366)
+ current = (now or datetime.now(timezone.utc)).astimezone(tz)
+ end_local = current
+ start_local = current - timedelta(days=days)
+ start, end = start_local.astimezone(timezone.utc), end_local.astimezone(timezone.utc)
+ if start > end or end - start > timedelta(days=366):
+ raise CollectorError("E_CONFIG", "Configured selection interval is invalid.")
+ include_raw = selection["include_types"]
+ if not isinstance(include_raw, list) or not include_raw or len(include_raw) != len(set(include_raw)):
+ raise CollectorError("E_CONFIG", "selection.include_types must be a non-empty unique list.")
+ include_types = frozenset(include_raw)
+ if not include_types.issubset(ALLOWED_TYPES):
+ raise CollectorError("E_CONFIG", "selection.include_types contains an unsupported type.")
+ policy = rerun["policy"]
+ if policy not in {"verify_only", "verify_or_append"}:
+ raise CollectorError("E_CONFIG", "rerun.policy is unsupported.")
+
+ return CollectorConfig(
+ path=path,
+ raw_bytes=raw,
+ sha256=hashlib.sha256(raw).hexdigest().upper(),
+ creator_uid=uid,
+ creator_name=name,
+ dynamic_url=dynamic_url,
+ profile_url=profile_url,
+ output_root=output_root,
+ manifest_path=output_root / "manifest.jsonl",
+ intake_root=intake_root,
+ summary_path=summary_path,
+ timezone_name=timezone_name,
+ date_start=start,
+ date_end=end,
+ include_types=include_types,
+ deadline_seconds=_bounded_int(readiness["deadline_seconds"], "readiness.deadline_seconds", 5, 600),
+ observation_interval_ms=_bounded_int(readiness["observation_interval_ms"], "readiness.observation_interval_ms", 100, 10000),
+ stable_observations=_bounded_int(readiness["stable_observations"], "readiness.stable_observations", 2, 5),
+ rerun_policy=policy,
+ max_items=_bounded_int(limits["max_items"], "limits.max_items", 1, 1000),
+ max_body_bytes=_bounded_int(limits["max_body_bytes"], "limits.max_body_bytes", 1, 8 * 1024 * 1024),
+ max_images_per_item=_bounded_int(limits["max_images_per_item"], "limits.max_images_per_item", 0, 50),
+ max_image_bytes=_bounded_int(limits["max_image_bytes"], "limits.max_image_bytes", 1, 50 * 1024 * 1024),
+ )
+
+
+def _capture_page_identity(value: Mapping[str, Any], config: CollectorConfig) -> None:
+ if value.get("creator_uid") != config.creator_uid or value.get("creator_name") != config.creator_name:
+ raise CollectorError("E_CREATOR_IDENTITY", "Capture creator differs from config.", safety=True)
+ if value.get("dynamic_url") != config.dynamic_url or value.get("profile_url") != config.profile_url:
+ raise CollectorError("E_PAGE_IDENTITY", "Capture page proof differs from config.", safety=True)
+
+
+def _accepted_snapshot_sha256(items: Sequence[Mapping[str, Any]]) -> str:
+ canonical_items = []
+ for item in items:
+ published = item["published_at"]
+ if not isinstance(published, datetime):
+ raise CollectorError("E_READINESS_DIGEST", "Accepted snapshot timestamp is not normalized.", safety=True)
+ if published.microsecond % 1000:
+ raise CollectorError("E_READINESS_DIGEST", "Accepted snapshot timestamp exceeds browser millisecond precision.", safety=True)
+ canonical_items.append({
+ "body_complete": True,
+ "body_text": item["body"].decode("utf-8", errors="strict").rstrip("\n"),
+ "image_count": len(item["images"]),
+ "item_type": item["item_type"],
+ "published_at_epoch_ms": int(published.timestamp() * 1000),
+ "source_url": item["source_url"],
+ "stable_id": item["stable_id"],
+ "title": item["title"],
+ })
+ canonical_items.sort(key=lambda item: item["stable_id"])
+ snapshot = {"items": canonical_items, "schema_version": 1}
+ return hashlib.sha256(_canonical_payload(snapshot)).hexdigest().upper()
+
+
+def _stable_readiness(observations: Any, config: CollectorConfig, expected_fingerprint: str) -> tuple[int, int, str]:
+ if not isinstance(observations, list) or not observations:
+ raise CollectorError("E_READINESS", "Capture requires readiness observations.")
+ streak = 0
+ prior_fingerprint: str | None = None
+ previous_elapsed = -1
+ for index, raw in enumerate(observations):
+ value = _exact_keys(raw, {"elapsed_ms", "state", "reason", "snapshot_sha256"}, f"observations[{index}]")
+ elapsed = value["elapsed_ms"]
+ state = value["state"]
+ reason = value["reason"]
+ fingerprint = value["snapshot_sha256"]
+ if not isinstance(elapsed, int) or isinstance(elapsed, bool) or elapsed <= previous_elapsed or elapsed > config.deadline_seconds * 1000:
+ raise CollectorError("E_READINESS", "Observation clock is invalid.", safety=True)
+ previous_elapsed = elapsed
+ if state in ACCESS_STATES:
+ raise CollectorError("E_ACCESS_CONTROL", "Visible page reported an access-control stop.", safety=True)
+ if state in PENDING_STATES:
+ if reason != state or fingerprint is not None:
+ raise CollectorError("E_READINESS", "Pending observation shape is invalid.", safety=True)
+ streak, prior_fingerprint = 0, None
+ continue
+ if state != READY_STATE or reason != "READY" or not isinstance(fingerprint, str) or SHA256.fullmatch(fingerprint) is None:
+ raise CollectorError("E_READINESS", "Observation state is unsupported.", safety=True)
+ if fingerprint != expected_fingerprint:
+ raise CollectorError("E_READINESS_DIGEST", "READY fingerprint does not bind the accepted items snapshot.", safety=True)
+ if fingerprint == prior_fingerprint:
+ streak += 1
+ else:
+ streak, prior_fingerprint = 1, fingerprint
+ if streak < config.stable_observations:
+ raise CollectorError("E_READINESS_TIMEOUT", "Stable READY evidence is not the terminal observation suffix.", safety=True)
+ return len(observations), previous_elapsed, expected_fingerprint
+
+
+def validate_capture(config: CollectorConfig, capture_path: Path) -> dict[str, Any]:
+ capture_path = capture_path.resolve(strict=True)
+ if not _within(capture_path, PROJECT_ROOT):
+ raise CollectorError("E_PATH_ESCAPE", "capture path must remain inside the project root.", safety=True)
+ _ordinary_file(capture_path)
+ root, raw = _strict_json(capture_path, "capture")
+ value = _exact_keys(root, {"schema_version", "creator_uid", "creator_name", "dynamic_url", "profile_url", "observations", "items"}, "capture")
+ if value["schema_version"] != CAPTURE_SCHEMA:
+ raise CollectorError("E_CAPTURE_SCHEMA", "capture.schema_version differs.")
+ _capture_page_identity(value, config)
+ items = value["items"]
+ if not isinstance(items, list) or len(items) > config.max_items:
+ raise CollectorError("E_CAPTURE_SCHEMA", "capture.items exceeds the configured bound.")
+ normalized: list[dict[str, Any]] = []
+ seen: set[str] = set()
+ for index, raw_item in enumerate(items):
+ item = _exact_keys(raw_item, {"stable_id", "item_type", "title", "source_url", "published_at", "body_text", "body_complete", "images"}, f"items[{index}]")
+ stable_id = item["stable_id"]
+ item_type = item["item_type"]
+ if not isinstance(stable_id, str) or ITEM_ID.fullmatch(stable_id) is None or stable_id in seen:
+ raise CollectorError("E_ITEM_IDENTITY", "Item stable identity is invalid or duplicated.", safety=True)
+ seen.add(stable_id)
+ if item_type not in config.include_types:
+ raise CollectorError("E_ITEM_TYPE", "Capture item type is outside configured include_types.", safety=True)
+ source = urlsplit(str(item["source_url"]))
+ if source.scheme != "https" or source.hostname != "www.bilibili.com" or source.query or source.fragment or source.path.rstrip("/") != f"/opus/{stable_id}":
+ raise CollectorError("E_ITEM_IDENTITY", "Item source URL does not bind the stable ID.", safety=True)
+ published = _parse_datetime(item["published_at"], f"items[{index}].published_at")
+ if published < config.date_start or published > config.date_end:
+ raise CollectorError("E_ITEM_WINDOW", "Capture item is outside the configured interval.", safety=True)
+ title = item["title"]
+ body = item["body_text"]
+ if not isinstance(title, str) or not title.strip() or CONTROL.search(title) or not isinstance(body, str) or not body.strip() or not item["body_complete"]:
+ raise CollectorError("E_CONTENT_INCOMPLETE", "Item title/body is incomplete.", safety=True)
+ body_bytes = body.replace("\r\n", "\n").replace("\r", "\n").rstrip("\n").encode("utf-8") + b"\n"
+ if len(body_bytes) > config.max_body_bytes:
+ raise CollectorError("E_CONTENT_LIMIT", "Item body exceeds the configured limit.")
+ images_raw = item["images"]
+ if not isinstance(images_raw, list) or len(images_raw) > config.max_images_per_item:
+ raise CollectorError("E_CONTENT_LIMIT", "Item images exceed the configured limit.")
+ if item_type == "image" and not images_raw:
+ raise CollectorError("E_CONTENT_INCOMPLETE", "Image item requires at least one original image.")
+ images: list[dict[str, Any]] = []
+ for sequence, raw_image in enumerate(images_raw, 1):
+ image = _exact_keys(raw_image, {"path", "bytes", "sha256", "extension"}, f"items[{index}].images[{sequence - 1}]")
+ relative = PurePosixPath(str(image["path"]))
+ if relative.is_absolute() or ".." in relative.parts or not relative.parts:
+ raise CollectorError("E_ARTIFACT_PATH", "Image intake path is unsafe.", safety=True)
+ source_path = (config.intake_root / Path(*relative.parts)).resolve(strict=False)
+ if not _within(source_path, config.intake_root):
+ raise CollectorError("E_ARTIFACT_PATH", "Image intake path escapes its root.", safety=True)
+ _ordinary_file(source_path)
+ extension = image["extension"]
+ if extension not in {".jpg", ".jpeg", ".png", ".webp"} or source_path.suffix.lower() != extension:
+ raise CollectorError("E_ARTIFACT", "Image extension is unsupported.")
+ size = image["bytes"]
+ digest = image["sha256"]
+ if not isinstance(size, int) or size <= 0 or size > config.max_image_bytes or not isinstance(digest, str) or SHA256.fullmatch(digest) is None:
+ raise CollectorError("E_ARTIFACT", "Image identity is invalid.")
+ if source_path.stat().st_size != size or _sha256(source_path) != digest:
+ raise CollectorError("E_ARTIFACT_HASH", "Image identity differs from intake bytes.", safety=True)
+ head = source_path.read_bytes()[:12]
+ if extension in {".jpg", ".jpeg"} and not head.startswith(b"\xff\xd8\xff"):
+ raise CollectorError("E_ARTIFACT", "JPEG magic differs.")
+ if extension == ".png" and not head.startswith(b"\x89PNG\r\n\x1a\n"):
+ raise CollectorError("E_ARTIFACT", "PNG magic differs.")
+ if extension == ".webp" and not (head.startswith(b"RIFF") and head[8:12] == b"WEBP"):
+ raise CollectorError("E_ARTIFACT", "WebP magic differs.")
+ images.append({"sequence": sequence, "source": source_path, "bytes": size, "sha256": digest, "extension": extension})
+ normalized.append({
+ "stable_id": stable_id,
+ "item_type": item_type,
+ "title": title.strip(),
+ "source_url": urlunsplit(("https", "www.bilibili.com", f"/opus/{stable_id}", "", "")),
+ "published_at": published,
+ "body": body_bytes,
+ "images": images,
+ })
+ fingerprint = _accepted_snapshot_sha256(normalized)
+ attempts, elapsed, fingerprint = _stable_readiness(value["observations"], config, fingerprint)
+ return {
+ "capture_bytes": len(raw),
+ "capture_sha256": hashlib.sha256(raw).hexdigest().upper(),
+ "readiness_attempts": attempts,
+ "readiness_elapsed_ms": elapsed,
+ "snapshot_sha256": fingerprint,
+ "items": normalized,
+ }
+
+
+def _same_file_identity(left: os.stat_result, right: os.stat_result) -> bool:
+ left_inode = (getattr(left, "st_dev", 0), getattr(left, "st_ino", 0))
+ right_inode = (getattr(right, "st_dev", 0), getattr(right, "st_ino", 0))
+ return left_inode == right_inode and stat.S_ISREG(left.st_mode) and stat.S_ISREG(right.st_mode)
+
+
+def _freeze_image_payload(image: Mapping[str, Any], max_bytes: int) -> bytes:
+ path = image["source"]
+ _ordinary_file(path)
+ before = path.lstat()
+ flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
+ try:
+ descriptor = os.open(path, flags)
+ except OSError as exc:
+ raise CollectorError("E_ARTIFACT_DRIFT", "Image intake cannot be opened without following indirection.", safety=True) from exc
+ try:
+ opened = os.fstat(descriptor)
+ if not _same_file_identity(before, opened) or _is_reparse(path):
+ raise CollectorError("E_ARTIFACT_DRIFT", "Image intake identity changed before precommit.", safety=True)
+ chunks: list[bytes] = []
+ total = 0
+ while True:
+ chunk = os.read(descriptor, min(1024 * 1024, max_bytes + 1 - total))
+ if not chunk:
+ break
+ chunks.append(chunk)
+ total += len(chunk)
+ if total > max_bytes:
+ raise CollectorError("E_ARTIFACT_DRIFT", "Image intake exceeds its configured bound at precommit.", safety=True)
+ payload = b"".join(chunks)
+ finally:
+ os.close(descriptor)
+ after = path.lstat()
+ if (
+ not _same_file_identity(opened, after)
+ or _is_reparse(path)
+ or opened.st_size != after.st_size
+ or getattr(opened, "st_mtime_ns", None) != getattr(after, "st_mtime_ns", None)
+ ):
+ raise CollectorError("E_ARTIFACT_DRIFT", "Image intake changed during the final precommit read.", safety=True)
+ digest = hashlib.sha256(payload).hexdigest().upper()
+ if len(payload) != image["bytes"] or digest != image["sha256"]:
+ raise CollectorError("E_ARTIFACT_DRIFT", "Image intake bytes differ from the validated identity at precommit.", safety=True)
+ extension = image["extension"]
+ head = payload[:12]
+ if extension in {".jpg", ".jpeg"} and not head.startswith(b"\xff\xd8\xff"):
+ raise CollectorError("E_ARTIFACT_DRIFT", "JPEG magic differs at precommit.", safety=True)
+ if extension == ".png" and not head.startswith(b"\x89PNG\r\n\x1a\n"):
+ raise CollectorError("E_ARTIFACT_DRIFT", "PNG magic differs at precommit.", safety=True)
+ if extension == ".webp" and not (head.startswith(b"RIFF") and head[8:12] == b"WEBP"):
+ raise CollectorError("E_ARTIFACT_DRIFT", "WebP magic differs at precommit.", safety=True)
+ return payload
+
+
+def _read_manifest(path: Path) -> list[dict[str, Any]]:
+ if not path.exists():
+ return []
+ _ordinary_file(path)
+ raw = path.read_bytes()
+ if raw.startswith(b"\xef\xbb\xbf") or b"\r" in raw or (raw and not raw.endswith(b"\n")):
+ raise CollectorError("E_MANIFEST", "Manifest encoding is invalid.", safety=True)
+ events: list[dict[str, Any]] = []
+ for line_number, line in enumerate(raw.splitlines(), 1):
+ if not line:
+ raise CollectorError("E_MANIFEST", "Manifest contains a blank line.", safety=True)
+ try:
+ value = json.loads(line.decode("utf-8", errors="strict"))
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+ raise CollectorError("E_MANIFEST", f"Manifest line {line_number} is invalid.", safety=True) from exc
+ if not isinstance(value, dict):
+ raise CollectorError("E_MANIFEST", "Manifest row is not an object.", safety=True)
+ events.append(value)
+ return events
+
+
+def _manifest_snapshot(config: CollectorConfig) -> tuple[int, str]:
+ if not config.manifest_path.exists():
+ return 0, hashlib.sha256(b"").hexdigest().upper()
+ raw = config.manifest_path.read_bytes()
+ return len(raw), hashlib.sha256(raw).hexdigest().upper()
+
+
+def _create_new(path: Path, payload: bytes) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ _safe_existing_chain(path.parent)
+ try:
+ with path.open("xb") as handle:
+ handle.write(payload)
+ handle.flush()
+ os.fsync(handle.fileno())
+ except FileExistsError as exc:
+ raise CollectorError("E_TARGET_EXISTS", "CreateNew target already exists.", safety=True) from exc
+
+
+def _terminal_write(path: Path | None, value: Mapping[str, Any]) -> str:
+ payload = _canonical_bytes(value)
+ if path is None:
+ return "STDOUT_ONLY"
+ target = path.resolve(strict=False)
+ if not _within(target, PROJECT_ROOT):
+ raise CollectorError("E_PATH_ESCAPE", "terminal path must remain inside the project root.", safety=True)
+ _safe_existing_chain(target, allow_missing_leaf=True)
+ if target.exists():
+ _ordinary_file(target)
+ if target.read_bytes() != payload:
+ raise CollectorError("E_TERMINAL_CONFLICT", "Terminal path exists with different bytes.", safety=True)
+ return "REUSED"
+ _create_new(target, payload)
+ return "CREATED"
+
+
+def _owned_recovery_evidence(config: CollectorConfig) -> list[Path]:
+ if not config.output_root.exists():
+ return []
+ if not config.output_root.is_dir() or _is_reparse(config.output_root):
+ raise CollectorError("E_PATH", "Output root is not an ordinary directory.", safety=True)
+ evidence: list[Path] = []
+ for candidate in config.output_root.iterdir():
+ if OWNED_PENDING.fullmatch(candidate.name):
+ _ordinary_file(candidate)
+ evidence.append(candidate)
+ return sorted(evidence, key=lambda item: item.name)
+
+
+def collect(config: CollectorConfig, capture_path: Path, terminal_path: Path | None) -> dict[str, Any]:
+ if config.rerun_policy != "verify_or_append":
+ raise CollectorError("E_RERUN_POLICY", "collect requires rerun.policy=verify_or_append.", safety=True)
+ if _owned_recovery_evidence(config):
+ raise CollectorError("E_RECOVERY_REQUIRED", "Owned pending evidence requires separate recovery.", safety=True)
+ capture = validate_capture(config, capture_path)
+ events = _read_manifest(config.manifest_path)
+ latest = {str(row.get("stable_id")): row for row in events if isinstance(row.get("stable_id"), str)}
+ plan: list[dict[str, Any]] = []
+ for item in capture["items"]:
+ prior = latest.get(item["stable_id"])
+ if prior and prior.get("status") == "SAVED":
+ continue
+ local = item["published_at"].astimezone(config.tz)
+ stem = f"{local:%Y%m%d-%H%M%S}_{item['item_type']}_{_safe_component(item['title'], max_length=48)}_{item['stable_id']}"
+ text_name = f"{stem}.txt"
+ images = [f"{stem}_{index:02d}{image['extension']}" for index, image in enumerate(item["images"], 1)]
+ targets = [config.output_root / text_name, *(config.output_root / name for name in images)]
+ if any(target.exists() for target in targets):
+ raise CollectorError("E_TARGET_EXISTS", "A planned output target already exists.", safety=True)
+ plan.append({"item": item, "text_name": text_name, "image_names": images, "targets": targets})
+ if not plan:
+ terminal = {
+ "schema_version": 1,
+ "status": "NO_NEW_ITEMS",
+ "creator_uid": config.creator_uid,
+ "creator_name": config.creator_name,
+ "config_sha256": config.sha256,
+ "capture_sha256": capture["capture_sha256"],
+ "input_items": len(capture["items"]),
+ "new_items": 0,
+ "mutation_count": 0,
+ }
+ terminal["terminal_disposition"] = _terminal_write(terminal_path, terminal)
+ return terminal
+ manifest_before = _manifest_snapshot(config)
+ for planned in plan:
+ planned["frozen_images"] = [
+ {"identity": image, "payload": _freeze_image_payload(image, config.max_image_bytes)}
+ for image in planned["item"]["images"]
+ ]
+ if _manifest_snapshot(config) != manifest_before or any(target.exists() for planned in plan for target in planned["targets"]):
+ raise CollectorError("E_PRECOMMIT_DRIFT", "Manifest or target state changed during image intake freeze.", safety=True)
+ config.output_root.mkdir(parents=True, exist_ok=True)
+ _safe_existing_chain(config.output_root)
+ created: list[Path] = []
+ rows: list[bytes] = []
+ pending_path = config.output_root / f".bili-article-image.pending.{uuid.uuid4().hex}.json"
+ pending = {
+ "schema_version": 1,
+ "status": "PUBLISH_PENDING",
+ "config_sha256": config.sha256,
+ "capture_sha256": capture["capture_sha256"],
+ "manifest_bytes": manifest_before[0],
+ "manifest_sha256": manifest_before[1],
+ "targets": [target.name for planned in plan for target in planned["targets"]],
+ }
+ _create_new(pending_path, _canonical_bytes(pending))
+ try:
+ for planned in plan:
+ item = planned["item"]
+ _create_new(planned["targets"][0], item["body"])
+ created.append(planned["targets"][0])
+ image_refs: list[dict[str, Any]] = []
+ for frozen, name, target in zip(planned["frozen_images"], planned["image_names"], planned["targets"][1:]):
+ image = frozen["identity"]
+ _create_new(target, frozen["payload"])
+ created.append(target)
+ image_refs.append({"path": name, "bytes": image["bytes"], "sha256": image["sha256"]})
+ row = {
+ "schema_version": MANIFEST_SCHEMA,
+ "creator": config.creator_name,
+ "creator_uid": config.creator_uid,
+ "item_type": item["item_type"],
+ "stable_id": item["stable_id"],
+ "title": item["title"],
+ "source_url": item["source_url"],
+ "published_at": item["published_at"].isoformat(),
+ "collected_at": datetime.now(timezone.utc).isoformat(),
+ "status": "SAVED",
+ "path": planned["text_name"],
+ "bytes": len(item["body"]),
+ "sha256": hashlib.sha256(item["body"]).hexdigest().upper(),
+ "images": image_refs,
+ "capture_method": "authenticated_visible_dom_config_bound",
+ "readiness_observation_count": capture["readiness_attempts"],
+ "config_sha256": config.sha256,
+ }
+ rows.append(_canonical_bytes(row))
+ if _manifest_snapshot(config) != manifest_before:
+ raise CollectorError("E_PRECOMMIT_DRIFT", "Manifest changed before append.", safety=True)
+ with config.manifest_path.open("ab") as handle:
+ for row in rows:
+ handle.write(row)
+ handle.flush()
+ os.fsync(handle.fileno())
+ appended = b"".join(rows)
+ manifest_after = config.manifest_path.read_bytes()
+ if not manifest_after.endswith(appended) or len(manifest_after) != manifest_before[0] + len(appended):
+ raise CollectorError("E_RECOVERY_REQUIRED", "Manifest append readback is ambiguous.", safety=True)
+ pending_path.unlink()
+ except Exception as failure:
+ try:
+ manifest_unchanged = _manifest_snapshot(config) == manifest_before
+ except Exception:
+ manifest_unchanged = False
+ cleanup_ok = manifest_unchanged
+ if manifest_unchanged:
+ for path in reversed(created):
+ try:
+ path.unlink()
+ except OSError:
+ cleanup_ok = False
+ if cleanup_ok:
+ try:
+ pending_path.unlink()
+ except OSError:
+ cleanup_ok = False
+ if not cleanup_ok:
+ raise CollectorError("E_RECOVERY_REQUIRED", "Publish state is ambiguous; owned evidence was preserved.", safety=True) from failure
+ raise
+ terminal = {
+ "schema_version": 1,
+ "status": "CONTENT_SAVED",
+ "creator_uid": config.creator_uid,
+ "creator_name": config.creator_name,
+ "config_sha256": config.sha256,
+ "capture_sha256": capture["capture_sha256"],
+ "input_items": len(capture["items"]),
+ "new_items": len(plan),
+ "artifact_count": len(created),
+ "mutation_count": len(created) + 1,
+ "manifest_bytes": config.manifest_path.stat().st_size,
+ "manifest_sha256": _sha256(config.manifest_path),
+ }
+ terminal["terminal_disposition"] = _terminal_write(terminal_path, terminal)
+ return terminal
+
+
+def _validate_artifact(root: Path, relative: Any, expected_bytes: Any, expected_sha: Any) -> Path:
+ if not isinstance(relative, str):
+ raise CollectorError("E_MANIFEST", "Artifact path is missing.", safety=True)
+ parsed = PurePosixPath(relative)
+ if parsed.is_absolute() or ".." in parsed.parts or not parsed.parts:
+ raise CollectorError("E_MANIFEST", "Artifact path is unsafe.", safety=True)
+ path = (root / Path(*parsed.parts)).resolve(strict=False)
+ if not _within(path, root):
+ raise CollectorError("E_MANIFEST", "Artifact path escapes output root.", safety=True)
+ _ordinary_file(path)
+ if not isinstance(expected_bytes, int) or expected_bytes < 1 or not isinstance(expected_sha, str) or SHA256.fullmatch(expected_sha) is None:
+ raise CollectorError("E_MANIFEST", "Artifact identity is invalid.", safety=True)
+ if path.stat().st_size != expected_bytes or _sha256(path) != expected_sha:
+ raise CollectorError("E_MANIFEST_DRIFT", "Artifact bytes or SHA-256 drifted.", safety=True)
+ return path
+
+
+def _verify_summary(config: CollectorConfig, manifest_latest: Mapping[str, Mapping[str, Any]]) -> tuple[int, int, int, int]:
+ if config.summary_path is None:
+ raise CollectorError("E_SUMMARY", "Summary path is absent.", safety=True)
+ root, _ = _strict_json(config.summary_path, "verification summary")
+ value = _exact_keys(
+ root,
+ {
+ "schema_version", "creator", "creator_uid", "date_window", "generated_at", "item_count",
+ "article_count", "text_count", "image_count", "body_bytes_total", "manifest", "rows",
+ },
+ "verification summary",
+ )
+ if value["schema_version"] != 1 or value["creator"] != config.creator_name or str(value["creator_uid"]) != config.creator_uid:
+ raise CollectorError("E_SUMMARY_IDENTITY", "Verification summary creator differs from config.", safety=True)
+ window = _exact_keys(value["date_window"], {"start", "end"}, "verification summary date_window")
+ if _parse_datetime(window["start"], "summary.date_window.start") != config.date_start or _parse_datetime(window["end"], "summary.date_window.end") != config.date_end:
+ raise CollectorError("E_SUMMARY_IDENTITY", "Verification summary interval differs from config.", safety=True)
+ manifest = _exact_keys(
+ value["manifest"],
+ {"path", "prefix_lines", "prefix_sha256", "appended_rows", "appended_block_sha256", "final_lines", "final_bytes", "final_sha256"},
+ "verification summary manifest",
+ )
+ manifest_raw = config.manifest_path.read_bytes()
+ manifest_lines = manifest_raw.splitlines(keepends=True)
+ final_lines = manifest.get("final_lines")
+ if not isinstance(final_lines, int) or isinstance(final_lines, bool) or final_lines < 0 or final_lines > len(manifest_lines):
+ raise CollectorError("E_SUMMARY_MANIFEST_DRIFT", "Verification summary manifest prefix length is invalid.", safety=True)
+ frozen_prefix = b"".join(manifest_lines[:final_lines])
+ if (
+ manifest["path"] != "manifest.jsonl"
+ or manifest["final_bytes"] != len(frozen_prefix)
+ or manifest["final_sha256"] != hashlib.sha256(frozen_prefix).hexdigest().upper()
+ ):
+ raise CollectorError("E_SUMMARY_MANIFEST_DRIFT", "Verification summary does not bind the immutable manifest prefix.", safety=True)
+ rows = value["rows"]
+ if not isinstance(rows, list) or len(rows) != value["item_count"] or len(rows) > config.max_items:
+ raise CollectorError("E_SUMMARY", "Verification summary item count is invalid.", safety=True)
+ seen: set[str] = set()
+ article_count = 0
+ text_count = 0
+ image_paths: set[str] = set()
+ for index, raw_row in enumerate(rows):
+ row = _exact_keys(
+ raw_row,
+ {"opus_id", "title", "content_type", "published_at", "source_url", "body_bytes", "body_sha256", "text", "images"},
+ f"verification summary rows[{index}]",
+ )
+ stable_id = row["opus_id"]
+ if not isinstance(stable_id, str) or ITEM_ID.fullmatch(stable_id) is None or stable_id in seen:
+ raise CollectorError("E_SUMMARY", "Verification summary has an invalid or duplicate opus ID.", safety=True)
+ seen.add(stable_id)
+ item_type = row["content_type"]
+ if item_type not in {"article", "text"} or item_type not in config.include_types:
+ raise CollectorError("E_SUMMARY", "Verification summary item type is outside config.", safety=True)
+ published = _parse_datetime(row["published_at"], f"summary.rows[{index}].published_at")
+ source = urlsplit(str(row["source_url"]))
+ if (
+ published < config.date_start
+ or published > config.date_end
+ or source.scheme != "https"
+ or source.hostname != "www.bilibili.com"
+ or source.query
+ or source.fragment
+ or source.path.rstrip("/") != f"/opus/{stable_id}"
+ ):
+ raise CollectorError("E_SUMMARY_IDENTITY", "Verification summary item proof is invalid.", safety=True)
+ if not isinstance(row["body_bytes"], int) or row["body_bytes"] < 1 or not isinstance(row["body_sha256"], str) or SHA256.fullmatch(row["body_sha256"]) is None:
+ raise CollectorError("E_SUMMARY", "Verification summary body identity is invalid.", safety=True)
+ text = _exact_keys(row["text"], {"path", "bytes", "sha256"}, f"verification summary rows[{index}].text")
+ _validate_artifact(config.output_root, text["path"], text["bytes"], text["sha256"])
+ manifest_row = manifest_latest.get(stable_id)
+ if (
+ manifest_row is None
+ or manifest_row.get("status") != "SAVED"
+ or manifest_row.get("path") != text["path"]
+ or manifest_row.get("bytes") != text["bytes"]
+ or manifest_row.get("sha256") != text["sha256"]
+ ):
+ raise CollectorError("E_SUMMARY_MANIFEST_DRIFT", "Verification summary item differs from append-only manifest.", safety=True)
+ if item_type == "article":
+ article_count += 1
+ else:
+ text_count += 1
+ images = row["images"]
+ if not isinstance(images, list) or len(images) > config.max_images_per_item:
+ raise CollectorError("E_SUMMARY", "Verification summary images are invalid.", safety=True)
+ for image_index, raw_image in enumerate(images):
+ image = _exact_keys(raw_image, {"path", "bytes", "sha256", "source_url"}, f"verification summary rows[{index}].images[{image_index}]")
+ source_image = urlsplit(str(image["source_url"]))
+ if source_image.scheme != "https" or source_image.hostname not in {"i0.hdslb.com", "i1.hdslb.com", "i2.hdslb.com"} or source_image.query or source_image.fragment or not source_image.path.startswith("/bfs/"):
+ raise CollectorError("E_SUMMARY_IDENTITY", "Verification summary image source proof is invalid.", safety=True)
+ path = _validate_artifact(config.output_root, image["path"], image["bytes"], image["sha256"])
+ image_paths.add(path.name)
+ if (
+ value["article_count"] != article_count
+ or value["text_count"] != text_count
+ or value["image_count"] != len(image_paths)
+ or value["item_count"] != article_count + text_count
+ ):
+ raise CollectorError("E_SUMMARY", "Verification summary aggregate counts differ from rows.", safety=True)
+ return len(rows), article_count, text_count, len(image_paths)
+
+
+def verify(config: CollectorConfig, terminal_path: Path | None) -> dict[str, Any]:
+ if _owned_recovery_evidence(config):
+ raise CollectorError("E_RECOVERY_REQUIRED", "Owned pending evidence requires separate recovery.", safety=True)
+ events = _read_manifest(config.manifest_path)
+ latest: dict[str, dict[str, Any]] = {}
+ for row in events:
+ if row.get("creator") != config.creator_name:
+ continue
+ row_uid = row.get("creator_uid")
+ if row_uid is not None and (isinstance(row_uid, bool) or str(row_uid) != config.creator_uid):
+ raise CollectorError("E_CREATOR_IDENTITY", "Manifest creator UID conflicts with config.", safety=True)
+ item_type = row.get("item_type")
+ if item_type not in config.include_types:
+ continue
+ _reject_secrets(row, "$selected_manifest")
+ published = _parse_datetime(row.get("published_at"), "manifest.published_at")
+ if published < config.date_start or published > config.date_end:
+ continue
+ stable_id = row.get("stable_id")
+ legacy_image = LEGACY_IMAGE_ID.fullmatch(stable_id) if isinstance(stable_id, str) else None
+ if not isinstance(stable_id, str) or (ITEM_ID.fullmatch(stable_id) is None and legacy_image is None):
+ raise CollectorError("E_MANIFEST", "Manifest stable ID is invalid.", safety=True)
+ source = urlsplit(str(row.get("source_url", "")))
+ if legacy_image is not None:
+ parent_id = row.get("source_parent_stable_id")
+ if (
+ row.get("item_type") != "image"
+ or parent_id != legacy_image.group(1)
+ or source.scheme != "https"
+ or source.hostname not in {"i0.hdslb.com", "i1.hdslb.com", "i2.hdslb.com"}
+ or source.query
+ or source.fragment
+ or not source.path.startswith("/bfs/")
+ ):
+ raise CollectorError("E_MANIFEST", "Legacy image source proof is invalid.", safety=True)
+ elif source.scheme != "https" or source.hostname != "www.bilibili.com" or source.query or source.fragment or source.path.rstrip("/") != f"/opus/{stable_id}":
+ raise CollectorError("E_MANIFEST", "Manifest source URL is invalid.", safety=True)
+ latest[stable_id] = row
+ if config.summary_path is not None:
+ item_count, article_count, non_article_count, image_count = _verify_summary(config, latest)
+ else:
+ article_count = 0
+ non_article_count = 0
+ image_paths: set[str] = set()
+ item_count = 0
+ for stable_id, row in ([] if config.summary_path is not None else latest.items()):
+ if row.get("status") != "SAVED":
+ raise CollectorError("E_CORPUS_INCOMPLETE", "Latest in-window item is not SAVED.", safety=True)
+ _validate_artifact(config.output_root, row.get("path"), row.get("bytes"), row.get("sha256"))
+ if row.get("item_type") == "article":
+ article_count += 1
+ else:
+ non_article_count += 1
+ images = row.get("images")
+ if images is not None:
+ if not isinstance(images, list):
+ raise CollectorError("E_MANIFEST", "Manifest images is not a list.", safety=True)
+ for image in images:
+ if not isinstance(image, Mapping):
+ raise CollectorError("E_MANIFEST", "Manifest image is not an object.", safety=True)
+ path = _validate_artifact(config.output_root, image.get("path"), image.get("bytes"), image.get("sha256"))
+ image_paths.add(path.name)
+ elif row.get("image_path") is not None:
+ path = _validate_artifact(config.output_root, row.get("image_path"), row.get("image_bytes"), row.get("image_sha256"))
+ image_paths.add(path.name)
+ elif row.get("item_type") == "image":
+ image_paths.add(Path(str(row["path"])).name)
+ if config.summary_path is None:
+ item_count = len(latest)
+ image_count = len(image_paths)
+ manifest_bytes, manifest_sha = _manifest_snapshot(config)
+ terminal = {
+ "schema_version": 1,
+ "status": "CORPUS_VERIFIED",
+ "creator_uid": config.creator_uid,
+ "creator_name": config.creator_name,
+ "config_sha256": config.sha256,
+ "date_start": config.date_start.isoformat(),
+ "date_end": config.date_end.isoformat(),
+ "include_types": sorted(config.include_types),
+ "item_count": item_count,
+ "article_count": article_count,
+ "text_image_dynamic_count": non_article_count,
+ "original_image_count": image_count,
+ "manifest_bytes": manifest_bytes,
+ "manifest_sha256": manifest_sha,
+ "mutation_count": 0,
+ }
+ terminal["terminal_disposition"] = _terminal_write(terminal_path, terminal)
+ return terminal
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Generic logged-session Bilibili article/image collector")
+ parser.add_argument("--config", required=True, type=Path, help="Strict UTF-8 JSON config")
+ subparsers = parser.add_subparsers(dest="command", required=True)
+ validate = subparsers.add_parser("validate-capture", help="Validate one sanitized browser capture without publishing")
+ validate.add_argument("--capture", required=True, type=Path)
+ collect_parser = subparsers.add_parser("collect", help="CreateNew-publish one validated capture")
+ collect_parser.add_argument("--capture", required=True, type=Path)
+ collect_parser.add_argument("--terminal", type=Path)
+ verify_parser = subparsers.add_parser("verify", help="Read-only verify the configured corpus")
+ verify_parser.add_argument("--terminal", type=Path)
+ return parser
+
+
+def run(argv: Sequence[str] | None = None) -> tuple[int, dict[str, Any]]:
+ args = build_parser().parse_args(argv)
+ try:
+ config = load_config(args.config)
+ if args.command == "validate-capture":
+ capture = validate_capture(config, args.capture)
+ result = {
+ "schema_version": 1,
+ "status": "CAPTURE_VALID",
+ "config_sha256": config.sha256,
+ "capture_sha256": capture["capture_sha256"],
+ "item_count": len(capture["items"]),
+ "readiness_attempts": capture["readiness_attempts"],
+ "mutation_count": 0,
+ }
+ elif args.command == "collect":
+ result = collect(config, args.capture, args.terminal)
+ else:
+ result = verify(config, args.terminal)
+ return 0, result
+ except CollectorError as exc:
+ return (3 if exc.safety else 2), {
+ "schema_version": 1,
+ "status": "SAFETY_STOP" if exc.safety else "INPUT_ERROR",
+ "error_code": exc.code,
+ "message": exc.message,
+ "mutation_count": 0,
+ }
+ except KeyboardInterrupt:
+ return 130, {"schema_version": 1, "status": "INTERRUPTED", "mutation_count": 0}
+ except Exception:
+ return 1, {"schema_version": 1, "status": "INTERNAL_ERROR", "error_code": "E_INTERNAL", "mutation_count": 0}
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ code, result = run(argv)
+ print(json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":")))
+ return code
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/dev/project-dev/bili_article_image_native_host.py b/dev/project-dev/bili_article_image_native_host.py
new file mode 100644
index 0000000..df9f05c
--- /dev/null
+++ b/dev/project-dev/bili_article_image_native_host.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Minimal native-messaging boundary for the generic article/image collector."""
+
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from pathlib import Path
+from typing import Any, BinaryIO, Mapping
+
+import bili_article_image_collector as collector
+
+
+MAX_REQUEST_BYTES = 16 * 1024 * 1024
+MAX_RESPONSE_BYTES = 1024 * 1024
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+
+
+def _exact_keys(value: Any, expected: set[str]) -> Mapping[str, Any]:
+ if not isinstance(value, Mapping) or set(value) != expected:
+ raise collector.CollectorError("E_HOST_SCHEMA", "Native Host request keys differ.", safety=True)
+ collector._reject_secrets(value)
+ return value
+
+
+def _project_path(value: Any, *, required: bool) -> Path | None:
+ if value is None and not required:
+ return None
+ if not isinstance(value, str) or not value.strip():
+ raise collector.CollectorError("E_HOST_PATH", "Native Host path is invalid.", safety=True)
+ path = Path(value)
+ path = (path if path.is_absolute() else PROJECT_ROOT / path).resolve(strict=False)
+ if not collector._within(path, PROJECT_ROOT):
+ raise collector.CollectorError("E_HOST_PATH", "Native Host path escapes the project root.", safety=True)
+ collector._safe_existing_chain(path, allow_missing_leaf=not required)
+ if required:
+ collector._ordinary_file(path)
+ return path
+
+
+def run_request(raw: Any) -> tuple[int, dict[str, Any]]:
+ try:
+ request = _exact_keys(raw, {"schema_version", "action", "config_path", "capture_path", "terminal_path"})
+ if request["schema_version"] != 1:
+ raise collector.CollectorError("E_HOST_SCHEMA", "Native Host schema_version differs.", safety=True)
+ action = request["action"]
+ if action not in {"validate_capture", "collect", "verify"}:
+ raise collector.CollectorError("E_HOST_ACTION", "Native Host action is unsupported.", safety=True)
+ config_path = _project_path(request["config_path"], required=True)
+ capture_required = action != "verify"
+ capture_path = _project_path(request["capture_path"], required=capture_required)
+ terminal_path = _project_path(request["terminal_path"], required=False)
+ if not capture_required and capture_path is not None:
+ raise collector.CollectorError("E_HOST_SCHEMA", "verify forbids capture_path.", safety=True)
+ argv = ["--config", str(config_path)]
+ if action == "validate_capture":
+ if terminal_path is not None:
+ raise collector.CollectorError("E_HOST_SCHEMA", "validate_capture forbids terminal_path.", safety=True)
+ argv.extend(["validate-capture", "--capture", str(capture_path)])
+ elif action == "collect":
+ argv.extend(["collect", "--capture", str(capture_path)])
+ if terminal_path is not None:
+ argv.extend(["--terminal", str(terminal_path)])
+ else:
+ argv.append("verify")
+ if terminal_path is not None:
+ argv.extend(["--terminal", str(terminal_path)])
+ return collector.run(argv)
+ except collector.CollectorError as exc:
+ return (3 if exc.safety else 2), {
+ "schema_version": 1,
+ "status": "SAFETY_STOP" if exc.safety else "INPUT_ERROR",
+ "error_code": exc.code,
+ "message": exc.message,
+ "mutation_count": 0,
+ }
+ except Exception:
+ return 1, {"schema_version": 1, "status": "INTERNAL_ERROR", "error_code": "E_INTERNAL", "mutation_count": 0}
+
+
+def _read_frame(stream: BinaryIO) -> Any:
+ header = stream.read(4)
+ if len(header) != 4:
+ raise EOFError
+ size = struct.unpack("<I", header)[0]
+ if size < 2 or size > MAX_REQUEST_BYTES:
+ raise collector.CollectorError("E_HOST_FRAME", "Native Host request frame size is invalid.", safety=True)
+ payload = stream.read(size)
+ if len(payload) != size:
+ raise collector.CollectorError("E_HOST_FRAME", "Native Host request frame is truncated.", safety=True)
+ return json.loads(payload.decode("utf-8", errors="strict"))
+
+
+def _write_frame(stream: BinaryIO, value: Mapping[str, Any]) -> None:
+ payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ if len(payload) > MAX_RESPONSE_BYTES:
+ payload = b'{"error_code":"E_HOST_RESPONSE_LIMIT","mutation_count":0,"schema_version":1,"status":"INTERNAL_ERROR"}'
+ stream.write(struct.pack("<I", len(payload)))
+ stream.write(payload)
+ stream.flush()
+
+
+def main() -> int:
+ try:
+ request = _read_frame(sys.stdin.buffer)
+ code, result = run_request(request)
+ except EOFError:
+ return 0
+ except Exception:
+ code, result = 3, {"schema_version": 1, "status": "SAFETY_STOP", "error_code": "E_HOST_FRAME", "mutation_count": 0}
+ _write_frame(sys.stdout.buffer, result)
+ return code
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/dev/project-dev/bili_article_image_source_manifest.json b/dev/project-dev/bili_article_image_source_manifest.json
new file mode 100644
index 0000000..355fcfd
--- /dev/null
+++ b/dev/project-dev/bili_article_image_source_manifest.json
@@ -0,0 +1,149 @@
+{
+ "schema": 1,
+ "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "scope": "generic-bilibili-article-image-collector-source",
+ "root": "dev/project-dev",
+ "files": [
+ {
+ "path": "bili_article_image_collector.py",
+ "bytes": 53497,
+ "sha256": "C8B9BB3EAA8BE580DF11A20DCA94B507DCE1414D62D65F42D09BDAAA5348BB93"
+ },
+ {
+ "path": "bili_article_image_collector.example.json",
+ "bytes": 912,
+ "sha256": "4A905F806BD0F5BD0C71C75A5D7C7F47466CBBD070BC0DBCD5306D1A221DA030"
+ },
+ {
+ "path": "bili_article_image_capture.js",
+ "bytes": 11629,
+ "sha256": "3A947C1F543F4509AC9ED650974C0CD8BBD716EA0A98E19266B90FA9602D1404"
+ },
+ {
+ "path": "bili_article_image_native_host.py",
+ "bytes": 5011,
+ "sha256": "F284BB138C621622F90F4FEBBB9BBD10A55FA76BE2766AA92B3C328C6A8F5C84"
+ },
+ {
+ "path": "bili_article_image_source_validator.py",
+ "bytes": 4160,
+ "sha256": "56FC6A6A8AC995689A5C7ABD57D7BC37889CBF7ED6FC48D993656D799463E73C"
+ },
+ {
+ "path": "bili_authenticated_extension/queue_producer.py",
+ "bytes": 70120,
+ "sha256": "50579347FE8B6576FC141E590163A770D855D3456155F97E4158AB6B8D9AF88F"
+ },
+ {
+ "path": "bili_authenticated_extension/source-artifact-manifest.json",
+ "bytes": 3770,
+ "sha256": "132B637634550A43B0EDD5E8E74C439205DE4BFECCE0A9276D8EBEF78445ECFC"
+ },
+ {
+ "path": "bili_authenticated_extension_unpacked_validator.py",
+ "bytes": 23919,
+ "sha256": "C2A450288DAE14532EDA4A5F83A035369160D3A7758177459BDDC383C81E536A"
+ },
+ {
+ "path": "bili_dynamic_collector.py",
+ "bytes": 72068,
+ "sha256": "5F535111FA84E8DDD06FF3D512331E2E7EF8124A350960C0BB209251FBA8DBE6"
+ },
+ {
+ "path": "bili_dynamic_refresh_extension/manifest.json",
+ "bytes": 821,
+ "sha256": "0DD4704E93A9989E1C44748888B0F779AD9A5030420F25C9DEDFB2EEC0B7FE86"
+ },
+ {
+ "path": "bili_dynamic_refresh_extension/package.json",
+ "bytes": 23,
+ "sha256": "3CA9D4AFD21425087CF31893B8F9F63C81B0B8408DB5E343CA76E5F8AA26AB9A"
+ },
+ {
+ "path": "bili_dynamic_refresh_extension/page_extract.js",
+ "bytes": 4541,
+ "sha256": "9A2FA6CA23E41BBE74A25429E36413E52EB330671354A493A22D3C11BE989B80"
+ },
+ {
+ "path": "bili_dynamic_refresh_extension/protocol.js",
+ "bytes": 4345,
+ "sha256": "F15D2AA3834E1212932F1E91C33FB1D3F7BE760AE6ECFEC1EF1230DAC21B6E0D"
+ },
+ {
+ "path": "bili_dynamic_refresh_extension/runtime.js",
+ "bytes": 20803,
+ "sha256": "8F83133202A93DFDC018CC59FB27C60A256A065FAF5F28732C3DFF5C78F65FAA"
+ },
+ {
+ "path": "bili_dynamic_refresh_extension/service_worker.js",
+ "bytes": 2204,
+ "sha256": "AE026F233E1C0D58A4D669508142806F03D7BC4396918BEE65FE824F30D56D8B"
+ },
+ {
+ "path": "bili_dynamic_refresh_extension/source-artifact-manifest.json",
+ "bytes": 1228,
+ "sha256": "6D98DE5B770244F180A50685D6F22EB82829BE59103DD64FEBA47190A920702C"
+ },
+ {
+ "path": "bili_dynamic_refresh_native_host/constants.py",
+ "bytes": 520,
+ "sha256": "072EFA0B7D555366B68C358DD3DCC5E4E2F9DBD3A9D26C0EED2722DD87CD271C"
+ },
+ {
+ "path": "bili_dynamic_refresh_native_host/native_host.py",
+ "bytes": 2983,
+ "sha256": "507595BD40EA5F760ABA29D1522EFB2658D3797FF30CBC44519334D8F8165989"
+ },
+ {
+ "path": "bili_dynamic_refresh_native_host/protocol.py",
+ "bytes": 8840,
+ "sha256": "C8CDB4BA5FCE931CB33A8628C80F9ECB8E8100DAF2FCBAEC9C3E31A05692BBC2"
+ },
+ {
+ "path": "bili_dynamic_refresh_native_host/source-artifact-manifest.json",
+ "bytes": 1538,
+ "sha256": "2959004238B6CB76E6F42785DDBFAEECD7BBE40A46B88EE7BB4609A94EE1261E"
+ },
+ {
+ "path": "test/test_bili_article_image_collector.py",
+ "bytes": 23082,
+ "sha256": "2BA384FB5ECCB0486FC80E294FB3036D2EEE0DFC39BFDA2BF650AE9F8ACD9E75"
+ },
+ {
+ "path": "test/test_bili_article_image_capture.mjs",
+ "bytes": 4319,
+ "sha256": "EB0E16808B8CF60C856E48C5CCEE456A2084A816D794C664A6022BC76E1BDC8A"
+ },
+ {
+ "path": "test/bili_authenticated_extension/test_successor_trust_gate.py",
+ "bytes": 55866,
+ "sha256": "04B00A195B9538650CC643FBC76086184A365B8BB17AF9A370A6B8FDAFBAFE6C"
+ },
+ {
+ "path": "test/bili_authenticated_extension/test_unpacked_projection.py",
+ "bytes": 13228,
+ "sha256": "DE8CD274E4C0D9BDF3FB0662093ABB00FBB534CD37DAE91CF980AE457C7CAD04"
+ },
+ {
+ "path": "test/bili_dynamic_refresh_trusted_adapter/test_v009_contract.py",
+ "bytes": 16956,
+ "sha256": "A00C4236FBB5E36BCC8B4520DF3FC19819F76C9B2F381975CE931DB5016C23C6"
+ },
+ {
+ "path": "test/bili_dynamic_refresh_trusted_adapter/js_contract.mjs",
+ "bytes": 3897,
+ "sha256": "0BAE277258922F0D288CC5B647CD4F6FF4F5D77641AB181601692CA2917784C5"
+ },
+ {
+ "path": "test/bili_dynamic_refresh_trusted_adapter/runtime_lifecycle.mjs",
+ "bytes": 19561,
+ "sha256": "EEB0836A141F12669E7F82CE8AC512BBD9FCDC8F3F536D33C4DC991CC123C61D"
+ },
+ {
+ "path": "test/bili_dynamic_refresh_trusted_adapter/codex_stdio_race_repro.mjs",
+ "bytes": 1717,
+ "sha256": "D714D2842B4A338BD7A77031921159C4D694FE330106C861A15FB9137F772494"
+ }
+ ],
+ "tree_sha256": "9E61A57C23ECAFBD607F80E5C084ADC8013E0005707F04E632C25BC8D560B066"
+}
diff --git a/dev/project-dev/bili_article_image_source_validator.py b/dev/project-dev/bili_article_image_source_validator.py
new file mode 100644
index 0000000..3fb973e
--- /dev/null
+++ b/dev/project-dev/bili_article_image_source_validator.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Validate the mechanically bound source set for the generic article/image collector."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import stat
+import sys
+from pathlib import Path
+from typing import Any
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_MANIFEST = Path(__file__).with_name("bili_article_image_source_manifest.json")
+SHA256 = __import__("re").compile(r"[0-9A-F]{64}")
+
+
+def _reparse(path: Path) -> bool:
+ info = path.lstat()
+ return stat.S_ISLNK(info.st_mode) or bool(getattr(info, "st_file_attributes", 0) & 0x400)
+
+
+def _strict_json(path: Path) -> tuple[Any, bytes]:
+ raw = path.read_bytes()
+ if raw.startswith(b"\xef\xbb\xbf") or b"\r" in raw or not raw.endswith(b"\n") or raw.endswith(b"\n\n"):
+ raise ValueError("manifest encoding differs")
+ return json.loads(raw[:-1].decode("utf-8", errors="strict")), raw
+
+
+def validate(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]:
+ manifest_path = Path(os.path.abspath(manifest_path))
+ value, raw = _strict_json(manifest_path)
+ if not isinstance(value, dict) or set(value) != {"schema", "task_id", "scope", "root", "files", "tree_sha256"}:
+ raise ValueError("manifest schema differs")
+ if value["schema"] != 1 or value["task_id"] != "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001" or value["scope"] != "generic-bilibili-article-image-collector-source":
+ raise ValueError("manifest identity differs")
+ root = (PROJECT_ROOT / value["root"]).resolve(strict=True)
+ if root != PROJECT_ROOT / "dev" / "project-dev" or _reparse(root):
+ raise ValueError("manifest root differs")
+ files = value["files"]
+ if not isinstance(files, list) or not files:
+ raise ValueError("manifest files differ")
+ seen: set[str] = set()
+ actual: list[dict[str, Any]] = []
+ for entry in files:
+ if not isinstance(entry, dict) or set(entry) != {"path", "bytes", "sha256"}:
+ raise ValueError("manifest entry schema differs")
+ relative = entry["path"]
+ if not isinstance(relative, str) or not relative or "\\" in relative or relative.startswith("/") or ".." in Path(relative).parts or relative in seen:
+ raise ValueError("manifest entry path differs")
+ seen.add(relative)
+ candidate = (root / Path(*relative.split("/"))).resolve(strict=True)
+ if candidate.parent == root and candidate == manifest_path:
+ raise ValueError("manifest cannot self-bind")
+ if not candidate.is_file() or _reparse(candidate):
+ raise ValueError("source file is missing or unsafe")
+ payload = candidate.read_bytes()
+ fact = {"path": relative, "bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest().upper()}
+ if entry != fact or SHA256.fullmatch(str(entry["sha256"])) is None:
+ raise ValueError("source file identity differs")
+ actual.append(fact)
+ material = bytearray()
+ for entry in sorted(actual, key=lambda item: item["path"]):
+ material.extend(entry["path"].encode("utf-8"))
+ material.extend(b"\0")
+ material.extend(str(entry["bytes"]).encode("ascii"))
+ material.extend(b"\0")
+ material.extend(entry["sha256"].encode("ascii"))
+ material.extend(b"\n")
+ tree = hashlib.sha256(material).hexdigest().upper()
+ if value["tree_sha256"] != tree:
+ raise ValueError("source tree identity differs")
+ return {
+ "schema_version": 1,
+ "status": "SOURCE_VALID",
+ "file_count": len(actual),
+ "tree_sha256": tree,
+ "manifest_bytes": len(raw),
+ "manifest_sha256": hashlib.sha256(raw).hexdigest().upper(),
+ "mutation_count": 0,
+ }
+
+
+def main() -> int:
+ try:
+ result = validate()
+ code = 0
+ except Exception:
+ result = {"schema_version": 1, "status": "SOURCE_INVALID", "error_code": "E_SOURCE_IDENTITY", "mutation_count": 0}
+ code = 3
+ print(json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":")))
+ return code
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/dev/project-dev/bili_authenticated_extension/__init__.py b/dev/project-dev/bili_authenticated_extension/__init__.py
index 5d6f4b3..afb538a 100644
--- a/dev/project-dev/bili_authenticated_extension/__init__.py
+++ b/dev/project-dev/bili_authenticated_extension/__init__.py
@@ -1,9 +1,9 @@
-"""Exact-BVID authenticated Bilibili extension ingress.
+"""Locally queued authenticated Bilibili extension ingress.
-The package is deliberately narrow: it only supports the frozen
-``BV1HA3o6oEJJ`` task and never imports yt-dlp in the broker process.
+The stdlib-only broker accepts only creator-allowlisted generic jobs and never
+imports yt-dlp in the broker process.
"""
-from .constants import CANONICAL_URL, TARGET_BVID
+from .constants import canonical_url, stable_job_id, validate_bvid
-__all__ = ["CANONICAL_URL", "TARGET_BVID"]
+__all__ = ["canonical_url", "stable_job_id", "validate_bvid"]
diff --git a/dev/project-dev/bili_authenticated_extension/background.js b/dev/project-dev/bili_authenticated_extension/background.js
index 049ae99..bef33c4 100644
--- a/dev/project-dev/bili_authenticated_extension/background.js
+++ b/dev/project-dev/bili_authenticated_extension/background.js
@@ -1,388 +1,1636 @@
-const TARGET = "BV1HA3o6oEJJ";
-const CANONICAL_URL = "https://www.bilibili.com/video/BV1HA3o6oEJJ";
-const TARGET_PATH = "/video/BV1HA3o6oEJJ";
-const HOST = "com.project_info.bili_auth_ingress";
-const SCHEMA = 2;
-const EXTENSION_BUILD = "project-info-bili-auth-ingress/1.0.0+20260805.v002";
-const HOST_BUILD = "project-info-bili-auth-native-host/1.0.0+20260805.v002";
-const SIDEPANEL_URL = chrome.runtime.getURL("sidepanel.html");
-const EXPECTED_DURATION_MS = 3133950;
-const DURATION_TOLERANCE_MS = 3134;
+const HOST_NAME = "com.project_info.bili_auth_ingress";
+const SCHEMA = 3;
+const EXTENSION_BUILD = "project-info-bili-auth-ingress/1.2.25+20260829.generic.v027";
+const OWNED_TAB_STORAGE_KEY = "bili_auth_job_owned_tab_v1";
+const OWNED_TAB_SCHEMA = 1;
+const BVID_RE = /^BV1[1-9A-HJ-NP-Za-km-z]{9}$/;
+const CREATOR_RE = /^[1-9][0-9]{0,19}$/;
+const HEX32_RE = /^[0-9a-f]{32}$/;
+const HEX64_RE = /^[0-9a-f]{64}$/;
+const UPPER_HEX64_RE = /^[0-9A-F]{64}$/;
+const MESSAGE_ID_RE = /^msg_[0-9]{17}_[0-9a-f]{8}$/;
+const HANDOFF_ID_RE = /^HANDOFF-[A-Z0-9-]{8,240}$/;
+const AUDIT_ID_RE = /^DEV-AUDIT-[A-Z0-9-]{8,240}$/;
+const ERROR_CODE_RE = /^E_[A-Z0-9_]{1,48}$/;
+const SAFE_PHASES = new Set([
+ "IDLE", "READY", "CHECKING", "DOWNLOADING", "MERGING", "VALIDATING",
+ "PUBLISHING", "MEDIA_COMPLETE", "POSTPROCESS_PENDING", "POSTPROCESS_FAILED",
+ "COMPLETE", "FAILED", "CANCELED", "RELOAD_REQUIRED"
+]);
+const RESPONSE_KEYS = [
+ "error_code", "formal_filename", "host_build", "job", "lease_id", "maintenance",
+ "mapping_filename", "phase", "prepare_id", "progress", "schema", "type"
+];
+const JOB_KEYS = ["bvid", "canonical_url", "creator_uid", "expected_duration_ms", "job_id", "published_at", "title"];
+const LINEAGE_KEYS = [
+ "authorization_handoff_id", "authorization_message_id", "authorization_sha256",
+ "predecessor_job_id", "predecessor_terminal_error_code", "repair_audit_bytes",
+ "repair_audit_id", "repair_audit_sha256", "repair_review_result_message_id", "retry_generation"
+];
+const PREPARELESS_TERMINAL_DIAGNOSTIC_CONTRACT = new Map([
+ ["E_DRM", ["PAGE_REJECTED", new Set(["PAGE_DRM_REJECTED"])]],
+ ["E_DURATION", ["PAGE_REJECTED", new Set(["PAGE_DURATION_REJECTED"])]],
+ ["E_MULTI_PART", ["PAGE_REJECTED", new Set(["PAGE_MULTIPART_REJECTED"])]],
+ ["E_OWNER", ["PAGE_REJECTED", new Set(["PAGE_IDENTITY_REJECTED", "PAGE_OWNER_REJECTED"])]],
+ ["E_PAGE_ACCESS_CONTROL", ["PAGE_REJECTED", new Set(["PAGE_ACCESS_REJECTED"])]],
+ ["E_PAGE_DUPLICATE_TAB", ["TAB_IDENTITY", new Set(["TAB_IDENTITY_DRIFT"])]],
+ ["E_PAGE_DIMENSIONS_UNAVAILABLE", ["DIMENSIONS_PENDING", new Set(["VIDEO_DIMENSIONS_PENDING"])]],
+ ["E_PAGE_IDENTITY_UNAVAILABLE", ["IDENTITY_PENDING", new Set([
+ "PAGE_DOM_IDENTITY_PENDING", "PAGE_NAVIGATION_PENDING", "PAGE_SCRIPTING_PENDING",
+ "PAGE_TAB_ACTIVATION_PENDING", "PAGE_URL_PENDING", "PAGE_VISIBILITY_PENDING",
+ "PAGE_WINDOW_FOCUS_PENDING", "PAGE_WINDOW_MINIMIZED_PENDING", "PAGE_WINDOW_STATE_PENDING"
+ ])]],
+ ["E_PAGE_METADATA_UNAVAILABLE", ["METADATA_NOT_READY", new Set([
+ "INITIAL_STATE_ASSIGNMENT_ABSENT", "INITIAL_STATE_JSON_INVALID",
+ "INITIAL_STATE_VIDEO_DATA_ABSENT"
+ ])]],
+ ["E_PAGE_OWNER_ANCHOR_UNAVAILABLE", ["OWNER_PENDING", new Set([
+ "OWNER_ANCHOR_ABSENT", "OWNER_ANCHOR_AMBIGUOUS", "OWNER_ANCHOR_MISMATCH"
+ ])]],
+ ["E_PAGE_PROOF", ["PAGE_REJECTED", new Set([
+ "PAGE_BVID_REJECTED", "PAGE_IDENTITY_REJECTED", "PAGE_ORIGIN_REJECTED", "PAGE_URL_REJECTED"
+ ])]],
+ ["E_PAGE_SCRIPT_TIMEOUT", ["SCRIPT_TIMEOUT", new Set(["SCRIPT_INVOCATION_FAILED"])]],
+ ["E_PAGE_STABILITY_TIMEOUT", ["READY_UNSTABLE", new Set(["STABLE_SNAPSHOTS_PENDING"])]],
+ ["E_PAGE_TAB_DRIFT", ["TAB_IDENTITY", new Set(["TAB_IDENTITY_DRIFT"])]],
+ ["E_PAGE_VIDEO_ABSENT", ["VIDEO_ABSENT", new Set(["VIDEO_ELEMENT_ABSENT"])]],
+]);
+const PREPARELESS_REJECT_CODES = new Set(PREPARELESS_TERMINAL_DIAGNOSTIC_CONTRACT.keys());
+const PAGE_PROOF_DEADLINE_MS = 60000;
+const PAGE_EXTENSION_ACTIVATION_GRACE_MS = 5000;
+const PAGE_PROOF_INVOCATION_TIMEOUT_MS = 5000;
+const PAGE_PROOF_POLL_MS = 500;
+const PAGE_PROOF_STABLE_SNAPSHOTS = 3;
+const PAGE_PROOF_KEYS = [
+ "bvid", "canonical_url", "creator_uid", "eme_present", "job_id", "observed_at_unix_ms",
+ "metadata_source", "observed_duration_ms", "ready_state", "task_nonce", "video_height", "video_width"
+];
+const PAGE_PENDING_STATES = new Set([
+ "DIMENSIONS_PENDING", "IDENTITY_PENDING", "METADATA_NOT_READY", "OWNER_PENDING", "VIDEO_ABSENT"
+]);
+const PAGE_DIAGNOSTIC_STATES = new Set([
+ ...PAGE_PENDING_STATES, "PAGE_REJECTED", "READY_UNSTABLE", "SCRIPT_TIMEOUT", "TAB_IDENTITY"
+]);
+const PAGE_DIAGNOSTIC_REASONS = new Set([
+ "DEADLINE_EXHAUSTED", "METADATA_FIELDS_PENDING", "OWNER_ANCHOR_ABSENT",
+ "OWNER_ANCHOR_AMBIGUOUS", "OWNER_ANCHOR_MISMATCH",
+ "INITIAL_STATE_ASSIGNMENT_ABSENT", "INITIAL_STATE_JSON_INVALID",
+ "INITIAL_STATE_VIDEO_DATA_ABSENT",
+ "PAGE_ACCESS_REJECTED", "PAGE_BVID_REJECTED", "PAGE_DOM_IDENTITY_PENDING",
+ "PAGE_DRM_REJECTED", "PAGE_DURATION_REJECTED", "PAGE_IDENTITY_REJECTED",
+ "PAGE_MULTIPART_REJECTED", "PAGE_NAVIGATION_PENDING", "PAGE_ORIGIN_REJECTED",
+ "PAGE_OWNER_REJECTED", "PAGE_SCRIPTING_PENDING", "PAGE_TAB_ACTIVATION_PENDING",
+ "PAGE_URL_PENDING", "PAGE_URL_REJECTED", "PAGE_VISIBILITY_PENDING",
+ "PAGE_WINDOW_FOCUS_PENDING", "PAGE_WINDOW_MINIMIZED_PENDING", "PAGE_WINDOW_STATE_PENDING",
+ "SCRIPT_INVOCATION_FAILED", "STABLE_SNAPSHOTS_PENDING",
+ "TAB_IDENTITY_DRIFT", "VIDEO_DIMENSIONS_PENDING", "VIDEO_ELEMENT_ABSENT"
+]);
+const PAGE_METADATA_UNAVAILABLE_REASONS = new Set([
+ "INITIAL_STATE_ASSIGNMENT_ABSENT", "INITIAL_STATE_JSON_INVALID",
+ "INITIAL_STATE_VIDEO_DATA_ABSENT"
+]);
+const PAGE_PENDING_TERMINAL_CONTRACT = new Map([
+ ["E_PAGE_DIMENSIONS_UNAVAILABLE", ["DIMENSIONS_PENDING", new Set(["VIDEO_DIMENSIONS_PENDING"])]],
+ ["E_PAGE_IDENTITY_UNAVAILABLE", ["IDENTITY_PENDING", new Set([
+ "PAGE_DOM_IDENTITY_PENDING", "PAGE_NAVIGATION_PENDING", "PAGE_SCRIPTING_PENDING",
+ "PAGE_TAB_ACTIVATION_PENDING", "PAGE_URL_PENDING", "PAGE_VISIBILITY_PENDING",
+ "PAGE_WINDOW_FOCUS_PENDING", "PAGE_WINDOW_MINIMIZED_PENDING", "PAGE_WINDOW_STATE_PENDING"
+ ])]],
+ ["E_PAGE_METADATA_NOT_READY", ["METADATA_NOT_READY", PAGE_METADATA_UNAVAILABLE_REASONS]],
+ ["E_PAGE_OWNER_ANCHOR_UNAVAILABLE", ["OWNER_PENDING", new Set([
+ "OWNER_ANCHOR_ABSENT", "OWNER_ANCHOR_AMBIGUOUS", "OWNER_ANCHOR_MISMATCH"
+ ])]],
+ ["E_PAGE_VIDEO_ABSENT", ["VIDEO_ABSENT", new Set(["VIDEO_ELEMENT_ABSENT"])]],
+]);
+const MAX_COOKIE_COUNT = 128;
+const COOKIE_KEYS = [
+ "domain", "expiration_unix", "host_only", "http_only", "name", "partition_key",
+ "path", "same_site", "secure", "session", "store_id", "value"
+];
+const COOKIE_SAME_SITE = new Set(["no_restriction", "lax", "strict", "unspecified"]);
+const COOKIE_CONTROL_RE = /\p{C}/u;
+const STORE_ID_RE = /^[0-9]{1,8}$/;
+const AUTH_COOKIE_NAME = "SESSDATA";
+const REQUIRED_AUTH_COOKIE_NAMES = new Set([AUTH_COOKIE_NAME]);
+const TRANSFER_COOKIE_NAMES = new Set([AUTH_COOKIE_NAME]);
+const COOKIE_ACCESS_REASONS = new Set([
+ "COOKIE_API_ERROR", "COOKIE_API_SHAPE", "COOKIE_AUTH_SET_EMPTY",
+ "COOKIE_EQUAL_PRECEDENCE_CONFLICT",
+ "COOKIE_RELEVANCE_AMBIGUOUS", "COOKIE_SELECTED_INVALID",
+ "COOKIE_SELECTION_EXCEPTION", "COOKIE_TRANSFER_INVALID",
+ "COOKIE_TRANSFER_OVERFLOW"
+]);
+let drainLease = false;
+let lastDrainErrorCode = null;
+let lastCookieAccessReason = null;
+let lastTabDriftReason = null;
+let lastPageReadinessDiagnostic = null;
+let lastExtensionDiagnostic = null;
+let lastBrowserLifecycleDiagnostic = null;
-let currentProof = null;
-let currentTabId = null;
-let currentTaskNonce = null;
-let nativePort = null;
-let nativeReady = false;
-let nativeRevision = 0;
-let lastNativeType = null;
-let preparedTaskNonce = null;
-let preparedLeaseId = null;
-let activeStartLease = null;
-const prepareResponses = new Map();
-let safeState = {
- phase: "IDLE",
- progress: 0,
- error_code: null,
- formal_filename: null,
- mapping_filename: null
-};
+const BROWSER_LIFECYCLE_DIAGNOSTIC_CONTRACT = new Map([
+ ["OWNED_TAB_RECORDED", "BROWSER_OWNERSHIP"],
+ ["OWNED_TAB_ADOPTED", "BROWSER_OWNERSHIP"],
+ ["OWNED_TAB_RECORD_INVALID", "BROWSER_OWNERSHIP"],
+ ["OWNED_TAB_STORAGE_FAILED", "BROWSER_OWNERSHIP"],
+ ["OWNED_TAB_CLOSED", "BROWSER_CLEANUP"],
+ ["OWNED_TAB_ALREADY_CLOSED", "BROWSER_CLEANUP"],
+ ["OWNED_TAB_IDENTITY_DRIFT", "BROWSER_CLEANUP"],
+ ["OWNED_TAB_CLOSE_FAILED", "BROWSER_CLEANUP"],
+ ["OWNED_TAB_AUTOPLAY_STOPPED", "BROWSER_CLEANUP"]
+]);
-chrome.runtime.onInstalled.addListener(() => {
- chrome.sidePanel.setPanelBehavior({openPanelOnActionClick: true});
-});
+const EXTENSION_DIAGNOSTIC_CONTRACT = new Map([
+ ["DOCUMENT_FOCUS_FALSE", "SCRIPTING"],
+ ["DOCUMENT_FOCUS_TRUE", "SCRIPTING"],
+ ["DOCUMENT_FOCUS_UNAVAILABLE", "SCRIPTING"],
+ ["NATIVE_CONNECT_FAILED", "NATIVE_MESSAGING"],
+ ["NATIVE_DISCONNECT_FAILED", "NATIVE_MESSAGING"],
+ ["SCRIPTING_EXECUTE_FAILED", "SCRIPTING"],
+ ["SERVICE_WORKER_ERROR", "SERVICE_WORKER"],
+ ["SERVICE_WORKER_UNHANDLED_REJECTION", "SERVICE_WORKER"],
+ ["TABS_CREATE_FAILED", "TABS"],
+ ["TABS_GET_FAILED", "TABS"],
+ ["TABS_QUERY_FAILED", "TABS"],
+ ["TABS_UPDATE_FAILED", "TABS"],
+ ["WINDOWS_GET_FAILED", "WINDOWS"],
+ ["WINDOWS_UPDATE_FAILED", "WINDOWS"],
+ ["WINDOW_FOCUS_FALSE", "WINDOWS"],
+ ["WINDOW_FOCUS_TRUE", "WINDOWS"],
+ ["DOCUMENT_VISIBILITY_HIDDEN", "SCRIPTING"],
+ ["DOCUMENT_VISIBILITY_VISIBLE", "SCRIPTING"],
+ ["OS_FOREGROUND_LAUNCH_FAILED", "NATIVE_FOREGROUND"],
+ ["OS_FOREGROUND_LOCKED", "NATIVE_FOREGROUND"],
+ ["OS_FOREGROUND_PLATFORM_UNSUPPORTED", "NATIVE_FOREGROUND"],
+ ["OS_FOREGROUND_READY", "NATIVE_FOREGROUND"],
+ ["OS_FOREGROUND_REPLAY", "NATIVE_FOREGROUND"],
+ ["OS_FOREGROUND_WINDOW_ABSENT", "NATIVE_FOREGROUND"],
+ ["OS_FOREGROUND_WINDOW_AMBIGUOUS", "NATIVE_FOREGROUND"]
+]);
+const DOCUMENT_FOCUS_DIAGNOSTICS = new Set([
+ "DOCUMENT_FOCUS_FALSE", "DOCUMENT_FOCUS_TRUE", "DOCUMENT_FOCUS_UNAVAILABLE"
+]);
+const DOCUMENT_VISIBILITY_DIAGNOSTICS = new Set([
+ "DOCUMENT_VISIBILITY_HIDDEN", "DOCUMENT_VISIBILITY_VISIBLE"
+]);
+const FOREGROUND_ERROR_DIAGNOSTICS = new Map([
+ ["E_FOREGROUND_LAUNCH_FAILED", "OS_FOREGROUND_LAUNCH_FAILED"],
+ ["E_FOREGROUND_LOCKED", "OS_FOREGROUND_LOCKED"],
+ ["E_FOREGROUND_PLATFORM_UNSUPPORTED", "OS_FOREGROUND_PLATFORM_UNSUPPORTED"],
+ ["E_FOREGROUND_REPLAY", "OS_FOREGROUND_REPLAY"],
+ ["E_FOREGROUND_WINDOW_ABSENT", "OS_FOREGROUND_WINDOW_ABSENT"],
+ ["E_FOREGROUND_WINDOW_AMBIGUOUS", "OS_FOREGROUND_WINDOW_AMBIGUOUS"]
+]);
+const FOREGROUND_ELIGIBLE_CODES = new Set([
+ "E_PAGE_DIMENSIONS_UNAVAILABLE", "E_PAGE_IDENTITY_UNAVAILABLE",
+ "E_PAGE_METADATA_UNAVAILABLE", "E_PAGE_OWNER_ANCHOR_UNAVAILABLE",
+ "E_PAGE_SCRIPT_TIMEOUT", "E_PAGE_STABILITY_TIMEOUT", "E_PAGE_VIDEO_ABSENT"
+]);
-function randomNonce() {
- const bytes = new Uint8Array(16);
- crypto.getRandomValues(bytes);
- return Array.from(bytes, value => value.toString(16).padStart(2, "0")).join("");
+function recordExtensionDiagnostic(reason) {
+ const phase = EXTENSION_DIAGNOSTIC_CONTRACT.get(reason);
+ if (typeof phase !== "string") return;
+ lastExtensionDiagnostic = Object.freeze({phase, reason});
}
-function fixedError(code) {
+function recordBrowserLifecycleDiagnostic(reason) {
+ const phase = BROWSER_LIFECYCLE_DIAGNOSTIC_CONTRACT.get(reason);
+ if (typeof phase !== "string") return;
+ lastBrowserLifecycleDiagnostic = Object.freeze({phase, reason});
+}
+
+function exactOwnedTabRecord(value) {
+ const keys = [
+ "schema", "job_id", "lease_id", "bvid", "tab_id", "window_id", "cookie_store_id"
+ ];
+ if (!isPlainObject(value) || !exactKeys(value, keys) || value.schema !== OWNED_TAB_SCHEMA ||
+ !HEX64_RE.test(value.job_id || "") || !HEX32_RE.test(value.lease_id || "") ||
+ !BVID_RE.test(value.bvid || "") || !Number.isInteger(value.tab_id) || value.tab_id < 0 ||
+ !Number.isInteger(value.window_id) || value.window_id < 0 ||
+ !STORE_ID_RE.test(value.cookie_store_id || "")) return null;
+ return Object.freeze({...value});
+}
+
+async function storageSessionCall(methodName, args) {
+ if (!chrome.storage?.session) throw new Error("E_OWNED_TAB_STORAGE");
+ return chromeApiCall(
+ chrome.storage.session, methodName, args,
+ "E_OWNED_TAB_STORAGE", "OWNED_TAB_STORAGE_FAILED"
+ );
+}
+
+async function loadOwnedTabRecord() {
+ try {
+ const value = await storageSessionCall("get", [OWNED_TAB_STORAGE_KEY]);
+ const raw = isPlainObject(value) ? value[OWNED_TAB_STORAGE_KEY] : null;
+ if (raw === undefined || raw === null) return null;
+ const record = exactOwnedTabRecord(raw);
+ if (record !== null) return record;
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_RECORD_INVALID");
+ await storageSessionCall("remove", [OWNED_TAB_STORAGE_KEY]);
+ } catch {
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_STORAGE_FAILED");
+ }
+ return null;
+}
+
+async function persistOwnedTabRecord(job, leaseId, binding) {
+ const record = exactOwnedTabRecord({
+ schema: OWNED_TAB_SCHEMA, job_id: job.job_id, lease_id: leaseId, bvid: job.bvid,
+ tab_id: binding.tabId, window_id: binding.windowId, cookie_store_id: binding.cookieStoreId
+ });
+ if (record === null || binding.managed !== true) throw new Error("E_OWNED_TAB_STORAGE");
+ try {
+ await storageSessionCall("set", [{[OWNED_TAB_STORAGE_KEY]: record}]);
+ } catch {
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_STORAGE_FAILED");
+ throw new Error("E_OWNED_TAB_STORAGE");
+ }
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_RECORDED");
+ return record;
+}
+
+async function clearOwnedTabRecord() {
+ try {
+ await storageSessionCall("remove", [OWNED_TAB_STORAGE_KEY]);
+ return true;
+ } catch {
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_STORAGE_FAILED");
+ return false;
+ }
+}
+
+async function cleanupOwnedTab(recordValue = null) {
+ const record = exactOwnedTabRecord(recordValue) || await loadOwnedTabRecord();
+ if (record === null) return true;
+ let tab;
+ try {
+ tab = await chromeApiCall(
+ chrome.tabs, "get", [record.tab_id], "E_OWNED_TAB_CLOSE", "TABS_GET_FAILED"
+ );
+ } catch {
+ await clearOwnedTabRecord();
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_ALREADY_CLOSED");
+ return true;
+ }
+ if (!isPlainObject(tab) || tab.id !== record.tab_id || tab.windowId !== record.window_id ||
+ String(tab.cookieStoreId ?? "0") !== record.cookie_store_id ||
+ (!locationMatches(tab.url, record.bvid) && !locationMatches(tab.pendingUrl, record.bvid) &&
+ !isManagedIdentityPendingUrl(tab.url, record.bvid))) {
+ await clearOwnedTabRecord();
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_IDENTITY_DRIFT");
+ return false;
+ }
+ // Retire the durable ownership capability before the non-atomic remove. A
+ // failed remove must never leave a record that a restarted service worker
+ // could apply to a same-id/same-BVID user replacement.
+ if (!await clearOwnedTabRecord()) {
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_CLOSE_FAILED");
+ return false;
+ }
+ try {
+ await chromeApiCall(
+ chrome.tabs, "remove", [record.tab_id], "E_OWNED_TAB_CLOSE", "TABS_REMOVE_FAILED"
+ );
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_CLOSED");
+ return true;
+ } catch {
+ // tabs.remove is not an atomic compare-and-close operation. Once it fails,
+ // the numeric tab id may already name a replacement user tab, including a
+ // replacement on the same BVID. No subsequent tabs.update/remove mutation
+ // can prove object ownership. The durable capability was already retired,
+ // so surface only a fixed, sanitized lifecycle result and never retry it.
+ recordBrowserLifecycleDiagnostic("OWNED_TAB_CLOSE_FAILED");
+ return false;
+ }
+}
+
+function extensionApiError(code, reason) {
+ recordExtensionDiagnostic(reason);
+ return new Error(code);
+}
+
+function chromeApiCall(receiver, methodName, args, code, reason) {
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const settle = (error, value) => {
+ if (settled) return;
+ settled = true;
+ if (error) reject(extensionApiError(code, reason)); else resolve(value);
+ };
+ const callback = value => {
+ let failed = false;
+ try { failed = chrome.runtime.lastError != null; } catch { failed = true; }
+ settle(failed, value);
+ };
+ let returned;
+ try {
+ returned = receiver[methodName](...args, callback);
+ } catch {
+ settle(true);
+ return;
+ }
+ if (returned && typeof returned.then === "function") {
+ returned.then(value => settle(false, value), () => settle(true));
+ }
+ });
+}
+
+if (typeof globalThis.addEventListener === "function") {
+ globalThis.addEventListener("error", () => recordExtensionDiagnostic("SERVICE_WORKER_ERROR"));
+ globalThis.addEventListener("unhandledrejection", () =>
+ recordExtensionDiagnostic("SERVICE_WORKER_UNHANDLED_REJECTION"));
+}
+
+function isPlainObject(value) {
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+ const prototype = Object.getPrototypeOf(value);
+ return prototype === Object.prototype || prototype === null;
+}
+
+function exactKeys(value, keys) {
+ return isPlainObject(value) && Object.keys(value).sort().join("\0") === [...keys].sort().join("\0");
+}
+
+function safeErrorCode(error, fallback = "E_EXTENSION") {
+ const candidate = typeof error?.message === "string" ? error.message : "";
+ return ERROR_CODE_RE.test(candidate) ? candidate : fallback;
+}
+
+function canonicalUrl(bvid) {
+ return `https://www.bilibili.com/video/${bvid}`;
+}
+
+function locationMatches(rawUrl, bvid) {
+ if (!BVID_RE.test(bvid) || typeof rawUrl !== "string") return false;
+ let parsed;
+ try { parsed = new URL(rawUrl); } catch { return false; }
+ const path = `/video/${bvid}`;
+ return parsed.origin === "https://www.bilibili.com" &&
+ (parsed.pathname === path || parsed.pathname === `${path}/`) &&
+ (parsed.search === "" || /^\?vd_source=[0-9a-f]{32}$/.test(parsed.search)) &&
+ parsed.hash === "";
+}
+
+function isManagedIdentityPendingUrl(rawUrl, bvid) {
+ if (rawUrl === "about:blank") return true;
+ if (typeof rawUrl !== "string" || !BVID_RE.test(bvid)) return false;
+ let parsed;
+ try { parsed = new URL(rawUrl); } catch { return true; }
+ if (parsed.origin !== "https://www.bilibili.com") return false;
+ const videoMatch = /^\/video\/(BV1[1-9A-HJ-NP-Za-km-z]{9})\/?$/.exec(parsed.pathname);
+ return videoMatch === null || videoMatch[1] === bvid;
+}
+
+function durationToleranceMs(expected) {
+ return Math.max(1500, Math.min(10000, Math.ceil(expected / 1000)));
+}
+
+async function sha256Hex(text) {
+ const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+ return Array.from(new Uint8Array(digest), value => value.toString(16).padStart(2, "0")).join("");
+}
+
+async function validateJob(job) {
+ const successor = isPlainObject(job) && Object.prototype.hasOwnProperty.call(job, "lineage");
+ if (!exactKeys(job, successor ? [...JOB_KEYS, "lineage"] : JOB_KEYS) || !HEX64_RE.test(job.job_id) || !BVID_RE.test(job.bvid) ||
+ !CREATOR_RE.test(job.creator_uid) || job.canonical_url !== canonicalUrl(job.bvid) ||
+ !Number.isSafeInteger(job.expected_duration_ms) || job.expected_duration_ms < 1000 ||
+ job.expected_duration_ms > 86400000 || typeof job.title !== "string" || !job.title.trim() ||
+ new TextEncoder().encode(job.title).length > 600 || typeof job.published_at !== "string" ||
+ job.published_at.length > 64 || !Number.isFinite(Date.parse(job.published_at))) return false;
+ let stable;
+ if (successor) {
+ const lineage = job.lineage;
+ if (!exactKeys(lineage, LINEAGE_KEYS) || !HEX64_RE.test(lineage.predecessor_job_id) ||
+ !Number.isSafeInteger(lineage.retry_generation) || lineage.retry_generation < 1 ||
+ lineage.retry_generation > 1000000 || !ERROR_CODE_RE.test(lineage.predecessor_terminal_error_code) ||
+ !MESSAGE_ID_RE.test(lineage.authorization_message_id) ||
+ !HANDOFF_ID_RE.test(lineage.authorization_handoff_id) ||
+ !UPPER_HEX64_RE.test(lineage.authorization_sha256) ||
+ !MESSAGE_ID_RE.test(lineage.repair_review_result_message_id) ||
+ !AUDIT_ID_RE.test(lineage.repair_audit_id) ||
+ !Number.isSafeInteger(lineage.repair_audit_bytes) || lineage.repair_audit_bytes < 1 ||
+ !UPPER_HEX64_RE.test(lineage.repair_audit_sha256)) return false;
+ stable = await sha256Hex([
+ "bili-auth-job-v2", job.creator_uid, job.bvid, lineage.predecessor_job_id,
+ String(lineage.retry_generation), lineage.predecessor_terminal_error_code,
+ lineage.authorization_message_id, lineage.authorization_handoff_id,
+ lineage.authorization_sha256, lineage.repair_review_result_message_id,
+ lineage.repair_audit_id, String(lineage.repair_audit_bytes), lineage.repair_audit_sha256
+ ].join("\0"));
+ } else {
+ stable = await sha256Hex(`bili-auth-job-v1\0${job.creator_uid}\0${job.bvid}`);
+ }
+ return stable === job.job_id;
+}
+
+function validateMaintenance(value) {
+ return exactKeys(value, ["reload_required", "reload_token", "required_extension_build", "retry_after_unix_ms"]) &&
+ typeof value.required_extension_build === "string" && value.required_extension_build.length <= 128 &&
+ typeof value.reload_required === "boolean" &&
+ (value.reload_token === null || HEX32_RE.test(value.reload_token)) &&
+ Number.isSafeInteger(value.retry_after_unix_ms) && value.retry_after_unix_ms >= 0;
+}
+
+async function validateResponse(value, expectedType) {
+ if (!exactKeys(value, RESPONSE_KEYS) || value.schema !== SCHEMA || value.type !== expectedType ||
+ typeof value.host_build !== "string" || !SAFE_PHASES.has(value.phase) ||
+ !Number.isInteger(value.progress) || value.progress < 0 || value.progress > 100 ||
+ (value.error_code !== null && !/^E_[A-Z0-9_]{1,48}$/.test(value.error_code)) ||
+ !validateMaintenance(value.maintenance)) return false;
+ if (value.job !== null && !(await validateJob(value.job))) return false;
+ if (value.lease_id !== null && !HEX32_RE.test(value.lease_id)) return false;
+ for (const name of ["formal_filename", "mapping_filename", "prepare_id"]) {
+ if (value[name] !== null && typeof value[name] !== "string") return false;
+ }
+ return true;
+}
+
+function nativeSession() {
+ let port;
+ try {
+ port = chrome.runtime.connectNative(HOST_NAME);
+ } catch {
+ throw extensionApiError("E_HOST_CONNECT", "NATIVE_CONNECT_FAILED");
+ }
+ if (!isPlainObject(port) && (port === null || typeof port !== "object")) {
+ throw extensionApiError("E_HOST_CONNECT", "NATIVE_CONNECT_FAILED");
+ }
+ const messages = [];
+ const waiters = [];
+ let disconnected = false;
+ let closeRequested = false;
+ let lastDeterministicError = null;
+ const rejectWaiters = code => {
+ while (waiters.length) {
+ const waiter = waiters.shift();
+ clearTimeout(waiter.timer);
+ waiter.reject(new Error(code));
+ }
+ };
+ port.onMessage.addListener(message => {
+ if (isPlainObject(message) && ERROR_CODE_RE.test(message.error_code || "")) {
+ lastDeterministicError = message.error_code;
+ }
+ const waiter = waiters.shift();
+ if (waiter) {
+ clearTimeout(waiter.timer);
+ waiter.resolve(message);
+ } else messages.push(message);
+ });
+ port.onDisconnect.addListener(() => {
+ if (disconnected) return;
+ disconnected = true;
+ let runtimeFailure = false;
+ try { runtimeFailure = chrome.runtime.lastError != null; } catch { runtimeFailure = true; }
+ if (runtimeFailure) recordExtensionDiagnostic("NATIVE_DISCONNECT_FAILED");
+ rejectWaiters(lastDeterministicError || "E_HOST_DISCONNECT");
+ });
+ const next = timeoutMs => new Promise((resolve, reject) => {
+ if (messages.length) { resolve(messages.shift()); return; }
+ if (disconnected) { reject(new Error(lastDeterministicError || "E_HOST_DISCONNECT")); return; }
+ const waiter = {resolve, reject, timer: null};
+ waiters.push(waiter);
+ waiter.timer = setTimeout(() => {
+ const index = waiters.indexOf(waiter);
+ if (index >= 0) {
+ waiters.splice(index, 1);
+ reject(new Error("E_HOST_TIMEOUT"));
+ }
+ }, timeoutMs);
+ });
+ return {
+ async send(message, timeoutMs = 20000) {
+ if (disconnected) throw new Error(lastDeterministicError || "E_HOST_DISCONNECT");
+ const response = next(timeoutMs);
+ try {
+ port.postMessage(message);
+ } catch (error) {
+ rejectWaiters(safeErrorCode(error, "E_HOST_SEND"));
+ }
+ return response;
+ },
+ disconnect() {
+ if (closeRequested) return;
+ closeRequested = true;
+ if (!disconnected) {
+ try { port.disconnect(); } catch {}
+ }
+ }
+ };
+}
+
+function randomHex(bytes) {
+ const data = new Uint8Array(bytes);
+ crypto.getRandomValues(data);
+ return Array.from(data, value => value.toString(16).padStart(2, "0")).join("");
+}
+
+function isPageReadinessDiagnostic(value) {
+ return exactKeys(value, ["attempts", "elapsed_ms", "reason", "state"]) &&
+ Number.isInteger(value.attempts) && value.attempts >= 0 && value.attempts <= 10000 &&
+ Number.isInteger(value.elapsed_ms) && value.elapsed_ms >= 0 && value.elapsed_ms <= 7200000 &&
+ PAGE_DIAGNOSTIC_STATES.has(value.state) && PAGE_DIAGNOSTIC_REASONS.has(value.reason);
+}
+
+function isCanonicalPreparelessTerminal(code, diagnostic) {
+ if (!isPageReadinessDiagnostic(diagnostic)) return false;
+ const contract = PREPARELESS_TERMINAL_DIAGNOSTIC_CONTRACT.get(code);
+ return Array.isArray(contract) && contract.length === 2 && diagnostic.state === contract[0] &&
+ contract[1] instanceof Set && contract[1].has(diagnostic.reason);
+}
+
+function pageReadinessError(code, attempts, startedAtUnixMs, state, reason) {
+ const diagnostic = {
+ attempts,
+ elapsed_ms: Math.max(0, Math.min(7200000, Date.now() - startedAtUnixMs)),
+ state,
+ reason
+ };
+ if (!isCanonicalPreparelessTerminal(code, diagnostic)) return new Error("E_REJECT");
+ lastPageReadinessDiagnostic = diagnostic;
const error = new Error(code);
- error.code = code;
+ error.page_readiness_diagnostic = diagnostic;
return error;
}
-async function activeTargetTab() {
- const tabs = await chrome.tabs.query({active: true, currentWindow: true});
- if (tabs.length !== 1 || !Number.isInteger(tabs[0].id)) {
- throw fixedError("E_PAGE_PROOF");
- }
- const parsed = new URL(tabs[0].url || "about:blank");
- if (parsed.protocol !== "https:" || parsed.hostname !== "www.bilibili.com" || parsed.pathname !== TARGET_PATH) {
- throw fixedError("E_PAGE_PROOF");
- }
- return tabs[0];
+function fallbackPageReadinessDiagnostic(code) {
+ const contract = PREPARELESS_TERMINAL_DIAGNOSTIC_CONTRACT.get(code);
+ if (!Array.isArray(contract) || contract.length !== 2 || !(contract[1] instanceof Set) ||
+ contract[1].size !== 1) return null;
+ const value = {attempts: 0, elapsed_ms: 0, state: contract[0], reason: [...contract[1]][0]};
+ lastPageReadinessDiagnostic = value;
+ return value;
}
-async function observePage(tab, nonce) {
- const results = await chrome.scripting.executeScript({
- target: {tabId: tab.id},
- world: "ISOLATED",
- args: [TARGET, CANONICAL_URL, nonce],
- func: (target, canonicalUrl, taskNonce) => {
- const videos = Array.from(document.querySelectorAll("video")).filter(video =>
- video.readyState >= 1 && video.videoWidth > 0 && video.videoHeight > 0
- );
- if (location.hostname !== "www.bilibili.com" || location.pathname !== `/video/${target}` || videos.length !== 1) {
- return null;
- }
- const video = videos[0];
- return {
- target,
- canonical_url: canonicalUrl,
- task_nonce: taskNonce,
- observed_at_unix_ms: Date.now(),
- observed_duration_ms: Math.round(video.duration * 1000),
- video_width: video.videoWidth,
- video_height: video.videoHeight,
- ready_state: video.readyState,
- eme_present: video.mediaKeys !== null
- };
- }
+function tabDriftError(reason) {
+ lastTabDriftReason = reason;
+ const error = new Error("E_PAGE_TAB_DRIFT");
+ error.reason_code = reason;
+ return error;
+}
+
+function tabIdentityReason(tab, binding) {
+ if (!isPlainObject(tab) || !Number.isInteger(tab.id)) return "TAB_ID_INVALID";
+ if (tab.id !== binding.tabId) return "TAB_ID_CHANGED";
+ if (!Number.isInteger(tab.windowId) || tab.windowId !== binding.windowId) return "TAB_WINDOW_CHANGED";
+ if (String(tab.cookieStoreId ?? "0") !== binding.cookieStoreId) return "TAB_STORE_CHANGED";
+ return null;
+}
+
+function bindTab(tab, bvid, managed) {
+ if (!isPlainObject(tab) || !Number.isInteger(tab.id) || !Number.isInteger(tab.windowId)) {
+ throw tabDriftError("TAB_IDENTITY_INVALID");
+ }
+ const exactUrl = locationMatches(tab.url, bvid);
+ const exactPendingUrl = managed === true && locationMatches(tab.pendingUrl, bvid);
+ if (!exactUrl && !exactPendingUrl) throw tabDriftError("TAB_URL_UNBOUND");
+ return Object.freeze({
+ tabId: tab.id,
+ windowId: tab.windowId,
+ cookieStoreId: String(tab.cookieStoreId ?? "0"),
+ managed: managed === true
});
- if (results.length !== 1 || !results[0].result) {
- throw fixedError("E_PAGE_PROOF");
- }
- const proof = results[0].result;
- const now = Date.now();
- const exactKeys = [
- "canonical_url", "eme_present", "observed_at_unix_ms", "observed_duration_ms",
- "ready_state", "target", "task_nonce", "video_height", "video_width"
- ];
- if (Object.keys(proof).sort().join("|") !== exactKeys.sort().join("|") ||
- proof.target !== TARGET || proof.canonical_url !== CANONICAL_URL ||
- proof.task_nonce !== nonce || proof.eme_present !== false ||
- !Number.isSafeInteger(proof.observed_at_unix_ms) ||
- proof.observed_at_unix_ms < now - 60000 || proof.observed_at_unix_ms > now + 5000 ||
- !Number.isSafeInteger(proof.observed_duration_ms) ||
- Math.abs(proof.observed_duration_ms - EXPECTED_DURATION_MS) > DURATION_TOLERANCE_MS ||
- !Number.isInteger(proof.video_width) || proof.video_width < 1 || proof.video_width > 7680 ||
- !Number.isInteger(proof.video_height) || proof.video_height < 1 || proof.video_height > 4320 ||
- !Number.isInteger(proof.ready_state) || proof.ready_state < 1 || proof.ready_state > 4) {
- throw fixedError(proof.eme_present ? "E_DRM" : "E_PAGE_PROOF");
- }
- return proof;
}
-async function validateCurrentPage({leaseId = null} = {}) {
- if (activeStartLease !== null && activeStartLease !== leaseId) {
- throw fixedError("E_BUSY");
+async function revalidateBoundTab(binding, bvid) {
+ let current;
+ try {
+ current = await chromeApiCall(
+ chrome.tabs, "get", [binding.tabId], "E_PAGE_TAB_DRIFT", "TABS_GET_FAILED"
+ );
+ } catch {
+ throw tabDriftError("TAB_REMOVED_OR_UNAVAILABLE");
}
- const tab = await activeTargetTab();
- const nonce = randomNonce();
- const proof = await observePage(tab, nonce);
- currentProof = proof;
- currentTabId = tab.id;
- if (nativePort) {
- const previousPort = nativePort;
- nativePort = null;
- nativeReady = false;
- preparedTaskNonce = null;
- preparedLeaseId = null;
- prepareResponses.clear();
- previousPort.disconnect();
- }
- await awaitNativeReady();
- safeState = {...safeState, phase: "READY", error_code: null};
- return safeState;
+ const identityReason = tabIdentityReason(current, binding);
+ if (identityReason !== null) throw tabDriftError(identityReason);
+ if (!locationMatches(current.url, bvid)) throw tabDriftError("TAB_URL_CHANGED");
+ return current;
}
-function connectNative() {
- if (nativePort) {
- return;
+async function activateBoundTab(binding, tab, bvid, startedAtUnixMs) {
+ const identityReason = tabIdentityReason(tab, binding);
+ if (identityReason !== null) throw tabDriftError(identityReason);
+ let focusedWindow;
+ try {
+ focusedWindow = await chromeApiCall(
+ chrome.windows, "update", [binding.windowId, {state: "normal", focused: true}],
+ "E_PAGE_IDENTITY_UNAVAILABLE", "WINDOWS_UPDATE_FAILED"
+ );
+ } catch {
+ throw pageReadinessError(
+ "E_PAGE_IDENTITY_UNAVAILABLE", 0, startedAtUnixMs,
+ "IDENTITY_PENDING", "PAGE_WINDOW_FOCUS_PENDING"
+ );
}
- nativeReady = false;
- preparedTaskNonce = null;
- preparedLeaseId = null;
- prepareResponses.clear();
- const port = chrome.runtime.connectNative(HOST);
- nativePort = port;
- port.onMessage.addListener(message => {
- const safeKeys = new Set([
- "schema", "type", "host_build", "target", "phase", "progress", "error_code",
- "formal_filename", "mapping_filename", "prepare_id"
- ]);
- const basename = value => value === null || (typeof value === "string" && value.length > 0 && !/[\\/]/.test(value));
- const prepareIdValid = message?.prepare_id === null ||
- (typeof message?.prepare_id === "string" && /^[0-9a-f]{32}$/.test(message.prepare_id));
- if (!message || Object.keys(message).some(key => !safeKeys.has(key)) ||
- message.schema !== SCHEMA || message.host_build !== HOST_BUILD || message.target !== TARGET ||
- !["IDLE", "READY", "CHECKING", "DOWNLOADING", "MERGING", "VALIDATING", "PUBLISHING", "COMPLETE", "FAILED", "CANCELED"].includes(message.phase) ||
- !Number.isInteger(message.progress) || message.progress < 0 || message.progress > 100 ||
- !basename(message.formal_filename) || !basename(message.mapping_filename) || !prepareIdValid ||
- (message.type === "prepare" ? message.prepare_id === null : message.prepare_id !== null)) {
- safeState = {...safeState, phase: "FAILED", error_code: "E_PROTOCOL"};
- return;
- }
- nativeReady = nativeReady || (message.type === "hello" && message.phase === "READY");
- lastNativeType = message.type;
- if (message.type === "prepare") {
- prepareResponses.set(message.prepare_id, {
- phase: message.phase,
- error_code: message.error_code
- });
- }
- safeState = {
- phase: message.phase,
- progress: message.progress,
- error_code: message.error_code,
- formal_filename: message.formal_filename,
- mapping_filename: message.mapping_filename
+ if (!isPlainObject(focusedWindow) || focusedWindow.id !== binding.windowId) {
+ throw tabDriftError("TAB_WINDOW_CHANGED");
+ }
+ let updated;
+ try {
+ updated = await chromeApiCall(
+ chrome.tabs, "update", [binding.tabId, {active: true}],
+ "E_PAGE_IDENTITY_UNAVAILABLE", "TABS_UPDATE_FAILED"
+ );
+ } catch {
+ throw pageReadinessError(
+ "E_PAGE_IDENTITY_UNAVAILABLE", 0, startedAtUnixMs,
+ "IDENTITY_PENDING", "PAGE_TAB_ACTIVATION_PENDING"
+ );
+ }
+ const updatedIdentityReason = tabIdentityReason(updated, binding);
+ if (updatedIdentityReason !== null) throw tabDriftError(updatedIdentityReason);
+ if (!locationMatches(updated.url, bvid) &&
+ !(binding.managed && (locationMatches(updated.pendingUrl, bvid) ||
+ isManagedIdentityPendingUrl(updated.url, bvid)))) {
+ throw tabDriftError("TAB_MANAGED_URL_DRIFT");
+ }
+ return updated;
+}
+
+async function waitForComplete(binding, tab, bvid, startedAtUnixMs, deadlineUnixMs) {
+ const identityReason = tabIdentityReason(tab, binding);
+ if (identityReason !== null) throw tabDriftError(identityReason);
+ if (tab.status === "complete" && locationMatches(tab.url, bvid)) {
+ return revalidateBoundTab(binding, bvid);
+ }
+ if (!locationMatches(tab.url, bvid) &&
+ !(binding.managed && (locationMatches(tab.pendingUrl, bvid) ||
+ isManagedIdentityPendingUrl(tab.url, bvid)))) {
+ throw tabDriftError("TAB_URL_UNBOUND");
+ }
+ return new Promise((resolve, reject) => {
+ let done = false;
+ let timer = null;
+ const finish = (error, value) => {
+ if (done) return;
+ done = true;
+ chrome.tabs.onUpdated.removeListener(onUpdated);
+ chrome.tabs.onRemoved.removeListener(onRemoved);
+ if (timer !== null) clearTimeout(timer);
+ if (error) reject(error); else resolve(value);
};
- nativeRevision += 1;
- });
- port.onDisconnect.addListener(() => {
- if (nativePort === port) {
- nativePort = null;
- nativeReady = false;
- preparedTaskNonce = null;
- preparedLeaseId = null;
- prepareResponses.clear();
- if (!["COMPLETE", "FAILED", "CANCELED"].includes(safeState.phase)) {
- safeState = {...safeState, phase: "FAILED", error_code: "E_NATIVE_DISCONNECT"};
+ const onUpdated = (tabId, change, current) => {
+ if (tabId !== binding.tabId) return;
+ const currentIdentityReason = tabIdentityReason(current, binding);
+ if (currentIdentityReason !== null) {
+ finish(tabDriftError(currentIdentityReason));
+ return;
}
+ if (typeof change?.url === "string" && !locationMatches(change.url, bvid) &&
+ !(binding.managed && isManagedIdentityPendingUrl(change.url, bvid))) {
+ finish(tabDriftError("TAB_MANAGED_URL_DRIFT"));
+ return;
+ }
+ if (locationMatches(current.url, bvid) && current.status === "complete") {
+ finish(null, current);
+ return;
+ }
+ if (!locationMatches(current.url, bvid) &&
+ !(binding.managed && (locationMatches(current.pendingUrl, bvid) ||
+ isManagedIdentityPendingUrl(current.url, bvid)))) {
+ finish(tabDriftError("TAB_URL_CHANGED"));
+ }
+ };
+ const onRemoved = tabId => {
+ if (tabId === binding.tabId) finish(tabDriftError("TAB_REMOVED_OR_UNAVAILABLE"));
+ };
+ chrome.tabs.onUpdated.addListener(onUpdated);
+ chrome.tabs.onRemoved.addListener(onRemoved);
+ timer = setTimeout(() => finish(pageReadinessError(
+ "E_PAGE_IDENTITY_UNAVAILABLE", 0, startedAtUnixMs,
+ "IDENTITY_PENDING", "PAGE_URL_PENDING"
+ )), Math.max(1, deadlineUnixMs - Date.now()));
+ });
+}
+
+async function waitForActiveNonMinimizedBoundTab(binding, bvid, startedAtUnixMs, deadlineUnixMs) {
+ let pendingReason = "PAGE_WINDOW_STATE_PENDING";
+ while (Date.now() < deadlineUnixMs) {
+ const current = await revalidateBoundTab(binding, bvid);
+ let currentWindow;
+ try {
+ currentWindow = await chromeApiCall(
+ chrome.windows, "get", [binding.windowId],
+ "E_PAGE_IDENTITY_UNAVAILABLE", "WINDOWS_GET_FAILED"
+ );
+ } catch {
+ currentWindow = null;
}
- });
- port.postMessage({
- schema: SCHEMA,
- type: "hello",
- extension_build: EXTENSION_BUILD,
- target: TARGET
- });
-}
-
-async function freshNativeStatus() {
- const prior = nativeRevision;
- nativePort.postMessage({schema: SCHEMA, type: "status", target: TARGET});
- const deadline = Date.now() + 5000;
- while (nativeRevision === prior && Date.now() < deadline) {
- await new Promise(resolve => setTimeout(resolve, 25));
- }
- if (nativeRevision === prior) {
- throw fixedError("E_NATIVE_CONNECT");
- }
- return safeState;
-}
-
-async function awaitNativeReady() {
- connectNative();
- const deadline = Date.now() + 5000;
- while (!nativeReady && Date.now() < deadline) {
- if (safeState.error_code) {
- throw fixedError(safeState.error_code);
+ if (!isPlainObject(currentWindow) || currentWindow.id !== binding.windowId) {
+ pendingReason = "PAGE_WINDOW_STATE_PENDING";
+ } else if (typeof currentWindow.state !== "string") {
+ pendingReason = "PAGE_WINDOW_STATE_PENDING";
+ } else if (currentWindow.state === "minimized") {
+ pendingReason = "PAGE_WINDOW_MINIMIZED_PENDING";
+ } else if (!["normal", "maximized", "fullscreen", "locked-fullscreen"].includes(currentWindow.state)) {
+ pendingReason = "PAGE_WINDOW_STATE_PENDING";
+ } else if (current.active !== true) {
+ pendingReason = "PAGE_TAB_ACTIVATION_PENDING";
+ } else {
+ recordExtensionDiagnostic(currentWindow.focused === true ?
+ "WINDOW_FOCUS_TRUE" : "WINDOW_FOCUS_FALSE");
+ return current;
}
- await new Promise(resolve => setTimeout(resolve, 25));
+ const remainingMs = deadlineUnixMs - Date.now();
+ if (remainingMs <= 0) break;
+ await new Promise(resolve => setTimeout(resolve, Math.min(PAGE_PROOF_POLL_MS, remainingMs)));
}
- if (!nativeReady) {
- throw fixedError("E_NATIVE_CONNECT");
+ throw pageReadinessError(
+ "E_PAGE_IDENTITY_UNAVAILABLE", 0, startedAtUnixMs,
+ "IDENTITY_PENDING", pendingReason
+ );
+}
+
+function foregroundFallbackEligible(error) {
+ return FOREGROUND_ELIGIBLE_CODES.has(error?.message) &&
+ isPageReadinessDiagnostic(error?.page_readiness_diagnostic);
+}
+
+function exactWindowBounds(value) {
+ if (!isPlainObject(value)) return null;
+ const bounds = {};
+ for (const key of ["left", "top", "width", "height"]) {
+ if (!Number.isSafeInteger(value[key])) return null;
+ bounds[key] = value[key];
+ }
+ if (bounds.width < 1 || bounds.height < 1) return null;
+ return bounds;
+}
+
+async function foregroundAndRebind(session, job, leaseId, selected, flow, sourceError) {
+ if (flow.foregroundUsed) throw sourceError;
+ flow.foregroundUsed = true;
+ let windowValue;
+ try {
+ windowValue = await chromeApiCall(
+ chrome.windows, "get", [selected.binding.windowId],
+ "E_PAGE_IDENTITY_UNAVAILABLE", "WINDOWS_GET_FAILED"
+ );
+ } catch {
+ throw sourceError;
+ }
+ const windowBounds = exactWindowBounds(windowValue);
+ if (windowBounds === null) throw sourceError;
+ const response = await session.send({
+ schema: SCHEMA, type: "foreground", job_id: job.job_id, lease_id: leaseId,
+ tab_id: selected.binding.tabId, window_id: selected.binding.windowId,
+ window_bounds: windowBounds
+ }, 20000);
+ if (!(await validateResponse(response, "foreground")) ||
+ response.job?.job_id !== job.job_id || response.lease_id !== leaseId) {
+ throw sourceError;
+ }
+ if (response.phase !== "READY" || response.error_code !== null) {
+ const diagnosticReason = FOREGROUND_ERROR_DIAGNOSTICS.get(response.error_code);
+ if (typeof diagnosticReason === "string") recordExtensionDiagnostic(diagnosticReason);
+ throw sourceError;
+ }
+ recordExtensionDiagnostic("OS_FOREGROUND_READY");
+ let pendingError = sourceError;
+ while (Date.now() < flow.deadlineUnixMs) {
+ let tabs;
+ try {
+ tabs = await chromeApiCall(
+ chrome.tabs, "query", [{url: ["https://www.bilibili.com/video/*"], windowType: "normal"}],
+ "E_PAGE_IDENTITY_UNAVAILABLE", "TABS_QUERY_FAILED"
+ );
+ } catch {
+ tabs = null;
+ }
+ if (Array.isArray(tabs)) {
+ const exactCandidates = tabs.filter(item =>
+ item?.incognito !== true && locationMatches(item?.url, job.bvid));
+ if (exactCandidates.length > 1) throw new Error("E_PAGE_DUPLICATE_TAB");
+ if (exactCandidates.length === 1) {
+ const preservesOwnership = selected.binding.managed === true &&
+ exactCandidates[0].id === selected.binding.tabId &&
+ exactCandidates[0].windowId === selected.binding.windowId;
+ const rebound = {
+ binding: bindTab(exactCandidates[0], job.bvid, preservesOwnership), tab: exactCandidates[0]
+ };
+ const active = await waitForActiveNonMinimizedBoundTab(
+ rebound.binding, job.bvid, flow.startedAtUnixMs, flow.deadlineUnixMs
+ );
+ return {binding: rebound.binding, tab: active, activeNonMinimized: active};
+ }
+ pendingError = pageReadinessError(
+ "E_PAGE_IDENTITY_UNAVAILABLE", 0, flow.startedAtUnixMs,
+ "IDENTITY_PENDING", "PAGE_URL_PENDING"
+ );
+ }
+ const remainingMs = flow.deadlineUnixMs - Date.now();
+ if (remainingMs <= 0) break;
+ await new Promise(resolve => setTimeout(resolve, Math.min(PAGE_PROOF_POLL_MS, remainingMs)));
+ }
+ throw pendingError;
+}
+
+async function targetTab(session, job, leaseId, flow) {
+ const startedAtUnixMs = Date.now();
+ let tabs;
+ try {
+ tabs = await chromeApiCall(
+ chrome.tabs, "query", [{
+ url: ["https://www.bilibili.com/video/*"], windowType: "normal"
+ }], "E_PAGE_IDENTITY_UNAVAILABLE", "TABS_QUERY_FAILED"
+ );
+ } catch {
+ throw pageReadinessError(
+ "E_PAGE_IDENTITY_UNAVAILABLE", 0, startedAtUnixMs,
+ "IDENTITY_PENDING", "PAGE_DOM_IDENTITY_PENDING"
+ );
+ }
+ if (!Array.isArray(tabs)) throw new Error("E_PAGE_PROOF");
+ const storedOwnership = await loadOwnedTabRecord();
+ const exactCandidates = [];
+ for (const candidate of tabs.filter(item => item?.incognito !== true && locationMatches(item?.url, job.bvid))) {
+ const adopted = storedOwnership !== null && storedOwnership.job_id === job.job_id &&
+ storedOwnership.lease_id === leaseId && storedOwnership.bvid === job.bvid &&
+ storedOwnership.tab_id === candidate.id && storedOwnership.window_id === candidate.windowId &&
+ storedOwnership.cookie_store_id === String(candidate.cookieStoreId ?? "0");
+ const provisional = bindTab(candidate, job.bvid, adopted);
+ const current = await revalidateBoundTab(provisional, job.bvid);
+ exactCandidates.push({binding: bindTab(current, job.bvid, adopted), tab: current});
+ if (adopted) recordBrowserLifecycleDiagnostic("OWNED_TAB_ADOPTED");
+ }
+ if (exactCandidates.length > 1) throw new Error("E_PAGE_DUPLICATE_TAB");
+ let selected;
+ if (exactCandidates.length === 1) {
+ selected = exactCandidates[0];
+ } else {
+ let created;
+ try {
+ created = await chromeApiCall(
+ chrome.tabs, "create", [{url: canonicalUrl(job.bvid), active: false}],
+ "E_PAGE_IDENTITY_UNAVAILABLE", "TABS_CREATE_FAILED"
+ );
+ } catch {
+ throw pageReadinessError(
+ "E_PAGE_IDENTITY_UNAVAILABLE", 0, startedAtUnixMs,
+ "IDENTITY_PENDING", "PAGE_NAVIGATION_PENDING"
+ );
+ }
+ selected = {binding: bindTab(created, job.bvid, true), tab: created};
+ await persistOwnedTabRecord(job, leaseId, selected.binding);
+ }
+ try {
+ const activated = await activateBoundTab(
+ selected.binding, selected.tab, job.bvid, startedAtUnixMs
+ );
+ const activationDeadline = Math.min(
+ flow.deadlineUnixMs, Date.now() + PAGE_EXTENSION_ACTIVATION_GRACE_MS
+ );
+ const ready = await waitForComplete(
+ selected.binding, activated, job.bvid, startedAtUnixMs, activationDeadline
+ );
+ const activeNonMinimized = await waitForActiveNonMinimizedBoundTab(
+ selected.binding, job.bvid, startedAtUnixMs, activationDeadline
+ );
+ return {binding: selected.binding, tab: ready, activeNonMinimized};
+ } catch (error) {
+ if (!foregroundFallbackEligible(error)) throw error;
+ return foregroundAndRebind(session, job, leaseId, selected, flow, error);
}
}
-async function freshNativePrepare(proof, leaseId) {
- await awaitNativeReady();
- if (activeStartLease !== leaseId) {
- throw fixedError("E_BUSY");
+async function acquirePageProof(session, job, leaseId, selected, flow) {
+ const remainingMs = flow.deadlineUnixMs - Date.now();
+ if (remainingMs <= 0) {
+ throw pageReadinessError(
+ "E_PAGE_IDENTITY_UNAVAILABLE", 0, flow.startedAtUnixMs,
+ "IDENTITY_PENDING", "PAGE_URL_PENDING"
+ );
}
- preparedTaskNonce = null;
- preparedLeaseId = null;
- const prepareId = randomNonce();
- prepareResponses.delete(prepareId);
- nativePort.postMessage({
- schema: SCHEMA,
- type: "prepare",
- extension_build: EXTENSION_BUILD,
- target: TARGET,
- prepare_id: prepareId,
- page_proof: proof
- });
- const deadline = Date.now() + 120000;
- while (!prepareResponses.has(prepareId) && Date.now() < deadline) {
- await new Promise(resolve => setTimeout(resolve, 25));
+ const initialBudgetMs = flow.foregroundUsed ? remainingMs :
+ Math.min(PAGE_EXTENSION_ACTIVATION_GRACE_MS, remainingMs);
+ try {
+ return await observe(job, selected.binding, initialBudgetMs);
+ } catch (error) {
+ if (flow.foregroundUsed || !foregroundFallbackEligible(error)) throw error;
+ const rebound = await foregroundAndRebind(session, job, leaseId, selected, flow, error);
+ const finalBudgetMs = flow.deadlineUnixMs - Date.now();
+ if (finalBudgetMs <= 0) throw error;
+ return observe(job, rebound.binding, finalBudgetMs);
}
- const response = prepareResponses.get(prepareId);
- prepareResponses.delete(prepareId);
- if (!response || response.phase !== "READY" || response.error_code) {
- throw fixedError(response?.error_code || "E_PREPARE");
- }
- preparedTaskNonce = proof.task_nonce;
- preparedLeaseId = leaseId;
- return prepareId;
}
-async function currentCookieStore(tabId) {
- const stores = await chrome.cookies.getAllCookieStores();
- const matches = stores.filter(store => Array.isArray(store.tabIds) && store.tabIds.includes(tabId));
- if (matches.length !== 1 || !/^[0-9]{1,8}$/.test(matches[0].id)) {
- throw fixedError("E_SECRET_INPUT");
+function injectedObservation(job, taskNonce) {
+ const path = `/video/${job.bvid}`;
+ if (location.href === "about:blank" ||
+ (typeof document.readyState === "string" && document.readyState !== "complete")) {
+ return {status: "PENDING", terminal_error_code: "E_PAGE_IDENTITY_UNAVAILABLE",
+ state: "IDENTITY_PENDING", reason: "PAGE_URL_PENDING"};
}
- return matches[0].id;
-}
-
-function projectCookie(cookie, storeId) {
- if (cookie.partitionKey !== undefined) {
- throw fixedError("E_SECRET_INPUT");
+ const documentVisibilityDiagnostic = document.visibilityState === "visible" ?
+ "DOCUMENT_VISIBILITY_VISIBLE" : "DOCUMENT_VISIBILITY_HIDDEN";
+ const documentFocusDiagnostic = typeof document.hasFocus !== "function" ?
+ "DOCUMENT_FOCUS_UNAVAILABLE" :
+ (document.hasFocus() === true ? "DOCUMENT_FOCUS_TRUE" : "DOCUMENT_FOCUS_FALSE");
+ if (location.origin !== "https://www.bilibili.com") {
+ return {status: "REJECT", error_code: "E_PAGE_PROOF",
+ state: "PAGE_REJECTED", reason: "PAGE_ORIGIN_REJECTED"};
}
- if (!cookie.name || !cookie.value) {
- return null;
+ const observedBvid = /^\/video\/(BV1[1-9A-HJ-NP-Za-km-z]{9})\/?$/.exec(location.pathname)?.[1] ?? null;
+ if (observedBvid !== null && observedBvid !== job.bvid) {
+ return {status: "REJECT", error_code: "E_PAGE_PROOF",
+ state: "PAGE_REJECTED", reason: "PAGE_BVID_REJECTED"};
}
- const session = Boolean(cookie.session);
- const expiration = session ? null : Math.floor(cookie.expirationDate);
- if (!session && (!Number.isSafeInteger(expiration) || expiration <= 0)) {
- throw fixedError("E_SECRET_INPUT");
+ if ((location.pathname !== path && location.pathname !== `${path}/`) ||
+ (location.search !== "" && !/^\?vd_source=[0-9a-f]{32}$/.test(location.search)) ||
+ location.hash !== "") {
+ return {status: "REJECT", error_code: "E_PAGE_PROOF",
+ state: "PAGE_REJECTED", reason: "PAGE_URL_REJECTED"};
+ }
+ if (document.querySelectorAll(
+ ".error-container,.error-body,[data-bili-access-state='denied']"
+ ).length > 0) {
+ return {status: "REJECT", error_code: "E_PAGE_ACCESS_CONTROL",
+ state: "PAGE_REJECTED", reason: "PAGE_ACCESS_REJECTED"};
+ }
+ const videos = Array.from(document.querySelectorAll("video"));
+ if (videos.length === 0) return {status: "PENDING", terminal_error_code: "E_PAGE_VIDEO_ABSENT",
+ state: "VIDEO_ABSENT", reason: "VIDEO_ELEMENT_ABSENT"};
+ if (videos.length > 1) return {status: "REJECT", error_code: "E_MULTI_PART",
+ state: "PAGE_REJECTED", reason: "PAGE_MULTIPART_REJECTED"};
+ const video = videos[0];
+ if (video.mediaKeys !== null) return {status: "REJECT", error_code: "E_DRM",
+ state: "PAGE_REJECTED", reason: "PAGE_DRM_REJECTED"};
+ let initialVideo = null;
+ let initialStateReason = "INITIAL_STATE_ASSIGNMENT_ABSENT";
+ for (const script of Array.from(document.querySelectorAll("script"))) {
+ const raw = script?.textContent;
+ if (typeof raw !== "string" || raw.length < 30 || raw.length > 2_000_000) continue;
+ const trimmed = raw.trim();
+ const assignment = /^window\.__INITIAL_STATE__\s*=\s*/.exec(trimmed);
+ if (assignment === null) continue;
+ initialStateReason = "INITIAL_STATE_JSON_INVALID";
+ let encoded = trimmed.slice(assignment[0].length);
+ const runtimeMarker = encoded.lastIndexOf(";(function(){");
+ if (runtimeMarker >= 0) encoded = encoded.slice(0, runtimeMarker);
+ else if (encoded.endsWith(";")) encoded = encoded.slice(0, -1);
+ try {
+ const parsed = JSON.parse(encoded);
+ const candidate = parsed?.videoData;
+ initialStateReason = "INITIAL_STATE_VIDEO_DATA_ABSENT";
+ if (candidate && typeof candidate === "object" && !Array.isArray(candidate)) {
+ if (initialVideo !== null) return {status: "REJECT", error_code: "E_PAGE_PROOF",
+ state: "PAGE_REJECTED", reason: "PAGE_URL_REJECTED"};
+ initialVideo = candidate;
+ initialStateReason = "METADATA_FIELDS_PENDING";
+ }
+ } catch {}
+ }
+ const ownerAnchorIdentities = new Set();
+ const ownerProofRoots = document.querySelectorAll(
+ "#v_upinfo,.up-info-container,.up-panel-container,.video-owner"
+ );
+ for (const root of ownerProofRoots) {
+ if (root === null || typeof root.querySelectorAll !== "function") continue;
+ for (const anchor of root.querySelectorAll("a[href]")) {
+ try {
+ const href = new URL(anchor.href, location.href);
+ if (href.origin !== "https://space.bilibili.com" ||
+ !/^\/[1-9][0-9]{0,19}\/?$/.test(href.pathname)) {
+ continue;
+ }
+ const ownerUid = href.pathname.replace(/^\//, "").replace(/\/$/, "");
+ ownerAnchorIdentities.add(`https://space.bilibili.com/${ownerUid}`);
+ } catch {}
+ }
+ }
+ const canonicalOwnerUrl = `https://space.bilibili.com/${job.creator_uid}`;
+ const ownerAnchorExact = ownerAnchorIdentities.size === 1 &&
+ ownerAnchorIdentities.has(canonicalOwnerUrl);
+ const ownerAnchorReason = ownerAnchorExact ? null :
+ (ownerAnchorIdentities.size === 0 ? "OWNER_ANCHOR_ABSENT" :
+ (ownerAnchorIdentities.size === 1 ?
+ "OWNER_ANCHOR_MISMATCH" : "OWNER_ANCHOR_AMBIGUOUS"));
+ let initialMetadata = null;
+ if (initialVideo !== null) {
+ if (initialVideo.bvid !== job.bvid) return {status: "REJECT", error_code: "E_PAGE_PROOF",
+ state: "PAGE_REJECTED", reason: "PAGE_BVID_REJECTED"};
+ if (!Array.isArray(initialVideo.pages) || initialVideo.pages.length === 0) {
+ return {status: "REJECT", error_code: "E_PAGE_PROOF",
+ state: "PAGE_REJECTED", reason: "PAGE_URL_REJECTED"};
+ }
+ if (initialVideo.pages.length > 1) {
+ return {status: "REJECT", error_code: "E_MULTI_PART",
+ state: "PAGE_REJECTED", reason: "PAGE_MULTIPART_REJECTED"};
+ }
+ const initialPage = initialVideo.pages[0];
+ const initialDimension = initialVideo.dimension;
+ const pageDimension = initialPage?.dimension;
+ const positiveSafeInteger = value => typeof value === "number" &&
+ Number.isSafeInteger(value) && value > 0;
+ const positiveFiniteNumber = value => typeof value === "number" &&
+ Number.isFinite(value) && value > 0;
+ if (!initialPage || typeof initialPage !== "object" || Array.isArray(initialPage) ||
+ !Object.prototype.hasOwnProperty.call(initialPage, "cid") ||
+ !Object.prototype.hasOwnProperty.call(initialPage, "page") ||
+ !Object.prototype.hasOwnProperty.call(initialPage, "duration") ||
+ !Object.prototype.hasOwnProperty.call(initialPage, "dimension") ||
+ !initialDimension || typeof initialDimension !== "object" || Array.isArray(initialDimension) ||
+ !pageDimension || typeof pageDimension !== "object" || Array.isArray(pageDimension) ||
+ !positiveSafeInteger(initialVideo.cid) || !positiveSafeInteger(initialPage.cid) ||
+ initialPage.cid !== initialVideo.cid || !positiveSafeInteger(initialPage.page) ||
+ initialPage.page !== 1 || !positiveFiniteNumber(initialVideo.duration) ||
+ !positiveFiniteNumber(initialPage.duration) || initialPage.duration !== initialVideo.duration ||
+ !positiveSafeInteger(initialDimension.width) ||
+ !positiveSafeInteger(initialDimension.height) ||
+ !positiveSafeInteger(pageDimension.width) ||
+ !positiveSafeInteger(pageDimension.height) ||
+ pageDimension.width !== initialDimension.width ||
+ pageDimension.height !== initialDimension.height) {
+ return {status: "REJECT", error_code: "E_PAGE_PROOF",
+ state: "PAGE_REJECTED", reason: "PAGE_URL_REJECTED"};
+ }
+ const initialOwner = String(initialVideo.owner?.mid ?? "");
+ if (initialOwner !== "" && initialOwner !== job.creator_uid) {
+ return {status: "REJECT", error_code: "E_OWNER",
+ state: "PAGE_REJECTED", reason: "PAGE_OWNER_REJECTED"};
+ }
+ initialMetadata = {
+ duration: initialVideo.duration,
+ width: initialDimension.width,
+ height: initialDimension.height,
+ owner: initialOwner
+ };
+ }
+ let metadataSource = "HTML_MEDIA_ELEMENT";
+ let durationSeconds = video.duration;
+ let videoWidth = video.videoWidth;
+ let videoHeight = video.videoHeight;
+ let readyState = video.readyState;
+ if (initialMetadata?.owner === "" && !ownerAnchorExact) {
+ return {status: "PENDING", terminal_error_code: "E_PAGE_OWNER_ANCHOR_UNAVAILABLE",
+ state: "OWNER_PENDING", reason: ownerAnchorReason};
+ }
+ if (!Number.isFinite(durationSeconds) || durationSeconds <= 0 || readyState < 1 ||
+ videoWidth <= 0 || videoHeight <= 0) {
+ if (initialMetadata !== null &&
+ (initialMetadata.owner === job.creator_uid || (initialMetadata.owner === "" && ownerAnchorExact))) {
+ metadataSource = "BILIBILI_INITIAL_STATE";
+ durationSeconds = initialMetadata.duration;
+ videoWidth = initialMetadata.width;
+ videoHeight = initialMetadata.height;
+ readyState = 1;
+ } else if (!Number.isFinite(durationSeconds) || durationSeconds <= 0 || readyState < 1) {
+ if (!PAGE_METADATA_UNAVAILABLE_REASONS.has(initialStateReason)) {
+ return {status: "REJECT", error_code: "E_PAGE_PROOF",
+ state: "PAGE_REJECTED", reason: "PAGE_URL_REJECTED"};
+ }
+ return {status: "PENDING", terminal_error_code: "E_PAGE_METADATA_NOT_READY",
+ state: "METADATA_NOT_READY", reason: initialStateReason};
+ } else {
+ return {status: "PENDING", terminal_error_code: "E_PAGE_DIMENSIONS_UNAVAILABLE",
+ state: "DIMENSIONS_PENDING", reason: "VIDEO_DIMENSIONS_PENDING"};
+ }
+ }
+ if (!ownerAnchorExact && String(initialVideo?.owner?.mid ?? "") !== job.creator_uid) {
+ return {status: "PENDING", terminal_error_code: "E_PAGE_OWNER_ANCHOR_UNAVAILABLE",
+ state: "OWNER_PENDING", reason: ownerAnchorReason};
+ }
+ const durationMs = Math.round(durationSeconds * 1000);
+ const tolerance = Math.max(1500, Math.min(10000, Math.ceil(job.expected_duration_ms / 1000)));
+ if (Math.abs(durationMs - job.expected_duration_ms) > tolerance) {
+ return {status: "REJECT", error_code: "E_DURATION",
+ state: "PAGE_REJECTED", reason: "PAGE_DURATION_REJECTED"};
}
return {
- name: String(cookie.name),
- value: String(cookie.value),
- domain: String(cookie.domain),
- host_only: Boolean(cookie.hostOnly),
- path: String(cookie.path),
- secure: Boolean(cookie.secure),
- http_only: Boolean(cookie.httpOnly),
- same_site: String(cookie.sameSite || "unspecified"),
- session,
- expiration_unix: expiration,
- store_id: storeId,
- partition_key: null
+ status: "READY",
+ document_focus_diagnostic: documentFocusDiagnostic,
+ document_visibility_diagnostic: documentVisibilityDiagnostic,
+ proof: {
+ job_id: job.job_id,
+ bvid: job.bvid,
+ creator_uid: job.creator_uid,
+ canonical_url: job.canonical_url,
+ task_nonce: taskNonce,
+ observed_at_unix_ms: Date.now(),
+ metadata_source: metadataSource,
+ observed_duration_ms: durationMs,
+ video_width: videoWidth,
+ video_height: videoHeight,
+ ready_state: readyState,
+ eme_present: video.mediaKeys !== null
+ }
};
}
-async function startExactTask({retry = false} = {}) {
- if (activeStartLease !== null) {
- throw fixedError("E_BUSY");
- }
- const leaseId = randomNonce();
- activeStartLease = leaseId;
- try {
- if (retry) {
- await validateCurrentPage({leaseId});
+async function observe(job, binding, deadlineMs = PAGE_PROOF_DEADLINE_MS) {
+ const taskNonce = randomHex(16);
+ const startedAtUnixMs = Date.now();
+ const deadlineUnixMs = startedAtUnixMs + deadlineMs;
+ let pendingTerminalCode = "E_PAGE_PROOF";
+ let pendingState = "PAGE_REJECTED";
+ let pendingReason = "PAGE_IDENTITY_REJECTED";
+ let attempts = 0;
+ let stableSnapshots = 0;
+ let stableSignature = null;
+ let stableRejectSnapshots = 0;
+ let stableRejectSignature = null;
+ const deadlineCode = () => pendingTerminalCode === "E_PAGE_METADATA_NOT_READY" ?
+ "E_PAGE_METADATA_UNAVAILABLE" : pendingTerminalCode;
+ while (true) {
+ await revalidateBoundTab(binding, job.bvid);
+ const remainingBeforeInvocationMs = deadlineUnixMs - Date.now();
+ if (remainingBeforeInvocationMs <= 0) {
+ throw pageReadinessError(
+ deadlineCode(), attempts, startedAtUnixMs, pendingState, pendingReason
+ );
}
- const tab = await activeTargetTab();
- if (!currentProof || currentTabId !== tab.id || Date.now() - currentProof.observed_at_unix_ms > 60000) {
- throw fixedError("E_PAGE_PROOF");
- }
- const proof = currentProof;
- const prepareId = await freshNativePrepare(proof, leaseId);
- if (activeStartLease !== leaseId || preparedLeaseId !== leaseId || preparedTaskNonce !== proof.task_nonce) {
- throw fixedError("E_PREPARE");
- }
- const storeId = await currentCookieStore(tab.id);
- const rawCookies = await chrome.cookies.getAll({url: "https://www.bilibili.com/", storeId});
- let records = [];
+ const invocationBudgetMs = Math.min(PAGE_PROOF_INVOCATION_TIMEOUT_MS, remainingBeforeInvocationMs);
+ let deadlineTimer = null;
+ let results;
+ attempts += 1;
try {
- records = rawCookies.map(cookie => projectCookie(cookie, storeId)).filter(Boolean);
- if (records.length < 1 || records.length > 128) {
- throw fixedError("E_SECRET_INPUT");
+ results = await Promise.race([
+ chromeApiCall(chrome.scripting, "executeScript", [{
+ target: {tabId: binding.tabId, frameIds: [0]}, world: "ISOLATED", func: injectedObservation,
+ args: [job, taskNonce]
+ }], "E_PAGE_IDENTITY_UNAVAILABLE", "SCRIPTING_EXECUTE_FAILED"),
+ new Promise((resolve, reject) => {
+ deadlineTimer = setTimeout(() => reject(new Error("E_PAGE_SCRIPT_TIMEOUT")), invocationBudgetMs);
+ })
+ ]);
+ } catch (error) {
+ if (error?.message === "E_PAGE_SCRIPT_TIMEOUT") {
+ pendingTerminalCode = "E_PAGE_SCRIPT_TIMEOUT";
+ pendingState = "SCRIPT_TIMEOUT";
+ pendingReason = "SCRIPT_INVOCATION_FAILED";
+ } else {
+ pendingTerminalCode = "E_PAGE_IDENTITY_UNAVAILABLE";
+ pendingState = "IDENTITY_PENDING";
+ pendingReason = "PAGE_SCRIPTING_PENDING";
}
- nativePort.postMessage({
- schema: SCHEMA,
- type: "start",
- extension_build: EXTENSION_BUILD,
- target: TARGET,
- canonical_url: CANONICAL_URL,
- cookie_store_id: storeId,
- prepare_id: prepareId,
- page_proof: proof,
- cookies: records
- });
- currentTaskNonce = proof.task_nonce;
- preparedTaskNonce = null;
- preparedLeaseId = null;
- safeState = {...safeState, phase: "CHECKING", progress: 0, error_code: null};
- currentProof = null;
- currentTabId = null;
- return safeState;
+ stableSnapshots = 0;
+ stableSignature = null;
+ stableRejectSnapshots = 0;
+ stableRejectSignature = null;
+ continue;
} finally {
- for (const record of records) {
- if (record) {
- record.name = "";
- record.value = "";
- }
+ if (deadlineTimer !== null) clearTimeout(deadlineTimer);
+ }
+ if (Date.now() >= deadlineUnixMs) {
+ throw pageReadinessError(
+ deadlineCode(), attempts, startedAtUnixMs, pendingState, pendingReason
+ );
+ }
+ if (!Array.isArray(results) || results.length !== 1 || !isPlainObject(results[0]) ||
+ !isPlainObject(results[0].result)) {
+ pendingTerminalCode = "E_PAGE_IDENTITY_UNAVAILABLE";
+ pendingState = "IDENTITY_PENDING";
+ pendingReason = "PAGE_DOM_IDENTITY_PENDING";
+ stableSnapshots = 0;
+ stableSignature = null;
+ stableRejectSnapshots = 0;
+ stableRejectSignature = null;
+ continue;
+ }
+ const result = results[0].result;
+ if (result.status === "READY" &&
+ exactKeys(result, ["document_focus_diagnostic", "document_visibility_diagnostic", "proof", "status"]) &&
+ DOCUMENT_FOCUS_DIAGNOSTICS.has(result.document_focus_diagnostic) &&
+ DOCUMENT_VISIBILITY_DIAGNOSTICS.has(result.document_visibility_diagnostic) &&
+ exactKeys(result.proof, PAGE_PROOF_KEYS)) {
+ if (result.document_focus_diagnostic !== "DOCUMENT_FOCUS_TRUE") {
+ recordExtensionDiagnostic(result.document_focus_diagnostic);
}
- records.fill(null);
- rawCookies.fill(null);
+ if (result.document_visibility_diagnostic !== "DOCUMENT_VISIBILITY_VISIBLE") {
+ recordExtensionDiagnostic(result.document_visibility_diagnostic);
+ }
+ const signature = JSON.stringify(PAGE_PROOF_KEYS.filter(
+ key => key !== "observed_at_unix_ms" && key !== "task_nonce"
+ ).map(key => result.proof[key]));
+ stableSnapshots = signature === stableSignature ? stableSnapshots + 1 : 1;
+ stableSignature = signature;
+ stableRejectSnapshots = 0;
+ stableRejectSignature = null;
+ if (stableSnapshots >= PAGE_PROOF_STABLE_SNAPSHOTS) return result.proof;
+ pendingTerminalCode = "E_PAGE_STABILITY_TIMEOUT";
+ pendingState = "READY_UNSTABLE";
+ pendingReason = "STABLE_SNAPSHOTS_PENDING";
+ } else if (result.status === "REJECT" &&
+ exactKeys(result, ["error_code", "reason", "state", "status"]) &&
+ PREPARELESS_REJECT_CODES.has(result.error_code) &&
+ isCanonicalPreparelessTerminal(result.error_code, {
+ attempts, elapsed_ms: Math.max(0, Math.min(7200000, Date.now() - startedAtUnixMs)),
+ state: result.state, reason: result.reason
+ })) {
+ const stableIdentityReject = new Set([
+ "E_OWNER", "E_PAGE_ACCESS_CONTROL", "E_PAGE_PROOF"
+ ]).has(result.error_code);
+ if (!stableIdentityReject) {
+ throw pageReadinessError(
+ result.error_code, attempts, startedAtUnixMs, result.state, result.reason
+ );
+ }
+ pendingTerminalCode = result.error_code;
+ pendingState = result.state;
+ pendingReason = result.reason;
+ stableSnapshots = 0;
+ stableSignature = null;
+ const rejectSignature = `${result.error_code}\0${result.state}\0${result.reason}`;
+ stableRejectSnapshots = rejectSignature === stableRejectSignature ?
+ stableRejectSnapshots + 1 : 1;
+ stableRejectSignature = rejectSignature;
+ if (stableRejectSnapshots >= PAGE_PROOF_STABLE_SNAPSHOTS) {
+ throw pageReadinessError(
+ result.error_code, attempts, startedAtUnixMs, result.state, result.reason
+ );
+ }
+ } else if (result.status !== "PENDING" ||
+ !exactKeys(result, ["reason", "state", "status", "terminal_error_code"]) ||
+ !PAGE_PENDING_TERMINAL_CONTRACT.has(result.terminal_error_code)) {
+ throw new Error("E_PAGE_PROOF");
+ } else {
+ const [expectedState, allowedReasons] = PAGE_PENDING_TERMINAL_CONTRACT.get(result.terminal_error_code);
+ if (!PAGE_PENDING_STATES.has(result.state) || result.state !== expectedState ||
+ !PAGE_DIAGNOSTIC_REASONS.has(result.reason) || !allowedReasons.has(result.reason)) {
+ throw new Error("E_PAGE_PROOF");
+ }
+ pendingTerminalCode = result.terminal_error_code;
+ pendingState = result.state;
+ pendingReason = result.reason;
+ stableSnapshots = 0;
+ stableSignature = null;
+ stableRejectSnapshots = 0;
+ stableRejectSignature = null;
}
- } finally {
- if (activeStartLease === leaseId) {
- activeStartLease = null;
+ const remainingMs = deadlineUnixMs - Date.now();
+ if (remainingMs <= 0) {
+ throw pageReadinessError(
+ deadlineCode(), attempts, startedAtUnixMs, pendingState, pendingReason
+ );
}
+ await new Promise(resolve => setTimeout(resolve, Math.min(PAGE_PROOF_POLL_MS, remainingMs)));
}
}
-async function requestStatus() {
- await awaitNativeReady();
- return freshNativeStatus();
-}
-
-async function cancelTask() {
- if (!currentTaskNonce) {
- throw fixedError("E_CANCEL");
- }
- await awaitNativeReady();
- nativePort.postMessage({schema: SCHEMA, type: "cancel", target: TARGET, task_nonce: currentTaskNonce});
- return safeState;
-}
-
-chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
- if (sender.id !== chrome.runtime.id || sender.url !== SIDEPANEL_URL || !message || typeof message.action !== "string") {
- return false;
- }
- const actions = {
- validate: () => validateCurrentPage(),
- start: () => startExactTask(),
- retry: () => startExactTask({retry: true}),
- status: () => requestStatus(),
- cancel: () => cancelTask()
+function normalizeCookie(cookie, storeId) {
+ return {
+ name: cookie.name,
+ value: cookie.value,
+ domain: cookie.domain,
+ host_only: cookie.hostOnly,
+ path: cookie.path,
+ secure: cookie.secure,
+ http_only: cookie.httpOnly,
+ same_site: cookie.sameSite,
+ session: cookie.session,
+ expiration_unix: cookie.session === true ? null : Math.floor(cookie.expirationDate),
+ store_id: cookie.storeId,
+ partition_key: cookie.partitionKey === undefined ? null : cookie.partitionKey
};
- const action = actions[message.action];
- if (!action) {
- sendResponse({ok: false, error_code: "E_PROTOCOL"});
+}
+
+function cookiePathMatches(path, bvid) {
+ const wanted = `/video/${bvid}`;
+ return path === "/" || (wanted.startsWith(path) &&
+ (path.endsWith("/") || path.length === wanted.length || wanted[path.length] === "/"));
+}
+
+function safeCookieString(value, minimum, maximumUtf8) {
+ if (typeof value !== "string" || COOKIE_CONTROL_RE.test(value)) return false;
+ const bytes = new TextEncoder().encode(value).length;
+ return bytes >= minimum && bytes <= maximumUtf8;
+}
+
+function validBrowserCookie(cookie, storeId, proof, bvid, partitioned) {
+ return exactKeys(cookie, COOKIE_KEYS) &&
+ safeCookieString(cookie.name, 1, 256) && safeCookieString(cookie.value, 1, 4096) &&
+ typeof cookie.host_only === "boolean" &&
+ cookie.domain === (cookie.host_only ? "www.bilibili.com" : ".bilibili.com") &&
+ safeCookieString(cookie.path, 1, 1024) &&
+ cookie.path.startsWith("/") && cookiePathMatches(cookie.path, bvid) &&
+ typeof cookie.secure === "boolean" && typeof cookie.http_only === "boolean" &&
+ typeof cookie.session === "boolean" && COOKIE_SAME_SITE.has(cookie.same_site) &&
+ STORE_ID_RE.test(cookie.store_id) && cookie.store_id === storeId &&
+ (partitioned ? validPartitionKeyDescriptor(cookie.partition_key) : cookie.partition_key === null) &&
+ (cookie.session
+ ? cookie.expiration_unix === null
+ : Number.isSafeInteger(cookie.expiration_unix) && cookie.expiration_unix >= 1 &&
+ cookie.expiration_unix > Math.floor(proof.observed_at_unix_ms / 1000));
+}
+
+function validCookie(cookie, storeId, proof, bvid) {
+ return validBrowserCookie(cookie, storeId, proof, bvid, false);
+}
+
+function validCookieSet(cookies, storeId, proof, bvid) {
+ return Array.isArray(cookies) && cookies.length >= 1 && cookies.length <= MAX_COOKIE_COUNT &&
+ cookies.every(cookie => validCookie(cookie, storeId, proof, bvid));
+}
+
+function validPartitionKeyDescriptor(value) {
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+ const keys = Object.keys(value).sort().join("\0");
+ if (!["", "hasCrossSiteAncestor", "topLevelSite", "hasCrossSiteAncestor\0topLevelSite"].includes(keys)) {
return false;
}
- action().then(
- state => sendResponse({ok: true, state}),
- error => sendResponse({ok: false, error_code: error.code || "E_EXTENSION"})
+ if (Object.prototype.hasOwnProperty.call(value, "hasCrossSiteAncestor") &&
+ typeof value.hasCrossSiteAncestor !== "boolean") return false;
+ if (!Object.prototype.hasOwnProperty.call(value, "topLevelSite")) return true;
+ if (!safeCookieString(value.topLevelSite, 1, 2048)) return false;
+ try {
+ const site = new URL(value.topLevelSite);
+ return (site.protocol === "https:" || site.protocol === "http:") &&
+ site.origin === value.topLevelSite && site.username === "" && site.password === "" &&
+ site.pathname === "/" && site.search === "" && site.hash === "";
+ } catch {
+ return false;
+ }
+}
+
+function validRawSessionExpiration(cookie, proof) {
+ if (typeof cookie.session !== "boolean") return false;
+ if (cookie.session) return cookie.expirationDate === undefined;
+ if (typeof cookie.expirationDate !== "number" || !Number.isFinite(cookie.expirationDate)) return false;
+ const expirationUnix = Math.floor(cookie.expirationDate);
+ return Number.isSafeInteger(expirationUnix) && expirationUnix >= 1 &&
+ expirationUnix > Math.floor(proof.observed_at_unix_ms / 1000);
+}
+
+function selectedCookieRelevance(cookie, storeId, bvid, proof) {
+ if (cookie === null || typeof cookie !== "object" || Array.isArray(cookie) ||
+ !safeCookieString(cookie.name, 1, 256)) return "AMBIGUOUS";
+ if (!TRANSFER_COOKIE_NAMES.has(cookie.name)) return "IRRELEVANT_NAME";
+ if (cookie.partitionKey !== undefined) {
+ if (!validRawSessionExpiration(cookie, proof)) return "AMBIGUOUS";
+ let normalized = null;
+ try {
+ normalized = normalizeCookie(cookie, storeId);
+ return validBrowserCookie(normalized, storeId, proof, bvid, true)
+ ? "IRRELEVANT_PARTITIONED" : "AMBIGUOUS";
+ } catch {
+ return "AMBIGUOUS";
+ } finally {
+ if (normalized !== null) {
+ normalized.name = "";
+ normalized.value = "";
+ }
+ }
+ }
+ return "SELECTED";
+}
+
+function selectTransferCookies(rawCookies, storeId, proof, bvid) {
+ const selected = [];
+ const requiredNames = new Set();
+ if (!Array.isArray(rawCookies)) return {ok: false, reason: "COOKIE_API_SHAPE", cookies: selected};
+ if (rawCookies.length > MAX_COOKIE_COUNT) {
+ return {ok: false, reason: "COOKIE_TRANSFER_OVERFLOW", cookies: selected};
+ }
+ for (const rawCookie of rawCookies) {
+ const relevance = selectedCookieRelevance(rawCookie, storeId, bvid, proof);
+ if (relevance === "IRRELEVANT_NAME" || relevance === "IRRELEVANT_PARTITIONED") continue;
+ if (relevance !== "SELECTED") {
+ return {ok: false, reason: "COOKIE_RELEVANCE_AMBIGUOUS", cookies: selected};
+ }
+ const cookie = normalizeCookie(rawCookie, storeId);
+ selected.push(cookie);
+ if (!validCookie(cookie, storeId, proof, bvid)) {
+ return {ok: false, reason: "COOKIE_SELECTED_INVALID", cookies: selected};
+ }
+ if (REQUIRED_AUTH_COOKIE_NAMES.has(cookie.name)) requiredNames.add(cookie.name);
+ }
+ if (selected.length === 0) return {ok: false, reason: "COOKIE_AUTH_SET_EMPTY", cookies: selected};
+ for (const requiredName of REQUIRED_AUTH_COOKIE_NAMES) {
+ if (!requiredNames.has(requiredName)) {
+ return {ok: false, reason: "COOKIE_AUTH_SET_EMPTY", cookies: selected};
+ }
+ }
+ const pathSpecificity = cookie => new TextEncoder().encode(cookie.path).length;
+ const bestSpecificity = Math.max(...selected.map(pathSpecificity));
+ const highestPrecedence = selected.filter(cookie => pathSpecificity(cookie) === bestSpecificity);
+ const firstValue = highestPrecedence[0].value;
+ if (highestPrecedence.some(cookie => cookie.value !== firstValue)) {
+ return {ok: false, reason: "COOKIE_EQUAL_PRECEDENCE_CONFLICT", cookies: selected};
+ }
+ highestPrecedence.sort((left, right) =>
+ Number(right.host_only) - Number(left.host_only) ||
+ left.domain.localeCompare(right.domain) || left.path.localeCompare(right.path) ||
+ left.same_site.localeCompare(right.same_site) || Number(right.secure) - Number(left.secure) ||
+ Number(right.http_only) - Number(left.http_only) || Number(right.session) - Number(left.session)
);
- return true;
+ const chosen = highestPrecedence[0];
+ for (const cookie of selected) {
+ if (cookie !== chosen) {
+ cookie.name = "";
+ cookie.value = "";
+ }
+ }
+ const transfer = [chosen];
+ selected.length = 0;
+ const valid = validCookieSet(transfer, storeId, proof, bvid);
+ return {ok: valid, reason: valid ? "COOKIE_TRANSFER_READY" : "COOKIE_TRANSFER_INVALID", cookies: transfer};
+}
+
+function zeroizeCookies(rawCookies, cookies) {
+ for (const cookie of Array.isArray(cookies) ? cookies : []) {
+ cookie.name = "";
+ cookie.value = "";
+ }
+ for (const cookie of Array.isArray(rawCookies) ? rawCookies : []) {
+ if (cookie !== null && typeof cookie === "object") {
+ try { cookie.name = ""; } catch {}
+ try { cookie.value = ""; } catch {}
+ }
+ }
+ if (Array.isArray(cookies)) cookies.length = 0;
+ if (Array.isArray(rawCookies)) rawCookies.length = 0;
+}
+
+async function abortPreparedForCookieAccess(session, job, leaseId, prepareId, rawCookies, cookies, reason) {
+ lastCookieAccessReason = COOKIE_ACCESS_REASONS.has(reason) ? reason : "COOKIE_SELECTION_EXCEPTION";
+ zeroizeCookies(rawCookies, cookies);
+ const aborted = await session.send({
+ schema: SCHEMA, type: "abort_prepare", job_id: job.job_id, lease_id: leaseId,
+ prepare_id: prepareId, error_code: "E_COOKIE_ACCESS", error_reason: lastCookieAccessReason
+ });
+ if (!(await validateResponse(aborted, "abort_prepare")) || aborted.phase !== "FAILED" ||
+ aborted.error_code !== "E_COOKIE_ACCESS" || aborted.prepare_id !== prepareId ||
+ aborted.job?.job_id !== job.job_id || aborted.lease_id !== leaseId) {
+ throw new Error("E_PREPARE");
+ }
+ return aborted;
+}
+
+async function runJobCore(session, job, leaseId, lifecycle) {
+ lastTabDriftReason = null;
+ lastCookieAccessReason = null;
+ lastPageReadinessDiagnostic = null;
+ let selectedTab;
+ let proof;
+ const flowStartedAtUnixMs = Date.now();
+ const foregroundFlow = {
+ startedAtUnixMs: flowStartedAtUnixMs,
+ deadlineUnixMs: flowStartedAtUnixMs + PAGE_PROOF_DEADLINE_MS,
+ foregroundUsed: false
+ };
+ try {
+ selectedTab = await targetTab(session, job, leaseId, foregroundFlow);
+ lifecycle.binding = selectedTab.binding;
+ proof = await acquirePageProof(session, job, leaseId, selectedTab, foregroundFlow);
+ await revalidateBoundTab(selectedTab.binding, job.bvid);
+ } catch (error) {
+ const code = typeof error?.message === "string" && PREPARELESS_REJECT_CODES.has(error.message)
+ ? error.message : null;
+ if (code === null) throw error;
+ const candidate = isPageReadinessDiagnostic(error?.page_readiness_diagnostic)
+ ? error.page_readiness_diagnostic
+ : isPageReadinessDiagnostic(lastPageReadinessDiagnostic)
+ ? lastPageReadinessDiagnostic
+ : fallbackPageReadinessDiagnostic(code);
+ const diagnostic = isCanonicalPreparelessTerminal(code, candidate)
+ ? candidate : fallbackPageReadinessDiagnostic(code);
+ if (!isCanonicalPreparelessTerminal(code, diagnostic)) throw new Error("E_REJECT");
+ const rejected = await session.send({
+ schema: SCHEMA, type: "reject", job_id: job.job_id, lease_id: leaseId,
+ error_code: code, diagnostic
+ });
+ if (!(await validateResponse(rejected, "reject")) || rejected.phase !== "FAILED" ||
+ rejected.error_code !== code || rejected.job?.job_id !== job.job_id || rejected.lease_id !== leaseId) {
+ throw new Error("E_REJECT");
+ }
+ return rejected;
+ }
+ const prepareId = randomHex(16);
+ const prepared = await session.send({
+ schema: SCHEMA, type: "prepare", extension_build: EXTENSION_BUILD,
+ lease_id: leaseId, prepare_id: prepareId, job, page_proof: proof
+ }, 150000);
+ const preparedValid = await validateResponse(prepared, "prepare");
+ if (preparedValid && prepared.phase === "COMPLETE" && prepared.prepare_id === prepareId &&
+ prepared.job?.job_id === job.job_id && prepared.lease_id === leaseId &&
+ prepared.progress === 100 && prepared.error_code === null &&
+ prepared.formal_filename === `${job.bvid}.mkv` &&
+ prepared.mapping_filename === `${job.bvid}.download.json`) {
+ return prepared;
+ }
+ if (!preparedValid || prepared.phase !== "READY" || prepared.prepare_id !== prepareId) {
+ const code = preparedValid && prepared.phase === "FAILED" && ERROR_CODE_RE.test(prepared.error_code || "")
+ ? prepared.error_code : "E_PREPARE";
+ throw new Error(code);
+ }
+ const storeId = selectedTab.binding.cookieStoreId;
+ let rawCookies = [];
+ let cookies = [];
+ try {
+ rawCookies = await chrome.cookies.getAll({
+ url: canonicalUrl(job.bvid),
+ name: AUTH_COOKIE_NAME,
+ storeId
+ });
+ } catch {
+ return abortPreparedForCookieAccess(session, job, leaseId, prepareId, rawCookies, cookies, "COOKIE_API_ERROR");
+ }
+ let selection;
+ try {
+ selection = selectTransferCookies(rawCookies, storeId, proof, job.bvid);
+ } catch {
+ return abortPreparedForCookieAccess(
+ session, job, leaseId, prepareId, rawCookies, cookies, "COOKIE_SELECTION_EXCEPTION"
+ );
+ }
+ cookies = selection.cookies;
+ if (!selection.ok) {
+ return abortPreparedForCookieAccess(
+ session, job, leaseId, prepareId, rawCookies, cookies, selection.reason
+ );
+ }
+ let started;
+ try {
+ started = await session.send({
+ schema: SCHEMA, type: "start", extension_build: EXTENSION_BUILD,
+ lease_id: leaseId, prepare_id: prepareId, job, cookie_store_id: storeId,
+ page_proof: proof, cookies
+ }, 30000);
+ } finally {
+ zeroizeCookies(rawCookies, cookies);
+ }
+ if (!(await validateResponse(started, "status")) || started.job?.job_id !== job.job_id) {
+ throw new Error("E_START");
+ }
+ let state = started;
+ while (!["COMPLETE", "FAILED", "POSTPROCESS_FAILED", "CANCELED"].includes(state.phase)) {
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ state = await session.send({schema: SCHEMA, type: "status", job_id: job.job_id, lease_id: leaseId});
+ if (!(await validateResponse(state, "status")) || state.job?.job_id !== job.job_id) throw new Error("E_STATUS");
+ }
+ return state;
+}
+
+async function runJob(session, job, leaseId) {
+ const lifecycle = {binding: null};
+ try {
+ return await runJobCore(session, job, leaseId, lifecycle);
+ } finally {
+ // The durable session record is the authority. This also closes a tab when
+ // a restart/rebind path lost its in-memory binding after Host start.
+ await cleanupOwnedTab();
+ }
+}
+
+async function drainQueue() {
+ if (drainLease) return;
+ drainLease = true;
+ lastDrainErrorCode = null;
+ let session;
+ try {
+ session = nativeSession();
+ const hello = await session.send({schema: SCHEMA, type: "hello", extension_build: EXTENSION_BUILD});
+ const helloValid = await validateResponse(hello, "hello");
+ if (!helloValid) throw new Error("E_HELLO");
+ if (hello.phase === "FAILED") throw new Error(hello.error_code || "E_HOST");
+ if (hello.maintenance.reload_required) {
+ const token = hello.maintenance.reload_token;
+ if (!HEX32_RE.test(token || "")) return;
+ const response = await session.send({schema: SCHEMA, type: "reload_begin", extension_build: EXTENSION_BUILD, reload_token: token});
+ if (!(await validateResponse(response, "reload_begin")) || response.phase !== "RELOAD_REQUIRED") return;
+ session.disconnect();
+ session = null;
+ chrome.runtime.reload();
+ return;
+ }
+ const polled = await session.send({schema: SCHEMA, type: "poll"});
+ const pollValid = await validateResponse(polled, "poll");
+ if (!pollValid) throw new Error("E_POLL");
+ const owned = await loadOwnedTabRecord();
+ const continuesOwnedJob = polled.phase === "READY" && polled.job && polled.lease_id &&
+ owned !== null && owned.job_id === polled.job.job_id && owned.lease_id === polled.lease_id;
+ if (owned !== null && !continuesOwnedJob) await cleanupOwnedTab(owned);
+ if (["FAILED", "POSTPROCESS_FAILED"].includes(polled.phase)) {
+ throw new Error(polled.error_code || "E_HOST");
+ }
+ if (polled.phase === "COMPLETE") {
+ if (!polled.job || !polled.lease_id || polled.progress !== 100 || polled.error_code !== null ||
+ polled.formal_filename !== `${polled.job.bvid}.mkv` ||
+ polled.mapping_filename !== `${polled.job.bvid}.download.json`) {
+ throw new Error("E_POLL");
+ }
+ return;
+ }
+ if (polled.phase !== "READY" || !polled.job || !polled.lease_id) return;
+ await runJob(session, polled.job, polled.lease_id);
+ } catch (error) {
+ // Memory-only safe code: never persist secrets, URLs, protocol payloads, or Host diagnostics here.
+ lastDrainErrorCode = safeErrorCode(error);
+ } finally {
+ if (session) session.disconnect();
+ drainLease = false;
+ }
+}
+
+function schedule() {
+ chrome.alarms.create("bili-auth-queue", {periodInMinutes: 1});
+ void drainQueue();
+}
+
+chrome.runtime.onInstalled.addListener(schedule);
+chrome.runtime.onStartup.addListener(schedule);
+chrome.alarms.onAlarm.addListener(alarm => {
+ if (alarm?.name === "bili-auth-queue") void drainQueue();
});
diff --git a/dev/project-dev/bili_authenticated_extension/build_host.ps1 b/dev/project-dev/bili_authenticated_extension/build_host.ps1
index 5cf66bd..07eb7d9 100644
--- a/dev/project-dev/bili_authenticated_extension/build_host.ps1
+++ b/dev/project-dev/bili_authenticated_extension/build_host.ps1
@@ -52,6 +52,26 @@
}
}
+function Assert-ExactSourceSnapshot([string]$Root, [string]$ManifestPath, [string[]]$ExpectedFiles, [object]$Manifest) {
+ Assert-ExactSourceTree $Root $ManifestPath $ExpectedFiles
+ foreach ($entry in $Manifest.files) {
+ if ($entry.path -notmatch '^[A-Za-z0-9._/-]+$' -or $entry.path.Contains('..') -or
+ $entry.sha256 -notmatch '^[A-F0-9]{64}$' -or $entry.bytes -lt 1) {
+ throw 'Source artifact manifest contains an invalid entry.'
+ }
+ $candidate = [System.IO.Path]::GetFullPath((Join-Path $Root $entry.path))
+ if (-not $candidate.StartsWith($Root + [System.IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) {
+ throw 'Source artifact path escaped its root.'
+ }
+ $item = Get-Item -LiteralPath $candidate
+ if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
+ $item.Length -ne $entry.bytes -or
+ (Get-FileHash -Algorithm SHA256 -LiteralPath $candidate).Hash -cne $entry.sha256) {
+ throw 'Source artifact hash mismatch.'
+ }
+ }
+}
+
$resolvedOutput = [System.IO.Path]::GetFullPath($OutputRoot)
if (-not [System.IO.Path]::IsPathRooted($resolvedOutput) -or $resolvedOutput.StartsWith('\\')) {
throw 'OutputRoot must be an absolute local path.'
@@ -83,18 +103,19 @@
'__init__.py', 'background.js', 'build_host.ps1', 'config.example.json',
'constants.py', 'dependencies/dependency-artifact-manifest.json',
'dependencies/yt_dlp-2026.7.4-py3-none-any.whl',
- 'install_native_host.ps1', 'job.py', 'manifest.json',
+ 'formal_legacy_identity_manifest.py', 'install_native_host.ps1', 'job.py', 'manifest.json',
'native-host-manifest.template.json', 'native_host.py', 'protocol.py',
+ 'queue-producer.example.json', 'queue_producer.py', 'queue_state.py',
'sidepanel.css', 'sidepanel.html', 'sidepanel.js', 'worker.py'
)
Assert-ExactSourceTree $sourceRoot $resolvedSourceManifest $expectedSourceFiles
$dependencyManifestPath = Join-Path $sourceRoot 'dependencies/dependency-artifact-manifest.json'
$dependencyManifestItem = Get-Item -LiteralPath $dependencyManifestPath
$sourceManifest = Get-StrictJson $resolvedSourceManifest
-if ($sourceManifest.schema -ne 1 -or $sourceManifest.target -cne 'BV1HA3o6oEJJ' -or
+if ($sourceManifest.schema -ne 1 -or $sourceManifest.scope -cne 'generic-bilibili-queue' -or
$sourceManifest.extension_id -cne 'oidmclckpdmpabbfedplkbdplmfcenbb' -or
- $sourceManifest.extension_build -cne 'project-info-bili-auth-ingress/1.0.0+20260805.v002' -or
- $sourceManifest.host_build -cne 'project-info-bili-auth-native-host/1.0.0+20260805.v002' -or
+ $sourceManifest.extension_build -cne 'project-info-bili-auth-ingress/1.2.25+20260829.generic.v027' -or
+ $sourceManifest.host_build -cne 'project-info-bili-auth-native-host/1.2.25+20260829.generic.v027' -or
$sourceManifest.dependency_artifact_manifest_bytes -ne $dependencyManifestItem.Length -or
$sourceManifest.dependency_artifact_manifest_sha256 -cne (Get-FileHash -Algorithm SHA256 -LiteralPath $dependencyManifestPath).Hash) {
throw 'Source artifact manifest identity mismatch.'
@@ -124,22 +145,7 @@
if (Compare-Object -CaseSensitive ($expectedSourceFiles | Sort-Object) $actualSourceFiles) {
throw 'Source artifact manifest file set mismatch.'
}
-foreach ($entry in $sourceManifest.files) {
- if ($entry.path -notmatch '^[A-Za-z0-9._/-]+$' -or $entry.path.Contains('..') -or
- $entry.sha256 -notmatch '^[A-F0-9]{64}$' -or $entry.bytes -lt 1) {
- throw 'Source artifact manifest contains an invalid entry.'
- }
- $candidate = [System.IO.Path]::GetFullPath((Join-Path $sourceRoot $entry.path))
- if (-not $candidate.StartsWith($sourceRoot + [System.IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) {
- throw 'Source artifact path escaped its root.'
- }
- $item = Get-Item -LiteralPath $candidate
- if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
- $item.Length -ne $entry.bytes -or
- (Get-FileHash -Algorithm SHA256 -LiteralPath $candidate).Hash -cne $entry.sha256) {
- throw 'Source artifact hash mismatch.'
- }
-}
+Assert-ExactSourceSnapshot $sourceRoot $resolvedSourceManifest $expectedSourceFiles $sourceManifest
$dependencyManifest = Get-StrictJson $dependencyManifestPath
$expectedDependencyKeys = @(
@@ -229,6 +235,7 @@
foreach ($name in @($pythonEnvironment.Keys)) {
Remove-Item -LiteralPath "Env:$name" -ErrorAction SilentlyContinue
}
+ $env:PYTHONDONTWRITEBYTECODE = '1'
$pyInstallerVersion = & $resolvedPython -I -B -c "import importlib.metadata; print(importlib.metadata.version('PyInstaller'))"
if ($LASTEXITCODE -ne 0 -or $pyInstallerVersion.Trim() -cne '6.15.0') {
throw 'PyInstaller 6.15.0 is required; no unpinned builder is allowed.'
@@ -673,10 +680,10 @@
$hostItem = Get-Item -LiteralPath $hostExecutable
$buildManifest = [ordered]@{
schema = 2
- target = 'BV1HA3o6oEJJ'
+ scope = 'generic-bilibili-queue'
extension_id = 'oidmclckpdmpabbfedplkbdplmfcenbb'
- extension_build = 'project-info-bili-auth-ingress/1.0.0+20260805.v002'
- host_build = 'project-info-bili-auth-native-host/1.0.0+20260805.v002'
+ extension_build = 'project-info-bili-auth-ingress/1.2.25+20260829.generic.v027'
+ host_build = 'project-info-bili-auth-native-host/1.2.25+20260829.generic.v027'
packaging = 'pyinstaller-onefile'
pyinstaller_version = '6.15.0'
yt_dlp_version = '2026.7.4'
@@ -712,12 +719,20 @@
($buildManifest | ConvertTo-Json -Depth 5),
$utf8NoBom
)
+ Assert-ExactSourceSnapshot $sourceRoot $resolvedSourceManifest $expectedSourceFiles $sourceManifest
} catch {
+ $caught = $_
if ($outputCreated -and (Test-Path -LiteralPath $resolvedOutput)) {
Remove-Item -LiteralPath $resolvedOutput -Recurse -Force
}
- throw
+ try {
+ Assert-ExactSourceSnapshot $sourceRoot $resolvedSourceManifest $expectedSourceFiles $sourceManifest
+ } catch {
+ throw "Post-build source snapshot drifted: $($_.Exception.Message)"
+ }
+ throw $caught
} finally {
+ Remove-Item -LiteralPath 'Env:PYTHONDONTWRITEBYTECODE' -ErrorAction SilentlyContinue
foreach ($name in @($pythonEnvironment.Keys)) {
Set-Item -LiteralPath "Env:$name" -Value $pythonEnvironment[$name]
}
diff --git a/dev/project-dev/bili_authenticated_extension/config.example.json b/dev/project-dev/bili_authenticated_extension/config.example.json
index 036dcb5..3bb99b4 100644
--- a/dev/project-dev/bili_authenticated_extension/config.example.json
+++ b/dev/project-dev/bili_authenticated_extension/config.example.json
@@ -1,7 +1,14 @@
{
- "schema": 1,
- "target": "BV1HA3o6oEJJ",
- "canonical_url": "https://www.bilibili.com/video/BV1HA3o6oEJJ",
+ "schema": 2,
+ "creator_allowlist": [
+ "1420210197"
+ ],
+ "queue_path": "C:\\ABSOLUTE\\bili-auth-queue.jsonl",
+ "queue_state_path": "C:\\ABSOLUTE\\bili-auth-queue-state.jsonl",
+ "queue_lock_path": "C:\\ABSOLUTE\\bili-auth-queue.lock",
+ "reload_state_path": "C:\\ABSOLUTE\\bili-auth-reload-state.jsonl",
+ "reload_generation": "bili-auth-generic-v027",
+ "required_extension_build": "project-info-bili-auth-ingress/1.2.25+20260829.generic.v027",
"ffmpeg": "C:\\ABSOLUTE\\ffmpeg.exe",
"ffmpeg_sha256": "REQUIRED_SHA256",
"ffprobe": "C:\\ABSOLUTE\\ffprobe.exe",
@@ -9,10 +16,11 @@
"bridge_python": "C:\\ABSOLUTE\\python.exe",
"bridge_python_sha256": "REQUIRED_SHA256",
"bridge_script": "C:\\ABSOLUTE\\bili_video_download_bridge.py",
- "bridge_script_sha256": "749FC486B0F42315BD463F11771FE2A7C71CAB53DD9AC2E411CE82E1175DFF13",
- "batch_json": "C:\\ABSOLUTE\\exact-batch.json",
- "batch_json_sha256": "REQUIRED_SHA256",
+ "bridge_script_sha256": "00F11DAF8387160DB863C89F0B33AB8480422233FF42199189222C989C7ED07E",
"yt_dlp_executable": "C:\\ABSOLUTE\\yt-dlp.exe",
"yt_dlp_executable_sha256": "REQUIRED_SHA256",
- "destination": "F:\\video\\青枫浦上Q"
+ "destination": "F:\\video\\REGISTERED_CREATOR_NAME",
+ "creator_name": "REGISTERED_CREATOR_NAME",
+ "formal_manifest_path": "C:\\ABSOLUTE\\project-info\\ana-data\\registered-creator\\manifest.jsonl",
+ "processing_handoff_path": "C:\\ABSOLUTE\\project-info\\ana-data\\registered-creator\\video-processing-handoffs.jsonl"
}
diff --git a/dev/project-dev/bili_authenticated_extension/constants.py b/dev/project-dev/bili_authenticated_extension/constants.py
index 36a3d66..d3b5b55 100644
--- a/dev/project-dev/bili_authenticated_extension/constants.py
+++ b/dev/project-dev/bili_authenticated_extension/constants.py
@@ -1,17 +1,123 @@
-"""Frozen identities and limits for the authenticated ingress."""
+"""Generic identities, validators, and limits for authenticated Bilibili jobs."""
from __future__ import annotations
-TARGET_BVID = "BV1HA3o6oEJJ"
-CANONICAL_URL = "https://www.bilibili.com/video/BV1HA3o6oEJJ"
-TARGET_PATH = "/video/BV1HA3o6oEJJ"
-EXPECTED_DURATION_MS = 3_133_950
-DURATION_TOLERANCE_MS = 3_134
+import hashlib
+import re
-SCHEMA_VERSION = 2
+BVID_RE = re.compile(r"BV1[1-9A-HJ-NP-Za-km-z]{9}\Z")
+CREATOR_UID_RE = re.compile(r"[1-9][0-9]{0,19}\Z")
+JOB_ID_RE = re.compile(r"[0-9a-f]{64}\Z")
+MESSAGE_ID_RE = re.compile(r"msg_[0-9]{17}_[0-9a-f]{8}\Z")
+HANDOFF_ID_RE = re.compile(r"HANDOFF-[A-Z0-9-]{8,240}\Z")
+AUDIT_ID_RE = re.compile(r"DEV-AUDIT-[A-Z0-9-]{8,240}\Z")
+ERROR_CODE_RE = re.compile(r"E_[A-Z0-9_]{1,48}\Z")
+UPPER_SHA256_RE = re.compile(r"[0-9A-F]{64}\Z")
+
+# Lineage-only marker for an already COMPLETE job whose exact published pair
+# still needs the governed formal-manifest and processing-handoff closure. It
+# is never a durable runtime terminal code.
+COMPLETION_CLOSURE_REQUIRED = "E_COMPLETION_CLOSURE_REQUIRED"
+
+
+def validate_bvid(value: object) -> str:
+ if not isinstance(value, str) or not BVID_RE.fullmatch(value):
+ raise ValueError("invalid BVID")
+ return value
+
+
+def validate_creator_uid(value: object) -> str:
+ if not isinstance(value, str) or not CREATOR_UID_RE.fullmatch(value):
+ raise ValueError("invalid creator UID")
+ return value
+
+
+def canonical_url(bvid: str) -> str:
+ return f"https://www.bilibili.com/video/{validate_bvid(bvid)}"
+
+
+def target_path(bvid: str) -> str:
+ return f"/video/{validate_bvid(bvid)}"
+
+
+def stable_job_id(creator_uid: str, bvid: str) -> str:
+ creator = validate_creator_uid(creator_uid)
+ target = validate_bvid(bvid)
+ return hashlib.sha256(f"bili-auth-job-v1\0{creator}\0{target}".encode("ascii")).hexdigest()
+
+
+def stable_successor_job_id(
+ creator_uid: str,
+ bvid: str,
+ predecessor_job_id: str,
+ retry_generation: int,
+ predecessor_terminal_error_code: str,
+ authorization_message_id: str,
+ authorization_handoff_id: str,
+ authorization_sha256: str,
+ repair_review_result_message_id: str,
+ repair_audit_id: str,
+ repair_audit_bytes: int,
+ repair_audit_sha256: str,
+) -> str:
+ """Derive the exact schema-2 successor identity frozen by V001."""
+ creator = validate_creator_uid(creator_uid)
+ target = validate_bvid(bvid)
+ fields: tuple[tuple[object, re.Pattern[str] | None], ...] = (
+ (predecessor_job_id, JOB_ID_RE),
+ (retry_generation, None),
+ (predecessor_terminal_error_code, ERROR_CODE_RE),
+ (authorization_message_id, MESSAGE_ID_RE),
+ (authorization_handoff_id, HANDOFF_ID_RE),
+ (authorization_sha256, UPPER_SHA256_RE),
+ (repair_review_result_message_id, MESSAGE_ID_RE),
+ (repair_audit_id, AUDIT_ID_RE),
+ (repair_audit_bytes, None),
+ (repair_audit_sha256, UPPER_SHA256_RE),
+ )
+ for value, pattern in fields:
+ if pattern is not None and (not isinstance(value, str) or not pattern.fullmatch(value)):
+ raise ValueError("invalid successor identity")
+ if (
+ isinstance(retry_generation, bool) or not isinstance(retry_generation, int)
+ or not 1 <= retry_generation <= 1_000_000
+ or isinstance(repair_audit_bytes, bool) or not isinstance(repair_audit_bytes, int)
+ or repair_audit_bytes <= 0
+ ):
+ raise ValueError("invalid successor identity")
+ payload = "\0".join((
+ "bili-auth-job-v2",
+ creator,
+ target,
+ predecessor_job_id,
+ str(retry_generation),
+ predecessor_terminal_error_code,
+ authorization_message_id,
+ authorization_handoff_id,
+ authorization_sha256,
+ repair_review_result_message_id,
+ repair_audit_id,
+ str(repair_audit_bytes),
+ repair_audit_sha256,
+ ))
+ return hashlib.sha256(payload.encode("ascii")).hexdigest()
+
+
+def duration_tolerance_ms(expected_duration_ms: int) -> int:
+ if isinstance(expected_duration_ms, bool) or not isinstance(expected_duration_ms, int):
+ raise ValueError("invalid duration")
+ if not 1_000 <= expected_duration_ms <= 86_400_000:
+ raise ValueError("invalid duration")
+ return max(1_500, min(10_000, (expected_duration_ms + 999) // 1_000))
+
+
+SCHEMA_VERSION = 3
+QUEUE_SCHEMA_VERSION = 1
+SUCCESSOR_QUEUE_SCHEMA_VERSION = 2
HOST_NAME = "com.project_info.bili_auth_ingress"
-EXTENSION_BUILD = "project-info-bili-auth-ingress/1.0.0+20260805.v002"
-HOST_BUILD = "project-info-bili-auth-native-host/1.0.0+20260805.v002"
+EXTENSION_BUILD = "project-info-bili-auth-ingress/1.2.25+20260829.generic.v027"
+HOST_BUILD = "project-info-bili-auth-native-host/1.2.25+20260829.generic.v027"
+RELOAD_GENERATION = "bili-auth-generic-v027"
EXPECTED_EXTENSION_ID = "oidmclckpdmpabbfedplkbdplmfcenbb"
EXPECTED_ORIGIN = f"chrome-extension://{EXPECTED_EXTENSION_ID}/"
PUBLIC_KEY_DER_SHA256 = (
@@ -30,6 +136,267 @@
MAX_OUTPUT_FRAME = 16_384
MAX_COOKIE_COUNT = 128
MAX_SAFE_INTEGER = (1 << 53) - 1
+QUEUE_LEASE_SECONDS = 300
+MAX_CLAIM_ATTEMPTS = 2
+PRESTART_ABORT_CODES = frozenset({
+ "E_COOKIE_ACCESS",
+})
+COOKIE_ACCESS_REASONS = frozenset({
+ "COOKIE_API_ERROR",
+ "COOKIE_API_SHAPE",
+ "COOKIE_AUTH_SET_EMPTY",
+ "COOKIE_EQUAL_PRECEDENCE_CONFLICT",
+ "COOKIE_RELEVANCE_AMBIGUOUS",
+ "COOKIE_SELECTED_INVALID",
+ "COOKIE_SELECTION_EXCEPTION",
+ "COOKIE_TRANSFER_INVALID",
+ "COOKIE_TRANSFER_OVERFLOW",
+})
+COOKIE_ACCESS_TERMINAL_CODES = {
+ reason: f"E_{reason}" for reason in COOKIE_ACCESS_REASONS
+}
+RUNTIME_DIAGNOSTIC_STATES = frozenset({
+ "BRIDGE_EXIT",
+ "BRIDGE_FFPROBE",
+ "BRIDGE_INVOCATION",
+ "BRIDGE_METADATA_BINDING",
+ "BRIDGE_OUTPUT_SCHEMA",
+ "BRIDGE_SOURCE_STABILITY",
+ "CREATE_NEW_PUBLISH",
+ "DIMENSIONS_PENDING",
+ "DURATION_SHA_VERIFICATION",
+ "IDENTITY_PENDING",
+ "MAPPING_READBACK",
+ "MEDIA_MAPPING_PRESENCE",
+ "METADATA_NOT_READY",
+ "OWNER_PENDING",
+ "PAGE_REJECTED",
+ "READY_UNSTABLE",
+ "SCRIPT_TIMEOUT",
+ "TAB_IDENTITY",
+ "VIDEO_ABSENT",
+})
+RUNTIME_DIAGNOSTIC_REASONS = frozenset({
+ "BATCH_RECEIPT_CREATE_FAILED",
+ "BRIDGE_REPORTED_PUBLISH_FAILURE",
+ "DEADLINE_EXHAUSTED",
+ "DURATION_OR_SHA_MISMATCH",
+ "MEDIA_OR_MAPPING_MISSING",
+ "METADATA_FIELDS_PENDING",
+ "INITIAL_STATE_ASSIGNMENT_ABSENT",
+ "INITIAL_STATE_JSON_INVALID",
+ "INITIAL_STATE_VIDEO_DATA_ABSENT",
+ "NONZERO_EXIT",
+ "EXPECTED_METADATA_MISMATCH",
+ "LOCAL_MEDIA_PROBE_FAILED",
+ "OUTPUT_SCHEMA_INVALID",
+ "OWNER_ANCHOR_ABSENT",
+ "OWNER_ANCHOR_AMBIGUOUS",
+ "OWNER_ANCHOR_MISMATCH",
+ "PAGE_DRM_REJECTED",
+ "PAGE_DURATION_REJECTED",
+ "PAGE_ACCESS_REJECTED",
+ "PAGE_BVID_REJECTED",
+ "PAGE_DOM_IDENTITY_PENDING",
+ "PAGE_IDENTITY_REJECTED",
+ "PAGE_MULTIPART_REJECTED",
+ "PAGE_NAVIGATION_PENDING",
+ "PAGE_ORIGIN_REJECTED",
+ "PAGE_OWNER_REJECTED",
+ "PAGE_SCRIPTING_PENDING",
+ "PAGE_TAB_ACTIVATION_PENDING",
+ "PAGE_URL_PENDING",
+ "PAGE_URL_REJECTED",
+ "PAGE_VISIBILITY_PENDING",
+ "PAGE_WINDOW_FOCUS_PENDING",
+ "PAGE_WINDOW_MINIMIZED_PENDING",
+ "PAGE_WINDOW_STATE_PENDING",
+ "PERSISTED_MAPPING_INVALID",
+ "SOURCE_FILE_INVALID",
+ "SCRIPT_INVOCATION_FAILED",
+ "STABLE_SNAPSHOTS_PENDING",
+ "SUBPROCESS_INVOCATION_FAILED",
+ "SUBPROCESS_TIMEOUT",
+ "TAB_IDENTITY_DRIFT",
+ "VIDEO_DIMENSIONS_PENDING",
+ "VIDEO_ELEMENT_ABSENT",
+})
+
+PREPARELESS_TERMINAL_DIAGNOSTIC_CONTRACT = {
+ "E_DRM": {
+ "PAGE_REJECTED": frozenset({"PAGE_DRM_REJECTED"}),
+ },
+ "E_DURATION": {
+ "PAGE_REJECTED": frozenset({"PAGE_DURATION_REJECTED"}),
+ },
+ "E_MULTI_PART": {
+ "PAGE_REJECTED": frozenset({"PAGE_MULTIPART_REJECTED"}),
+ },
+ "E_OWNER": {
+ "PAGE_REJECTED": frozenset({"PAGE_IDENTITY_REJECTED", "PAGE_OWNER_REJECTED"}),
+ },
+ "E_PAGE_ACCESS_CONTROL": {
+ "PAGE_REJECTED": frozenset({"PAGE_ACCESS_REJECTED"}),
+ },
+ "E_PAGE_DUPLICATE_TAB": {
+ "TAB_IDENTITY": frozenset({"TAB_IDENTITY_DRIFT"}),
+ },
+ "E_PAGE_DIMENSIONS_UNAVAILABLE": {
+ "DIMENSIONS_PENDING": frozenset({"VIDEO_DIMENSIONS_PENDING"}),
+ },
+ "E_PAGE_METADATA_UNAVAILABLE": {
+ "METADATA_NOT_READY": frozenset({
+ "INITIAL_STATE_ASSIGNMENT_ABSENT",
+ "INITIAL_STATE_JSON_INVALID",
+ "INITIAL_STATE_VIDEO_DATA_ABSENT",
+ }),
+ },
+ "E_PAGE_IDENTITY_UNAVAILABLE": {
+ "IDENTITY_PENDING": frozenset({
+ "PAGE_DOM_IDENTITY_PENDING",
+ "PAGE_NAVIGATION_PENDING",
+ "PAGE_SCRIPTING_PENDING",
+ "PAGE_TAB_ACTIVATION_PENDING",
+ "PAGE_URL_PENDING",
+ "PAGE_VISIBILITY_PENDING",
+ "PAGE_WINDOW_FOCUS_PENDING",
+ "PAGE_WINDOW_MINIMIZED_PENDING",
+ "PAGE_WINDOW_STATE_PENDING",
+ }),
+ },
+ "E_PAGE_OWNER_ANCHOR_UNAVAILABLE": {
+ "OWNER_PENDING": frozenset({
+ "OWNER_ANCHOR_ABSENT",
+ "OWNER_ANCHOR_AMBIGUOUS",
+ "OWNER_ANCHOR_MISMATCH",
+ }),
+ },
+ "E_PAGE_PROOF": {
+ "PAGE_REJECTED": frozenset({
+ "PAGE_BVID_REJECTED",
+ "PAGE_IDENTITY_REJECTED",
+ "PAGE_ORIGIN_REJECTED",
+ "PAGE_URL_REJECTED",
+ }),
+ },
+ "E_PAGE_SCRIPT_TIMEOUT": {
+ "SCRIPT_TIMEOUT": frozenset({"SCRIPT_INVOCATION_FAILED"}),
+ },
+ "E_PAGE_STABILITY_TIMEOUT": {
+ "READY_UNSTABLE": frozenset({"STABLE_SNAPSHOTS_PENDING"}),
+ },
+ "E_PAGE_TAB_DRIFT": {
+ "TAB_IDENTITY": frozenset({"TAB_IDENTITY_DRIFT"}),
+ },
+ "E_PAGE_VIDEO_ABSENT": {
+ "VIDEO_ABSENT": frozenset({"VIDEO_ELEMENT_ABSENT"}),
+ },
+}
+PREPARELESS_REJECT_CODES = frozenset(PREPARELESS_TERMINAL_DIAGNOSTIC_CONTRACT)
+INTERNAL_ONLY_PAGE_PENDING_CODES = frozenset({"E_PAGE_METADATA_NOT_READY"})
+
+# One bounded compatibility contract for the immutable pre-V017 journal. The
+# legacy code remains forbidden on every live write/wire boundary; only these
+# exact raw lines inside the exact governed prefix may be decoded during
+# append-only state replay.
+LEGACY_PAGE_METADATA_REPLAY_STATE_SUFFIX = (
+ "project-info", "bili-auth-generic-runtime", "queue-state.jsonl",
+)
+LEGACY_PAGE_METADATA_REPLAY_PREFIX_BYTES = 27_899
+LEGACY_PAGE_METADATA_REPLAY_PREFIX_SHA256 = (
+ "43CB0D7994AC349038026D202DBC85B897E8BF322A3B0EEFD536997FFCBE32DB"
+)
+LEGACY_PAGE_METADATA_REPLAY_LINES = (
+ (68, 307, "4F0AF7297B700E045C66E91812C470335447833A9248898C7A746521D6DB688D"),
+ (70, 307, "10B4B3A77CA2F11D5DA2E2A1503006182C9B2EA616E7E265AC504B88329179C8"),
+ (75, 415, "C3CA763D02CE4A45E23131759CE4F2B8449885975E0848AABBD0C5D290076C76"),
+ (80, 415, "F62B30976883F027D1F9AF4BA883BE7DAA9E055328DB345E73B71CFA98249CE1"),
+ (82, 415, "B889763C7B9C65140CA8403B26628A58E8F9BF0FA58FFE7D64637017CF15A2DF"),
+ (86, 415, "AE93EDC05478CCC0699ACA31BA957FE9537A067E9409F71FF88768F08561E5BA"),
+ (88, 415, "A4D748BEF18B15A75A65DA7F9D88D40772421CEA4041AC604579D40EF822DFC5"),
+ (93, 415, "DA83626D0C7D45D29CD55A5556F8214E71C2781D52A91FACCCB695EF37AF11A1"),
+)
+LEGACY_MISSING_DIAGNOSTIC_REPLAY_LINES = (
+ (2, 294, "474B56CD9D57ABCD11F663D1FECE38087CE5CC6F18A4C02837F867548EF42242", "E_PAGE_PROOF"),
+ (4, 294, "C9C38D1559A519661E9C00EC2B2C8DD56073F117726CB477343283860D52324F", "E_PAGE_PROOF"),
+ (6, 294, "A4DB42C3223611CD14B4EF25F4A028964DBABF93903BA65931398166E1936424", "E_PAGE_PROOF"),
+ (14, 294, "F69E2BFF4A2EF51AA614E5C510FE7E9A965B47BCC97ED671E572D62AD2FB875C", "E_PAGE_PROOF"),
+ (16, 294, "40D03BF0F736B8207A61DFFAF807E75BDED2F645F8CED9C586B2C5257AD86E43", "E_PAGE_PROOF"),
+ (22, 294, "8D07285C1364D147DFD4060CB3B6E6BCADD057D686F64C5BC3D2A7AD097635EE", "E_PAGE_PROOF"),
+ (24, 294, "42D3A599A00D37F5C65B2DC86CEFDC2E971C32FFFA163C7C2407D5E0CA95E20D", "E_PAGE_PROOF"),
+ (26, 298, "1477BA661C0F060149D6725D2E44D1F1A419714C9350AD266FFDF6344C02EEAF", "E_PAGE_TAB_DRIFT"),
+ (30, 298, "DA135304B6F29B4A3937C9DF9C8E6861C179F71675414ED49931EEF461E2890E", "E_PAGE_TAB_DRIFT"),
+)
+
+
+def validate_runtime_diagnostic(value: object) -> dict[str, object]:
+ """Validate the only durable, non-secret runtime diagnostic shape."""
+
+ if not isinstance(value, dict) or set(value) != {"attempts", "elapsed_ms", "state", "reason"}:
+ raise ValueError("invalid runtime diagnostic")
+ attempts = value["attempts"]
+ elapsed_ms = value["elapsed_ms"]
+ if (
+ isinstance(attempts, bool) or not isinstance(attempts, int) or not 0 <= attempts <= 10_000
+ or isinstance(elapsed_ms, bool) or not isinstance(elapsed_ms, int)
+ or not 0 <= elapsed_ms <= 7_200_000
+ or value["state"] not in RUNTIME_DIAGNOSTIC_STATES
+ or value["reason"] not in RUNTIME_DIAGNOSTIC_REASONS
+ ):
+ raise ValueError("invalid runtime diagnostic")
+ return value
+
+
+POSTPROCESS_TERMINAL_DIAGNOSTIC_CONTRACT = {
+ "E_BRIDGE_OUTPUT_SCHEMA": {
+ "BRIDGE_OUTPUT_SCHEMA": frozenset({"OUTPUT_SCHEMA_INVALID"}),
+ },
+}
+
+
+def validate_postprocess_terminal(
+ error_code: object, diagnostic: object,
+) -> dict[str, object] | None:
+ """Validate the canonical diagnostic matrix for a post-media terminal."""
+
+ if not isinstance(error_code, str) or ERROR_CODE_RE.fullmatch(error_code) is None:
+ raise ValueError("invalid postprocess terminal")
+ if (
+ error_code == COMPLETION_CLOSURE_REQUIRED
+ or error_code in INTERNAL_ONLY_PAGE_PENDING_CODES
+ or error_code in PREPARELESS_REJECT_CODES
+ ):
+ raise ValueError("invalid postprocess terminal")
+ contract = POSTPROCESS_TERMINAL_DIAGNOSTIC_CONTRACT.get(error_code)
+ if diagnostic is None:
+ if contract is not None:
+ raise ValueError("invalid postprocess terminal")
+ return None
+ if contract is None:
+ raise ValueError("invalid postprocess terminal")
+ validated = validate_runtime_diagnostic(diagnostic)
+ reasons = contract.get(validated["state"])
+ if reasons is None or validated["reason"] not in reasons:
+ raise ValueError("invalid postprocess terminal")
+ return validated
+
+
+def validate_prepareless_terminal(
+ error_code: object, diagnostic: object,
+) -> dict[str, object]:
+ """Validate one canonical durable pre-secret terminal and its typed diagnostic."""
+
+ if not isinstance(error_code, str):
+ raise ValueError("invalid prepareless terminal")
+ contract = PREPARELESS_TERMINAL_DIAGNOSTIC_CONTRACT.get(error_code)
+ if contract is None:
+ raise ValueError("invalid prepareless terminal")
+ validated = validate_runtime_diagnostic(diagnostic)
+ allowed_reasons = contract.get(validated["state"])
+ if allowed_reasons is None or validated["reason"] not in allowed_reasons:
+ raise ValueError("invalid prepareless terminal")
+ return validated
+RELOAD_BACKOFF_SECONDS = 3_600
SOCKET_TIMEOUT_SECONDS = 20
EXTRACTOR_RETRIES = 1
@@ -70,14 +437,7 @@
}
SAFE_PHASES = {
- "IDLE",
- "READY",
- "CHECKING",
- "DOWNLOADING",
- "MERGING",
- "VALIDATING",
- "PUBLISHING",
- "COMPLETE",
- "FAILED",
- "CANCELED",
+ "IDLE", "READY", "CHECKING", "DOWNLOADING", "MERGING", "VALIDATING",
+ "PUBLISHING", "MEDIA_COMPLETE", "POSTPROCESS_PENDING", "POSTPROCESS_FAILED",
+ "COMPLETE", "FAILED", "CANCELED", "RELOAD_REQUIRED",
}
diff --git a/dev/project-dev/bili_authenticated_extension/formal_legacy_identity_manifest.py b/dev/project-dev/bili_authenticated_extension/formal_legacy_identity_manifest.py
new file mode 100644
index 0000000..2ab8eb7
--- /dev/null
+++ b/dev/project-dev/bili_authenticated_extension/formal_legacy_identity_manifest.py
@@ -0,0 +1,55 @@
+"""Exact immutable formal-prefix identities allowed by the legacy reader.
+
+This is data, not a shape-based migration rule. A row is legacy-compatible
+only when the whole governed prefix and the row's exact ordinal/bytes/hash and
+business identity all match this manifest.
+"""
+
+from __future__ import annotations
+
+
+FORMAL_PREFIX_BYTES = 103_766
+FORMAL_PREFIX_LINES = 119
+FORMAL_PREFIX_SHA256 = "C29312B4B21DFFA469F54CD57D44413DB4B4D896FBD7E7D6D918180C578EF18F"
+FORMAL_LEGACY_CREATOR_UID = "1420210197"
+
+# line ordinal, raw row byte count (excluding LF), raw row SHA-256,
+# stable_id, source_url, published_at, item_type, allowed status.
+FORMAL_LEGACY_ROWS = (
+ (2, 485, "D88A743ECD96E2961960F9E36746ECB535E4146835DE9A023193B03441858CEB", "BV1HA3o6oEJJ", "https://www.bilibili.com/video/BV1HA3o6oEJJ", "2026-08-02T21:11:11+08:00", "video", "VIDEO_DOWNLOAD_PENDING_EXTENSION"),
+ (7, 486, "EE4C0FF67B790771794DA1393EA4CBC757869DFC38664E1E41B6135BBFD23230", "BV1DVMX6XEPq", "https://www.bilibili.com/video/BV1DVMX6XEPq", "2026-08-03T19:39:03+08:00", "video", "VIDEO_DOWNLOAD_PENDING_EXTENSION"),
+ (18, 754, "9127AA7F8F16773440C35DEB50ACB5822AFC2FC3E276CA031611BD75436CED58", "BV1DVMX6XEPq", "https://www.bilibili.com/video/BV1DVMX6XEPq", "2026-08-03T19:39:03+08:00", "video", "VIDEO_DOWNLOADED_COMPLETE_HANDOFF_SENT"),
+ (19, 965, "AF8004A662C8FD3E5A0755DDFEFFC0275678BEA4C56D2EE429262A0A5FDEC319", "BV1HA3o6oEJJ", "https://www.bilibili.com/video/BV1HA3o6oEJJ", "2026-08-02T21:11:11+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_AUTH_REQUIRED"),
+ (24, 900, "A9D27D3C4D89FDF6381134C6A69D4F1A4A911B1BAE200DF986251CAC7AC56E00", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_RUNTIME_STABILITY_GATE"),
+ (25, 1117, "240BE244BA21B5CEA918F982134DCB374111D4C895E5B6297B39BE0BDB979F11", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_AUTH_SESSION_SOURCE_REVIEW_HOLD2"),
+ (27, 1661, "D3EE288F8AD6D982C7B2381BCFA2020BD404580CAC21CC7DD37EA350782B99D9", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_REPLACEMENT_BUILD_REPAIR_PENDING"),
+ (28, 1407, "58EEF3735DB1CE0C41819FC76C4DA7C71D71E3760CB8D41F1F8AB747CCEB2625", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_METADATA_REPAIR_SOURCE_REREVIEW_PENDING"),
+ (29, 1687, "C9F32063CEC413FC01EA8ECDA1AEEF8EEA59460866E7847E6A473664F63C9AAD", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_REPLACEMENT_BUILD_002_ARCHIVE_METADATA_MISSING"),
+ (30, 1207, "5AAE520AAA6224C54759EB46B4C8B9040F9D94CE32666B2FB68E339AC5A2EE96", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_ARCHIVE_METADATA_CONTRACT_REPAIR_PENDING"),
+ (31, 1447, "6CB2F549E7BEEE19CBFBF2C5014C40C804B50B46C1CB603BB4419EA7AE9ABE91", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_ARCHIVE_METADATA_SOURCE_REVIEW_CAPACITY"),
+ (32, 1347, "058E5316B9F70ED46DDE325FD7C0772160689EDF11278C1BB557EFE15F31CF26", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_ARCHIVE_METADATA_SOURCE_REVIEW_RESUME_SCHEDULED"),
+ (33, 1569, "A4793A7FBC698B4ADAF3E006460468817AD7AC84EEEDE873B6DF53464E27AEC6", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_CONTROLLED_BUILD003_IN_PROGRESS"),
+ (34, 1704, "E4F00FB508A099C9F292FEF11F0A366C543B902F3B026B94A397932957CBF7BE", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_BUILD003_ARCHIVE_METADATA_TYPE_FALSE_REJECTION"),
+ (35, 1501, "1303EB306236D7CA5DBDFC400BC8C72586096C80E684E957B0704966012BC1E0", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_TYPE_CONTRACT_REPAIR_SOURCE_REVIEW_PENDING"),
+ (36, 1915, "E0C9B172ED332EC37CC5A2CEE4107D1829847C0F73D1A2DC959D00F7A1AF7870", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_TYPE_CONTRACT_REPAIR_COMPLETE_SOURCE_REREVIEW_PENDING"),
+ (37, 1734, "3344432EF46265AC5F2338454340773286E47AD83EB4551EE8A96162741719A2", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_TYPE_CONTRACT_SOURCE_PASS_BUILD004_AUTHORIZED_PENDING"),
+ (38, 1202, "3F2888834283F026628B6080B1CAD398B4C9386B312F3462E3A0CC30F975AEA3", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_CONTROLLED_BUILD004_IN_PROGRESS"),
+ (39, 1614, "3EBC4F60B8532307301CB1BFFD55AD532B210EAA2944128CA8181245451C6584", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_BUILD004_STATIC_PASS_HASH_ONLY_REVIEW_PENDING"),
+ (40, 1824, "09B7CC22774E996DE59560AA8D3E3A4F6793AD76A4821318FD33325658E14471", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_BUILD004_NOT_INSTALLABLE_TREE_HASH_MISMATCH"),
+ (41, 1835, "E2245063069F88B76852E0FE1DB20638DDC59598352A552C0E5CB09B33F1D5D3", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_TREE_HASH_CONTRACT_REPAIR_SOURCE_REVIEW_PENDING"),
+ (42, 1984, "429B23DA59037EABA01DA3E488A0798AC081D358CC58B0F6A544EAA67F170882", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_CANONICAL_TREE_HASH_SOURCE_PASS_BUILD005_AUTHORIZED_PENDING"),
+ (43, 1837, "FAAA9642C97AA97D9AAAB46224AE7CAA832D4220DEF9254A6DC6D0BDB3D90E2E", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_BUILD005_STATIC_PASS_HASH_ONLY_REVIEW_PENDING"),
+ (44, 1712, "17BA394A0DEFBB064FB362E78D956592E241AF7BE5524E0349EEF7EA97DBCF18", "BV1AkuH64EZh", "https://www.bilibili.com/video/BV1AkuH64EZh", "2026-08-06T22:30:10+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_YTDLP_BUILD005_EXACT_PASS_INSTALL_SCHEDULING_PENDING"),
+ (45, 2050, "29F7AD92C29CEA2F031208CF1549E2812860C1FB3A6AD65F7365A66FE53A4668", "BV1HA3o6oEJJ", "https://www.bilibili.com/video/BV1HA3o6oEJJ", "2026-08-02T21:11:11+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_EXTENSION_IDENTITY_VISIBILITY"),
+ (46, 1722, "A3F2943F12853176D6C79BCA267F59D347522D4C42FBE1C1C0CFB4FBD8EB8A0E", "BV1HA3o6oEJJ", "https://www.bilibili.com/video/BV1HA3o6oEJJ", "2026-08-02T21:11:11+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_EXTENSION_NOT_LOADED"),
+)
+
+# One historical row used a JSON integer for creator_uid. This is a byte-exact
+# compatibility identity, not a type-coercion rule: all other current/future
+# rows continue to require a JSON string creator_uid.
+# line ordinal, raw row byte count (excluding LF), raw row SHA-256,
+# creator_uid integer, stable_id, source_url, published_at, item_type,
+# allowed status.
+FORMAL_LEGACY_INTEGER_UID_ROWS = (
+ (47, 1475, "70981CB1B6B6EF370C32654E132B414F5DC0EF45F267E07C2BB582D821018054", 1420210197, "BV189u16KEPe", "https://www.bilibili.com/video/BV189u16KEPe", "2026-08-09T14:08:09+08:00", "video", "VIDEO_DOWNLOAD_BLOCKED_EXTENSION_NOT_LOADED"),
+)
diff --git a/dev/project-dev/bili_authenticated_extension/install_native_host.ps1 b/dev/project-dev/bili_authenticated_extension/install_native_host.ps1
index b491b9f..04d4936 100644
--- a/dev/project-dev/bili_authenticated_extension/install_native_host.ps1
+++ b/dev/project-dev/bili_authenticated_extension/install_native_host.ps1
@@ -22,7 +22,11 @@
[Parameter(DontShow = $true)]
[string]$TestRegistryRoot,
[Parameter(DontShow = $true)]
- [ValidateSet('none', 'after-root', 'after-payload', 'after-config', 'after-manifest', 'after-registry-key', 'after-registry-value')]
+ [string]$TestPreviousInstallRoot,
+ [Parameter(DontShow = $true)]
+ [string]$TestPreviousArtifactReceipt,
+ [Parameter(DontShow = $true)]
+ [ValidateSet('none', 'after-root', 'after-payload', 'after-config', 'after-manifest', 'after-registry-key', 'after-registry-value', 'after-registry-value-mixed')]
[string]$InjectFailure = 'none'
)
@@ -31,9 +35,17 @@
$expectedOrigin = 'chrome-extension://oidmclckpdmpabbfedplkbdplmfcenbb/'
$expectedHostName = 'com.project_info.bili_auth_ingress'
$expectedPublicDerHash = 'E83C2B2AF3CF011543FBA13FBC524D1122EEA68548F9F27B9F7A82B5D594666C'
-$expectedExtensionBuild = 'project-info-bili-auth-ingress/1.0.0+20260805.v002'
-$expectedHostBuild = 'project-info-bili-auth-native-host/1.0.0+20260805.v002'
+$expectedExtensionBuild = 'project-info-bili-auth-ingress/1.2.25+20260829.generic.v027'
+$expectedHostBuild = 'project-info-bili-auth-native-host/1.2.25+20260829.generic.v027'
$expectedHostExecutable = 'project-info-bili-auth-native-host.exe'
+$expectedPreviousExtensionBuild = 'project-info-bili-auth-ingress/1.2.24+20260829.generic.v026'
+$expectedPreviousVersion = '1.2.24+20260829.generic.v026'
+$expectedPreviousManifestBytes = 381
+$expectedPreviousManifestSha256 = '9FFAC5073A5839F030ED321840C5715ECBE2CC7276DD057A7236B3CF78AA5C38'
+$expectedPreviousHostBytes = 21452778
+$expectedPreviousHostSha256 = '24E0C15B9E8269F55F25E55A0614DD96BEF58CB0F7EE069C41F86AF95CD19762'
+$expectedPreviousConfigBytes = 1870
+$expectedPreviousConfigSha256 = 'ACA285C700CD970A846A824BD32E6826195B0204640F8445C288C7476EC475E3'
$publicKey = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt2dT1HGYaI0DXM7zZwOTNBWXTKlMBJMpyDVjRUc+v6bUotmLyoraC+ay2scy9UQluSZVYq0tS8qvQNNvuZOlc5w2bOExm4TH2IIKvaVO8nVthHBnNz2kXdiM8ItN0vPZEmS+8gpTCI1+6wPTuUglMoXpqYBYhii8fJ5RkENRF3PRJBBigGt8soqdBFRY1QZUmpQv9dYw4dRq4L2C4QtBgClUg4bQpuCppiVZ9LHbePi9IAjc9r9R93KLzpaBuXdJpfVRE5w/6YHnxP8ovXxBdl7XktmrdH3xj7mWT7Q7ZBxkDNwn2RkruD45XgDD3yuNxOYSkLFMkseN+Ua69gSS4wIDAQAB'
function Stop-Injected([string]$Point) {
@@ -86,6 +98,26 @@
}
}
+function Assert-ExactSourceSnapshot([string]$Root, [string]$ManifestPath, [string[]]$ExpectedFiles, [object]$Manifest) {
+ Assert-ExactSourceTree $Root $ManifestPath $ExpectedFiles
+ foreach ($entry in $Manifest.files) {
+ if ($entry.path -notmatch '^[A-Za-z0-9._/-]+$' -or $entry.path.Contains('..') -or
+ $entry.sha256 -notmatch '^[A-F0-9]{64}$' -or $entry.bytes -lt 1) {
+ throw 'Source artifact manifest contains an invalid entry.'
+ }
+ $candidate = [System.IO.Path]::GetFullPath((Join-Path $Root $entry.path))
+ if (-not $candidate.StartsWith($Root + [System.IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) {
+ throw 'Source artifact path escaped its root.'
+ }
+ $item = Get-Item -LiteralPath $candidate
+ if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
+ $item.Length -ne $entry.bytes -or
+ (Get-FileHash -Algorithm SHA256 -LiteralPath $candidate).Hash -cne $entry.sha256) {
+ throw 'Source artifact hash mismatch.'
+ }
+ }
+}
+
function Copy-CreateNew([string]$Source, [string]$Destination) {
$input = [System.IO.File]::Open($Source, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::Read)
try {
@@ -110,6 +142,339 @@
} finally {
$stream.Dispose()
}
+}
+
+function Get-FileIdentity([string]$Path) {
+ $item = Get-Item -LiteralPath $Path -Force
+ if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+ throw 'Expected a regular non-reparse file.'
+ }
+ return [pscustomobject]@{
+ Bytes = $item.Length
+ Sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash
+ }
+}
+
+function ConvertTo-StrictPositiveInt64 {
+ [CmdletBinding(PositionalBinding = $false)]
+ param(
+ [Parameter(Mandatory = $true)]
+ [AllowNull()]
+ [object]$Value,
+
+ [Parameter(Mandatory = $true)]
+ [ValidateSet(
+ 'previous native host manifest bytes',
+ 'previous Host executable bytes',
+ 'previous config bytes'
+ )]
+ [string]$IdentityName
+ )
+ $isIntegralScalar =
+ $Value -is [byte] -or $Value -is [sbyte] -or
+ $Value -is [int16] -or $Value -is [uint16] -or
+ $Value -is [int32] -or $Value -is [uint32] -or
+ $Value -is [int64]
+ if (-not $isIntegralScalar -or [long]$Value -lt 1) {
+ throw "Pinned $IdentityName must be a positive integer scalar."
+ }
+ return [long]$Value
+}
+
+function ConvertTo-StrictSha256 {
+ [CmdletBinding(PositionalBinding = $false)]
+ param(
+ [Parameter(Mandatory = $true)]
+ [AllowNull()]
+ [object]$Value,
+
+ [Parameter(Mandatory = $true)]
+ [ValidateSet(
+ 'previous native host manifest SHA-256',
+ 'previous Host executable SHA-256',
+ 'previous config SHA-256'
+ )]
+ [string]$IdentityName
+ )
+ if ($Value -isnot [string] -or $Value -cnotmatch '^[A-F0-9]{64}$') {
+ throw "Pinned $IdentityName must be an uppercase 64-character hexadecimal string."
+ }
+ return [string]$Value
+}
+
+function Assert-ExactPreviousInstall {
+ [CmdletBinding(PositionalBinding = $false)]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$PreviousRoot,
+
+ [Parameter(Mandatory = $true)]
+ [AllowNull()]
+ [object]$ManifestBytes,
+
+ [Parameter(Mandatory = $true)]
+ [AllowNull()]
+ [object]$ManifestSha256,
+
+ [Parameter(Mandatory = $true)]
+ [AllowNull()]
+ [object]$HostBytes,
+
+ [Parameter(Mandatory = $true)]
+ [AllowNull()]
+ [object]$HostSha256,
+
+ [Parameter(Mandatory = $true)]
+ [AllowNull()]
+ [object]$ConfigBytes,
+
+ [Parameter(Mandatory = $true)]
+ [AllowNull()]
+ [object]$ConfigSha256
+ )
+ $pinnedManifestBytes = ConvertTo-StrictPositiveInt64 -Value $ManifestBytes -IdentityName 'previous native host manifest bytes'
+ $pinnedManifestSha256 = ConvertTo-StrictSha256 -Value $ManifestSha256 -IdentityName 'previous native host manifest SHA-256'
+ $pinnedHostBytes = ConvertTo-StrictPositiveInt64 -Value $HostBytes -IdentityName 'previous Host executable bytes'
+ $pinnedHostSha256 = ConvertTo-StrictSha256 -Value $HostSha256 -IdentityName 'previous Host executable SHA-256'
+ $pinnedConfigBytes = ConvertTo-StrictPositiveInt64 -Value $ConfigBytes -IdentityName 'previous config bytes'
+ $pinnedConfigSha256 = ConvertTo-StrictSha256 -Value $ConfigSha256 -IdentityName 'previous config SHA-256'
+
+ $rootItem = Get-Item -LiteralPath $PreviousRoot -Force
+ if (-not $rootItem.PSIsContainer -or ($rootItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+ throw 'Previous install root must be an ordinary non-reparse directory.'
+ }
+ $children = @(Get-ChildItem -LiteralPath $PreviousRoot -Force)
+ if ($children.Count -ne 3 -or @($children | Where-Object { $_.PSIsContainer }).Count -ne 0 -or
+ (Compare-Object -CaseSensitive @('config.json', 'native-host-manifest.json', $expectedHostExecutable) @($children.Name | Sort-Object))) {
+ throw 'Previous install root does not contain the exact three-file preimage.'
+ }
+ foreach ($child in $children) {
+ if ($child.Attributes -band [IO.FileAttributes]::ReparsePoint) {
+ throw 'Previous install contains a reparse path.'
+ }
+ }
+ $manifestPath = Join-Path $PreviousRoot 'native-host-manifest.json'
+ $hostPath = Join-Path $PreviousRoot $expectedHostExecutable
+ $configPath = Join-Path $PreviousRoot 'config.json'
+ $manifestIdentity = Get-FileIdentity $manifestPath
+ $hostIdentity = Get-FileIdentity $hostPath
+ $configIdentity = Get-FileIdentity $configPath
+ if ($manifestIdentity.Bytes -ne $pinnedManifestBytes) {
+ throw 'Previous native host manifest bytes mismatch.'
+ }
+ if ($manifestIdentity.Sha256 -cne $pinnedManifestSha256) {
+ throw 'Previous native host manifest SHA-256 mismatch.'
+ }
+ if ($hostIdentity.Bytes -ne $pinnedHostBytes) {
+ throw 'Previous Host executable bytes mismatch.'
+ }
+ if ($hostIdentity.Sha256 -cne $pinnedHostSha256) {
+ throw 'Previous Host executable SHA-256 mismatch.'
+ }
+ if ($configIdentity.Bytes -ne $pinnedConfigBytes) {
+ throw 'Previous config bytes mismatch.'
+ }
+ if ($configIdentity.Sha256 -cne $pinnedConfigSha256) {
+ throw 'Previous config SHA-256 mismatch.'
+ }
+ $previousManifest = Get-StrictJson $manifestPath
+ if ((Compare-Object @('allowed_origins', 'description', 'name', 'path', 'type') @($previousManifest.PSObject.Properties.Name | Sort-Object)) -or
+ $previousManifest.name -cne $expectedHostName -or
+ $previousManifest.path -cne $hostPath -or
+ $previousManifest.type -cne 'stdio' -or
+ @($previousManifest.allowed_origins).Count -ne 1 -or
+ $previousManifest.allowed_origins[0] -cne $expectedOrigin) {
+ throw 'Previous native host manifest identity mismatch.'
+ }
+ $previousConfig = Get-StrictJson $configPath
+ if ($previousConfig.schema -ne 2 -or
+ $previousConfig.required_extension_build -cne $expectedPreviousExtensionBuild -or
+ @($previousConfig.creator_allowlist).Count -ne 1 -or
+ $previousConfig.creator_allowlist[0] -cne '1420210197') {
+ throw 'Previous host config identity mismatch.'
+ }
+ return [pscustomobject]@{
+ Root = $PreviousRoot
+ ManifestPath = $manifestPath
+ HostPath = $hostPath
+ ConfigPath = $configPath
+ }
+}
+
+function Get-RegistrySnapshot([string]$Path, [bool]$UseFileProvider) {
+ if (-not (Test-Path -LiteralPath $Path)) {
+ return [pscustomobject]@{
+ Exists = $false
+ ValueNames = @()
+ SubKeyNames = @()
+ DefaultKind = $null
+ DefaultValue = $null
+ }
+ }
+ if ($UseFileProvider) {
+ $item = Get-Item -LiteralPath $Path -Force
+ if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+ throw 'Test registry key must be an ordinary non-reparse directory.'
+ }
+ $children = @(Get-ChildItem -LiteralPath $Path -Force)
+ if (@($children | Where-Object { $_.Attributes -band [IO.FileAttributes]::ReparsePoint }).Count -ne 0) {
+ throw 'Test registry contains a reparse path.'
+ }
+ $files = @($children | Where-Object { -not $_.PSIsContainer })
+ $subkeys = @($children | Where-Object { $_.PSIsContainer } | ForEach-Object { $_.Name })
+ $valueNames = @($files | ForEach-Object { if ($_.Name -ceq 'default.value') { '' } else { $_.Name } })
+ $defaultPath = Join-Path $Path 'default.value'
+ $defaultValue = $null
+ $defaultKind = $null
+ if (Test-Path -LiteralPath $defaultPath) {
+ $defaultValue = [System.IO.File]::ReadAllText($defaultPath, [System.Text.UTF8Encoding]::new($false, $true))
+ $defaultKind = 'String'
+ }
+ return [pscustomobject]@{
+ Exists = $true
+ ValueNames = $valueNames
+ SubKeyNames = $subkeys
+ DefaultKind = $defaultKind
+ DefaultValue = $defaultValue
+ }
+ }
+ $key = Get-Item -LiteralPath $Path -Force
+ $valueNames = @($key.GetValueNames())
+ $subKeyNames = @($key.GetSubKeyNames())
+ $defaultValue = $null
+ $defaultKind = $null
+ if ($valueNames -contains '') {
+ $defaultValue = $key.GetValue('', $null, [Microsoft.Win32.RegistryValueOptions]::DoNotExpandEnvironmentNames)
+ $defaultKind = $key.GetValueKind('').ToString()
+ }
+ return [pscustomobject]@{
+ Exists = $true
+ ValueNames = $valueNames
+ SubKeyNames = $subKeyNames
+ DefaultKind = $defaultKind
+ DefaultValue = $defaultValue
+ }
+}
+
+function Test-RegistrySnapshotExact([object]$Snapshot, [string]$ExpectedDefault) {
+ return $Snapshot.Exists -and
+ @($Snapshot.ValueNames).Count -eq 1 -and $Snapshot.ValueNames[0] -ceq '' -and
+ @($Snapshot.SubKeyNames).Count -eq 0 -and
+ $Snapshot.DefaultKind -ceq 'String' -and
+ $Snapshot.DefaultValue -ceq $ExpectedDefault
+}
+
+function Test-RegistrySnapshotEmpty([object]$Snapshot) {
+ return $Snapshot.Exists -and @($Snapshot.ValueNames).Count -eq 0 -and @($Snapshot.SubKeyNames).Count -eq 0
+}
+
+function Set-RegistryDefaultAtomic([string]$Path, [string]$Value, [bool]$UseFileProvider) {
+ if ($UseFileProvider) {
+ $defaultPath = Join-Path $Path 'default.value'
+ if (-not (Test-Path -LiteralPath $defaultPath)) {
+ Write-Utf8CreateNew $defaultPath $Value
+ return
+ }
+ $pending = Join-Path $Path ('.default.pending.' + [Guid]::NewGuid().ToString('N'))
+ $backup = Join-Path $Path ('.default.backup.' + [Guid]::NewGuid().ToString('N'))
+ try {
+ Write-Utf8CreateNew $pending $Value
+ [System.IO.File]::Replace($pending, $defaultPath, $backup, $true)
+ } finally {
+ if (Test-Path -LiteralPath $pending) {
+ Remove-Item -LiteralPath $pending -Force
+ }
+ if (Test-Path -LiteralPath $backup) {
+ Remove-Item -LiteralPath $backup -Force
+ }
+ }
+ return
+ }
+ Set-Item -LiteralPath $Path -Value $Value -ErrorAction Stop
+}
+
+function Assert-NewInstallRoot(
+ [string]$InstallRootPath,
+ [string]$InstalledHostPath,
+ [string]$InstalledConfigPath,
+ [string]$InstalledManifestPath,
+ [object]$ExpectedBuildEntry,
+ [object]$ExpectedConfigIdentity
+) {
+ $rootItem = Get-Item -LiteralPath $InstallRootPath -Force
+ if (-not $rootItem.PSIsContainer -or ($rootItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+ throw 'Installed root must be an ordinary non-reparse directory.'
+ }
+ $children = @(Get-ChildItem -LiteralPath $InstallRootPath -Force)
+ if ($children.Count -ne 3 -or @($children | Where-Object { $_.PSIsContainer }).Count -ne 0 -or
+ (Compare-Object -CaseSensitive @('config.json', 'native-host-manifest.json', $expectedHostExecutable) @($children.Name | Sort-Object))) {
+ throw 'Installed root does not contain the exact three-file set.'
+ }
+ foreach ($child in $children) {
+ if ($child.Attributes -band [IO.FileAttributes]::ReparsePoint) {
+ throw 'Installed root contains a reparse path.'
+ }
+ }
+ $hostIdentity = Get-FileIdentity $InstalledHostPath
+ $configIdentity = Get-FileIdentity $InstalledConfigPath
+ if ($hostIdentity.Bytes -ne $ExpectedBuildEntry.bytes -or $hostIdentity.Sha256 -cne $ExpectedBuildEntry.sha256 -or
+ $configIdentity.Bytes -ne $ExpectedConfigIdentity.Bytes -or $configIdentity.Sha256 -cne $ExpectedConfigIdentity.Sha256) {
+ throw 'Installed payload identity mismatch.'
+ }
+ $persistedManifest = Get-StrictJson $InstalledManifestPath
+ if ((Compare-Object @('allowed_origins', 'description', 'name', 'path', 'type') @($persistedManifest.PSObject.Properties.Name | Sort-Object)) -or
+ $persistedManifest.name -cne $expectedHostName -or
+ $persistedManifest.path -cne $InstalledHostPath -or
+ $persistedManifest.type -cne 'stdio' -or
+ @($persistedManifest.allowed_origins).Count -ne 1 -or
+ $persistedManifest.allowed_origins[0] -cne $expectedOrigin) {
+ throw 'Installed native host manifest identity mismatch.'
+ }
+}
+
+function Remove-ProvenOwnedInstallRoot(
+ [string]$InstallRootPath,
+ [string]$InstalledHostPath,
+ [string]$InstalledConfigPath,
+ [string]$InstalledManifestPath,
+ [object]$ExpectedBuildEntry,
+ [object]$ExpectedConfigIdentity
+) {
+ if (-not (Test-Path -LiteralPath $InstallRootPath)) {
+ return
+ }
+ $rootItem = Get-Item -LiteralPath $InstallRootPath -Force
+ if (-not $rootItem.PSIsContainer -or ($rootItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+ throw 'RECOVERY_REQUIRED: target root is not a proven owned ordinary directory.'
+ }
+ $children = @(Get-ChildItem -LiteralPath $InstallRootPath -Force)
+ if (@($children | Where-Object { $_.PSIsContainer -or ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) }).Count -ne 0 -or
+ @($children | Where-Object { $_.Name -cnotin @('config.json', 'native-host-manifest.json', $expectedHostExecutable) }).Count -ne 0) {
+ throw 'RECOVERY_REQUIRED: target root contains an unknown or reparse object.'
+ }
+ if (Test-Path -LiteralPath $InstalledHostPath) {
+ $identity = Get-FileIdentity $InstalledHostPath
+ if ($identity.Bytes -ne $ExpectedBuildEntry.bytes -or $identity.Sha256 -cne $ExpectedBuildEntry.sha256) {
+ throw 'RECOVERY_REQUIRED: target host identity is ambiguous.'
+ }
+ }
+ if (Test-Path -LiteralPath $InstalledConfigPath) {
+ $identity = Get-FileIdentity $InstalledConfigPath
+ if ($identity.Bytes -ne $ExpectedConfigIdentity.Bytes -or $identity.Sha256 -cne $ExpectedConfigIdentity.Sha256) {
+ throw 'RECOVERY_REQUIRED: target config identity is ambiguous.'
+ }
+ }
+ if (Test-Path -LiteralPath $InstalledManifestPath) {
+ $persistedManifest = Get-StrictJson $InstalledManifestPath
+ if ($persistedManifest.name -cne $expectedHostName -or
+ $persistedManifest.path -cne $InstalledHostPath -or
+ $persistedManifest.type -cne 'stdio' -or
+ @($persistedManifest.allowed_origins).Count -ne 1 -or
+ $persistedManifest.allowed_origins[0] -cne $expectedOrigin) {
+ throw 'RECOVERY_REQUIRED: target manifest identity is ambiguous.'
+ }
+ }
+ Remove-Item -LiteralPath $InstallRootPath -Recurse -Force
}
if ($ObservedExtensionId -cne $expectedId) {
@@ -195,14 +560,15 @@
'__init__.py', 'background.js', 'build_host.ps1', 'config.example.json',
'constants.py', 'dependencies/dependency-artifact-manifest.json',
'dependencies/yt_dlp-2026.7.4-py3-none-any.whl',
- 'install_native_host.ps1', 'job.py', 'manifest.json',
+ 'formal_legacy_identity_manifest.py', 'install_native_host.ps1', 'job.py', 'manifest.json',
'native-host-manifest.template.json', 'native_host.py', 'protocol.py',
+ 'queue-producer.example.json', 'queue_producer.py', 'queue_state.py',
'sidepanel.css', 'sidepanel.html', 'sidepanel.js', 'worker.py'
)
-Assert-ExactSourceTree $sourceRoot $sourceManifestPath $expectedSourceFiles
+Assert-ExactSourceSnapshot $sourceRoot $sourceManifestPath $expectedSourceFiles $sourceManifest
$dependencyManifestPath = Join-Path $sourceRoot 'dependencies/dependency-artifact-manifest.json'
$dependencyManifestItem = Get-Item -LiteralPath $dependencyManifestPath
-if ($sourceManifest.schema -ne 1 -or $sourceManifest.target -cne 'BV1HA3o6oEJJ' -or
+if ($sourceManifest.schema -ne 1 -or $sourceManifest.scope -cne 'generic-bilibili-queue' -or
$sourceManifest.extension_id -cne $expectedId -or
$sourceManifest.extension_build -cne $expectedExtensionBuild -or
$sourceManifest.host_build -cne $expectedHostBuild -or
@@ -253,7 +619,7 @@
$buildManifest = Get-StrictJson $buildManifestPath
$buildScript = Join-Path $sourceRoot 'build_host.ps1'
-if ($buildManifest.schema -ne 2 -or $buildManifest.target -cne 'BV1HA3o6oEJJ' -or
+if ($buildManifest.schema -ne 2 -or $buildManifest.scope -cne 'generic-bilibili-queue' -or
$buildManifest.extension_id -cne $expectedId -or
$buildManifest.extension_build -cne $expectedExtensionBuild -or
$buildManifest.host_build -cne $expectedHostBuild -or
@@ -315,21 +681,26 @@
if (($configItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $configItem.PSIsContainer) {
throw 'Config file must be a regular non-reparse file.'
}
+$configIdentity = Get-FileIdentity $config
$configObject = Get-StrictJson $config
$expectedConfigKeys = @(
- 'schema', 'target', 'canonical_url', 'ffmpeg', 'ffmpeg_sha256', 'ffprobe',
- 'ffprobe_sha256', 'bridge_python', 'bridge_python_sha256', 'bridge_script',
- 'bridge_script_sha256', 'batch_json', 'batch_json_sha256', 'yt_dlp_executable',
- 'yt_dlp_executable_sha256', 'destination'
+ 'schema', 'creator_allowlist', 'queue_path', 'queue_state_path', 'queue_lock_path',
+ 'reload_state_path', 'reload_generation', 'required_extension_build',
+ 'ffmpeg', 'ffmpeg_sha256', 'ffprobe', 'ffprobe_sha256', 'bridge_python',
+ 'bridge_python_sha256', 'bridge_script', 'bridge_script_sha256',
+ 'yt_dlp_executable', 'yt_dlp_executable_sha256', 'destination',
+ 'creator_name', 'formal_manifest_path', 'processing_handoff_path'
)
$actualConfigKeys = @($configObject.PSObject.Properties.Name | Sort-Object)
if ((Compare-Object ($expectedConfigKeys | Sort-Object) $actualConfigKeys) -or
- $configObject.schema -ne 1 -or $configObject.target -cne 'BV1HA3o6oEJJ' -or
- $configObject.canonical_url -cne 'https://www.bilibili.com/video/BV1HA3o6oEJJ' -or
- $configObject.bridge_script_sha256 -cne '749FC486B0F42315BD463F11771FE2A7C71CAB53DD9AC2E411CE82E1175DFF13') {
+ $configObject.schema -ne 2 -or
+ $configObject.required_extension_build -cne $expectedExtensionBuild -or
+ $null -eq $configObject.creator_allowlist -or @($configObject.creator_allowlist).Count -lt 1 -or
+ $configObject.reload_generation -cne 'bili-auth-generic-v027' -or
+ $configObject.bridge_script_sha256 -cne '00F11DAF8387160DB863C89F0B33AB8480422233FF42199189222C989C7ED07E') {
throw 'Host config identity mismatch.'
}
-foreach ($name in @('ffmpeg', 'ffprobe', 'bridge_python', 'bridge_script', 'batch_json', 'yt_dlp_executable')) {
+foreach ($name in @('ffmpeg', 'ffprobe', 'bridge_python', 'bridge_script', 'yt_dlp_executable')) {
$value = $configObject.$name
$expectedHash = $configObject."${name}_sha256"
if (-not [System.IO.Path]::IsPathRooted($value) -or $value.StartsWith('\\') -or
@@ -351,30 +722,100 @@
if (-not $destinationItem.PSIsContainer -or ($destinationItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
throw 'Host config destination must be an existing non-reparse directory.'
}
-if (Get-ChildItem -LiteralPath $resolvedDestination -File | Where-Object { $_.Name.StartsWith('BV1HA3o6oEJJ.', [StringComparison]::OrdinalIgnoreCase) }) {
- throw 'Formal output already exists; overwrite is forbidden.'
+if ([string]::IsNullOrWhiteSpace([string]$configObject.creator_name) -or
+ [Text.Encoding]::UTF8.GetByteCount([string]$configObject.creator_name) -gt 240) {
+ throw 'Host config creator_name is invalid.'
+}
+foreach ($name in @('formal_manifest_path', 'processing_handoff_path')) {
+ $value = [string]$configObject.$name
+ if (-not [System.IO.Path]::IsPathRooted($value) -or $value.StartsWith('\\')) {
+ throw 'Host governed output path must be absolute and local.'
+ }
+ $parent = [System.IO.Path]::GetDirectoryName($value)
+ $parentItem = Get-Item -LiteralPath $parent
+ if (-not $parentItem.PSIsContainer -or ($parentItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+ throw 'Host governed output parent must be an ordinary directory.'
+ }
+ if (Test-Path -LiteralPath $value) {
+ $item = Get-Item -LiteralPath $value
+ if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+ throw 'Host governed output must be an ordinary file.'
+ }
+ } elseif ($name -ceq 'formal_manifest_path') {
+ throw 'Host formal manifest must already exist.'
+ }
+}
+if ($configObject.formal_manifest_path -ceq $configObject.processing_handoff_path) {
+ throw 'Host governed output paths must be distinct.'
}
if (Test-Path -LiteralPath $root) {
throw 'InstallRoot already exists; overwrite is forbidden.'
}
$registryPath = "HKCU:\Software\Google\Chrome\NativeMessagingHosts\$expectedHostName"
+$previousRoot = [System.IO.Path]::GetFullPath((Join-Path $env:LOCALAPPDATA "project-info\bili-auth-native-host\$expectedPreviousVersion"))
if ($TestFileRegistryProvider) {
- if (-not $TestRegistryRoot) {
- throw 'TestRegistryRoot is required for the test file registry provider.'
+ if (-not $TestRegistryRoot -or -not $TestPreviousInstallRoot -or -not $TestPreviousArtifactReceipt) {
+ throw 'The test file registry provider requires its registry root and previous-install fixture inputs.'
}
$testRegistryBase = [System.IO.Path]::GetFullPath($TestRegistryRoot)
$tempBase = [System.IO.Path]::GetFullPath([System.IO.Path]::GetTempPath())
- if (-not $testRegistryBase.StartsWith($tempBase, [StringComparison]::OrdinalIgnoreCase)) {
- throw 'The test registry provider must be under the current temporary directory.'
+ $previousRoot = [System.IO.Path]::GetFullPath($TestPreviousInstallRoot)
+ $previousReceiptPath = (Resolve-Path -LiteralPath $TestPreviousArtifactReceipt).Path
+ if (-not $testRegistryBase.StartsWith($tempBase, [StringComparison]::OrdinalIgnoreCase) -or
+ -not $previousRoot.StartsWith($tempBase, [StringComparison]::OrdinalIgnoreCase) -or
+ -not $previousReceiptPath.StartsWith($tempBase, [StringComparison]::OrdinalIgnoreCase) -or
+ ($root -notlike ($tempBase.TrimEnd('\') + '\*'))) {
+ throw 'Test-only installer paths must be under the current temporary directory.'
}
+ $previousReceiptItem = Get-Item -LiteralPath $previousReceiptPath -Force
+ if ($previousReceiptItem.PSIsContainer -or ($previousReceiptItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+ throw 'Test previous-artifact receipt must be a regular non-reparse file.'
+ }
+ $previousReceipt = Get-StrictJson $previousReceiptPath
+ $expectedPreviousReceiptKeys = @(
+ 'schema', 'manifest_bytes', 'manifest_sha256', 'host_executable_bytes',
+ 'host_executable_sha256', 'config_bytes', 'config_sha256'
+ )
+ if ((Compare-Object ($expectedPreviousReceiptKeys | Sort-Object) @($previousReceipt.PSObject.Properties.Name | Sort-Object)) -or
+ $previousReceipt.schema -ne 1) {
+ throw 'Test previous-artifact receipt identity mismatch.'
+ }
+ $expectedPreviousManifestBytes = $previousReceipt.manifest_bytes
+ $expectedPreviousManifestSha256 = $previousReceipt.manifest_sha256
+ $expectedPreviousHostBytes = $previousReceipt.host_executable_bytes
+ $expectedPreviousHostSha256 = $previousReceipt.host_executable_sha256
+ $expectedPreviousConfigBytes = $previousReceipt.config_bytes
+ $expectedPreviousConfigSha256 = $previousReceipt.config_sha256
$registryPath = Join-Path $testRegistryBase $expectedHostName
-} elseif ($TestRegistryRoot -or $InjectFailure -cne 'none') {
+} elseif ($TestRegistryRoot -or $TestPreviousInstallRoot -or $TestPreviousArtifactReceipt -or $InjectFailure -cne 'none') {
throw 'Test-only controls require TestFileRegistryProvider.'
+} else {
+ $expectedInstallRoot = [System.IO.Path]::GetFullPath((Join-Path $env:LOCALAPPDATA 'project-info\bili-auth-native-host\1.2.25+20260829.generic.v027'))
+ if ($root -cne $expectedInstallRoot) {
+ throw 'Production InstallRoot does not match the pinned v027 location.'
+ }
}
-if (Test-Path -LiteralPath $registryPath) {
- throw 'Native Messaging registration already exists; overwrite is forbidden.'
+
+$previousIdentityParameters = @{
+ PreviousRoot = $previousRoot
+ ManifestBytes = $expectedPreviousManifestBytes
+ ManifestSha256 = $expectedPreviousManifestSha256
+ HostBytes = $expectedPreviousHostBytes
+ HostSha256 = $expectedPreviousHostSha256
+ ConfigBytes = $expectedPreviousConfigBytes
+ ConfigSha256 = $expectedPreviousConfigSha256
}
+
+$registryPreimage = Get-RegistrySnapshot $registryPath $TestFileRegistryProvider.IsPresent
+if (-not $registryPreimage.Exists) {
+ throw 'Exact v026 Native Messaging registration preimage is required.'
+}
+$previousInstall = Assert-ExactPreviousInstall @previousIdentityParameters
+if (-not (Test-RegistrySnapshotExact $registryPreimage $previousInstall.ManifestPath)) {
+ throw 'Existing Native Messaging registration does not match the pinned v026 preimage.'
+}
+$installMode = 'EXACT_V026_TO_V027_SWITCH'
if (-not $Install) {
[pscustomobject]@{
@@ -383,16 +824,18 @@
origin = $expectedOrigin
host_name = $expectedHostName
packaging = 'pyinstaller-onefile'
+ install_mode = $installMode
} | ConvertTo-Json -Compress
return
}
if ($PSCmdlet.ShouldProcess($root, 'Install exact-BVID Native Messaging host for current user')) {
+ $installedHost = Join-Path $root $expectedHostExecutable
+ $installedConfig = Join-Path $root 'config.json'
+ $manifestPath = Join-Path $root 'native-host-manifest.json'
try {
[System.IO.Directory]::CreateDirectory($root) | Out-Null
Stop-Injected 'after-root'
- $installedHost = Join-Path $root $expectedHostExecutable
- $installedConfig = Join-Path $root 'config.json'
Copy-CreateNew $resolvedHost $installedHost
if ((Get-Item -LiteralPath $installedHost).Length -ne $buildEntry.bytes -or
(Get-FileHash -Algorithm SHA256 -LiteralPath $installedHost).Hash -cne $buildEntry.sha256) {
@@ -405,7 +848,6 @@
throw 'Installed config reread verification failed.'
}
Stop-Injected 'after-config'
- $manifestPath = Join-Path $root 'native-host-manifest.json'
$manifest = [ordered]@{
name = $expectedHostName
description = 'project-info exact-BVID authenticated ingress'
@@ -414,38 +856,44 @@
allowed_origins = @($expectedOrigin)
}
Write-Utf8CreateNew $manifestPath ($manifest | ConvertTo-Json -Depth 3)
- $persistedManifest = Get-StrictJson $manifestPath
- if ($persistedManifest.name -cne $expectedHostName -or
- $persistedManifest.path -cne $installedHost -or
- $persistedManifest.type -cne 'stdio' -or
- $persistedManifest.allowed_origins.Count -ne 1 -or
- $persistedManifest.allowed_origins[0] -cne $expectedOrigin) {
- throw 'Native host manifest reread verification failed.'
- }
+ Assert-NewInstallRoot $root $installedHost $installedConfig $manifestPath $buildEntry $configIdentity
Stop-Injected 'after-manifest'
- if ($TestFileRegistryProvider) {
- [System.IO.Directory]::CreateDirectory($registryPath) | Out-Null
- } else {
- New-Item -Path $registryPath -ErrorAction Stop | Out-Null
+ $lastRegistryPreimage = Get-RegistrySnapshot $registryPath $TestFileRegistryProvider.IsPresent
+ $null = Assert-ExactPreviousInstall @previousIdentityParameters
+ if (-not (Test-RegistrySnapshotExact $lastRegistryPreimage $previousInstall.ManifestPath)) {
+ throw 'Native Messaging v026 preimage changed before the registry switch.'
}
Stop-Injected 'after-registry-key'
- if ($TestFileRegistryProvider) {
- Write-Utf8CreateNew (Join-Path $registryPath 'default.value') $manifestPath
- } else {
- Set-Item -LiteralPath $registryPath -Value $manifestPath -ErrorAction Stop
+ Set-RegistryDefaultAtomic $registryPath $manifestPath $TestFileRegistryProvider.IsPresent
+ if ($InjectFailure -ceq 'after-registry-value-mixed') {
+ Write-Utf8CreateNew (Join-Path $registryPath 'unexpected.value') 'ambiguous'
+ throw 'Injected ambiguous registry state after v026 switch.'
}
Stop-Injected 'after-registry-value'
+ $committedRegistry = Get-RegistrySnapshot $registryPath $TestFileRegistryProvider.IsPresent
+ if (-not (Test-RegistrySnapshotExact $committedRegistry $manifestPath)) {
+ throw 'Native Messaging v026 registry commit reread verification failed.'
+ }
+ Assert-NewInstallRoot $root $installedHost $installedConfig $manifestPath $buildEntry $configIdentity
} catch {
- # Both targets were proven absent before the transaction, so any
- # surviving object belongs to this attempt even if a provider threw
- # after partially creating it.
- if (Test-Path -LiteralPath $registryPath) {
- Remove-Item -LiteralPath $registryPath -Recurse -Force
+ $originalFailure = $_.Exception
+ try {
+ $rollbackRegistry = Get-RegistrySnapshot $registryPath $TestFileRegistryProvider.IsPresent
+ if (Test-RegistrySnapshotExact $rollbackRegistry $manifestPath) {
+ Set-RegistryDefaultAtomic $registryPath $previousInstall.ManifestPath $TestFileRegistryProvider.IsPresent
+ } elseif (-not (Test-RegistrySnapshotExact $rollbackRegistry $previousInstall.ManifestPath)) {
+ throw 'RECOVERY_REQUIRED: Native Messaging registration is neither the pinned v026 preimage nor this v027 attempt.'
+ }
+ $restoredRegistry = Get-RegistrySnapshot $registryPath $TestFileRegistryProvider.IsPresent
+ if (-not (Test-RegistrySnapshotExact $restoredRegistry $previousInstall.ManifestPath)) {
+ throw 'RECOVERY_REQUIRED: Native Messaging v026 registry preimage was not restored.'
+ }
+ $null = Assert-ExactPreviousInstall @previousIdentityParameters
+ Remove-ProvenOwnedInstallRoot $root $installedHost $installedConfig $manifestPath $buildEntry $configIdentity
+ } catch {
+ throw "RECOVERY_REQUIRED: $($_.Exception.Message) Original failure: $($originalFailure.Message)"
}
- if (Test-Path -LiteralPath $root) {
- Remove-Item -LiteralPath $root -Recurse -Force
- }
- throw
+ throw $originalFailure
}
}
diff --git a/dev/project-dev/bili_authenticated_extension/manifest.json b/dev/project-dev/bili_authenticated_extension/manifest.json
index 33a12d0..5776c2e 100644
--- a/dev/project-dev/bili_authenticated_extension/manifest.json
+++ b/dev/project-dev/bili_authenticated_extension/manifest.json
@@ -1,28 +1,25 @@
{
"manifest_version": 3,
"name": "project-info Bilibili 完整视频入口",
- "version": "1.0.0",
- "version_name": "1.0.0+20260805.v002",
- "description": "仅处理已授权的 BV1HA3o6oEJJ 完整视频任务。",
+ "version": "1.2.25",
+ "version_name": "1.2.25+20260829.generic.v027",
+ "description": "仅处理本机受控队列中 creator allowlist 授权的完整视频任务。",
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt2dT1HGYaI0DXM7zZwOTNBWXTKlMBJMpyDVjRUc+v6bUotmLyoraC+ay2scy9UQluSZVYq0tS8qvQNNvuZOlc5w2bOExm4TH2IIKvaVO8nVthHBnNz2kXdiM8ItN0vPZEmS+8gpTCI1+6wPTuUglMoXpqYBYhii8fJ5RkENRF3PRJBBigGt8soqdBFRY1QZUmpQv9dYw4dRq4L2C4QtBgClUg4bQpuCppiVZ9LHbePi9IAjc9r9R93KLzpaBuXdJpfVRE5w/6YHnxP8ovXxBdl7XktmrdH3xj7mWT7Q7ZBxkDNwn2RkruD45XgDD3yuNxOYSkLFMkseN+Ua69gSS4wIDAQAB",
"permissions": [
- "activeTab",
+ "alarms",
"cookies",
"nativeMessaging",
"scripting",
- "sidePanel"
+ "storage",
+ "tabs"
],
"host_permissions": [
- "https://www.bilibili.com/*"
+ "https://www.bilibili.com/*",
+ "https://*.bilibili.com/*"
],
"background": {
"service_worker": "background.js",
"type": "module"
},
- "side_panel": {
- "default_path": "sidepanel.html"
- },
- "action": {
- "default_title": "打开完整视频任务"
- }
+ "minimum_chrome_version": "120"
}
diff --git a/dev/project-dev/bili_authenticated_extension/native_host.py b/dev/project-dev/bili_authenticated_extension/native_host.py
index c5c2bb8..1dd791c 100644
--- a/dev/project-dev/bili_authenticated_extension/native_host.py
+++ b/dev/project-dev/bili_authenticated_extension/native_host.py
@@ -14,22 +14,30 @@
import os
import queue
import re
+import stat
+import subprocess
import sys
import threading
import time
+import uuid
from pathlib import Path
from typing import Any, BinaryIO
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from bili_authenticated_extension.constants import ( # type: ignore[import-not-found]
+ COMPLETION_CLOSURE_REQUIRED,
+ COOKIE_ACCESS_TERMINAL_CODES,
+ EXTENSION_BUILD,
EXPECTED_ORIGIN,
GRACEFUL_CANCEL_SECONDS,
JOB_WAIT_SECONDS,
MAX_INPUT_FRAME,
METADATA_TIMEOUT_SECONDS,
- TARGET_BVID,
+ RELOAD_GENERATION,
THREAD_JOIN_SECONDS,
+ validate_postprocess_terminal,
+ validate_runtime_diagnostic,
)
from bili_authenticated_extension.job import ( # type: ignore[import-not-found]
WindowsJob,
@@ -41,22 +49,32 @@
)
from bili_authenticated_extension.protocol import ( # type: ignore[import-not-found]
ProtocolError,
+ maintenance_state,
read_frame,
safe_response,
strict_json_loads,
validate_message,
validate_origin_argv,
+ validate_media_complete_ack,
+ validate_media_complete_identity,
+ validate_worker_prepare,
write_frame,
)
+ from bili_authenticated_extension.queue_state import QueueStore, ReloadStore # type: ignore[import-not-found]
else:
from .constants import (
+ COMPLETION_CLOSURE_REQUIRED,
+ COOKIE_ACCESS_TERMINAL_CODES,
+ EXTENSION_BUILD,
EXPECTED_ORIGIN,
GRACEFUL_CANCEL_SECONDS,
JOB_WAIT_SECONDS,
MAX_INPUT_FRAME,
METADATA_TIMEOUT_SECONDS,
- TARGET_BVID,
+ RELOAD_GENERATION,
THREAD_JOIN_SECONDS,
+ validate_postprocess_terminal,
+ validate_runtime_diagnostic,
)
from .job import (
WindowsJob,
@@ -68,13 +86,18 @@
)
from .protocol import (
ProtocolError,
+ maintenance_state,
read_frame,
safe_response,
strict_json_loads,
validate_message,
validate_origin_argv,
+ validate_media_complete_ack,
+ validate_media_complete_identity,
+ validate_worker_prepare,
write_frame,
)
+ from .queue_state import QueueStore, ReloadStore
def _config_path() -> Path:
@@ -90,12 +113,517 @@
return digest.hexdigest().upper()
-def preflight_configuration(path: Path) -> str | None:
- """Perform stdlib-only collision/config checks before any Cookie is read."""
+_RECOVERY_MODE_NONE = "NONE"
+_RECOVERY_MODE_EXACT_PAIR = "EXACT_PUBLISHED_PAIR"
+_FOREGROUND_ERROR_CODES = frozenset({
+ "E_FOREGROUND_LOCKED",
+ "E_FOREGROUND_WINDOW_ABSENT",
+ "E_FOREGROUND_WINDOW_AMBIGUOUS",
+ "E_FOREGROUND_LAUNCH_FAILED",
+ "E_FOREGROUND_PLATFORM_UNSUPPORTED",
+})
+_RECOVERABLE_MAPPING_KEYS = frozenset({
+ "schema_version", "bvid", "source", "published_at", "title", "local_file",
+ "bytes", "sha256", "duration_seconds", "remote_duration_seconds",
+ "local_duration_seconds", "duration_delta_seconds", "duration_tolerance_seconds",
+ "format_name", "video_codec", "audio_codec", "completed_at", "acquisition_mode",
+ "handoff_source_sha256",
+})
+
+
+def _native_parent_window(arguments: list[str]) -> int:
+ for argument in arguments[1:]:
+ if argument.startswith("--parent-window="):
+ try:
+ value = int(argument.split("=", 1)[1], 10)
+ except ValueError:
+ return 0
+ return value if value > 0 else 0
+ return 0
+
+
+class _WindowsKnownFolderId(ctypes.Structure):
+ _fields_ = [
+ ("Data1", ctypes.c_uint32),
+ ("Data2", ctypes.c_uint16),
+ ("Data3", ctypes.c_uint16),
+ ("Data4", ctypes.c_ubyte * 8),
+ ]
+
+ @classmethod
+ def from_text(cls, value: str) -> "_WindowsKnownFolderId":
+ return cls.from_buffer_copy(uuid.UUID(value).bytes_le)
+
+
+class _WindowsChromeForegroundAdapter:
+ """A title-free, profile-free adapter for one bounded Chrome foreground action."""
+
+ _SW_RESTORE = 9
+ _GA_ROOT = 2
+ _PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
+ _CHROME_RELATIVE_PATH = ("Google", "Chrome", "Application", "chrome.exe")
+ _KNOWN_FOLDER_IDS = (
+ "905e63b6-c1bf-494e-b29c-65b732d3d21a", # ProgramFiles
+ "7c5a40ef-a0fb-4bfc-874a-c0f2e0b9fa8e", # ProgramFilesX86
+ "f1b32785-6fba-4fcf-9d55-7b8e7f157091", # LocalAppData
+ )
+
+ def __init__(self) -> None:
+ if os.name != "nt":
+ raise OSError("unsupported platform")
+ from ctypes import wintypes
+
+ self._wintypes = wintypes
+ self._user32 = ctypes.WinDLL("user32", use_last_error=True)
+ self._kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ self._shell32 = ctypes.WinDLL("shell32", use_last_error=True)
+ self._ole32 = ctypes.WinDLL("ole32", use_last_error=True)
+ self._user32.IsWindow.argtypes = [wintypes.HWND]
+ self._user32.IsWindow.restype = wintypes.BOOL
+ self._user32.IsWindowVisible.argtypes = [wintypes.HWND]
+ self._user32.IsWindowVisible.restype = wintypes.BOOL
+ self._user32.GetAncestor.argtypes = [wintypes.HWND, wintypes.UINT]
+ self._user32.GetAncestor.restype = wintypes.HWND
+ self._user32.GetClassNameW.argtypes = [wintypes.HWND, wintypes.LPWSTR, ctypes.c_int]
+ self._user32.GetClassNameW.restype = ctypes.c_int
+ self._user32.GetWindowRect.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.RECT)]
+ self._user32.GetWindowRect.restype = wintypes.BOOL
+ self._user32.GetWindowThreadProcessId.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.DWORD)]
+ self._user32.GetWindowThreadProcessId.restype = wintypes.DWORD
+ self._user32.ShowWindowAsync.argtypes = [wintypes.HWND, ctypes.c_int]
+ self._user32.ShowWindowAsync.restype = wintypes.BOOL
+ self._user32.BringWindowToTop.argtypes = [wintypes.HWND]
+ self._user32.BringWindowToTop.restype = wintypes.BOOL
+ self._user32.SetForegroundWindow.argtypes = [wintypes.HWND]
+ self._user32.SetForegroundWindow.restype = wintypes.BOOL
+ self._user32.GetForegroundWindow.argtypes = []
+ self._user32.GetForegroundWindow.restype = wintypes.HWND
+ self._kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
+ self._kernel32.OpenProcess.restype = wintypes.HANDLE
+ self._kernel32.QueryFullProcessImageNameW.argtypes = [
+ wintypes.HANDLE, wintypes.DWORD, wintypes.LPWSTR, ctypes.POINTER(wintypes.DWORD),
+ ]
+ self._kernel32.QueryFullProcessImageNameW.restype = wintypes.BOOL
+ self._kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
+ self._kernel32.CloseHandle.restype = wintypes.BOOL
+ self._shell32.SHGetKnownFolderPath.argtypes = [
+ ctypes.POINTER(_WindowsKnownFolderId), wintypes.DWORD,
+ wintypes.HANDLE, ctypes.POINTER(ctypes.c_void_p),
+ ]
+ self._shell32.SHGetKnownFolderPath.restype = ctypes.c_long
+ self._ole32.CoTaskMemFree.argtypes = [ctypes.c_void_p]
+ self._ole32.CoTaskMemFree.restype = None
+
+ def _root(self, hwnd: int) -> int:
+ if hwnd <= 0 or not self._user32.IsWindow(hwnd):
+ return 0
+ return int(self._user32.GetAncestor(hwnd, self._GA_ROOT) or hwnd)
+
+ def _class_name(self, hwnd: int) -> str:
+ buffer = ctypes.create_unicode_buffer(256)
+ length = int(self._user32.GetClassNameW(hwnd, buffer, len(buffer)))
+ return buffer.value[:length] if length > 0 else ""
+
+ def _image_is_chrome(self, hwnd: int) -> bool:
+ process_id = self._wintypes.DWORD(0)
+ self._user32.GetWindowThreadProcessId(hwnd, ctypes.byref(process_id))
+ if process_id.value <= 0:
+ return False
+ handle = self._kernel32.OpenProcess(
+ self._PROCESS_QUERY_LIMITED_INFORMATION, False, process_id.value,
+ )
+ if not handle:
+ return False
+ try:
+ size = self._wintypes.DWORD(32768)
+ buffer = ctypes.create_unicode_buffer(size.value)
+ if not self._kernel32.QueryFullProcessImageNameW(handle, 0, buffer, ctypes.byref(size)):
+ return False
+ return Path(buffer.value).name.casefold() == "chrome.exe"
+ finally:
+ self._kernel32.CloseHandle(handle)
+
+ def is_chrome_window(self, hwnd: int) -> bool:
+ root = self._root(hwnd)
+ return bool(
+ root > 0
+ and self._user32.IsWindowVisible(root)
+ and self._class_name(root) == "Chrome_WidgetWin_1"
+ and self._image_is_chrome(root)
+ )
+
+ def list_windows(self) -> list[int]:
+ windows: list[int] = []
+ callback_type = ctypes.WINFUNCTYPE(
+ self._wintypes.BOOL, self._wintypes.HWND, self._wintypes.LPARAM,
+ )
+
+ @callback_type
+ def collect(hwnd: int, _parameter: int) -> bool:
+ value = int(hwnd)
+ if self.is_chrome_window(value):
+ windows.append(self._root(value))
+ return True
+
+ self._user32.EnumWindows(collect, 0)
+ return sorted(set(windows))
+
+ def bounds(self, hwnd: int) -> dict[str, int] | None:
+ rectangle = self._wintypes.RECT()
+ if not self._user32.GetWindowRect(hwnd, ctypes.byref(rectangle)):
+ return None
+ return {
+ "left": int(rectangle.left), "top": int(rectangle.top),
+ "width": int(rectangle.right - rectangle.left),
+ "height": int(rectangle.bottom - rectangle.top),
+ }
+
+ def focus(self, hwnd: int) -> bool:
+ root = self._root(hwnd)
+ if not self.is_chrome_window(root):
+ return False
+ self._user32.ShowWindowAsync(root, self._SW_RESTORE)
+ self._user32.BringWindowToTop(root)
+ requested = bool(self._user32.SetForegroundWindow(root))
+ deadline = time.monotonic() + 2.0
+ while time.monotonic() < deadline:
+ foreground = self._root(int(self._user32.GetForegroundWindow() or 0))
+ if foreground == root:
+ return True
+ time.sleep(0.05)
+ return requested and self._root(int(self._user32.GetForegroundWindow() or 0)) == root
+
+ def _known_folder_roots(self) -> tuple[Path, ...]:
+ roots: list[Path] = []
+ observed: set[str] = set()
+ for text in self._KNOWN_FOLDER_IDS:
+ folder_id = _WindowsKnownFolderId.from_text(text)
+ allocated = ctypes.c_void_p()
+ try:
+ result = int(self._shell32.SHGetKnownFolderPath(
+ ctypes.byref(folder_id), 0, None, ctypes.byref(allocated),
+ ))
+ if result != 0 or not allocated.value:
+ continue
+ root = Path(ctypes.wstring_at(allocated.value))
+ key = os.path.normcase(os.path.abspath(os.fspath(root)))
+ if key not in observed:
+ observed.add(key)
+ roots.append(root)
+ finally:
+ if allocated.value:
+ self._ole32.CoTaskMemFree(allocated)
+ return tuple(roots)
+
+ @staticmethod
+ def _local_absolute(path: Path) -> Path | None:
+ try:
+ value = Path(os.path.abspath(os.fspath(path)))
+ except (OSError, TypeError, ValueError):
+ return None
+ if not value.is_absolute() or value.anchor.startswith("\\\\"):
+ return None
+ if not re.fullmatch(r"[A-Za-z]:", value.drive):
+ return None
+ return value
+
+ @classmethod
+ def _validated_chrome_candidate(
+ cls, root: Path,
+ ) -> tuple[Path, Path, tuple[tuple[str, int, int, int, int, int, int], ...]] | None:
+ lexical_root = cls._local_absolute(root)
+ if lexical_root is None:
+ return None
+ lexical_candidate = lexical_root.joinpath(*cls._CHROME_RELATIVE_PATH)
+ try:
+ canonical_root = lexical_root.resolve(strict=True)
+ canonical_candidate = lexical_candidate.resolve(strict=True)
+ except OSError:
+ return None
+ if (
+ os.path.normcase(os.fspath(lexical_root))
+ != os.path.normcase(os.fspath(canonical_root))
+ or os.path.normcase(os.fspath(lexical_candidate))
+ != os.path.normcase(os.fspath(canonical_candidate))
+ ):
+ return None
+ try:
+ relative = canonical_candidate.relative_to(canonical_root)
+ except ValueError:
+ return None
+ if tuple(part.casefold() for part in relative.parts) != tuple(
+ part.casefold() for part in cls._CHROME_RELATIVE_PATH
+ ):
+ return None
+
+ chain: list[Path] = []
+ current = Path(canonical_root.anchor)
+ chain.append(current)
+ for part in canonical_root.parts[1:]:
+ current /= part
+ chain.append(current)
+ for part in cls._CHROME_RELATIVE_PATH:
+ current /= part
+ chain.append(current)
+ snapshots: list[tuple[str, int, int, int, int, int, int]] = []
+ for index, item in enumerate(chain):
+ try:
+ identity = item.lstat()
+ if item.is_symlink() or _is_reparse(item):
+ return None
+ final = index == len(chain) - 1
+ if final and not stat.S_ISREG(identity.st_mode):
+ return None
+ if not final and not stat.S_ISDIR(identity.st_mode):
+ return None
+ snapshots.append((
+ os.path.normcase(os.fspath(item)),
+ int(identity.st_dev), int(identity.st_ino), int(identity.st_mode),
+ int(identity.st_size), int(identity.st_mtime_ns), int(identity.st_ctime_ns),
+ ))
+ except OSError:
+ return None
+ return canonical_candidate, canonical_root, tuple(snapshots)
+
+ def _chrome_executable(
+ self,
+ ) -> tuple[Path, Path, tuple[tuple[str, int, int, int, int, int, int], ...]] | None:
+ for root in self._known_folder_roots():
+ validated = self._validated_chrome_candidate(root)
+ if validated is not None:
+ return validated
+ return None
+
+ def launch(self, canonical_target_url: str, prior_windows: set[int]) -> int | None:
+ discovered = self._chrome_executable()
+ if discovered is None:
+ return None
+ executable, trusted_root, identity = discovered
+ revalidated = self._validated_chrome_candidate(trusted_root)
+ if revalidated is None:
+ return None
+ current_executable, current_root, current_identity = revalidated
+ if (
+ os.path.normcase(os.fspath(current_executable))
+ != os.path.normcase(os.fspath(executable))
+ or os.path.normcase(os.fspath(current_root))
+ != os.path.normcase(os.fspath(trusted_root))
+ or current_identity != identity
+ ):
+ return None
+ try:
+ subprocess.Popen(
+ [str(current_executable), "--new-window", canonical_target_url],
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ close_fds=True,
+ )
+ except OSError:
+ return None
+ deadline = time.monotonic() + 10.0
+ while time.monotonic() < deadline:
+ created = [item for item in self.list_windows() if item not in prior_windows]
+ if len(created) == 1:
+ return created[0]
+ if len(created) > 1:
+ return -1
+ time.sleep(0.1)
+ return None
+
+
+def _window_bounds_match(observed: dict[str, int] | None, expected: dict[str, int]) -> bool:
+ if observed is None:
+ return False
+ return all(abs(observed[key] - expected[key]) <= 16 for key in ("left", "top", "width", "height"))
+
+
+def _foreground_chrome_window(
+ canonical_target_url: str,
+ expected_bounds: dict[str, int],
+ parent_window: int,
+ *,
+ adapter: Any | None = None,
+) -> str | None:
+ """Restore/foreground one exact Chrome window or launch one visible target.
+
+ Returns a fixed sanitized error code, or ``None`` on success. No title,
+ command line, URL query, profile, page, or secret data is inspected.
+ """
+
+ if adapter is None:
+ try:
+ adapter = _WindowsChromeForegroundAdapter()
+ except OSError:
+ return "E_FOREGROUND_PLATFORM_UNSUPPORTED"
+ windows = adapter.list_windows()
+ parent = adapter._root(parent_window) if parent_window > 0 else 0
+ if parent > 0 and parent in windows and adapter.is_chrome_window(parent):
+ candidates = [parent]
+ else:
+ candidates = [
+ hwnd for hwnd in windows if _window_bounds_match(adapter.bounds(hwnd), expected_bounds)
+ ]
+ if len(candidates) > 1:
+ return "E_FOREGROUND_WINDOW_AMBIGUOUS"
+ if len(candidates) == 1:
+ return None if adapter.focus(candidates[0]) else "E_FOREGROUND_LOCKED"
+ if windows:
+ return "E_FOREGROUND_WINDOW_ABSENT"
+ launched = adapter.launch(canonical_target_url, set(windows))
+ if launched == -1:
+ return "E_FOREGROUND_WINDOW_AMBIGUOUS"
+ if launched is None:
+ return "E_FOREGROUND_LAUNCH_FAILED"
+ return None if adapter.focus(launched) else "E_FOREGROUND_LOCKED"
+
+
+def _is_reparse(path: Path) -> bool:
+ try:
+ return bool(path.lstat().st_file_attributes & 0x400)
+ except AttributeError:
+ return path.is_symlink()
+
+
+def _ordinary_child(path: Path, parent: Path) -> os.stat_result:
+ value = path.lstat()
+ if (
+ not stat.S_ISREG(value.st_mode)
+ or path.is_symlink()
+ or _is_reparse(path)
+ or path.resolve(strict=True).parent != parent.resolve(strict=True)
+ ):
+ raise OSError("non-ordinary destination child")
+ return value
+
+
+def _read_small_ordinary_json(path: Path, parent: Path) -> dict[str, Any]:
+ before = _ordinary_child(path, parent)
+ if before.st_size <= 0 or before.st_size > 64 * 1024:
+ raise OSError("mapping size")
+ descriptor: int | None = None
+ handle: int | None = None
+ try:
+ if os.name == "nt":
+ import msvcrt # noqa: PLC0415
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ create_file = kernel32.CreateFileW
+ create_file.argtypes = (
+ ctypes.c_wchar_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p,
+ ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p,
+ )
+ create_file.restype = ctypes.c_void_p
+ handle = create_file(
+ str(path), 0x80000000, 0x00000001, None, 3,
+ 0x00000080 | 0x00200000 | 0x08000000, None,
+ )
+ if handle in (None, ctypes.c_void_p(-1).value):
+ raise OSError(ctypes.get_last_error(), "mapping open")
+ descriptor = msvcrt.open_osfhandle(
+ int(handle), os.O_RDONLY | getattr(os, "O_BINARY", 0)
+ )
+ handle = None
+ else:
+ flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
+ descriptor = os.open(path, flags)
+ with os.fdopen(descriptor, "rb", closefd=True) as source:
+ descriptor = None
+ handle_stat = os.fstat(source.fileno())
+ payload = source.read(64 * 1024 + 1)
+ after = _ordinary_child(path, parent)
+ finally:
+ if descriptor is not None:
+ os.close(descriptor)
+ if handle is not None:
+ ctypes.WinDLL("kernel32", use_last_error=True).CloseHandle(ctypes.c_void_p(handle))
+ identity = lambda value: (value.st_dev, value.st_ino, value.st_mode, value.st_size)
+ if (
+ len(payload) > 64 * 1024 or payload == b""
+ or identity(before) != identity(handle_stat) or identity(before) != identity(after)
+ ):
+ raise OSError("mapping drift")
+ value = strict_json_loads(payload)
+ if not isinstance(value, dict):
+ raise OSError("mapping root")
+ return value
+
+
+def _recovery_stage_is_clear(bvid: str) -> bool:
+ local = os.environ.get("LOCALAPPDATA")
+ if not local:
+ return False
+ local_root = Path(local)
+ if not local_root.is_absolute() or str(local_root).startswith("\\\\"):
+ return False
+ stage = local_root / "project-info" / "bili-auth-ingress" / bvid
+ if not stage.exists():
+ return True
+ return stage.is_dir() and not stage.is_symlink() and not _is_reparse(stage) and next(stage.iterdir(), None) is None
+
+
+def _classify_job_destination(config: dict[str, Any], job: dict[str, Any]) -> tuple[str | None, str]:
+ """Classify only an exact published pair as a recovery candidate.
+
+ This gate is deliberately not a success decision. It only lets the
+ worker's locked, handle-backed verifier observe the pair before any Cookie
+ or network access. Any ambiguity remains a collision.
+ """
+ try:
+ destination = Path(config["destination"]).resolve(strict=True)
+ prefix = f"{job['bvid']}.".casefold()
+ candidates = [child for child in destination.iterdir() if child.name.casefold().startswith(prefix)]
+ lineage = job.get("lineage")
+ closure_required = (
+ isinstance(lineage, dict)
+ and lineage.get("predecessor_terminal_error_code") == COMPLETION_CLOSURE_REQUIRED
+ )
+ if not candidates:
+ if closure_required:
+ return "E_EXISTS", _RECOVERY_MODE_NONE
+ return None, _RECOVERY_MODE_NONE
+ formal_name = f"{job['bvid']}.mkv"
+ mapping_name = f"{job['bvid']}.download.json"
+ if {child.name for child in candidates} != {formal_name, mapping_name} or len(candidates) != 2:
+ return "E_EXISTS", _RECOVERY_MODE_NONE
+ formal = destination / formal_name
+ mapping = destination / mapping_name
+ formal_stat = _ordinary_child(formal, destination)
+ persisted = _read_small_ordinary_json(mapping, destination)
+ if (
+ set(persisted) != _RECOVERABLE_MAPPING_KEYS
+ or persisted.get("schema_version") != "1.0"
+ or persisted.get("bvid") != job["bvid"]
+ or persisted.get("source") != job["canonical_url"]
+ or persisted.get("published_at") != job["published_at"]
+ or persisted.get("local_file") != formal_name
+ or persisted.get("acquisition_mode") != "authorized_browser_file_handoff"
+ or isinstance(persisted.get("bytes"), bool)
+ or persisted.get("bytes") != formal_stat.st_size
+ or not isinstance(persisted.get("sha256"), str)
+ or re.fullmatch(r"[0-9a-f]{64}", persisted["sha256"]) is None
+ or persisted.get("handoff_source_sha256") != persisted["sha256"]
+ or job.get("creator_uid") not in config.get("creator_allowlist", ())
+ or not _recovery_stage_is_clear(job["bvid"])
+ or _ordinary_child(formal, destination).st_size != formal_stat.st_size
+ ):
+ return "E_EXISTS", _RECOVERY_MODE_NONE
+ return None, _RECOVERY_MODE_EXACT_PAIR
+ except (KeyError, OSError, ProtocolError, TypeError, ValueError):
+ return "E_EXISTS", _RECOVERY_MODE_NONE
+
+
+def load_runtime_configuration(path: Path) -> dict[str, Any]:
+ """Load the generic local queue/tool configuration without importing the worker."""
expected = {
"schema",
- "target",
- "canonical_url",
+ "creator_allowlist",
+ "queue_path",
+ "queue_state_path",
+ "queue_lock_path",
+ "reload_state_path",
+ "reload_generation",
+ "required_extension_build",
"ffmpeg",
"ffmpeg_sha256",
"ffprobe",
@@ -104,45 +632,94 @@
"bridge_python_sha256",
"bridge_script",
"bridge_script_sha256",
- "batch_json",
- "batch_json_sha256",
"yt_dlp_executable",
"yt_dlp_executable_sha256",
"destination",
+ "creator_name",
+ "formal_manifest_path",
+ "processing_handoff_path",
}
+ raw = strict_json_loads(path.read_bytes())
+ if set(raw) != expected or raw["schema"] != 2:
+ raise ProtocolError("E_CONFIG")
+ creators = raw["creator_allowlist"]
+ if (
+ not isinstance(creators, list) or not creators or len(creators) > 64
+ or any(not isinstance(item, str) or not re.fullmatch(r"[1-9][0-9]{0,19}", item) for item in creators)
+ or creators != sorted(set(creators))
+ ):
+ raise ProtocolError("E_CONFIG")
+ if raw["required_extension_build"] != EXTENSION_BUILD or raw["reload_generation"] != RELOAD_GENERATION:
+ raise ProtocolError("E_CONFIG")
+ for name in (
+ "queue_path", "queue_state_path", "queue_lock_path", "reload_state_path",
+ "formal_manifest_path", "processing_handoff_path",
+ ):
+ value = raw[name]
+ if not isinstance(value, str):
+ raise ProtocolError("E_CONFIG")
+ candidate = Path(value)
+ if not candidate.is_absolute() or str(candidate).startswith("\\\\"):
+ raise ProtocolError("E_CONFIG")
+ parent = candidate.parent.resolve(strict=True)
+ if not parent.is_dir() or parent.is_symlink() or (candidate.exists() and (not candidate.is_file() or candidate.is_symlink())):
+ raise ProtocolError("E_CONFIG")
+ creator_name = raw["creator_name"]
+ if (
+ not isinstance(creator_name, str) or not creator_name.strip()
+ or len(creator_name.encode("utf-8")) > 240
+ or any(ord(ch) < 32 or ord(ch) == 127 for ch in creator_name)
+ or not Path(raw["formal_manifest_path"]).is_file()
+ or raw["formal_manifest_path"] == raw["processing_handoff_path"]
+ ):
+ raise ProtocolError("E_CONFIG")
+ raw[name] = str(candidate.resolve(strict=False))
try:
- raw = strict_json_loads(path.read_bytes())
- if set(raw) != expected or raw["schema"] != 1:
- return "E_CONFIG"
- if raw["target"] != TARGET_BVID or raw["canonical_url"] != "https://www.bilibili.com/video/BV1HA3o6oEJJ":
- return "E_CONFIG"
- for name in ("ffmpeg", "ffprobe", "bridge_python", "bridge_script", "batch_json", "yt_dlp_executable"):
+ for name in ("ffmpeg", "ffprobe", "bridge_python", "bridge_script", "yt_dlp_executable"):
value = raw[name]
expected_hash = raw[f"{name}_sha256"]
if not isinstance(value, str) or not isinstance(expected_hash, str):
- return "E_CONFIG"
+ raise ProtocolError("E_CONFIG")
candidate = Path(value)
if not candidate.is_absolute() or str(candidate).startswith("\\\\"):
- return "E_CONFIG"
+ raise ProtocolError("E_CONFIG")
candidate = candidate.resolve(strict=True)
if not candidate.is_file() or candidate.is_symlink() or _hash_file(candidate) != expected_hash.upper():
- return "E_CONFIG_HASH"
- if raw["bridge_script_sha256"].upper() != "749FC486B0F42315BD463F11771FE2A7C71CAB53DD9AC2E411CE82E1175DFF13":
- return "E_CONFIG_HASH"
+ raise ProtocolError("E_CONFIG_HASH")
+ raw[name] = str(candidate)
+ if raw["bridge_script_sha256"].upper() != "00F11DAF8387160DB863C89F0B33AB8480422233FF42199189222C989C7ED07E":
+ raise ProtocolError("E_CONFIG_HASH")
destination = Path(raw["destination"])
if not destination.is_absolute() or str(destination).startswith("\\\\"):
- return "E_CONFIG"
+ raise ProtocolError("E_CONFIG")
destination = destination.resolve(strict=True)
if not destination.is_dir() or destination.is_symlink():
- return "E_CONFIG"
- if any(
- child.is_file() and child.name.casefold().startswith(f"{TARGET_BVID}.".casefold())
- for child in destination.iterdir()
- ):
- return "E_EXISTS"
- except (OSError, ProtocolError, KeyError, TypeError, ValueError):
+ raise ProtocolError("E_CONFIG")
+ raw["destination"] = str(destination)
+ except OSError as exc:
+ raise ProtocolError("E_CONFIG") from exc
+ return raw
+
+
+def preflight_configuration(path: Path) -> str | None:
+ try:
+ load_runtime_configuration(path)
+ except ProtocolError as exc:
+ return exc.code
+ except (OSError, KeyError, TypeError, ValueError):
return "E_CONFIG"
return None
+
+
+def preflight_job(config: dict[str, Any], job: dict[str, Any]) -> str | None:
+ return _classify_job_destination(config, job)[0]
+
+
+def _queue_store(config: dict[str, Any]) -> QueueStore:
+ return QueueStore(
+ Path(config["queue_path"]), Path(config["queue_state_path"]),
+ Path(config["queue_lock_path"]), frozenset(config["creator_allowlist"]),
+ )
def _worker_command(
@@ -171,16 +748,25 @@
return int(msvcrt.get_osfhandle(fd))
+def _media_complete_ack_test_seam(_stage: str) -> None:
+ """No-op production seam for Host-exit/ACK-loss counterexamples."""
+
+ return None
+
+
class WorkerTask:
"""One worker, its task job, and sanitized control channel."""
- def __init__(self, config_path: Path | None = None) -> None:
+ def __init__(
+ self, config_path: Path | None = None, queue_store: QueueStore | None = None,
+ ) -> None:
self.job: WindowsJob | None = None
self.process: Any = None
self.cancel_handle = 0
self.commit_handle = 0
self.config_path = _config_path() if config_path is None else config_path.resolve(strict=True)
self.input_writer: BinaryIO | None = None
+ self.input_lock = threading.Lock()
self.control_reader: BinaryIO | None = None
self.control_queue: queue.Queue[dict[str, Any] | None] = queue.Queue()
self.control_thread: threading.Thread | None = None
@@ -189,6 +775,7 @@
self.error_code: str | None = None
self.formal_filename: str | None = None
self.mapping_filename: str | None = None
+ self.diagnostic: dict[str, object] | None = None
self.task_nonce: str | None = None
self.terminal = False
self.task_started_at: float | None = None
@@ -196,10 +783,41 @@
self.prepared = False
self.secret_started = False
self.prepare_id: str | None = None
+ self.job_spec: dict[str, Any] | None = None
+ self.lease_id: str | None = None
+ self.queue_terminal_recorded = False
+ self.media_complete = False
+ self.media_identity: dict[str, Any] | None = None
+ self.postprocess_recovery_binding: dict[str, Any] | None = None
+ self.queue_store = queue_store
+ self.closing = False
+ self.recovery_mode = _RECOVERY_MODE_NONE
- def prepare(self, page_proof: dict[str, Any], prepare_id: str) -> None:
+ def prepare(
+ self,
+ job_spec: dict[str, Any],
+ lease_id: str,
+ page_proof: dict[str, Any] | None,
+ prepare_id: str,
+ recovery_mode: str = _RECOVERY_MODE_NONE,
+ ) -> None:
if self.process is not None:
raise ProtocolError("E_BUSY")
+ if recovery_mode not in {_RECOVERY_MODE_NONE, _RECOVERY_MODE_EXACT_PAIR}:
+ raise ProtocolError("E_PREPARE")
+ if recovery_mode == _RECOVERY_MODE_NONE:
+ if not isinstance(page_proof, dict) or not isinstance(page_proof.get("task_nonce"), str):
+ raise ProtocolError("E_PREPARE")
+ self.task_nonce = page_proof["task_nonce"]
+ else:
+ # Exact-pair recovery is deliberately independent of page, browser,
+ # Cookie, and network state. A proof may be present for compatibility
+ # with an already-prepared caller, but is never required or consumed.
+ self.task_nonce = None
+ self.prepare_id = prepare_id
+ self.job_spec = dict(job_spec)
+ self.lease_id = lease_id
+ self.recovery_mode = recovery_mode
input_read_fd, input_write_fd = os.pipe()
control_read_fd, control_write_fd = os.pipe()
input_read_handle = _fd_handle(input_read_fd)
@@ -243,6 +861,12 @@
self.control_reader = os.fdopen(control_read_fd, "rb", buffering=0)
self.control_thread = threading.Thread(target=self._read_control, name="bili-auth-control", daemon=True)
self.control_thread.start()
+ with self.input_lock:
+ write_frame(
+ self.input_writer,
+ {"schema": 3, "type": "worker_prepare", "job": job_spec,
+ "lease_id": lease_id, "recovery_mode": recovery_mode},
+ )
deadline = time.monotonic() + METADATA_TIMEOUT_SECONDS
while time.monotonic() < deadline:
try:
@@ -254,19 +878,25 @@
if message is None:
break
if message.get("type") == "ready" and message.get("code") == "READY_PLUGIN_DISABLED":
- self.task_nonce = page_proof["task_nonce"]
- self.prepare_id = prepare_id
self.phase = "READY"
self.prepared = True
return
+ if message.get("type") == "ready" and message.get("code") == "E_PLUGIN_BOUNDARY":
+ self.error_code = "E_PLUGIN_BOUNDARY"
+ self.phase = "FAILED"
+ self.terminal = True
+ break
if message.get("type") == "terminal":
self._apply_control(message)
+ if self.phase == "COMPLETE":
+ return
break
- self.error_code = "E_PLUGIN_BOUNDARY"
+ error_code = self.error_code if isinstance(self.error_code, str) and re.fullmatch(r"E_[A-Z0-9_]{1,48}", self.error_code) else "E_PLUGIN_BOUNDARY"
+ self.error_code = error_code
self.phase = "FAILED"
self.terminal = True
self.terminate()
- raise ProtocolError("E_PLUGIN_BOUNDARY")
+ raise ProtocolError(error_code)
def start(self, start_message: dict[str, Any]) -> None:
if (
@@ -275,11 +905,12 @@
or self.input_writer is None
or start_message["page_proof"]["task_nonce"] != self.task_nonce
or start_message["prepare_id"] != self.prepare_id
+ or start_message["job"] != self.job_spec
+ or start_message["lease_id"] != self.lease_id
):
raise ProtocolError("E_PREPARE")
- write_frame(self.input_writer, start_message)
- self.input_writer.close()
- self.input_writer = None
+ with self.input_lock:
+ write_frame(self.input_writer, start_message)
self.secret_started = True
self.phase = "CHECKING"
self.task_started_at = time.monotonic()
@@ -294,15 +925,60 @@
if payload is None:
break
value = strict_json_loads(payload)
- if _valid_control_message(value):
- self.control_queue.put(value)
+ if _valid_control_message(value, self.job_spec, self.lease_id):
+ if value.get("type") == "progress" and value.get("phase") == "MEDIA_COMPLETE":
+ self._accept_media_complete(value)
+ else:
+ self.control_queue.put(value)
else:
self.control_queue.put({"schema": 1, "type": "terminal", "phase": "FAILED", "error_code": "E_CONTROL"})
break
except BaseException:
- self.control_queue.put({"schema": 1, "type": "terminal", "phase": "FAILED", "error_code": "E_CONTROL"})
+ if not self.closing:
+ self.control_queue.put({"schema": 1, "type": "terminal", "phase": "FAILED", "error_code": "E_CONTROL"})
finally:
self.control_queue.put(None)
+
+ def _accept_media_complete(self, value: dict[str, Any]) -> None:
+ """Durably persist and reread exact media identity before ACKing Worker."""
+
+ if (
+ self.queue_store is None or self.job_spec is None or self.lease_id is None
+ or self.input_writer is None
+ or value["job"] != self.job_spec or value["lease_id"] != self.lease_id
+ ):
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ media = validate_media_complete_identity(value["media"], self.job_spec)
+ persisted = self.queue_store.mark_media_complete(
+ self.job_spec, self.lease_id, int(time.time() * 1000), media,
+ )
+ if persisted.get("media") != media:
+ raise ProtocolError("E_QUEUE_WRITE")
+ self.media_identity = dict(media)
+ self.media_complete = True
+ self.phase = "MEDIA_COMPLETE"
+ self.progress = 100
+ _media_complete_ack_test_seam("BEFORE_ACK")
+ acknowledgement = {
+ "schema": 3,
+ "type": "media_complete_ack",
+ "job_id": self.job_spec["job_id"],
+ "lease_id": self.lease_id,
+ "media": media,
+ }
+ validate_media_complete_ack(
+ acknowledgement, self.job_spec, self.lease_id, media,
+ )
+ with self.input_lock:
+ if self.input_writer is None:
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ write_frame(self.input_writer, acknowledgement)
+ _media_complete_ack_test_seam("AFTER_ACK")
+ self.control_queue.put({
+ "schema": 1, "type": "progress", "phase": "MEDIA_COMPLETE",
+ "progress": 100, "job": self.job_spec, "lease_id": self.lease_id,
+ "media": media,
+ })
def _apply_control(self, value: dict[str, Any]) -> None:
message_type = value["type"]
@@ -311,15 +987,18 @@
self.phase_started_at = time.monotonic()
self.phase = value["phase"]
self.progress = value["progress"]
+ if value["phase"] == "MEDIA_COMPLETE":
+ self.media_complete = True
elif message_type == "terminal":
self.phase = value["phase"]
self.progress = 100 if self.phase == "COMPLETE" else self.progress
self.error_code = value.get("error_code")
self.formal_filename = value.get("formal_filename")
self.mapping_filename = value.get("mapping_filename")
+ self.diagnostic = value.get("diagnostic")
self.terminal = True
- def poll(self) -> None:
+ def _drain_control_queue(self) -> None:
while True:
try:
value = self.control_queue.get_nowait()
@@ -327,6 +1006,9 @@
break
if value is not None:
self._apply_control(value)
+
+ def poll(self) -> None:
+ self._drain_control_queue()
if self.process is not None and self.process.wait(0) and not self.terminal:
self.phase = "FAILED"
self.error_code = "E_WORKER_EXIT"
@@ -344,7 +1026,10 @@
if self.phase == "CHECKING" and self.phase_started_at is not None:
if now - self.phase_started_at >= METADATA_TIMEOUT_SECONDS:
return "E_METADATA_TIMEOUT"
- if self.phase in {"DOWNLOADING", "MERGING", "VALIDATING", "PUBLISHING"}:
+ if self.phase in {
+ "DOWNLOADING", "MERGING", "VALIDATING", "PUBLISHING",
+ "MEDIA_COMPLETE", "POSTPROCESS_PENDING",
+ }:
if now - self.task_started_at >= 7_200:
return "E_DOWNLOAD_TIMEOUT"
return None
@@ -375,43 +1060,88 @@
return True
def terminate(self) -> None:
+ if self.closing:
+ return
+ self._drain_control_queue()
+ if self.terminal:
+ self.close()
+ return
if self.cancel_handle:
try:
set_event(self.cancel_handle)
except BaseException:
pass
- if self.process is not None and not self.process.wait(GRACEFUL_CANCEL_SECONDS):
+ if self.input_writer is not None:
+ writer, self.input_writer = self.input_writer, None
+ try:
+ writer.close()
+ except (BrokenPipeError, OSError, ValueError):
+ pass
+ deadline = time.monotonic() + GRACEFUL_CANCEL_SECONDS
+ while self.process is not None and time.monotonic() < deadline:
+ self._drain_control_queue()
+ if self.terminal or self.process.wait(0):
+ break
+ time.sleep(0.02)
+ if self.process is not None and not self.process.wait(0):
if self.job is not None:
self.job.terminate()
if not self.process.wait(JOB_WAIT_SECONDS):
self.error_code = "E_PROCESS_OWNERSHIP"
+ if self.control_thread is not None and self.control_thread is not threading.current_thread():
+ self.control_thread.join(THREAD_JOIN_SECONDS)
+ self._drain_control_queue()
self.close()
def close(self) -> None:
+ if self.closing:
+ return
+ self.closing = True
if self.input_writer is not None:
- self.input_writer.close()
- self.input_writer = None
+ writer, self.input_writer = self.input_writer, None
+ try:
+ writer.close()
+ except (BrokenPipeError, OSError, ValueError):
+ pass
if self.control_reader is not None:
- self.control_reader.close()
- self.control_reader = None
+ reader, self.control_reader = self.control_reader, None
+ try:
+ reader.close()
+ except (BrokenPipeError, OSError, ValueError):
+ pass
if self.control_thread is not None and self.control_thread is not threading.current_thread():
- self.control_thread.join(THREAD_JOIN_SECONDS)
- self.control_thread = None
+ control_thread, self.control_thread = self.control_thread, None
+ control_thread.join(THREAD_JOIN_SECONDS)
if self.process is not None:
- self.process.close()
- self.process = None
+ process, self.process = self.process, None
+ try:
+ process.close()
+ except BaseException:
+ pass
if self.job is not None:
- self.job.close()
- self.job = None
+ job, self.job = self.job, None
+ try:
+ job.close()
+ except BaseException:
+ pass
if self.cancel_handle:
- close_handles(self.cancel_handle)
- self.cancel_handle = 0
+ cancel_handle, self.cancel_handle = self.cancel_handle, 0
+ try:
+ close_handles(cancel_handle)
+ except BaseException:
+ pass
if self.commit_handle:
- close_handles(self.commit_handle)
- self.commit_handle = 0
+ commit_handle, self.commit_handle = self.commit_handle, 0
+ try:
+ close_handles(commit_handle)
+ except BaseException:
+ pass
-def _valid_control_message(value: dict[str, Any]) -> bool:
+def _valid_control_message(
+ value: dict[str, Any], expected_job: dict[str, Any] | str | None = None,
+ expected_lease: str | None = None,
+) -> bool:
if value.get("schema") != 1 or value.get("type") not in {"ready", "progress", "terminal"}:
return False
if value["type"] == "ready":
@@ -420,9 +1150,26 @@
"E_PLUGIN_BOUNDARY",
}
if value["type"] == "progress":
+ if value.get("phase") == "MEDIA_COMPLETE":
+ if not isinstance(expected_job, dict):
+ return False
+ if set(value) != {
+ "schema", "type", "phase", "progress", "job", "lease_id", "media",
+ }:
+ return False
+ if value["job"] != expected_job or value["lease_id"] != expected_lease:
+ return False
+ try:
+ validate_media_complete_identity(value["media"], expected_job)
+ except ProtocolError:
+ return False
+ return value["progress"] == 100 and not isinstance(value["progress"], bool)
return (
set(value) == {"schema", "type", "phase", "progress"}
- and value["phase"] in {"CHECKING", "DOWNLOADING", "MERGING", "VALIDATING", "PUBLISHING"}
+ and value["phase"] in {
+ "CHECKING", "DOWNLOADING", "MERGING", "VALIDATING", "PUBLISHING",
+ "POSTPROCESS_PENDING",
+ }
and isinstance(value["progress"], int)
and not isinstance(value["progress"], bool)
and 0 <= value["progress"] <= 100
@@ -436,7 +1183,10 @@
"mapping_filename",
"cookie_stream_closed",
}
- if set(value) != allowed or value["phase"] not in {"COMPLETE", "FAILED", "CANCELED"}:
+ keys = set(value)
+ if keys not in (allowed, allowed | {"diagnostic"}) or value["phase"] not in {
+ "COMPLETE", "FAILED", "POSTPROCESS_FAILED", "CANCELED"
+ }:
return False
if not isinstance(value["cookie_stream_closed"], bool):
return False
@@ -445,12 +1195,31 @@
not isinstance(error_code, str) or not re.fullmatch(r"E_[A-Z0-9_]{1,48}", error_code)
):
return False
+ if error_code == COMPLETION_CLOSURE_REQUIRED:
+ return False
+ if "diagnostic" in value:
+ try:
+ if value["phase"] == "POSTPROCESS_FAILED":
+ validate_postprocess_terminal(error_code, value["diagnostic"])
+ else:
+ validate_runtime_diagnostic(value["diagnostic"])
+ except ValueError:
+ return False
+ if value["phase"] not in {"FAILED", "POSTPROCESS_FAILED"}:
+ return False
+ elif value["phase"] == "POSTPROCESS_FAILED":
+ try:
+ validate_postprocess_terminal(error_code, None)
+ except ValueError:
+ return False
if value["phase"] == "COMPLETE":
return (
+ expected_job is not None
+ and
error_code is None
and value["cookie_stream_closed"] is True
- and value["formal_filename"] == f"{TARGET_BVID}.mkv"
- and value["mapping_filename"] == f"{TARGET_BVID}.download.json"
+ and value["formal_filename"] == f"{expected_job['bvid'] if isinstance(expected_job, dict) else expected_job}.mkv"
+ and value["mapping_filename"] == f"{expected_job['bvid'] if isinstance(expected_job, dict) else expected_job}.download.json"
)
return (
value["formal_filename"] is None
@@ -491,6 +1260,7 @@
control = _open_inherited_handle(control_handle, "wb")
worker_input = _open_inherited_handle(input_handle, "rb")
cookie_closed = False
+ media_complete = False
prepared_run: Path | None = None
stage_root: Path | None = None
try:
@@ -502,6 +1272,7 @@
cleanup_run_directory,
fixed_stage_root,
prepare_run_directory,
+ recover_published_task,
run_authenticated_task,
sanitized_environment,
)
@@ -513,6 +1284,7 @@
cleanup_run_directory,
fixed_stage_root,
prepare_run_directory,
+ recover_published_task,
run_authenticated_task,
sanitized_environment,
)
@@ -526,7 +1298,66 @@
os.environ.clear()
os.environ.update(safe_environment)
config = HostConfig.load(config_path)
- stage_root = fixed_stage_root()
+ prepare_payload = read_frame(worker_input, MAX_INPUT_FRAME)
+ if prepare_payload is None:
+ raise CancelRequested()
+ worker_prepare = validate_worker_prepare(strict_json_loads(prepare_payload))
+ job_spec = worker_prepare["job"]
+ lease_id = worker_prepare["lease_id"]
+ recovery_mode = worker_prepare["recovery_mode"]
+
+ def report(
+ phase: str, progress: int, media_identity: dict[str, Any] | None = None,
+ ) -> None:
+ nonlocal media_complete
+ if phase == "MEDIA_COMPLETE":
+ media = validate_media_complete_identity(media_identity, job_spec)
+ write_frame(
+ control,
+ {
+ "schema": 1, "type": "progress", "phase": phase,
+ "progress": progress, "job": job_spec,
+ "lease_id": lease_id, "media": media,
+ },
+ )
+ acknowledgement_payload = read_frame(worker_input, MAX_INPUT_FRAME)
+ if acknowledgement_payload is None:
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ validate_media_complete_ack(
+ strict_json_loads(acknowledgement_payload),
+ job_spec, lease_id, media,
+ )
+ media_complete = True
+ return
+ if media_identity is not None:
+ raise ProtocolError("E_CONTROL")
+ write_frame(control, {
+ "schema": 1, "type": "progress", "phase": phase,
+ "progress": progress,
+ })
+
+ if recovery_mode == _RECOVERY_MODE_EXACT_PAIR:
+ formal, mapping = recover_published_task(
+ config,
+ job_spec,
+ cancel_check=lambda: is_event_set(cancel_handle),
+ report=report,
+ commit_begin=lambda: set_event(commit_handle),
+ )
+ write_frame(
+ control,
+ {
+ "schema": 1,
+ "type": "terminal",
+ "phase": "COMPLETE",
+ "error_code": None,
+ "formal_filename": formal,
+ "mapping_filename": mapping,
+ "cookie_stream_closed": True,
+ },
+ )
+ return 0
+ stage_root = fixed_stage_root(job_spec["bvid"])
prepared_run = prepare_run_directory(stage_root)
write_frame(control, {"schema": 1, "type": "ready", "code": "READY_PLUGIN_DISABLED"})
payload = read_frame(worker_input, MAX_INPUT_FRAME)
@@ -534,12 +1365,8 @@
raise CancelRequested()
start = strict_json_loads(payload)
validate_message(start)
-
- def report(phase: str, progress: int) -> None:
- write_frame(
- control,
- {"schema": 1, "type": "progress", "phase": phase, "progress": progress},
- )
+ if start["job"] != job_spec:
+ raise ProtocolError("E_JOB")
def closure_report(closed: bool) -> None:
nonlocal cookie_closed
@@ -554,6 +1381,7 @@
prepared_run_directory=prepared_run,
commit_begin=lambda: set_event(commit_handle),
closure_report=closure_report,
+ recovery_required=recovery_mode == _RECOVERY_MODE_EXACT_PAIR,
)
prepared_run = None
write_frame(
@@ -571,22 +1399,31 @@
return 0
except BaseException as exc:
error_code = getattr(exc, "code", None)
- phase = "CANCELED" if type(exc).__name__ == "CancelRequested" else "FAILED"
+ diagnostic = getattr(exc, "diagnostic", None)
+ phase = (
+ "CANCELED" if type(exc).__name__ == "CancelRequested"
+ else "POSTPROCESS_FAILED" if media_complete
+ else "FAILED"
+ )
if not isinstance(error_code, str) or not error_code.startswith("E_"):
error_code = None if phase == "CANCELED" else "E_WORKER"
try:
- write_frame(
- control,
- {
- "schema": 1,
- "type": "terminal",
- "phase": phase,
- "error_code": error_code,
- "formal_filename": None,
- "mapping_filename": None,
- "cookie_stream_closed": cookie_closed,
- },
- )
+ terminal = {
+ "schema": 1,
+ "type": "terminal",
+ "phase": phase,
+ "error_code": error_code,
+ "formal_filename": None,
+ "mapping_filename": None,
+ "cookie_stream_closed": cookie_closed,
+ }
+ if phase == "POSTPROCESS_FAILED":
+ validate_postprocess_terminal(error_code, diagnostic)
+ elif diagnostic is not None:
+ validate_runtime_diagnostic(diagnostic)
+ if diagnostic is not None:
+ terminal["diagnostic"] = diagnostic
+ write_frame(control, terminal)
except BaseException:
pass
return 32
@@ -657,11 +1494,15 @@
incoming.put(None)
-def _task_response(task: WorkerTask | None, preflight_error: str | None = None) -> dict[str, Any]:
+def _task_response(
+ task: WorkerTask | None,
+ preflight_error: str | None = None,
+ maintenance: dict[str, Any] | None = None,
+) -> dict[str, Any]:
if task is None:
if preflight_error:
- return safe_response("status", "FAILED", error_code=preflight_error)
- return safe_response("status", "READY")
+ return safe_response("status", "FAILED", error_code=preflight_error, maintenance=maintenance)
+ return safe_response("status", "READY", maintenance=maintenance)
task.poll()
return safe_response(
"status",
@@ -670,7 +1511,91 @@
error_code=task.error_code,
formal_filename=task.formal_filename,
mapping_filename=task.mapping_filename,
+ job=task.job_spec,
+ lease_id=task.lease_id,
+ maintenance=maintenance,
)
+
+
+def _presecret_poll_recovery(
+ queue_store: QueueStore,
+ runtime_config: dict[str, Any],
+ claimed_job: dict[str, Any],
+ claimed_lease: str,
+ maintenance: dict[str, Any] | None = None,
+) -> tuple[WorkerTask | None, dict[str, Any] | None]:
+ """Close an exact published pair before browser, Cookie, or network work."""
+ recovery_binding = queue_store.postprocess_recovery_claim(
+ claimed_job, claimed_lease,
+ )
+ collision_error, recovery_mode = _classify_job_destination(runtime_config, claimed_job)
+ if collision_error is not None:
+ if recovery_binding is None:
+ queue_store.mark_terminal(
+ claimed_job, claimed_lease, int(time.time() * 1000),
+ complete=False, error_code=collision_error,
+ )
+ phase = "FAILED"
+ else:
+ queue_store.mark_postprocess_failed(
+ claimed_job, claimed_lease, int(time.time() * 1000),
+ error_code=collision_error,
+ )
+ phase = "POSTPROCESS_FAILED"
+ return None, safe_response(
+ "poll", phase, error_code=collision_error, job=claimed_job,
+ lease_id=claimed_lease, maintenance=maintenance,
+ )
+ if recovery_mode != _RECOVERY_MODE_EXACT_PAIR:
+ if recovery_binding is not None:
+ queue_store.mark_postprocess_failed(
+ claimed_job, claimed_lease, int(time.time() * 1000),
+ error_code="E_EXISTS",
+ )
+ return None, safe_response(
+ "poll", "POSTPROCESS_FAILED", error_code="E_EXISTS",
+ job=claimed_job, lease_id=claimed_lease, maintenance=maintenance,
+ )
+ return None, None
+
+ task = WorkerTask(queue_store=queue_store)
+ task.postprocess_recovery_binding = recovery_binding
+ try:
+ if recovery_binding is None:
+ queue_store.mark_started(
+ claimed_job, claimed_lease, int(time.time() * 1000),
+ )
+ task.prepare(
+ claimed_job, claimed_lease, None, os.urandom(16).hex(),
+ _RECOVERY_MODE_EXACT_PAIR,
+ )
+ if not task.terminal:
+ task.phase = "FAILED"
+ task.error_code = "E_PLUGIN_BOUNDARY"
+ task.terminal = True
+ task.terminate()
+ response = safe_response(
+ "poll", task.phase, progress=task.progress,
+ error_code=task.error_code, formal_filename=task.formal_filename,
+ mapping_filename=task.mapping_filename, job=claimed_job,
+ lease_id=claimed_lease, maintenance=maintenance,
+ )
+ _record_task_terminal(queue_store, task)
+ return task, response
+ except BaseException as exc:
+ error_code = getattr(exc, "code", None)
+ if not isinstance(error_code, str) or not re.fullmatch(r"E_[A-Z0-9_]{1,48}", error_code):
+ error_code = "E_PLUGIN_BOUNDARY"
+ task.job_spec = dict(claimed_job)
+ task.lease_id = claimed_lease
+ task.phase = "FAILED"
+ task.error_code = error_code
+ task.terminal = True
+ _record_task_terminal(queue_store, task)
+ return task, safe_response(
+ "poll", "FAILED", error_code=error_code, job=claimed_job,
+ lease_id=claimed_lease, maintenance=maintenance,
+ )
def _start_prepared_task(task: WorkerTask | None, message: dict[str, Any]) -> str | None:
@@ -687,6 +1612,203 @@
return None
+def _reject_claimed_job(
+ queue_store: QueueStore | None,
+ claimed_job: dict[str, Any] | None,
+ claimed_lease: str | None,
+ message: dict[str, Any],
+ maintenance: dict[str, Any] | None = None,
+ *,
+ now_ms: int | None = None,
+) -> dict[str, Any]:
+ if (
+ queue_store is None or claimed_job is None or claimed_lease is None
+ or message["job_id"] != claimed_job["job_id"] or message["lease_id"] != claimed_lease
+ ):
+ return safe_response("reject", "FAILED", error_code="E_LEASE", maintenance=maintenance)
+ queue_store.reject_claim(
+ claimed_job,
+ claimed_lease,
+ int(time.time() * 1000) if now_ms is None else now_ms,
+ message["error_code"],
+ message.get("diagnostic"),
+ )
+ return safe_response(
+ "reject", "FAILED", error_code=message["error_code"],
+ job=claimed_job, lease_id=claimed_lease, maintenance=maintenance,
+ )
+
+
+def _foreground_claimed_job(
+ queue_store: QueueStore | None,
+ claimed_job: dict[str, Any] | None,
+ claimed_lease: str | None,
+ message: dict[str, Any],
+ maintenance: dict[str, Any] | None,
+ *,
+ attempted: bool,
+ parent_window: int,
+ now_ms: int | None = None,
+ activator: Any = _foreground_chrome_window,
+) -> tuple[dict[str, Any], bool]:
+ """Execute at most one claim-bound, pre-secret foreground request."""
+
+ if (
+ queue_store is None or claimed_job is None or claimed_lease is None
+ or message["job_id"] != claimed_job["job_id"]
+ or message["lease_id"] != claimed_lease
+ ):
+ error_code = "E_LEASE"
+ consumed = attempted
+ elif attempted:
+ error_code = "E_FOREGROUND_REPLAY"
+ consumed = True
+ else:
+ consumed = True
+ try:
+ queue_store.assert_claim(
+ claimed_job, claimed_lease,
+ int(time.time() * 1000) if now_ms is None else now_ms,
+ )
+ error_code = activator(
+ claimed_job["canonical_url"], message["window_bounds"], parent_window,
+ )
+ except ProtocolError:
+ error_code = "E_LEASE"
+ except BaseException:
+ error_code = "E_FOREGROUND_PLATFORM_UNSUPPORTED"
+ if error_code is not None and error_code not in _FOREGROUND_ERROR_CODES:
+ error_code = "E_FOREGROUND_PLATFORM_UNSUPPORTED"
+ return safe_response(
+ "foreground", "READY" if error_code is None else "FAILED",
+ error_code=error_code, job=claimed_job, lease_id=claimed_lease,
+ maintenance=maintenance,
+ ), consumed
+
+
+def _abort_prepared_task(
+ queue_store: QueueStore | None,
+ task: WorkerTask | None,
+ claimed_job: dict[str, Any] | None,
+ claimed_lease: str | None,
+ message: dict[str, Any],
+ maintenance: dict[str, Any] | None = None,
+ *,
+ now_ms: int | None = None,
+) -> dict[str, Any]:
+ """Close one prepared worker and persist the extension's safe pre-start terminal."""
+ if (
+ queue_store is None or task is None or claimed_job is None or claimed_lease is None
+ or message["job_id"] != claimed_job["job_id"]
+ or message["lease_id"] != claimed_lease
+ or message["prepare_id"] != task.prepare_id
+ or task.job_spec != claimed_job or task.lease_id != claimed_lease
+ or not task.prepared or task.secret_started or task.terminal
+ ):
+ return safe_response(
+ "abort_prepare", "FAILED", error_code="E_PREPARE",
+ prepare_id=message.get("prepare_id"), maintenance=maintenance,
+ )
+ # Terminate while the task is still nonterminal so the owned worker receives
+ # cancellation/EOF and cannot survive the durable queue terminal.
+ task.terminate()
+ earlier_error = (
+ task.error_code
+ if task.phase == "FAILED" and isinstance(task.error_code, str)
+ and re.fullmatch(r"E_[A-Z0-9_]{1,48}", task.error_code)
+ else None
+ )
+ task.phase = "FAILED"
+ task.error_code = earlier_error or COOKIE_ACCESS_TERMINAL_CODES[message["error_reason"]]
+ task.terminal = True
+ if not _record_task_terminal(
+ queue_store,
+ task,
+ now_ms=int(time.time() * 1000) if now_ms is None else now_ms,
+ ):
+ return safe_response(
+ "abort_prepare", "FAILED", error_code="E_PREPARE",
+ prepare_id=message["prepare_id"], maintenance=maintenance,
+ )
+ return safe_response(
+ "abort_prepare", "FAILED", error_code=earlier_error or message["error_code"],
+ prepare_id=message["prepare_id"], job=claimed_job,
+ lease_id=claimed_lease, maintenance=maintenance,
+ )
+
+
+def _record_task_terminal(
+ queue_store: QueueStore | None,
+ task: WorkerTask | None,
+ *,
+ fallback_error: str | None = None,
+ now_ms: int | None = None,
+) -> bool:
+ """Persist one task terminal, preserving an earlier deterministic worker error.
+
+ A Host/port disconnect is only a fallback. Control messages already queued
+ before teardown win; teardown-induced CANCELED/E_WORKER_EXIT states do not.
+ """
+ if (
+ queue_store is None or task is None or task.queue_terminal_recorded
+ or task.job_spec is None or task.lease_id is None
+ ):
+ return False
+ task.poll()
+ terminal_before_shutdown = task.terminal
+ if not task.terminal and fallback_error is not None:
+ task.terminate()
+ task.poll()
+ if (
+ not task.terminal
+ or task.phase == "CANCELED"
+ or task.error_code in {None, "E_WORKER_EXIT"}
+ ):
+ task.phase = "FAILED"
+ task.error_code = fallback_error
+ task.terminal = True
+ if not task.terminal:
+ return False
+ task_media_complete = bool(getattr(task, "media_complete", False))
+ postprocess_recovery = getattr(task, "postprocess_recovery_binding", None) is not None
+ if (task_media_complete or postprocess_recovery) and task.phase != "COMPLETE":
+ task.phase = "POSTPROCESS_FAILED"
+ complete = task.phase == "COMPLETE"
+ if complete:
+ error_code = None
+ elif task.phase == "CANCELED" and terminal_before_shutdown:
+ error_code = "E_CANCEL"
+ elif isinstance(task.error_code, str) and re.fullmatch(r"E_[A-Z0-9_]{1,48}", task.error_code):
+ error_code = task.error_code
+ else:
+ error_code = fallback_error or "E_WORKER"
+ terminal_now = int(time.time() * 1000) if now_ms is None else now_ms
+ if task_media_complete:
+ if task.media_identity is None:
+ raise ProtocolError("E_QUEUE_STATE")
+ queue_store.mark_media_complete(
+ task.job_spec, task.lease_id, terminal_now, task.media_identity,
+ )
+ if task.phase == "POSTPROCESS_FAILED":
+ queue_store.mark_postprocess_failed(
+ task.job_spec, task.lease_id, terminal_now,
+ error_code=error_code,
+ diagnostic=getattr(task, "diagnostic", None),
+ )
+ else:
+ queue_store.mark_terminal(
+ task.job_spec,
+ task.lease_id,
+ terminal_now,
+ complete=complete,
+ error_code=error_code,
+ diagnostic=getattr(task, "diagnostic", None),
+ )
+ task.queue_terminal_recorded = True
+ task.close()
+ return True
+
+
def broker_main(arguments: list[str]) -> int:
validate_origin_argv(arguments, EXPECTED_ORIGIN)
protocol_output = _duplicate_protocol_output()
@@ -699,70 +1821,215 @@
)
reader.start()
task: WorkerTask | None = None
+ claimed_job: dict[str, Any] | None = None
+ claimed_lease: str | None = None
+ foreground_attempted = False
hello_complete = False
+ current_extension_build: str | None = None
preflight_error = preflight_configuration(_config_path())
+ runtime_config: dict[str, Any] | None = None
+ queue_store: QueueStore | None = None
+ reload_store: ReloadStore | None = None
+ current_maintenance = maintenance_state()
+ if preflight_error is None:
+ runtime_config = load_runtime_configuration(_config_path())
+ queue_store = _queue_store(runtime_config)
+ reload_store = ReloadStore(Path(runtime_config["reload_state_path"]), runtime_config["reload_generation"])
try:
while True:
if task is not None:
- task.poll()
+ _record_task_terminal(queue_store, task)
try:
message = incoming.get(timeout=0.1)
except queue.Empty:
continue
if message is None:
- if task is not None and not task.terminal:
- task.terminate()
+ _record_task_terminal(queue_store, task, fallback_error="E_HOST_DISCONNECT")
return 0
message_type = message["type"]
if not hello_complete:
if message_type != "hello":
raise ProtocolError()
hello_complete = True
+ current_extension_build = message["extension_build"]
+ if reload_store is not None:
+ current_maintenance = reload_store.status(current_extension_build, int(time.time() * 1000))
write_frame(
protocol_output,
safe_response(
"hello",
"FAILED" if preflight_error else "READY",
error_code=preflight_error,
+ maintenance=current_maintenance,
),
)
continue
if message_type == "hello":
raise ProtocolError()
- if message_type == "status":
- current_preflight = preflight_configuration(_config_path())
- write_frame(protocol_output, _task_response(task, current_preflight))
- elif message_type == "start":
- start_error = _start_prepared_task(task, message)
- if start_error is None:
- write_frame(protocol_output, _task_response(task))
- else:
- write_frame(protocol_output, safe_response("start", "FAILED", error_code=start_error))
- elif message_type == "prepare":
- preflight_error = preflight_configuration(_config_path())
- if preflight_error:
- write_frame(protocol_output, safe_response("prepare", "FAILED", error_code=preflight_error, prepare_id=message["prepare_id"]))
+ if message_type == "reload_begin":
+ if reload_store is None or current_extension_build is None:
+ write_frame(protocol_output, safe_response("reload_begin", "FAILED", error_code="E_RELOAD", maintenance=current_maintenance))
+ continue
+ reload_store.begin(current_extension_build, message["reload_token"], int(time.time() * 1000))
+ write_frame(protocol_output, safe_response("reload_begin", "RELOAD_REQUIRED", maintenance=current_maintenance))
+ elif current_maintenance.get("reload_required"):
+ write_frame(protocol_output, safe_response(message_type, "FAILED", error_code="E_RELOAD_REQUIRED", maintenance=current_maintenance))
+ elif message_type == "poll":
+ if queue_store is None:
+ write_frame(protocol_output, safe_response("poll", "FAILED", error_code=preflight_error or "E_CONFIG", maintenance=current_maintenance))
continue
if task is not None and not task.terminal:
- write_frame(protocol_output, safe_response("prepare", "FAILED", error_code="E_BUSY", prepare_id=message["prepare_id"]))
+ write_frame(protocol_output, _task_response(task, maintenance=current_maintenance))
+ continue
+ claimed = queue_store.claim_next(int(time.time() * 1000))
+ if claimed is None:
+ claimed_job = None
+ claimed_lease = None
+ foreground_attempted = False
+ write_frame(protocol_output, safe_response("poll", "IDLE", maintenance=current_maintenance))
+ else:
+ claimed_job, claimed_lease = claimed
+ foreground_attempted = False
+ preflight_error = preflight_configuration(_config_path())
+ if preflight_error is not None or runtime_config is None:
+ recovery_binding = queue_store.postprocess_recovery_claim(
+ claimed_job, claimed_lease,
+ )
+ terminal_error = preflight_error or "E_CONFIG"
+ if recovery_binding is None:
+ queue_store.mark_terminal(
+ claimed_job, claimed_lease, int(time.time() * 1000),
+ complete=False, error_code=terminal_error,
+ )
+ terminal_phase = "FAILED"
+ else:
+ queue_store.mark_postprocess_failed(
+ claimed_job, claimed_lease, int(time.time() * 1000),
+ error_code=terminal_error,
+ )
+ terminal_phase = "POSTPROCESS_FAILED"
+ write_frame(protocol_output, safe_response(
+ "poll", terminal_phase, error_code=terminal_error,
+ job=claimed_job, lease_id=claimed_lease,
+ maintenance=current_maintenance,
+ ))
+ continue
+ task, recovery_response = _presecret_poll_recovery(
+ queue_store, runtime_config, claimed_job, claimed_lease,
+ current_maintenance,
+ )
+ if recovery_response is not None:
+ write_frame(protocol_output, recovery_response)
+ else:
+ write_frame(protocol_output, safe_response(
+ "poll", "READY", job=claimed_job, lease_id=claimed_lease,
+ maintenance=current_maintenance,
+ ))
+ elif message_type == "foreground":
+ if task is not None and not task.terminal:
+ foreground_response = safe_response(
+ "foreground", "FAILED", error_code="E_LEASE",
+ job=claimed_job, lease_id=claimed_lease,
+ maintenance=current_maintenance,
+ )
+ else:
+ foreground_response, foreground_attempted = _foreground_claimed_job(
+ queue_store, claimed_job, claimed_lease, message, current_maintenance,
+ attempted=foreground_attempted,
+ parent_window=_native_parent_window(arguments),
+ )
+ write_frame(protocol_output, foreground_response)
+ elif message_type == "reject":
+ rejected = _reject_claimed_job(
+ queue_store, claimed_job, claimed_lease, message, current_maintenance
+ )
+ write_frame(protocol_output, rejected)
+ if rejected["error_code"] != "E_LEASE":
+ claimed_job = None
+ claimed_lease = None
+ elif message_type == "abort_prepare":
+ aborted = _abort_prepared_task(
+ queue_store, task, claimed_job, claimed_lease, message, current_maintenance
+ )
+ write_frame(protocol_output, aborted)
+ if aborted["error_code"] != "E_PREPARE":
+ claimed_job = None
+ claimed_lease = None
+ elif message_type == "status":
+ if task is None or message["job_id"] != (task.job_spec or {}).get("job_id") or message["lease_id"] != task.lease_id:
+ write_frame(protocol_output, safe_response("status", "FAILED", error_code="E_LEASE", maintenance=current_maintenance))
+ continue
+ current_preflight = preflight_configuration(_config_path())
+ write_frame(protocol_output, _task_response(task, current_preflight, current_maintenance))
+ elif message_type == "start":
+ if queue_store is None or message["job"] != claimed_job or message["lease_id"] != claimed_lease:
+ write_frame(protocol_output, safe_response("start", "FAILED", error_code="E_LEASE", job=claimed_job, lease_id=claimed_lease, maintenance=current_maintenance))
+ continue
+ queue_store.mark_started(message["job"], message["lease_id"], int(time.time() * 1000))
+ start_error = _start_prepared_task(task, message)
+ if start_error is None:
+ write_frame(protocol_output, _task_response(task, maintenance=current_maintenance))
+ else:
+ queue_store.mark_terminal(message["job"], message["lease_id"], int(time.time() * 1000), complete=False, error_code=start_error)
+ write_frame(protocol_output, safe_response("start", "FAILED", error_code=start_error, job=claimed_job, lease_id=claimed_lease, maintenance=current_maintenance))
+ elif message_type == "prepare":
+ preflight_error = preflight_configuration(_config_path())
+ if message["job"] != claimed_job or message["lease_id"] != claimed_lease or queue_store is None or runtime_config is None:
+ write_frame(protocol_output, safe_response("prepare", "FAILED", error_code="E_LEASE", prepare_id=message["prepare_id"], job=claimed_job, lease_id=claimed_lease, maintenance=current_maintenance))
+ continue
+ queue_store.assert_claim(message["job"], message["lease_id"], int(time.time() * 1000))
+ recovery_mode = _RECOVERY_MODE_NONE
+ if preflight_error is None:
+ preflight_error, recovery_mode = _classify_job_destination(runtime_config, message["job"])
+ if preflight_error:
+ queue_store.mark_terminal(message["job"], message["lease_id"], int(time.time() * 1000), complete=False, error_code=preflight_error)
+ write_frame(protocol_output, safe_response("prepare", "FAILED", error_code=preflight_error, prepare_id=message["prepare_id"], job=claimed_job, lease_id=claimed_lease, maintenance=current_maintenance))
+ continue
+ if task is not None and not task.terminal:
+ write_frame(protocol_output, safe_response("prepare", "FAILED", error_code="E_BUSY", prepare_id=message["prepare_id"], job=claimed_job, lease_id=claimed_lease, maintenance=current_maintenance))
continue
if task is not None:
task.close()
- task = WorkerTask()
+ task = WorkerTask(queue_store=queue_store)
try:
- task.prepare(message["page_proof"], message["prepare_id"])
- write_frame(protocol_output, safe_response("prepare", "READY", prepare_id=message["prepare_id"]))
- except BaseException:
- write_frame(protocol_output, safe_response("prepare", "FAILED", error_code="E_PLUGIN_BOUNDARY", prepare_id=message["prepare_id"]))
+ if recovery_mode == _RECOVERY_MODE_EXACT_PAIR:
+ queue_store.mark_started(
+ message["job"], message["lease_id"], int(time.time() * 1000)
+ )
+ task.prepare(message["job"], message["lease_id"], message["page_proof"], message["prepare_id"], recovery_mode)
+ if task.terminal:
+ response = safe_response(
+ "prepare", task.phase, progress=task.progress,
+ error_code=task.error_code,
+ formal_filename=task.formal_filename,
+ mapping_filename=task.mapping_filename,
+ prepare_id=message["prepare_id"], job=claimed_job,
+ lease_id=claimed_lease, maintenance=current_maintenance,
+ )
+ _record_task_terminal(queue_store, task)
+ write_frame(protocol_output, response)
+ else:
+ write_frame(protocol_output, safe_response("prepare", "READY", prepare_id=message["prepare_id"], job=claimed_job, lease_id=claimed_lease, maintenance=current_maintenance))
+ except BaseException as exc:
+ error_code = getattr(exc, "code", None)
+ if not isinstance(error_code, str) or not re.fullmatch(r"E_[A-Z0-9_]{1,48}", error_code):
+ error_code = "E_PLUGIN_BOUNDARY"
+ task.phase = "FAILED"
+ task.error_code = error_code
+ task.terminal = True
+ _record_task_terminal(queue_store, task)
+ write_frame(protocol_output, safe_response("prepare", "FAILED", error_code=error_code, prepare_id=message["prepare_id"], job=claimed_job, lease_id=claimed_lease, maintenance=current_maintenance))
elif message_type == "cancel":
- canceled = task is not None and task.cancel(message["task_nonce"])
+ canceled = task is not None and message["job_id"] == (task.job_spec or {}).get("job_id") and message["lease_id"] == task.lease_id and task.cancel(message["task_nonce"])
write_frame(
protocol_output,
- safe_response("cancel", "CANCELED" if canceled else "FAILED", error_code=None if canceled else "E_CANCEL"),
+ safe_response("cancel", "CANCELED" if canceled else "FAILED", error_code=None if canceled else "E_CANCEL", job=None if task is None else task.job_spec, lease_id=None if task is None else task.lease_id, maintenance=current_maintenance),
)
except BaseException:
- if task is not None and not task.terminal:
- task.terminate()
+ try:
+ _record_task_terminal(queue_store, task, fallback_error="E_HOST")
+ except BaseException:
+ pass
try:
write_frame(protocol_output, safe_response("error", "FAILED", error_code="E_PROTOCOL"))
except BaseException:
diff --git a/dev/project-dev/bili_authenticated_extension/protocol.py b/dev/project-dev/bili_authenticated_extension/protocol.py
index de40d4e..3b5ffe3 100644
--- a/dev/project-dev/bili_authenticated_extension/protocol.py
+++ b/dev/project-dev/bili_authenticated_extension/protocol.py
@@ -1,7 +1,4 @@
-"""Strict Native Messaging wire validation.
-
-This module is stdlib-only and contains no downloader imports.
-"""
+"""Strict generic Native Messaging wire validation (stdlib-only)."""
from __future__ import annotations
@@ -10,33 +7,44 @@
import struct
import time
import unicodedata
+from datetime import datetime
from typing import Any, BinaryIO, Iterable
from .constants import (
- CANONICAL_URL,
- DURATION_TOLERANCE_MS,
- EXPECTED_DURATION_MS,
+ AUDIT_ID_RE,
+ COOKIE_ACCESS_REASONS,
EXTENSION_BUILD,
HOST_BUILD,
+ JOB_ID_RE,
+ HANDOFF_ID_RE,
+ MESSAGE_ID_RE,
+ PRESTART_ABORT_CODES,
MAX_COOKIE_COUNT,
MAX_INPUT_FRAME,
MAX_OUTPUT_FRAME,
MAX_SAFE_INTEGER,
SCHEMA_VERSION,
- TARGET_BVID,
- TARGET_PATH,
+ canonical_url,
+ duration_tolerance_ms,
+ stable_job_id,
+ stable_successor_job_id,
+ UPPER_SHA256_RE,
+ target_path,
+ validate_bvid,
+ validate_creator_uid,
+ validate_prepareless_terminal,
)
_NONCE_RE = re.compile(r"[0-9a-f]{32}\Z")
_PREPARE_ID_RE = re.compile(r"[0-9a-f]{32}\Z")
+_LEASE_ID_RE = re.compile(r"[0-9a-f]{32}\Z")
+_RELOAD_TOKEN_RE = re.compile(r"[0-9a-f]{32}\Z")
_STORE_RE = re.compile(r"[0-9]{1,8}\Z")
_PARENT_RE = re.compile(r"--parent-window=[0-9]+\Z")
_COOKIE_SAME_SITE = {"no_restriction", "lax", "strict", "unspecified"}
class ProtocolError(Exception):
- """A fixed-code input rejection that is safe to expose."""
-
def __init__(self, code: str = "E_PROTOCOL") -> None:
super().__init__(code)
self.code = code
@@ -57,9 +65,8 @@
def strict_json_loads(payload: bytes) -> dict[str, Any]:
try:
- text = payload.decode("utf-8", errors="strict")
value = json.loads(
- text,
+ payload.decode("utf-8", errors="strict"),
object_pairs_hook=_unique_object,
parse_constant=_reject_constant,
)
@@ -74,11 +81,7 @@
def encode_json(value: dict[str, Any], limit: int = MAX_OUTPUT_FRAME) -> bytes:
payload = json.dumps(
- value,
- ensure_ascii=True,
- allow_nan=False,
- separators=(",", ":"),
- sort_keys=True,
+ value, ensure_ascii=True, allow_nan=False, separators=(",", ":"), sort_keys=True
).encode("utf-8")
if len(payload) > limit:
raise ProtocolError()
@@ -113,9 +116,7 @@
def _integer(value: Any, minimum: int, maximum: int) -> int:
- if isinstance(value, bool) or not isinstance(value, int):
- raise ProtocolError()
- if not minimum <= value <= maximum:
+ if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
raise ProtocolError()
return value
@@ -137,6 +138,14 @@
return value
+def _schema_type(value: dict[str, Any], message_type: str, keys: Iterable[str]) -> None:
+ _exact_keys(value, keys)
+ if _integer(value.get("schema"), SCHEMA_VERSION, SCHEMA_VERSION) != SCHEMA_VERSION:
+ raise ProtocolError()
+ if value.get("type") != message_type:
+ raise ProtocolError()
+
+
def validate_origin_argv(arguments: list[str], expected_origin: str) -> None:
if len(arguments) not in (1, 2) or arguments[0] != expected_origin:
raise ProtocolError("E_ORIGIN")
@@ -144,76 +153,294 @@
raise ProtocolError("E_ORIGIN")
-def _validate_common(value: dict[str, Any], message_type: str, keys: Iterable[str]) -> None:
- _exact_keys(value, keys)
- if _integer(value.get("schema"), SCHEMA_VERSION, SCHEMA_VERSION) != SCHEMA_VERSION:
- raise ProtocolError()
- if value.get("type") != message_type or value.get("target") != TARGET_BVID:
- raise ProtocolError()
+def validate_job(value: Any) -> dict[str, Any]:
+ if not isinstance(value, dict):
+ raise ProtocolError("E_JOB")
+ base_keys = {"job_id", "bvid", "creator_uid", "canonical_url", "expected_duration_ms", "published_at", "title"}
+ successor = "lineage" in value
+ _exact_keys(value, base_keys | ({"lineage"} if successor else set()))
+ try:
+ bvid = validate_bvid(value["bvid"])
+ creator = validate_creator_uid(value["creator_uid"])
+ except ValueError as exc:
+ raise ProtocolError("E_JOB") from exc
+ if not isinstance(value["job_id"], str) or not JOB_ID_RE.fullmatch(value["job_id"]):
+ raise ProtocolError("E_JOB")
+ if value["canonical_url"] != canonical_url(bvid):
+ raise ProtocolError("E_JOB")
+ if successor:
+ lineage = value["lineage"]
+ lineage_keys = {
+ "predecessor_job_id", "retry_generation", "predecessor_terminal_error_code",
+ "authorization_message_id", "authorization_handoff_id", "authorization_sha256",
+ "repair_review_result_message_id", "repair_audit_id", "repair_audit_bytes",
+ "repair_audit_sha256",
+ }
+ if not isinstance(lineage, dict):
+ raise ProtocolError("E_JOB")
+ _exact_keys(lineage, lineage_keys)
+ try:
+ expected_job_id = stable_successor_job_id(
+ creator, bvid, lineage["predecessor_job_id"], lineage["retry_generation"],
+ lineage["predecessor_terminal_error_code"], lineage["authorization_message_id"],
+ lineage["authorization_handoff_id"], lineage["authorization_sha256"],
+ lineage["repair_review_result_message_id"], lineage["repair_audit_id"],
+ lineage["repair_audit_bytes"], lineage["repair_audit_sha256"],
+ )
+ except (KeyError, ValueError) as exc:
+ raise ProtocolError("E_JOB") from exc
+ if (
+ not JOB_ID_RE.fullmatch(lineage["predecessor_job_id"])
+ or not MESSAGE_ID_RE.fullmatch(lineage["authorization_message_id"])
+ or not HANDOFF_ID_RE.fullmatch(lineage["authorization_handoff_id"])
+ or not UPPER_SHA256_RE.fullmatch(lineage["authorization_sha256"])
+ or not MESSAGE_ID_RE.fullmatch(lineage["repair_review_result_message_id"])
+ or not AUDIT_ID_RE.fullmatch(lineage["repair_audit_id"])
+ or not UPPER_SHA256_RE.fullmatch(lineage["repair_audit_sha256"])
+ ):
+ raise ProtocolError("E_JOB")
+ else:
+ expected_job_id = stable_job_id(creator, bvid)
+ if value["job_id"] != expected_job_id:
+ raise ProtocolError("E_JOB")
+ _integer(value["expected_duration_ms"], 1_000, 86_400_000)
+ title = _safe_string(value["title"], 1, 600)
+ if not title.strip():
+ raise ProtocolError("E_JOB")
+ published = _safe_string(value["published_at"], 1, 64)
+ try:
+ parsed = datetime.fromisoformat(published)
+ except ValueError as exc:
+ raise ProtocolError("E_JOB") from exc
+ if parsed.utcoffset() is None:
+ raise ProtocolError("E_JOB")
+ return value
def validate_hello(value: dict[str, Any]) -> dict[str, Any]:
- _validate_common(value, "hello", {"schema", "type", "extension_build", "target"})
- if value["extension_build"] != EXTENSION_BUILD:
- raise ProtocolError("E_BUILD")
+ _schema_type(value, "hello", {"schema", "type", "extension_build"})
+ _safe_string(value["extension_build"], 1, 128)
+ return value
+
+
+def validate_poll(value: dict[str, Any]) -> dict[str, Any]:
+ _schema_type(value, "poll", {"schema", "type"})
+ return value
+
+
+def validate_foreground(value: dict[str, Any]) -> dict[str, Any]:
+ """Validate one pre-secret request to foreground the bound Chrome window.
+
+ Only opaque Chrome identifiers and screen geometry cross the wire. The
+ Native Host derives the canonical target URL from the already-claimed job;
+ callers cannot supply an arbitrary command, executable, URL, title, or
+ profile path.
+ """
+
+ _schema_type(
+ value,
+ "foreground",
+ {"schema", "type", "job_id", "lease_id", "tab_id", "window_id", "window_bounds"},
+ )
+ if not isinstance(value["job_id"], str) or not JOB_ID_RE.fullmatch(value["job_id"]):
+ raise ProtocolError("E_JOB")
+ _lease(value["lease_id"])
+ _integer(value["tab_id"], 0, MAX_SAFE_INTEGER)
+ _integer(value["window_id"], 0, MAX_SAFE_INTEGER)
+ bounds = value["window_bounds"]
+ if not isinstance(bounds, dict):
+ raise ProtocolError("E_FOREGROUND")
+ _exact_keys(bounds, {"left", "top", "width", "height"})
+ _integer(bounds["left"], -1_000_000, 1_000_000)
+ _integer(bounds["top"], -1_000_000, 1_000_000)
+ _integer(bounds["width"], 1, 1_000_000)
+ _integer(bounds["height"], 1, 1_000_000)
+ return value
+
+
+def validate_reject(value: dict[str, Any]) -> dict[str, Any]:
+ try:
+ _schema_type(
+ value,
+ "reject",
+ {"schema", "type", "job_id", "lease_id", "error_code", "diagnostic"},
+ )
+ except ProtocolError as exc:
+ raise ProtocolError("E_REJECT") from exc
+ if not isinstance(value["job_id"], str) or not JOB_ID_RE.fullmatch(value["job_id"]):
+ raise ProtocolError("E_JOB")
+ _lease(value["lease_id"])
+ try:
+ validate_prepareless_terminal(value["error_code"], value["diagnostic"])
+ except ValueError as exc:
+ raise ProtocolError("E_REJECT") from exc
+ return value
+
+
+def validate_abort_prepare(value: dict[str, Any]) -> dict[str, Any]:
+ _schema_type(
+ value,
+ "abort_prepare",
+ {"schema", "type", "job_id", "lease_id", "prepare_id", "error_code", "error_reason"},
+ )
+ if not isinstance(value["job_id"], str) or not JOB_ID_RE.fullmatch(value["job_id"]):
+ raise ProtocolError("E_JOB")
+ _lease(value["lease_id"])
+ if not isinstance(value["prepare_id"], str) or not _PREPARE_ID_RE.fullmatch(value["prepare_id"]):
+ raise ProtocolError("E_PREPARE")
+ if value["error_code"] not in PRESTART_ABORT_CODES:
+ raise ProtocolError("E_PREPARE")
+ if value["error_reason"] not in COOKIE_ACCESS_REASONS:
+ raise ProtocolError("E_PREPARE")
+ return value
+
+
+def validate_reload_begin(value: dict[str, Any]) -> dict[str, Any]:
+ _schema_type(value, "reload_begin", {"schema", "type", "extension_build", "reload_token"})
+ _safe_string(value["extension_build"], 1, 128)
+ if not isinstance(value["reload_token"], str) or not _RELOAD_TOKEN_RE.fullmatch(value["reload_token"]):
+ raise ProtocolError("E_RELOAD")
+ return value
+
+
+def _lease(value: Any) -> str:
+ if not isinstance(value, str) or not _LEASE_ID_RE.fullmatch(value):
+ raise ProtocolError("E_LEASE")
return value
def validate_status(value: dict[str, Any]) -> dict[str, Any]:
- _validate_common(value, "status", {"schema", "type", "target"})
+ _schema_type(value, "status", {"schema", "type", "job_id", "lease_id"})
+ if not isinstance(value["job_id"], str) or not JOB_ID_RE.fullmatch(value["job_id"]):
+ raise ProtocolError("E_JOB")
+ _lease(value["lease_id"])
return value
def validate_cancel(value: dict[str, Any]) -> dict[str, Any]:
- _validate_common(value, "cancel", {"schema", "type", "target", "task_nonce"})
+ _schema_type(value, "cancel", {"schema", "type", "job_id", "lease_id", "task_nonce"})
+ if not isinstance(value["job_id"], str) or not JOB_ID_RE.fullmatch(value["job_id"]):
+ raise ProtocolError("E_JOB")
+ _lease(value["lease_id"])
if not isinstance(value["task_nonce"], str) or not _NONCE_RE.fullmatch(value["task_nonce"]):
raise ProtocolError()
return value
def validate_prepare(value: dict[str, Any], now_ms: int | None = None) -> dict[str, Any]:
- _validate_common(
+ _schema_type(
value,
"prepare",
- {"schema", "type", "extension_build", "target", "prepare_id", "page_proof"},
+ {"schema", "type", "extension_build", "lease_id", "prepare_id", "job", "page_proof"},
)
if value["extension_build"] != EXTENSION_BUILD:
raise ProtocolError("E_BUILD")
+ _lease(value["lease_id"])
if not isinstance(value["prepare_id"], str) or not _PREPARE_ID_RE.fullmatch(value["prepare_id"]):
raise ProtocolError("E_PREPARE")
- validate_page_proof(value["page_proof"], now_ms=now_ms)
+ job = validate_job(value["job"])
+ validate_page_proof(value["page_proof"], job, now_ms=now_ms)
return value
-def validate_page_proof(value: Any, now_ms: int | None = None) -> dict[str, Any]:
+def validate_worker_prepare(value: dict[str, Any]) -> dict[str, Any]:
+ _schema_type(
+ value, "worker_prepare", {"schema", "type", "job", "lease_id", "recovery_mode"}
+ )
+ validate_job(value["job"])
+ _lease(value["lease_id"])
+ if value["recovery_mode"] not in {"NONE", "EXACT_PUBLISHED_PAIR"}:
+ raise ProtocolError("E_PREPARE")
+ return value
+
+
+_MEDIA_COMPLETE_KEYS = {
+ "formal_filename", "mapping_filename", "media_bytes", "media_sha256",
+ "mapping_bytes", "mapping_sha256", "duration_milliseconds",
+ "video_codec", "audio_codec",
+}
+
+
+def validate_media_complete_identity(value: Any, job: dict[str, Any]) -> dict[str, Any]:
+ """Validate the exact, non-secret identity durably ACKed before postprocess."""
+
+ if not isinstance(job, dict):
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ try:
+ bvid = validate_bvid(job.get("bvid"))
+ except (TypeError, ValueError) as exc:
+ raise ProtocolError("E_MEDIA_COMPLETE") from exc
+ if not isinstance(value, dict):
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ _exact_keys(value, _MEDIA_COMPLETE_KEYS)
+ if (
+ value["formal_filename"] != f"{bvid}.mkv"
+ or value["mapping_filename"] != f"{bvid}.download.json"
+ ):
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ _integer(value["media_bytes"], 1, MAX_SAFE_INTEGER)
+ _integer(value["mapping_bytes"], 1, 65_536)
+ _integer(value["duration_milliseconds"], 1, MAX_SAFE_INTEGER)
+ for name in ("media_sha256", "mapping_sha256"):
+ if not isinstance(value[name], str) or UPPER_SHA256_RE.fullmatch(value[name]) is None:
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ for name in ("video_codec", "audio_codec"):
+ if (
+ not isinstance(value[name], str)
+ or re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", value[name]) is None
+ ):
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ return value
+
+
+def validate_media_complete_ack(
+ value: dict[str, Any], job: dict[str, Any], lease_id: str,
+ media: dict[str, Any],
+) -> dict[str, Any]:
+ """Validate the Host's typed durable-state acknowledgement."""
+
+ _schema_type(
+ value, "media_complete_ack",
+ {"schema", "type", "job_id", "lease_id", "media"},
+ )
+ validated_job = validate_job(job)
+ _lease(value["lease_id"])
+ if value["job_id"] != validated_job["job_id"] or value["lease_id"] != lease_id:
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ validated_media = validate_media_complete_identity(value["media"], validated_job)
+ if validated_media != media:
+ raise ProtocolError("E_MEDIA_COMPLETE")
+ return value
+
+
+def validate_page_proof(value: Any, job: dict[str, Any], now_ms: int | None = None) -> dict[str, Any]:
if not isinstance(value, dict):
raise ProtocolError()
_exact_keys(
value,
{
- "target",
- "canonical_url",
- "task_nonce",
- "observed_at_unix_ms",
- "observed_duration_ms",
- "video_width",
- "video_height",
- "ready_state",
- "eme_present",
+ "job_id", "bvid", "creator_uid", "canonical_url", "task_nonce",
+ "observed_at_unix_ms", "observed_duration_ms", "video_width",
+ "video_height", "ready_state", "eme_present", "metadata_source",
},
)
- if value["target"] != TARGET_BVID or value["canonical_url"] != CANONICAL_URL:
- raise ProtocolError()
+ for key in ("job_id", "bvid", "creator_uid", "canonical_url"):
+ if value[key] != job[key]:
+ raise ProtocolError("E_PAGE_PROOF")
if not isinstance(value["task_nonce"], str) or not _NONCE_RE.fullmatch(value["task_nonce"]):
raise ProtocolError()
observed_at = _integer(value["observed_at_unix_ms"], 1, MAX_SAFE_INTEGER)
observed_duration = _integer(value["observed_duration_ms"], 1, MAX_SAFE_INTEGER)
_integer(value["video_width"], 1, 7680)
_integer(value["video_height"], 1, 4320)
- _integer(value["ready_state"], 1, 4)
+ ready_state = _integer(value["ready_state"], 1, 4)
+ if value["metadata_source"] not in {"HTML_MEDIA_ELEMENT", "BILIBILI_INITIAL_STATE"}:
+ raise ProtocolError("E_PAGE_PROOF")
+ if value["metadata_source"] == "BILIBILI_INITIAL_STATE" and ready_state != 1:
+ raise ProtocolError("E_PAGE_PROOF")
if _boolean(value["eme_present"]):
raise ProtocolError("E_DRM")
- if abs(observed_duration - EXPECTED_DURATION_MS) > DURATION_TOLERANCE_MS:
+ if abs(observed_duration - job["expected_duration_ms"]) > duration_tolerance_ms(job["expected_duration_ms"]):
raise ProtocolError("E_DURATION")
current = int(time.time() * 1000) if now_ms is None else now_ms
if observed_at < current - 60_000 or observed_at > current + 5_000:
@@ -221,98 +448,67 @@
return value
-def _cookie_path_matches(path: str) -> bool:
- if path == "/":
- return True
- if not TARGET_PATH.startswith(path):
- return False
- return path.endswith("/") or len(path) == len(TARGET_PATH) or TARGET_PATH[len(path)] == "/"
+def _cookie_path_matches(path: str, bvid: str) -> bool:
+ wanted = target_path(bvid)
+ return path == "/" or (
+ wanted.startswith(path) and (path.endswith("/") or len(path) == len(wanted) or wanted[len(path)] == "/")
+ )
-def validate_cookie(
- value: Any,
- store_id: str,
- observed_at_unix_ms: int,
-) -> dict[str, Any]:
+def validate_cookie(value: Any, store_id: str, observed_at_unix_ms: int, bvid: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise ProtocolError("E_SECRET_INPUT")
try:
_exact_keys(
value,
- {
- "name",
- "value",
- "domain",
- "host_only",
- "path",
- "secure",
- "http_only",
- "same_site",
- "session",
- "expiration_unix",
- "store_id",
- "partition_key",
- },
+ {"name", "value", "domain", "host_only", "path", "secure", "http_only",
+ "same_site", "session", "expiration_unix", "store_id", "partition_key"},
)
_safe_string(value["name"], 1, 256)
_safe_string(value["value"], 1, 4096)
host_only = _boolean(value["host_only"])
- expected_domain = "www.bilibili.com" if host_only else ".bilibili.com"
- if value["domain"] != expected_domain:
+ if value["domain"] != ("www.bilibili.com" if host_only else ".bilibili.com"):
raise ProtocolError()
path = _safe_string(value["path"], 1, 1024)
- if not path.startswith("/") or not _cookie_path_matches(path):
+ if not path.startswith("/") or not _cookie_path_matches(path, bvid):
raise ProtocolError()
_boolean(value["secure"])
_boolean(value["http_only"])
session = _boolean(value["session"])
- if value["same_site"] not in _COOKIE_SAME_SITE:
- raise ProtocolError()
- if value["store_id"] != store_id or value["partition_key"] is not None:
+ if value["same_site"] not in _COOKIE_SAME_SITE or value["store_id"] != store_id or value["partition_key"] is not None:
raise ProtocolError()
if session:
if value["expiration_unix"] is not None:
raise ProtocolError()
- else:
- expires = _integer(value["expiration_unix"], 1, MAX_SAFE_INTEGER)
- if expires <= observed_at_unix_ms // 1000:
- raise ProtocolError()
+ elif _integer(value["expiration_unix"], 1, MAX_SAFE_INTEGER) <= observed_at_unix_ms // 1000:
+ raise ProtocolError()
except ProtocolError as exc:
raise ProtocolError("E_SECRET_INPUT") from exc
return value
def validate_start(value: dict[str, Any], now_ms: int | None = None) -> dict[str, Any]:
- _validate_common(
+ _schema_type(
value,
"start",
- {
- "schema",
- "type",
- "extension_build",
- "target",
- "canonical_url",
- "cookie_store_id",
- "prepare_id",
- "page_proof",
- "cookies",
- },
+ {"schema", "type", "extension_build", "lease_id", "prepare_id", "job",
+ "cookie_store_id", "page_proof", "cookies"},
)
if value["extension_build"] != EXTENSION_BUILD:
raise ProtocolError("E_BUILD")
- if value["canonical_url"] != CANONICAL_URL:
- raise ProtocolError()
+ _lease(value["lease_id"])
if not isinstance(value["prepare_id"], str) or not _PREPARE_ID_RE.fullmatch(value["prepare_id"]):
raise ProtocolError("E_PREPARE")
+ job = validate_job(value["job"])
store_id = value["cookie_store_id"]
if not isinstance(store_id, str) or not _STORE_RE.fullmatch(store_id):
raise ProtocolError("E_SECRET_INPUT")
- proof = validate_page_proof(value["page_proof"], now_ms=now_ms)
+ proof = validate_page_proof(value["page_proof"], job, now_ms=now_ms)
cookies = value["cookies"]
if not isinstance(cookies, list) or not 1 <= len(cookies) <= MAX_COOKIE_COUNT:
raise ProtocolError("E_SECRET_INPUT")
for cookie in cookies:
- validate_cookie(cookie, store_id, proof["observed_at_unix_ms"])
+ validate_cookie(cookie, store_id, proof["observed_at_unix_ms"], job["bvid"])
return value
@@ -320,6 +516,16 @@
message_type = value.get("type")
if message_type == "hello":
return validate_hello(value)
+ if message_type == "poll":
+ return validate_poll(value)
+ if message_type == "foreground":
+ return validate_foreground(value)
+ if message_type == "reject":
+ return validate_reject(value)
+ if message_type == "abort_prepare":
+ return validate_abort_prepare(value)
+ if message_type == "reload_begin":
+ return validate_reload_begin(value)
if message_type == "status":
return validate_status(value)
if message_type == "cancel":
@@ -331,6 +537,18 @@
raise ProtocolError()
+def maintenance_state(
+ *, required_extension_build: str = EXTENSION_BUILD, reload_required: bool = False,
+ reload_token: str | None = None, retry_after_unix_ms: int = 0,
+) -> dict[str, Any]:
+ return {
+ "required_extension_build": required_extension_build,
+ "reload_required": bool(reload_required),
+ "reload_token": reload_token,
+ "retry_after_unix_ms": max(0, int(retry_after_unix_ms)),
+ }
+
+
def safe_response(
message_type: str,
phase: str,
@@ -340,16 +558,21 @@
formal_filename: str | None = None,
mapping_filename: str | None = None,
prepare_id: str | None = None,
+ job: dict[str, Any] | None = None,
+ lease_id: str | None = None,
+ maintenance: dict[str, Any] | None = None,
) -> dict[str, Any]:
return {
"schema": SCHEMA_VERSION,
"type": message_type,
"host_build": HOST_BUILD,
- "target": TARGET_BVID,
"phase": phase,
"progress": max(0, min(100, int(progress))),
"error_code": error_code,
"formal_filename": formal_filename,
"mapping_filename": mapping_filename,
"prepare_id": prepare_id,
+ "job": job,
+ "lease_id": lease_id,
+ "maintenance": maintenance if maintenance is not None else maintenance_state(),
}
diff --git a/dev/project-dev/bili_authenticated_extension/queue-producer.example.json b/dev/project-dev/bili_authenticated_extension/queue-producer.example.json
new file mode 100644
index 0000000..17e9db5
--- /dev/null
+++ b/dev/project-dev/bili_authenticated_extension/queue-producer.example.json
@@ -0,0 +1,14 @@
+{
+ "schema": 2,
+ "producer_id": "project-info-bili-auth-queue-producer/2",
+ "project_root": "C:\\ABSOLUTE\\project-info",
+ "host_config_path": "C:\\ABSOLUTE\\installed-host\\config.json",
+ "host_config_sha256": "REQUIRED_64_HEX",
+ "creator_name": "REGISTERED_CREATOR_NAME",
+ "creator_uid": "REGISTERED_CREATOR_UID",
+ "dynamic_manifest_path": "C:\\ABSOLUTE\\project-info\\ana-data\\registered-dynamic-manifest.jsonl",
+ "registered_catalog_path": null,
+ "registered_catalog_sha256": null,
+ "bvid_allowlist": null,
+ "successor_authorization_message_id": null
+}
diff --git a/dev/project-dev/bili_authenticated_extension/queue_producer.py b/dev/project-dev/bili_authenticated_extension/queue_producer.py
new file mode 100644
index 0000000..3290698
--- /dev/null
+++ b/dev/project-dev/bili_authenticated_extension/queue_producer.py
@@ -0,0 +1,1416 @@
+"""Governed dynamic-catalog to schema-1 authenticated queue producer.
+
+The command is intentionally separate from the Native Messaging host. It
+does not open Chrome, call the host, read credentials, or accept caller-owned
+job records. Its only mutation is an append to the configured queue through
+``QueueStore.append_ingress_jobs`` and that method's shared first-byte lock.
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import re
+import stat
+import sys
+from datetime import datetime
+from decimal import Decimal, InvalidOperation
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from .constants import (
+ EXTENSION_BUILD,
+ EXPECTED_EXTENSION_ID,
+ EXPECTED_ORIGIN,
+ HOST_BUILD,
+ MESSAGE_ID_RE,
+ RELOAD_GENERATION,
+ canonical_url,
+ stable_job_id,
+ validate_bvid,
+ validate_creator_uid,
+)
+from .protocol import ProtocolError, strict_json_loads
+from .queue_state import (
+ AuthorizedSuccessor,
+ QueueStore,
+ ReleaseApproval,
+ validate_ingress_record,
+)
+
+
+PRODUCER_ID = "project-info-bili-auth-queue-producer/2"
+_LEGACY_PRODUCER_ID = "project-info-bili-auth-queue-producer/1"
+_TASK_ID = "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001"
+_PROJECT_ID = "project-info"
+_HEX64 = re.compile(r"[0-9A-Fa-f]{64}\Z")
+_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
+_FORBIDDEN_KEYS = re.compile(r"(?:cookie|password|passwd|access_token|refresh_token|signed_url|local_?storage|captcha)", re.I)
+_HOST_KEYS = {
+ "schema", "creator_allowlist", "queue_path", "queue_state_path", "queue_lock_path",
+ "reload_state_path", "reload_generation", "required_extension_build", "ffmpeg",
+ "ffmpeg_sha256", "ffprobe", "ffprobe_sha256", "bridge_python",
+ "bridge_python_sha256", "bridge_script", "bridge_script_sha256",
+ "yt_dlp_executable", "yt_dlp_executable_sha256", "destination",
+ "creator_name", "formal_manifest_path", "processing_handoff_path",
+}
+_CONFIG_V1_KEYS = {
+ "schema", "producer_id", "project_root", "host_config_path", "host_config_sha256",
+ "creator_name", "creator_uid", "dynamic_manifest_path", "registered_catalog_path",
+ "registered_catalog_sha256", "bvid_allowlist",
+}
+_CONFIG_V2_KEYS = _CONFIG_V1_KEYS | {"successor_authorization_message_id"}
+_REGISTERED_KEYS = {"schema", "source", "creator_name", "creator_uid", "items"}
+_INGRESS_KEYS = {
+ "schema", "bvid", "creator_uid", "expected_duration_ms",
+ "discovered_at_unix_ms", "published_at", "title",
+}
+_TERMINAL_VIDEO_STATUSES = frozenset({
+ "VIDEO_DOWNLOADED_COMPLETE_HANDOFF_SENT",
+ "VIDEO_DOWNLOADED_COMPLETE",
+ "VIDEO_COMPLETE",
+})
+_SOURCE_MANIFEST_KEYS = {
+ "schema", "scope", "extension_id", "extension_build", "host_build",
+ "archive_metadata_contract", "dependency_artifact_manifest_bytes",
+ "dependency_artifact_manifest_sha256", "files",
+}
+_ARCHIVE_METADATA_KEYS = {
+ "schema", "root", "relative_files", "distribution_name", "distribution_version",
+ "allowed_type_codes", "source_date_epoch", "tree_hash_algorithm",
+ "canonical_tree_sha256",
+}
+_ARTIFACT_FILE_KEYS = {"path", "bytes", "sha256"}
+_BUILD_APPROVAL_KEYS = {
+ "schema", "task_id", "approval_scope", "approved_by_role", "status",
+ "source_artifact_manifest_bytes", "source_artifact_manifest_sha256",
+}
+_BUILD_RECEIPT_KEYS = {
+ "schema", "scope", "extension_id", "extension_build", "host_build", "packaging",
+ "pyinstaller_version", "yt_dlp_version", "builder_python_sha256",
+ "pyinstaller_executable_bytes", "pyinstaller_executable_sha256",
+ "builder_provision_receipt_bytes", "builder_provision_receipt_sha256",
+ "build_script_sha256", "source_artifact_manifest_bytes",
+ "source_artifact_manifest_sha256", "dependency_artifact_manifest_bytes",
+ "dependency_artifact_manifest_sha256", "yt_dlp_wheel_sha256",
+ "archive_verification", "files",
+}
+_ARCHIVE_VERIFICATION_KEYS = {
+ "status", "method", "required_modules", "metadata_entry", "metadata_files",
+ "metadata_type_codes", "metadata_tree_sha256",
+}
+_INSTALL_APPROVAL_KEYS = {
+ "schema", "task_id", "approval_scope", "approved_by_role", "status",
+ "source_artifact_manifest_sha256", "build_artifact_manifest_bytes",
+ "build_artifact_manifest_sha256", "host_executable_bytes", "host_executable_sha256",
+}
+_INSTALL_RECEIPT_KEYS = {
+ "schema", "task_id", "host_build", "required_extension_build", "extension_id",
+ "host_name", "installed_root", "installed_files",
+}
+_FAST_PATH_INSTALL_RECEIPT_KEYS = _INSTALL_RECEIPT_KEYS | {
+ "validation_scope", "validated_by_role", "status",
+ "continuous_authorization_handoff_id", "owner_ai_id", "owner_thread_id",
+ "owner_role_instance_id", "authorization_file_relative_path",
+ "authorization_file_bytes", "authorization_file_sha256", "successor_scope_sha256",
+ "implementation_review_audit_id", "implementation_review_audit_bytes",
+ "implementation_review_audit_sha256", "source_artifact_manifest_bytes",
+ "source_artifact_manifest_sha256", "source_receipt_bytes", "source_receipt_sha256",
+ "build_artifact_manifest_bytes", "build_artifact_manifest_sha256",
+ "build_receipt_bytes", "build_receipt_sha256", "host_executable_bytes",
+ "host_executable_sha256", "native_messaging_host_manifest",
+ "host_build_source_manifest", "host_source_binding_mode", "producer_only_changed_files",
+ "host_archive_excluded_modules",
+ "projection_contract_sha256", "projection_tree_sha256", "reload_state_path",
+ "reload_state_bytes", "reload_state_lines", "reload_state_sha256", "queue_path",
+ "queue_prefix_bytes", "queue_prefix_lines", "queue_prefix_sha256",
+ "queue_state_path", "queue_state_prefix_bytes", "queue_state_prefix_lines",
+ "queue_state_prefix_sha256", "formal_manifest_path", "formal_manifest_prefix_bytes",
+ "formal_manifest_prefix_lines", "formal_manifest_prefix_sha256", "secret_field_count",
+}
+_CONTINUOUS_FAST_PATH_HANDOFF_ID = (
+ "HANDOFF-INFOADMIN-INFODEV2-BILI-AUTH-V010-GENERATION9-PASS0-"
+ "CONTINUOUS-TARGET-FAST-PATH-RELEASE-RUNTIME-20260820-001"
+)
+_FAST_PATH_OWNER = (
+ "infodev-2",
+ "019fbcbb-bed7-7c90-83ab-f50610f80d3a",
+ "dev.developer.project.secondary",
+)
+
+
+def _is_reparse(stat_result: os.stat_result) -> bool:
+ attributes = int(getattr(stat_result, "st_file_attributes", 0))
+ reparse = int(getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400))
+ return stat.S_ISLNK(stat_result.st_mode) or bool(attributes & reparse)
+
+
+def _absolute_local_path(value: Any, code: str) -> Path:
+ if not isinstance(value, str) or not value or _CONTROL.search(value):
+ raise ProtocolError(code)
+ path = Path(value)
+ if not path.is_absolute() or str(path).startswith("\\\\"):
+ raise ProtocolError(code)
+ return Path(os.path.abspath(path))
+
+
+def _check_existing_chain(path: Path, *, require_file: bool | None) -> None:
+ current = Path(path.anchor)
+ for part in path.parts[1:]:
+ current = current / part
+ try:
+ info = os.lstat(current)
+ except FileNotFoundError:
+ if require_file:
+ raise ProtocolError("E_CONFIG")
+ return
+ if _is_reparse(info):
+ raise ProtocolError("E_CONFIG")
+ if require_file is True and not path.is_file():
+ raise ProtocolError("E_CONFIG")
+ if require_file is False and path.exists() and not path.is_dir():
+ raise ProtocolError("E_CONFIG")
+
+
+def _within(path: Path, root: Path) -> bool:
+ try:
+ return os.path.commonpath((str(path), str(root))) == str(root)
+ except ValueError:
+ return False
+
+
+def _reject_secret_keys(value: Any) -> None:
+ if isinstance(value, dict):
+ for key, child in value.items():
+ if not isinstance(key, str) or _FORBIDDEN_KEYS.search(key):
+ raise ProtocolError("E_SECRET_FIELD")
+ _reject_secret_keys(child)
+ elif isinstance(value, list):
+ for child in value:
+ _reject_secret_keys(child)
+
+
+def _safe_text(value: Any, maximum: int = 600) -> str:
+ if not isinstance(value, str) or not value.strip() or _CONTROL.search(value):
+ raise ProtocolError("E_CATALOG")
+ if len(value.encode("utf-8")) > maximum:
+ raise ProtocolError("E_CATALOG")
+ return value
+
+
+def _time_ms(value: Any) -> int:
+ if not isinstance(value, str) or _CONTROL.search(value):
+ raise ProtocolError("E_CATALOG")
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise ProtocolError("E_CATALOG") from exc
+ if parsed.utcoffset() is None:
+ raise ProtocolError("E_CATALOG")
+ return int(parsed.timestamp() * 1000)
+
+
+def _duration_ms(value: Any) -> int:
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ raise ProtocolError("E_CATALOG")
+ try:
+ milliseconds = Decimal(str(value)) * 1000
+ except InvalidOperation as exc:
+ raise ProtocolError("E_CATALOG") from exc
+ if milliseconds != milliseconds.to_integral_value():
+ raise ProtocolError("E_CATALOG")
+ result = int(milliseconds)
+ if not 1_000 <= result <= 86_400_000:
+ raise ProtocolError("E_CATALOG")
+ return result
+
+
+def _read_strict_payload(path: Path, maximum: int, code: str) -> bytes:
+ try:
+ _check_existing_chain(path, require_file=True)
+ with path.open("rb") as source:
+ payload = source.read(maximum + 1)
+ except (OSError, ProtocolError) as exc:
+ raise ProtocolError(code) from exc
+ if len(payload) > maximum:
+ raise ProtocolError(code)
+ return payload
+
+
+def _enumerate_exact_tree(root: Path, expected_files: set[str], code: str) -> None:
+ """Require the actual no-follow tree to equal an exact case-sensitive file set."""
+ try:
+ _check_existing_chain(root, require_file=False)
+ root_info = os.lstat(root)
+ except (OSError, ProtocolError) as exc:
+ raise ProtocolError(code) from exc
+ if _is_reparse(root_info) or not stat.S_ISDIR(root_info.st_mode):
+ raise ProtocolError(code)
+ expected_directories = {
+ PurePosixPath(path).parent.as_posix()
+ for path in expected_files
+ if PurePosixPath(path).parent.as_posix() != "."
+ }
+ actual_files: set[str] = set()
+ actual_directories: set[str] = set()
+ pending: list[tuple[Path, PurePosixPath]] = [(root, PurePosixPath("."))]
+ while pending:
+ directory, relative_directory = pending.pop()
+ try:
+ entries = list(os.scandir(directory))
+ except OSError as exc:
+ raise ProtocolError(code) from exc
+ for entry in entries:
+ try:
+ info = entry.stat(follow_symlinks=False)
+ except OSError as exc:
+ raise ProtocolError(code) from exc
+ if _is_reparse(info):
+ raise ProtocolError(code)
+ relative = PurePosixPath(entry.name) if relative_directory == PurePosixPath(".") else relative_directory / entry.name
+ normalized = relative.as_posix()
+ if stat.S_ISDIR(info.st_mode):
+ actual_directories.add(normalized)
+ pending.append((Path(entry.path), relative))
+ elif stat.S_ISREG(info.st_mode):
+ actual_files.add(normalized)
+ else:
+ raise ProtocolError(code)
+ if actual_files != expected_files or actual_directories != expected_directories:
+ raise ProtocolError(code)
+
+
+def _validate_source_tree(root: Path, source_payload: bytes, deployment: dict[str, Any]) -> dict[str, Any]:
+ code = "E_DEPLOYMENT_NOT_READY"
+ source = _plain_mapping(strict_json_loads(source_payload), _SOURCE_MANIFEST_KEYS, code)
+ if (
+ source["schema"] != 1 or source["scope"] != "generic-bilibili-queue"
+ or source["extension_id"] != EXPECTED_EXTENSION_ID
+ or source["extension_build"] != EXTENSION_BUILD or source["host_build"] != HOST_BUILD
+ ):
+ raise ProtocolError(code)
+ archive = _plain_mapping(source["archive_metadata_contract"], _ARCHIVE_METADATA_KEYS, code)
+ if (
+ archive["schema"] != 1 or archive["root"] != "yt_dlp-2026.7.4.dist-info"
+ or archive["relative_files"] != [
+ "INSTALLER", "METADATA", "RECORD", "REQUESTED", "WHEEL",
+ "entry_points.txt", "licenses/LICENSE",
+ ]
+ or archive["distribution_name"] != "yt-dlp" or archive["distribution_version"] != "2026.7.4"
+ or archive["allowed_type_codes"] != ["b", "x"]
+ or archive["source_date_epoch"] != 1786207924
+ or archive["tree_hash_algorithm"] != "sha256(path-utf8,nul,decimal-bytes-ascii,nul,payload-sha256-lower-hex-ascii,lf)-upper-hex-v1"
+ or archive["canonical_tree_sha256"] != "32F1DC23F6704966AE4511CB173318CD11FCBA031323028D268D9F2488589E70"
+ ):
+ raise ProtocolError(code)
+ dependency_size = _positive_integer(source["dependency_artifact_manifest_bytes"], code)
+ dependency_hash = _upper_sha(source["dependency_artifact_manifest_sha256"], code)
+ files = source["files"]
+ if not isinstance(files, list) or len(files) != 21:
+ raise ProtocolError(code)
+ source_root = root / "dev" / "project-dev" / "bili_authenticated_extension"
+ expected: dict[str, tuple[int, str]] = {}
+ for item in files:
+ item = _plain_mapping(item, _ARTIFACT_FILE_KEYS, code)
+ relative = _relative_path(item["path"], ("",), code)
+ size = _positive_integer(item["bytes"], code)
+ digest = _upper_sha(item["sha256"], code)
+ if relative == "source-artifact-manifest.json" or relative in expected:
+ raise ProtocolError(code)
+ expected[relative] = (size, digest)
+ if list(expected) != sorted(expected):
+ raise ProtocolError(code)
+ _enumerate_exact_tree(source_root, set(expected) | {"source-artifact-manifest.json"}, code)
+ for relative, (size, digest) in expected.items():
+ payload = _read_strict_payload(source_root / Path(relative), size + 1, code)
+ if len(payload) != size or hashlib.sha256(payload).hexdigest().upper() != digest:
+ raise ProtocolError(code)
+ dependency = expected.get("dependencies/dependency-artifact-manifest.json")
+ if dependency != (dependency_size, dependency_hash):
+ raise ProtocolError(code)
+ return source
+
+
+def _strict_file(path: Path, maximum: int, code: str = "E_CATALOG") -> Any:
+ payload = _read_strict_payload(path, maximum, code)
+ return strict_json_loads(payload)
+
+
+def load_producer_configuration(path: Path) -> dict[str, Any]:
+ path = Path(os.path.abspath(path))
+ _check_existing_chain(path, require_file=True)
+ raw = _strict_file(path, 128 * 1024, "E_CONFIG")
+ if not isinstance(raw, dict) or raw.get("schema") not in {1, 2}:
+ raise ProtocolError("E_CONFIG")
+ expected_keys = _CONFIG_V1_KEYS if raw["schema"] == 1 else _CONFIG_V2_KEYS
+ if set(raw) != expected_keys or raw.get("producer_id") not in {_LEGACY_PRODUCER_ID, PRODUCER_ID}:
+ raise ProtocolError("E_CONFIG")
+ if raw["schema"] == 2:
+ authorization_message_id = raw["successor_authorization_message_id"]
+ if authorization_message_id is not None and (
+ not isinstance(authorization_message_id, str)
+ or not MESSAGE_ID_RE.fullmatch(authorization_message_id)
+ ):
+ raise ProtocolError("E_CONFIG")
+ _reject_secret_keys(raw)
+ project_root = _absolute_local_path(raw["project_root"], "E_CONFIG")
+ _check_existing_chain(project_root, require_file=False)
+ if not project_root.is_dir() or not _within(path, project_root):
+ raise ProtocolError("E_CONFIG")
+ creator_name = _safe_text(raw["creator_name"], 128)
+ try:
+ creator_uid = validate_creator_uid(raw["creator_uid"])
+ except ValueError as exc:
+ raise ProtocolError("E_CONFIG") from exc
+ host_path = _absolute_local_path(raw["host_config_path"], "E_CONFIG")
+ _check_existing_chain(host_path, require_file=True)
+ expected_host_hash = raw["host_config_sha256"]
+ host_payload = _read_strict_payload(host_path, 128 * 1024, "E_CONFIG")
+ actual_host_hash = hashlib.sha256(host_payload).hexdigest().upper()
+ if not isinstance(expected_host_hash, str) or not _HEX64.fullmatch(expected_host_hash) or actual_host_hash != expected_host_hash.upper():
+ raise ProtocolError("E_CONFIG_HASH")
+ host = strict_json_loads(host_payload)
+ if not isinstance(host, dict) or set(host) != _HOST_KEYS or host.get("schema") != 2:
+ raise ProtocolError("E_CONFIG")
+ creators = host.get("creator_allowlist")
+ if not isinstance(creators, list) or creators != sorted(set(creators)) or creator_uid not in creators:
+ raise ProtocolError("E_ALLOWLIST")
+ if host.get("required_extension_build") != EXTENSION_BUILD:
+ raise ProtocolError("E_CONFIG")
+ if host.get("reload_generation") != RELOAD_GENERATION:
+ raise ProtocolError("E_CONFIG")
+ queue_paths = {
+ name: _absolute_local_path(host[name], "E_CONFIG")
+ for name in ("queue_path", "queue_state_path", "queue_lock_path", "reload_state_path")
+ }
+ if len(set(queue_paths.values())) != 4 or len({value.parent for value in queue_paths.values()}) != 1:
+ raise ProtocolError("E_CONFIG")
+ for candidate in queue_paths.values():
+ _check_existing_chain(candidate, require_file=None)
+ if candidate.exists() and not candidate.is_file():
+ raise ProtocolError("E_CONFIG")
+ dynamic_path = _absolute_local_path(raw["dynamic_manifest_path"], "E_CONFIG")
+ if not _within(dynamic_path, project_root):
+ raise ProtocolError("E_CONFIG")
+ _check_existing_chain(dynamic_path, require_file=True)
+ formal_path = _absolute_local_path(host["formal_manifest_path"], "E_CONFIG")
+ handoff_path = _absolute_local_path(host["processing_handoff_path"], "E_CONFIG")
+ if (
+ host["creator_name"] != creator_name or formal_path != dynamic_path
+ or not _within(handoff_path, project_root) or handoff_path == formal_path
+ ):
+ raise ProtocolError("E_CONFIG")
+ _check_existing_chain(handoff_path, require_file=None)
+ if handoff_path.exists() and not handoff_path.is_file():
+ raise ProtocolError("E_CONFIG")
+ registered_path = None
+ registered_payload = None
+ actual_registered_hash = None
+ registered_hash = raw["registered_catalog_sha256"]
+ if raw["registered_catalog_path"] is None:
+ if registered_hash is not None:
+ raise ProtocolError("E_CONFIG")
+ else:
+ registered_path = _absolute_local_path(raw["registered_catalog_path"], "E_CONFIG")
+ if not _within(registered_path, project_root):
+ raise ProtocolError("E_CONFIG")
+ _check_existing_chain(registered_path, require_file=True)
+ registered_payload = _read_strict_payload(registered_path, 4 * 1024 * 1024, "E_CATALOG")
+ actual_registered_hash = hashlib.sha256(registered_payload).hexdigest().upper()
+ if not isinstance(registered_hash, str) or not _HEX64.fullmatch(registered_hash) or actual_registered_hash != registered_hash.upper():
+ raise ProtocolError("E_CONFIG_HASH")
+ allowlist = raw["bvid_allowlist"]
+ if allowlist is not None:
+ if not isinstance(allowlist, list) or not allowlist or allowlist != sorted(set(allowlist)):
+ raise ProtocolError("E_CONFIG")
+ try:
+ allowlist = [validate_bvid(value) for value in allowlist]
+ except ValueError as exc:
+ raise ProtocolError("E_CONFIG") from exc
+ return {
+ **raw,
+ "project_root": project_root,
+ "host_config_path": host_path,
+ "dynamic_manifest_path": dynamic_path,
+ "registered_catalog_path": registered_path,
+ "creator_name": creator_name,
+ "creator_uid": creator_uid,
+ "bvid_allowlist": None if allowlist is None else frozenset(allowlist),
+ "queue_paths": queue_paths,
+ "_host_config_sha256": actual_host_hash,
+ "_registered_catalog_payload": registered_payload,
+ "_registered_catalog_sha256": actual_registered_hash,
+ }
+
+
+_RELEASE_KEYS = {
+ "schema", "scope", "project_id", "task_id", "approved_by_role",
+ "authorization_message_id", "authorization_handoff_id", "authorization_file",
+ "implementation_review", "deployment",
+}
+_FILE_ID_KEYS = {"relative_path", "bytes", "sha256"}
+_REVIEW_KEYS = {
+ "result_message_id", "audit_id", "audit_bytes", "audit_sha256",
+ "verdict", "blocking_findings",
+}
+_DEPLOYMENT_KEYS = {
+ "extension_build", "host_build", "reload_generation", "source_manifest_sha256",
+ "build_approval", "build_receipt", "exe", "install_approval", "install_receipt",
+ "installed_config_sha256", "installed_manifest_sha256", "installed_exe_sha256",
+ "projection_contract_sha256", "projection_tree_sha256", "extension_id",
+}
+_AUTH_KEYS = {
+ "schema", "scope", "task_id", "authorized_by_role", "authorization_message_id",
+ "authorization_handoff_id", "repair", "successors",
+}
+_AUTH_REPAIR_KEYS = {
+ "review_result_message_id", "audit_id", "audit_bytes", "audit_sha256",
+ "verdict", "blocking_findings",
+}
+_AUTH_SUCCESSOR_KEYS = {
+ "creator_uid", "bvid", "predecessor_job_id", "retry_generation",
+ "terminal_error_code",
+}
+
+
+def _project_root_from_source() -> Path:
+ root = Path(__file__).resolve().parents[3]
+ _check_existing_chain(root, require_file=False)
+ project_config = root / "mbx.project.yaml"
+ payload = _read_strict_payload(project_config, 4 * 1024 * 1024, "E_AUTH_TRUST")
+ text = payload.decode("utf-8", errors="strict")
+ if not re.search(r"(?m)^project:\s*\r?\n(?:^[ \t].*\r?\n)*?^ id: project-info\s*$", text) or not re.search(r"(?m)^ project_admin: project\.admin\s*$", text):
+ raise ProtocolError("E_AUTH_TRUST")
+ if not re.search(r"(?ms)^- id: project\.admin\s*$.*?^ private_dir: ai-infoadmin\s*$", text):
+ raise ProtocolError("E_AUTH_TRUST")
+ return root
+
+
+def _relative_path(value: Any, prefixes: tuple[str, ...], code: str) -> str:
+ if not isinstance(value, str) or not value or "\\" in value or _CONTROL.search(value):
+ raise ProtocolError(code)
+ path = PurePosixPath(value)
+ if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
+ raise ProtocolError(code)
+ normalized = path.as_posix()
+ if normalized != value or not any(normalized.startswith(prefix) for prefix in prefixes):
+ raise ProtocolError(code)
+ return normalized
+
+
+def _file_identity(root: Path, value: Any, prefixes: tuple[str, ...], code: str) -> tuple[Path, bytes]:
+ if not isinstance(value, dict) or set(value) != _FILE_ID_KEYS:
+ raise ProtocolError(code)
+ relative = _relative_path(value.get("relative_path"), prefixes, code)
+ size = value.get("bytes")
+ digest = value.get("sha256")
+ if (
+ isinstance(size, bool) or not isinstance(size, int) or size <= 0
+ or not isinstance(digest, str) or not _HEX64.fullmatch(digest)
+ or digest != digest.upper()
+ ):
+ raise ProtocolError(code)
+ path = root / Path(relative)
+ payload = _read_strict_payload(path, max(size, 1) + 1, code)
+ if len(payload) != size or hashlib.sha256(payload).hexdigest().upper() != digest:
+ raise ProtocolError(code)
+ return path, payload
+
+
+def _plain_mapping(value: Any, keys: set[str], code: str) -> dict[str, Any]:
+ if not isinstance(value, dict) or set(value) != keys:
+ raise ProtocolError(code)
+ return value
+
+
+def _upper_sha(value: Any, code: str) -> str:
+ if not isinstance(value, str) or not _HEX64.fullmatch(value) or value != value.upper():
+ raise ProtocolError(code)
+ return value
+
+
+def _positive_integer(value: Any, code: str) -> int:
+ if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+ raise ProtocolError(code)
+ return value
+
+
+def _extension_id_from_manifest_key(value: Any) -> str:
+ if not isinstance(value, str) or _CONTROL.search(value):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ try:
+ public_key = base64.b64decode(value, validate=True)
+ except (ValueError, TypeError) as exc:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY") from exc
+ alphabet = "abcdefghijklmnop"
+ prefix = hashlib.sha256(public_key).digest()[:16]
+ return "".join(alphabet[byte >> 4] + alphabet[byte & 15] for byte in prefix)
+
+
+def _projection_tree(entries: list[dict[str, Any]]) -> str:
+ builder = bytearray()
+ for item in sorted(entries, key=lambda candidate: candidate["path"]):
+ builder.extend(item["path"].encode("utf-8"))
+ builder.extend(b"\0")
+ builder.extend(str(item["bytes"]).encode("ascii"))
+ builder.extend(b"\0")
+ builder.extend(item["sha256"].encode("ascii"))
+ builder.extend(b"\n")
+ return hashlib.sha256(builder).hexdigest().upper()
+
+
+def _read_native_host_registry_default() -> str:
+ try:
+ import winreg
+
+ key_path = rf"Software\Google\Chrome\NativeMessagingHosts\com.project_info.bili_auth_ingress"
+ with winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path, 0, winreg.KEY_READ) as key:
+ value, kind = winreg.QueryValueEx(key, None)
+ if kind != winreg.REG_SZ or not isinstance(value, str):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ return value
+ except (OSError, ImportError) as exc:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY") from exc
+
+
+def _read_native_host_registry_exact() -> str:
+ """Return the sole default REG_SZ value; reject extra values or subkeys."""
+ try:
+ import winreg
+
+ key_path = rf"Software\Google\Chrome\NativeMessagingHosts\com.project_info.bili_auth_ingress"
+ with winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path, 0, winreg.KEY_READ) as key:
+ subkey_count, value_count, _ = winreg.QueryInfoKey(key)
+ if subkey_count != 0 or value_count != 1:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ name, value, kind = winreg.EnumValue(key, 0)
+ if name != "" or kind != winreg.REG_SZ or not isinstance(value, str):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ return value
+ except (OSError, ImportError) as exc:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY") from exc
+
+
+def _fast_path_canonical_paths() -> dict[str, Path]:
+ return {
+ "queue": Path(r"C:\Users\Cai\AppData\Local\project-info\bili-auth-generic-runtime\queue.jsonl"),
+ "queue_state": Path(r"C:\Users\Cai\AppData\Local\project-info\bili-auth-generic-runtime\queue-state.jsonl"),
+ "reload_state": Path(r"C:\Users\Cai\AppData\Local\project-info\bili-auth-generic-runtime\reload-state.jsonl"),
+ "formal_manifest": Path(r"E:\mb-ms-doc\project-info\ana-data\news-青枫浦上Q\manifest.jsonl"),
+ }
+
+
+def _successor_scope_sha256(successors: Any) -> str:
+ if not isinstance(successors, list):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ normalized: list[dict[str, Any]] = []
+ for value in successors:
+ value = _plain_mapping(value, _AUTH_SUCCESSOR_KEYS, "E_DEPLOYMENT_NOT_READY")
+ normalized.append({name: value[name] for name in sorted(_AUTH_SUCCESSOR_KEYS)})
+ normalized.sort(key=lambda value: (
+ value["creator_uid"], value["bvid"], value["predecessor_job_id"],
+ value["retry_generation"], value["terminal_error_code"],
+ ))
+ payload = json.dumps(
+ normalized, ensure_ascii=True, sort_keys=True, separators=(",", ":"),
+ ).encode("ascii")
+ return hashlib.sha256(payload).hexdigest().upper()
+
+
+def _validate_prefix_identity(
+ path: Path, receipt: dict[str, Any], prefix: str, *, exact: bool,
+) -> None:
+ code = "E_DEPLOYMENT_NOT_READY"
+ expected_path = _absolute_local_path(receipt[f"{prefix}_path"], code)
+ if path != expected_path:
+ raise ProtocolError(code)
+ size_name = f"{prefix}_bytes" if exact else f"{prefix}_prefix_bytes"
+ lines_name = f"{prefix}_lines" if exact else f"{prefix}_prefix_lines"
+ sha_name = f"{prefix}_sha256" if exact else f"{prefix}_prefix_sha256"
+ size = _positive_integer(receipt[size_name], code)
+ lines = _positive_integer(receipt[lines_name], code)
+ digest = _upper_sha(receipt[sha_name], code)
+ payload = _read_strict_payload(path, 32 * 1024 * 1024, code)
+ if exact:
+ candidate = payload
+ if len(payload) != size:
+ raise ProtocolError(code)
+ else:
+ if len(payload) < size:
+ raise ProtocolError(code)
+ candidate = payload[:size]
+ if (
+ not candidate.endswith(b"\n") or candidate.count(b"\n") != lines
+ or hashlib.sha256(candidate).hexdigest().upper() != digest
+ ):
+ raise ProtocolError(code)
+
+
+def _validate_review(value: Any, code: str) -> dict[str, Any]:
+ review = _plain_mapping(value, _REVIEW_KEYS, code)
+ if (
+ not isinstance(review["result_message_id"], str)
+ or not MESSAGE_ID_RE.fullmatch(review["result_message_id"])
+ or not isinstance(review["audit_id"], str) or not review["audit_id"].startswith("DEV-AUDIT-")
+ or _positive_integer(review["audit_bytes"], code) != review["audit_bytes"]
+ or _upper_sha(review["audit_sha256"], code) != review["audit_sha256"]
+ or review["verdict"] != "PASS/0" or review["blocking_findings"] != 0
+ ):
+ raise ProtocolError(code)
+ return review
+
+
+def _validate_audit_prefix(root: Path, review: dict[str, Any], code: str) -> None:
+ audit_path = root / "dev-doc" / "开发审计报告.md"
+ payload = _read_strict_payload(audit_path, 16 * 1024 * 1024, code)
+ size = review["audit_bytes"]
+ if len(payload) < size or hashlib.sha256(payload[:size]).hexdigest().upper() != review["audit_sha256"]:
+ raise ProtocolError(code)
+ try:
+ prefix = payload[:size].decode("utf-8", errors="strict")
+ except UnicodeError as exc:
+ raise ProtocolError(code) from exc
+ if review["audit_id"] not in prefix:
+ raise ProtocolError(code)
+
+
+def _validate_projection(root: Path, deployment: dict[str, Any]) -> None:
+ contract_path = root / "dev" / "project-dev" / "bili_authenticated_extension_unpacked_contract.json"
+ contract_payload = _read_strict_payload(contract_path, 128 * 1024, "E_DEPLOYMENT_NOT_READY")
+ if hashlib.sha256(contract_payload).hexdigest().upper() != deployment["projection_contract_sha256"]:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ contract = strict_json_loads(contract_payload)
+ required = {
+ "schema", "task_id", "project_id", "source_root", "projection_root",
+ "expected_extension_id", "public_key_der_sha256", "tree_hash_algorithm",
+ "tree_sha256", "files",
+ }
+ if not isinstance(contract, dict) or set(contract) != required or contract.get("schema") != 1:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ if (
+ contract["task_id"] != _TASK_ID or contract["project_id"] != _PROJECT_ID
+ or contract["expected_extension_id"] != EXPECTED_EXTENSION_ID
+ or contract["tree_sha256"] != deployment["projection_tree_sha256"]
+ or contract["tree_hash_algorithm"] != "path-nul-bytes-nul-sha256-upper-lf-v1"
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ source_root = root / Path(_relative_path(contract["source_root"], ("dev/project-dev/bili_authenticated_extension",), "E_DEPLOYMENT_NOT_READY"))
+ projection_root = root / Path(_relative_path(contract["projection_root"], ("dev/project-dev/bili_authenticated_extension_unpacked",), "E_DEPLOYMENT_NOT_READY"))
+ _check_existing_chain(source_root, require_file=False)
+ _check_existing_chain(projection_root, require_file=False)
+ files = contract["files"]
+ if not isinstance(files, list) or len(files) != 5:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ expected_paths = ["background.js", "manifest.json", "sidepanel.css", "sidepanel.html", "sidepanel.js"]
+ _enumerate_exact_tree(projection_root, set(expected_paths), "E_DEPLOYMENT_NOT_READY")
+ actual_paths: list[str] = []
+ entries: list[dict[str, Any]] = []
+ for spec in files:
+ if not isinstance(spec, dict) or set(spec) != {"path", "bytes", "sha256"}:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ name = spec["path"]
+ if name not in expected_paths or name in actual_paths:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ size = _positive_integer(spec["bytes"], "E_DEPLOYMENT_NOT_READY")
+ digest = _upper_sha(spec["sha256"], "E_DEPLOYMENT_NOT_READY")
+ payload = _read_strict_payload(projection_root / name, size + 1, "E_DEPLOYMENT_NOT_READY")
+ source_payload = _read_strict_payload(source_root / name, size + 1, "E_DEPLOYMENT_NOT_READY")
+ if len(payload) != size or payload != source_payload or hashlib.sha256(payload).hexdigest().upper() != digest:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ actual_paths.append(name)
+ entries.append({"path": name, "bytes": size, "sha256": digest})
+ if sorted(actual_paths) != expected_paths or _projection_tree(entries) != contract["tree_sha256"]:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ manifest = strict_json_loads((projection_root / "manifest.json").read_bytes())
+ if (
+ not isinstance(manifest, dict) or manifest.get("manifest_version") != 3
+ or manifest.get("version") != "1.2.25"
+ or manifest.get("version_name") != "1.2.25+20260829.generic.v027"
+ or _extension_id_from_manifest_key(manifest.get("key")) != EXPECTED_EXTENSION_ID
+ or EXTENSION_BUILD.encode("ascii") not in (projection_root / "background.js").read_bytes()
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+
+
+def _validate_reload_applied(path: Path) -> None:
+ payload = _read_strict_payload(path, 4 * 1024 * 1024, "E_DEPLOYMENT_NOT_READY")
+ if not payload.endswith(b"\n"):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ matching: list[dict[str, Any]] = []
+ for line in payload.splitlines():
+ value = strict_json_loads(line)
+ if not isinstance(value, dict) or set(value) != {
+ "schema", "generation", "event", "token", "from_build", "to_build", "at_unix_ms"
+ }:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ if value.get("generation") == RELOAD_GENERATION:
+ matching.append(value)
+ events = [value.get("event") for value in matching]
+ if events not in (["APPLIED"], ["OFFERED", "BEGIN", "APPLIED"]):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ last = matching[-1]
+ if last.get("from_build") != EXTENSION_BUILD or last.get("to_build") != EXTENSION_BUILD:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ if (
+ any(value.get("schema") != 1 for value in matching)
+ or any(not isinstance(value.get("token"), str) or not re.fullmatch(r"[0-9a-f]{32}", value["token"]) for value in matching)
+ or len({value["token"] for value in matching}) != 1
+ or any(value.get("to_build") != EXTENSION_BUILD for value in matching)
+ or any(isinstance(value.get("at_unix_ms"), bool) or not isinstance(value.get("at_unix_ms"), int) or value["at_unix_ms"] <= 0 for value in matching)
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+
+
+def _deployment_identity_branch(root: Path, deployment: dict[str, Any]) -> str:
+ code = "E_DEPLOYMENT_NOT_READY"
+ identities = {
+ name: _plain_mapping(deployment[name], _FILE_ID_KEYS, code)["relative_path"]
+ for name in ("build_approval", "install_approval", "install_receipt")
+ }
+ for relative in identities.values():
+ _relative_path(relative, ("ai-inforev/worklog/", "ai-infoadmin/worklog/"), code)
+ build_parent = PurePosixPath(identities["build_approval"]).parent.as_posix()
+ install_parent = PurePosixPath(identities["install_approval"]).parent.as_posix()
+ receipt_parent = PurePosixPath(identities["install_receipt"]).parent.as_posix()
+ if (
+ build_parent == "ai-inforev/worklog"
+ and install_parent == "ai-inforev/worklog"
+ and receipt_parent == "ai-infoadmin/worklog"
+ ):
+ return "legacy-reviewer"
+ if (
+ build_parent == "ai-infoadmin/worklog"
+ and install_parent == "ai-infoadmin/worklog"
+ and receipt_parent == "ai-infoadmin/worklog"
+ and PurePosixPath(identities["build_approval"]).name.endswith("-fast-path-source-receipt.json")
+ and PurePosixPath(identities["install_approval"]).name.endswith("-fast-path-build-receipt.json")
+ and PurePosixPath(identities["install_receipt"]).name.endswith("-fast-path-install-readiness-receipt.json")
+ ):
+ return "project-admin-fast-path"
+ raise ProtocolError(code)
+
+
+def _validate_fast_path_host_source(
+ root: Path,
+ receipt: dict[str, Any],
+ current_source: dict[str, Any],
+ current_source_payload: bytes,
+ build_receipt: dict[str, Any],
+ install_approval: dict[str, Any],
+) -> None:
+ code = "E_DEPLOYMENT_NOT_READY"
+ path, payload = _file_identity(
+ root, receipt["host_build_source_manifest"], ("ai-infoadmin/worklog/",), code,
+ )
+ if (
+ path.parent != root / "ai-infoadmin" / "worklog"
+ or not path.name.endswith("-fast-path-host-build-source-manifest.json")
+ ):
+ raise ProtocolError(code)
+ host_source = _plain_mapping(strict_json_loads(payload), _SOURCE_MANIFEST_KEYS, code)
+ for name in _SOURCE_MANIFEST_KEYS - {"files"}:
+ if host_source[name] != current_source[name]:
+ raise ProtocolError(code)
+ current_files = current_source["files"]
+ host_files = host_source["files"]
+ if not isinstance(current_files, list) or not isinstance(host_files, list) or len(host_files) != 21:
+ raise ProtocolError(code)
+ current_by_path: dict[str, dict[str, Any]] = {}
+ host_by_path: dict[str, dict[str, Any]] = {}
+ for target, values in ((current_by_path, current_files), (host_by_path, host_files)):
+ for raw in values:
+ item = _plain_mapping(raw, _ARTIFACT_FILE_KEYS, code)
+ relative = _relative_path(item["path"], ("",), code)
+ if relative in target:
+ raise ProtocolError(code)
+ _positive_integer(item["bytes"], code)
+ _upper_sha(item["sha256"], code)
+ target[relative] = item
+ if set(current_by_path) != set(host_by_path):
+ raise ProtocolError(code)
+ changed = sorted(
+ relative for relative in current_by_path
+ if current_by_path[relative] != host_by_path[relative]
+ )
+ mode = receipt["host_source_binding_mode"]
+ if mode == "SAME_SOURCE_HOST_BUILD":
+ expected_changed: list[str] = []
+ if payload != current_source_payload:
+ raise ProtocolError(code)
+ elif mode == "PRODUCER_ONLY_DELTA":
+ expected_changed = ["queue_producer.py"]
+ if payload == current_source_payload:
+ raise ProtocolError(code)
+ else:
+ raise ProtocolError(code)
+ host_manifest_hash = hashlib.sha256(payload).hexdigest().upper()
+ if (
+ changed != expected_changed
+ or receipt["producer_only_changed_files"] != expected_changed
+ or receipt["host_archive_excluded_modules"] != ["bili_authenticated_extension.queue_producer"]
+ or "bili_authenticated_extension.queue_producer"
+ in build_receipt["archive_verification"]["required_modules"]
+ or build_receipt["source_artifact_manifest_bytes"] != len(payload)
+ or build_receipt["source_artifact_manifest_sha256"] != host_manifest_hash
+ or install_approval["source_artifact_manifest_sha256"] != host_manifest_hash
+ ):
+ raise ProtocolError(code)
+
+
+def _validate_fast_path_receipt(
+ root: Path,
+ config: dict[str, Any],
+ approval: dict[str, Any],
+ authorization: dict[str, Any],
+ deployment: dict[str, Any],
+ source: dict[str, Any],
+ build_receipt: dict[str, Any],
+ install_approval: dict[str, Any],
+ source_payload: bytes,
+ source_receipt_payload: bytes,
+ build_receipt_payload: bytes,
+ build_validation_receipt_payload: bytes,
+ exe_payload: bytes,
+ receipt: dict[str, Any],
+ actual: dict[str, tuple[Path, bytes]],
+) -> None:
+ code = "E_DEPLOYMENT_NOT_READY"
+ owner_ai_id, owner_thread_id, owner_role = _FAST_PATH_OWNER
+ authorization_spec = _plain_mapping(approval["authorization_file"], _FILE_ID_KEYS, code)
+ review = _plain_mapping(approval["implementation_review"], _REVIEW_KEYS, code)
+ source_receipt_hash = hashlib.sha256(source_receipt_payload).hexdigest().upper()
+ build_validation_receipt_hash = hashlib.sha256(build_validation_receipt_payload).hexdigest().upper()
+ build_receipt_hash = hashlib.sha256(build_receipt_payload).hexdigest().upper()
+ exe_hash = hashlib.sha256(exe_payload).hexdigest().upper()
+ if (
+ receipt["schema"] != 2
+ or receipt["task_id"] != _TASK_ID
+ or receipt["validation_scope"] != "continuous-fast-path-installed-readiness-v1"
+ or receipt["validated_by_role"] != "project.admin"
+ or receipt["status"] != "VALIDATED"
+ or receipt["continuous_authorization_handoff_id"] != _CONTINUOUS_FAST_PATH_HANDOFF_ID
+ or receipt["owner_ai_id"] != owner_ai_id
+ or receipt["owner_thread_id"] != owner_thread_id
+ or receipt["owner_role_instance_id"] != owner_role
+ or receipt["authorization_file_relative_path"] != authorization_spec["relative_path"]
+ or receipt["authorization_file_bytes"] != authorization_spec["bytes"]
+ or receipt["authorization_file_sha256"] != authorization_spec["sha256"]
+ or receipt["successor_scope_sha256"] != _successor_scope_sha256(authorization["successors"])
+ or receipt["implementation_review_audit_id"] != review["audit_id"]
+ or receipt["implementation_review_audit_bytes"] != review["audit_bytes"]
+ or receipt["implementation_review_audit_sha256"] != review["audit_sha256"]
+ or receipt["source_artifact_manifest_bytes"] != len(source_payload)
+ or receipt["source_artifact_manifest_sha256"] != deployment["source_manifest_sha256"]
+ or receipt["source_receipt_bytes"] != len(source_receipt_payload)
+ or receipt["source_receipt_sha256"] != source_receipt_hash
+ or receipt["build_artifact_manifest_bytes"] != len(build_receipt_payload)
+ or receipt["build_artifact_manifest_sha256"] != build_receipt_hash
+ or receipt["build_receipt_bytes"] != len(build_validation_receipt_payload)
+ or receipt["build_receipt_sha256"] != build_validation_receipt_hash
+ or receipt["host_executable_bytes"] != len(exe_payload)
+ or receipt["host_executable_sha256"] != exe_hash
+ or receipt["native_messaging_host_manifest"] != str(actual["native-host-manifest.json"][0])
+ or receipt["projection_contract_sha256"] != deployment["projection_contract_sha256"]
+ or receipt["projection_tree_sha256"] != deployment["projection_tree_sha256"]
+ or receipt["secret_field_count"] != 0
+ ):
+ raise ProtocolError(code)
+ for name in (
+ "authorization_file_sha256", "successor_scope_sha256",
+ "implementation_review_audit_sha256", "source_artifact_manifest_sha256",
+ "source_receipt_sha256", "build_artifact_manifest_sha256", "build_receipt_sha256",
+ "host_executable_sha256", "projection_contract_sha256", "projection_tree_sha256",
+ ):
+ _upper_sha(receipt[name], code)
+ for name in (
+ "authorization_file_bytes", "implementation_review_audit_bytes",
+ "source_artifact_manifest_bytes", "source_receipt_bytes",
+ "build_artifact_manifest_bytes", "build_receipt_bytes", "host_executable_bytes",
+ ):
+ _positive_integer(receipt[name], code)
+ _reject_secret_keys(receipt)
+ _validate_fast_path_host_source(
+ root, receipt, source, source_payload, build_receipt, install_approval,
+ )
+ canonical = _fast_path_canonical_paths()
+ if (
+ config["queue_paths"]["queue_path"] != canonical["queue"]
+ or config["queue_paths"]["queue_state_path"] != canonical["queue_state"]
+ or config["queue_paths"]["reload_state_path"] != canonical["reload_state"]
+ ):
+ raise ProtocolError(code)
+ _validate_prefix_identity(canonical["reload_state"], receipt, "reload_state", exact=True)
+ _validate_prefix_identity(canonical["queue"], receipt, "queue", exact=False)
+ _validate_prefix_identity(canonical["queue_state"], receipt, "queue_state", exact=False)
+ _validate_prefix_identity(canonical["formal_manifest"], receipt, "formal_manifest", exact=False)
+ if Path(_read_native_host_registry_exact()) != actual["native-host-manifest.json"][0]:
+ raise ProtocolError(code)
+
+
+def _load_release_approval(config: dict[str, Any]) -> ReleaseApproval:
+ if config.get("schema") != 2:
+ raise ProtocolError("E_AUTH_TRUST")
+ authorization_message_id = config.get("successor_authorization_message_id")
+ if not isinstance(authorization_message_id, str) or not MESSAGE_ID_RE.fullmatch(authorization_message_id):
+ raise ProtocolError("E_AUTH_TRUST")
+ root = _project_root_from_source()
+ if config["project_root"] != root:
+ raise ProtocolError("E_AUTH_TRUST")
+ approval_path = root / "ai-infoadmin" / "worklog" / f"bili-auth-successor-release-approval-{authorization_message_id}.json"
+ approval_payload = _read_strict_payload(approval_path, 256 * 1024, "E_AUTH_TRUST")
+ approval = strict_json_loads(approval_payload)
+ _plain_mapping(approval, _RELEASE_KEYS, "E_AUTH_TRUST")
+ if (
+ approval["schema"] != 1 or approval["scope"] != "bili-auth-successor-exact-release-v1"
+ or approval["project_id"] != _PROJECT_ID or approval["task_id"] != _TASK_ID
+ or approval["approved_by_role"] != "project.admin"
+ or approval["authorization_message_id"] != authorization_message_id
+ or not isinstance(approval["authorization_handoff_id"], str)
+ or not approval["authorization_handoff_id"].startswith("HANDOFF-")
+ ):
+ raise ProtocolError("E_AUTH_TRUST")
+ implementation_review = _validate_review(approval["implementation_review"], "E_AUTH_TRUST")
+ _validate_audit_prefix(root, implementation_review, "E_AUTH_TRUST")
+ authorization_path, authorization_payload = _file_identity(
+ root, approval["authorization_file"], ("ai-infoadmin/worklog/",), "E_AUTH_TRUST"
+ )
+ if authorization_path.parent != approval_path.parent:
+ raise ProtocolError("E_AUTH_TRUST")
+ authorization = strict_json_loads(authorization_payload)
+ _plain_mapping(authorization, _AUTH_KEYS, "E_AUTH_TRUST")
+ if (
+ authorization["schema"] != 1 or authorization["scope"] != "bili-auth-successor-lineage-v1"
+ or authorization["task_id"] != _TASK_ID or authorization["authorized_by_role"] != "project.admin"
+ or authorization["authorization_message_id"] != authorization_message_id
+ or authorization["authorization_handoff_id"] != approval["authorization_handoff_id"]
+ ):
+ raise ProtocolError("E_AUTH_TRUST")
+ repair = _validate_review({
+ "result_message_id": authorization["repair"].get("review_result_message_id") if isinstance(authorization.get("repair"), dict) else None,
+ "audit_id": authorization["repair"].get("audit_id") if isinstance(authorization.get("repair"), dict) else None,
+ "audit_bytes": authorization["repair"].get("audit_bytes") if isinstance(authorization.get("repair"), dict) else None,
+ "audit_sha256": authorization["repair"].get("audit_sha256") if isinstance(authorization.get("repair"), dict) else None,
+ "verdict": authorization["repair"].get("verdict") if isinstance(authorization.get("repair"), dict) else None,
+ "blocking_findings": authorization["repair"].get("blocking_findings") if isinstance(authorization.get("repair"), dict) else None,
+ }, "E_AUTH_TRUST")
+ if not isinstance(authorization["repair"], dict) or set(authorization["repair"]) != _AUTH_REPAIR_KEYS:
+ raise ProtocolError("E_AUTH_TRUST")
+ _validate_audit_prefix(root, repair, "E_AUTH_TRUST")
+ raw_successors = authorization["successors"]
+ if not isinstance(raw_successors, list) or not 1 <= len(raw_successors) <= 100:
+ raise ProtocolError("E_AUTH_TRUST")
+ successors: list[AuthorizedSuccessor] = []
+ for raw in raw_successors:
+ _plain_mapping(raw, _AUTH_SUCCESSOR_KEYS, "E_AUTH_TRUST")
+ successors.append(AuthorizedSuccessor(
+ creator_uid=raw["creator_uid"], bvid=raw["bvid"],
+ predecessor_job_id=raw["predecessor_job_id"],
+ retry_generation=raw["retry_generation"], terminal_error_code=raw["terminal_error_code"],
+ ))
+ release = ReleaseApproval(
+ authorization_message_id=authorization_message_id,
+ authorization_handoff_id=approval["authorization_handoff_id"],
+ authorization_sha256=hashlib.sha256(authorization_payload).hexdigest().upper(),
+ repair_review_result_message_id=repair["result_message_id"],
+ repair_audit_id=repair["audit_id"], repair_audit_bytes=repair["audit_bytes"],
+ repair_audit_sha256=repair["audit_sha256"], successors=tuple(successors),
+ )
+ QueueStore._validate_release(release)
+ _validate_deployment(root, config, approval, authorization)
+ return release
+
+
+def _validate_deployment(
+ root: Path, config: dict[str, Any], approval: dict[str, Any], authorization: dict[str, Any]
+) -> None:
+ deployment = _plain_mapping(approval["deployment"], _DEPLOYMENT_KEYS, "E_DEPLOYMENT_NOT_READY")
+ if (
+ deployment["extension_build"] != EXTENSION_BUILD
+ or deployment["host_build"] != HOST_BUILD
+ or deployment["reload_generation"] != RELOAD_GENERATION
+ or deployment["extension_id"] != EXPECTED_EXTENSION_ID
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ for name in (
+ "source_manifest_sha256", "installed_config_sha256", "installed_manifest_sha256",
+ "installed_exe_sha256", "projection_contract_sha256", "projection_tree_sha256",
+ ):
+ _upper_sha(deployment[name], "E_DEPLOYMENT_NOT_READY")
+ source_manifest_path = root / "dev" / "project-dev" / "bili_authenticated_extension" / "source-artifact-manifest.json"
+ source_payload = _read_strict_payload(source_manifest_path, 256 * 1024, "E_DEPLOYMENT_NOT_READY")
+ if hashlib.sha256(source_payload).hexdigest().upper() != deployment["source_manifest_sha256"]:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ source = _validate_source_tree(root, source_payload, deployment)
+ branch = _deployment_identity_branch(root, deployment)
+ approval_prefix = ("ai-inforev/worklog/",) if branch == "legacy-reviewer" else ("ai-infoadmin/worklog/",)
+ _, build_approval_payload = _file_identity(root, deployment["build_approval"], approval_prefix, "E_DEPLOYMENT_NOT_READY")
+ _, build_receipt_payload = _file_identity(root, deployment["build_receipt"], ("dev/tmp/",), "E_DEPLOYMENT_NOT_READY")
+ _, exe_payload = _file_identity(root, deployment["exe"], ("dev/tmp/",), "E_DEPLOYMENT_NOT_READY")
+ _, install_approval_payload = _file_identity(root, deployment["install_approval"], approval_prefix, "E_DEPLOYMENT_NOT_READY")
+ _, install_receipt_payload = _file_identity(root, deployment["install_receipt"], ("ai-infoadmin/worklog/",), "E_DEPLOYMENT_NOT_READY")
+ build_approval = strict_json_loads(build_approval_payload)
+ build_receipt = strict_json_loads(build_receipt_payload)
+ install_approval = strict_json_loads(install_approval_payload)
+ install_receipt = strict_json_loads(install_receipt_payload)
+ build_approval = _plain_mapping(build_approval, _BUILD_APPROVAL_KEYS, "E_DEPLOYMENT_NOT_READY")
+ build_receipt = _plain_mapping(build_receipt, _BUILD_RECEIPT_KEYS, "E_DEPLOYMENT_NOT_READY")
+ install_approval = _plain_mapping(install_approval, _INSTALL_APPROVAL_KEYS, "E_DEPLOYMENT_NOT_READY")
+ install_receipt = _plain_mapping(
+ install_receipt,
+ _INSTALL_RECEIPT_KEYS if branch == "legacy-reviewer" else _FAST_PATH_INSTALL_RECEIPT_KEYS,
+ "E_DEPLOYMENT_NOT_READY",
+ )
+ exe_hash = hashlib.sha256(exe_payload).hexdigest().upper()
+ build_receipt_hash = hashlib.sha256(build_receipt_payload).hexdigest().upper()
+ expected_role = "dev.reviewer.project" if branch == "legacy-reviewer" else "project.admin"
+ if (
+ build_approval["schema"] != 1 or build_approval["task_id"] != _TASK_ID
+ or build_approval["approval_scope"] != "controlled-build-source-manifest"
+ or build_approval["approved_by_role"] != expected_role or build_approval["status"] != "APPROVED"
+ or build_approval["source_artifact_manifest_bytes"] != len(source_payload)
+ or build_approval["source_artifact_manifest_sha256"] != deployment["source_manifest_sha256"]
+ or build_receipt["schema"] != 2 or build_receipt["scope"] != "generic-bilibili-queue"
+ or build_receipt["extension_id"] != EXPECTED_EXTENSION_ID
+ or build_receipt["extension_build"] != EXTENSION_BUILD or build_receipt["host_build"] != HOST_BUILD
+ or build_receipt["packaging"] != "pyinstaller-onefile"
+ or build_receipt["pyinstaller_version"] != "6.15.0" or build_receipt["yt_dlp_version"] != "2026.7.4"
+ or (
+ branch == "legacy-reviewer"
+ and (
+ build_receipt["source_artifact_manifest_bytes"] != len(source_payload)
+ or build_receipt["source_artifact_manifest_sha256"] != deployment["source_manifest_sha256"]
+ )
+ )
+ or build_receipt["dependency_artifact_manifest_bytes"] != source["dependency_artifact_manifest_bytes"]
+ or build_receipt["dependency_artifact_manifest_sha256"] != source["dependency_artifact_manifest_sha256"]
+ or install_approval["schema"] != 1 or install_approval["task_id"] != _TASK_ID
+ or install_approval["approval_scope"] != "install-exact-build"
+ or install_approval["approved_by_role"] != expected_role or install_approval["status"] != "APPROVED"
+ or (
+ branch == "legacy-reviewer"
+ and install_approval["source_artifact_manifest_sha256"] != deployment["source_manifest_sha256"]
+ )
+ or install_approval["build_artifact_manifest_bytes"] != len(build_receipt_payload)
+ or install_approval["build_artifact_manifest_sha256"] != build_receipt_hash
+ or install_approval["host_executable_bytes"] != len(exe_payload)
+ or install_approval["host_executable_sha256"] != exe_hash
+ or install_receipt["schema"] != (1 if branch == "legacy-reviewer" else 2)
+ or install_receipt["task_id"] != _TASK_ID
+ or install_receipt["host_build"] != HOST_BUILD or install_receipt["required_extension_build"] != EXTENSION_BUILD
+ or install_receipt["extension_id"] != EXPECTED_EXTENSION_ID
+ or install_receipt["host_name"] != "com.project_info.bili_auth_ingress"
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ for name in (
+ "builder_python_sha256", "pyinstaller_executable_sha256",
+ "builder_provision_receipt_sha256", "build_script_sha256",
+ "yt_dlp_wheel_sha256",
+ ):
+ _upper_sha(build_receipt[name], "E_DEPLOYMENT_NOT_READY")
+ for name in (
+ "pyinstaller_executable_bytes", "builder_provision_receipt_bytes",
+ ):
+ _positive_integer(build_receipt[name], "E_DEPLOYMENT_NOT_READY")
+ archive = _plain_mapping(build_receipt["archive_verification"], _ARCHIVE_VERIFICATION_KEYS, "E_DEPLOYMENT_NOT_READY")
+ if (
+ archive["status"] != "PASS"
+ or not isinstance(archive["method"], str) or not archive["method"] or _CONTROL.search(archive["method"])
+ or archive["required_modules"] != [
+ "bili_authenticated_extension.worker", "yt_dlp", "yt_dlp.downloader",
+ "yt_dlp.globals", "yt_dlp.plugins", "yt_dlp.version",
+ ]
+ or archive["metadata_entry"] != "yt_dlp-2026.7.4.dist-info/METADATA"
+ or archive["metadata_files"] != 7
+ or not isinstance(archive["metadata_type_codes"], list)
+ or not archive["metadata_type_codes"]
+ or any(value not in {"b", "x"} for value in archive["metadata_type_codes"])
+ or len(set(archive["metadata_type_codes"])) != len(archive["metadata_type_codes"])
+ or archive["metadata_tree_sha256"] != source["archive_metadata_contract"]["canonical_tree_sha256"]
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ files = build_receipt.get("files")
+ if not isinstance(files, list) or len(files) != 1:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ build_file = _plain_mapping(files[0], _ARTIFACT_FILE_KEYS, "E_DEPLOYMENT_NOT_READY")
+ if (
+ build_file["path"] != "project-info-bili-auth-native-host.exe"
+ or build_file["bytes"] != len(exe_payload) or build_file["sha256"] != exe_hash
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ installed_root = _absolute_local_path(install_receipt.get("installed_root"), "E_DEPLOYMENT_NOT_READY")
+ installed_files = install_receipt.get("installed_files")
+ if not isinstance(installed_files, list) or {item.get("path") for item in installed_files if isinstance(item, dict)} != {
+ "project-info-bili-auth-native-host.exe", "config.json", "native-host-manifest.json"
+ }:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ actual: dict[str, tuple[Path, bytes]] = {}
+ for item in installed_files:
+ item = _plain_mapping(item, _ARTIFACT_FILE_KEYS, "E_DEPLOYMENT_NOT_READY")
+ size = _positive_integer(item["bytes"], "E_DEPLOYMENT_NOT_READY")
+ digest = _upper_sha(item["sha256"], "E_DEPLOYMENT_NOT_READY")
+ candidate = installed_root / item["path"]
+ payload = _read_strict_payload(candidate, size + 1, "E_DEPLOYMENT_NOT_READY")
+ if len(payload) != size or hashlib.sha256(payload).hexdigest().upper() != digest:
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ actual[item["path"]] = (candidate, payload)
+ if (
+ hashlib.sha256(actual["config.json"][1]).hexdigest().upper() != config.get("_host_config_sha256")
+ or hashlib.sha256(actual["config.json"][1]).hexdigest().upper() != deployment["installed_config_sha256"]
+ or hashlib.sha256(actual["native-host-manifest.json"][1]).hexdigest().upper() != deployment["installed_manifest_sha256"]
+ or hashlib.sha256(actual["project-info-bili-auth-native-host.exe"][1]).hexdigest().upper() != deployment["installed_exe_sha256"]
+ or actual["project-info-bili-auth-native-host.exe"][1] != exe_payload
+ or deployment["installed_exe_sha256"] != exe_hash
+ or actual["config.json"][0] != config["host_config_path"]
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ installed_config = _plain_mapping(
+ strict_json_loads(actual["config.json"][1]), _HOST_KEYS, "E_DEPLOYMENT_NOT_READY"
+ )
+ if (
+ installed_config["schema"] != 2
+ or installed_config["required_extension_build"] != EXTENSION_BUILD
+ or installed_config["reload_generation"] != RELOAD_GENERATION
+ or installed_config["creator_allowlist"] != sorted(set(installed_config["creator_allowlist"]))
+ or config.get("creator_uid") not in installed_config["creator_allowlist"]
+ or any(
+ installed_config[name] != str(config["queue_paths"][name])
+ for name in ("queue_path", "queue_state_path", "queue_lock_path", "reload_state_path")
+ )
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ native_manifest = strict_json_loads(actual["native-host-manifest.json"][1])
+ if (
+ not isinstance(native_manifest, dict) or native_manifest.get("name") != "com.project_info.bili_auth_ingress"
+ or native_manifest.get("type") != "stdio" or native_manifest.get("allowed_origins") != [EXPECTED_ORIGIN]
+ or Path(native_manifest.get("path", "")) != actual["project-info-bili-auth-native-host.exe"][0]
+ or Path(_read_native_host_registry_default()) != actual["native-host-manifest.json"][0]
+ ):
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ _validate_projection(root, deployment)
+ _validate_reload_applied(config["queue_paths"]["reload_state_path"])
+ if branch == "project-admin-fast-path":
+ _validate_fast_path_receipt(
+ root, config, approval, authorization, deployment, source, build_receipt,
+ install_approval, source_payload,
+ build_approval_payload, build_receipt_payload, install_approval_payload,
+ exe_payload, install_receipt, actual,
+ )
+
+
+def _formal_record(event: dict[str, Any], config: dict[str, Any]) -> tuple[str, dict[str, Any] | None, bool] | None:
+ schema = event.get("schema_version")
+ if schema == 1:
+ if event.get("creator") != config["creator_name"]:
+ raise ProtocolError("E_CATALOG")
+ uid = event.get("creator_uid")
+ if uid is not None and str(uid) != config["creator_uid"]:
+ raise ProtocolError("E_ALLOWLIST")
+ if event.get("item_type") != "video":
+ return None
+ bvid = event.get("stable_id")
+ status = event.get("status")
+ if not isinstance(status, str) or not status.startswith("VIDEO_"):
+ raise ProtocolError("E_CATALOG")
+ terminal = status in _TERMINAL_VIDEO_STATUSES or event.get("video_path") is not None
+ duration = event.get("expected_duration_seconds")
+ elif schema == 2:
+ if event.get("creator") != config["creator_name"] or str(event.get("creator_uid")) != config["creator_uid"]:
+ raise ProtocolError("E_ALLOWLIST")
+ if event.get("event_type") != "DYNAMIC_CONTENT_SAVED" or event.get("status") != "SAVED":
+ raise ProtocolError("E_CATALOG")
+ if event.get("content_type") != "video":
+ return None
+ bvid = event.get("bvid")
+ terminal = False
+ duration = event.get("duration_seconds")
+ else:
+ raise ProtocolError("E_CATALOG")
+ try:
+ bvid = validate_bvid(bvid)
+ except ValueError as exc:
+ raise ProtocolError("E_CATALOG") from exc
+ allowlist = config["bvid_allowlist"]
+ if allowlist is not None and bvid not in allowlist:
+ return bvid, None, terminal
+ if event.get("source_url") != canonical_url(bvid):
+ raise ProtocolError("E_CATALOG")
+ if terminal:
+ return bvid, None, True
+ required = (event.get("title"), event.get("published_at"), event.get("collected_at"), duration)
+ if any(value is None for value in required):
+ return bvid, None, terminal
+ record = {
+ "schema": 1,
+ "bvid": bvid,
+ "creator_uid": config["creator_uid"],
+ "expected_duration_ms": _duration_ms(duration),
+ "discovered_at_unix_ms": _time_ms(event["collected_at"]),
+ "published_at": _safe_text(event["published_at"], 64),
+ "title": _safe_text(event["title"]),
+ }
+ validate_ingress_record(record, frozenset({config["creator_uid"]}))
+ return bvid, record, terminal
+
+
+def load_dynamic_manifest(config: dict[str, Any]) -> tuple[dict[str, dict[str, Any]], set[str], str]:
+ path = config["dynamic_manifest_path"]
+ payload = path.read_bytes()
+ if len(payload) > 16 * 1024 * 1024 or (payload and not payload.endswith(b"\n")):
+ raise ProtocolError("E_CATALOG")
+ records: dict[str, dict[str, Any]] = {}
+ terminal: set[str] = set()
+ seen: set[str] = set()
+ for line in payload.splitlines():
+ if not line or len(line) > 1024 * 1024:
+ raise ProtocolError("E_CATALOG")
+ event = strict_json_loads(line)
+ if not isinstance(event, dict):
+ raise ProtocolError("E_CATALOG")
+ _reject_secret_keys(event)
+ normalized = _formal_record(event, config)
+ if normalized is None:
+ continue
+ bvid, record, is_terminal = normalized
+ seen.add(bvid)
+ if is_terminal:
+ terminal.add(bvid)
+ if record is None:
+ continue
+ previous = records.get(bvid)
+ if previous is None:
+ records[bvid] = record
+ continue
+ left = {key: value for key, value in previous.items() if key != "discovered_at_unix_ms"}
+ right = {key: value for key, value in record.items() if key != "discovered_at_unix_ms"}
+ if left != right:
+ raise ProtocolError("E_CATALOG_CONFLICT")
+ previous["discovered_at_unix_ms"] = min(previous["discovered_at_unix_ms"], record["discovered_at_unix_ms"])
+ return records, terminal, hashlib.sha256(payload).hexdigest().upper()
+
+
+def load_registered_catalog(config: dict[str, Any]) -> dict[str, dict[str, Any]]:
+ payload = config["_registered_catalog_payload"]
+ if payload is None:
+ return {}
+ raw = strict_json_loads(payload)
+ if not isinstance(raw, dict) or set(raw) != _REGISTERED_KEYS or raw.get("schema") != 1 or raw.get("source") != "bili-dynamic-collector-registered-v1":
+ raise ProtocolError("E_CATALOG")
+ _reject_secret_keys(raw)
+ if raw.get("creator_name") != config["creator_name"] or raw.get("creator_uid") != config["creator_uid"]:
+ raise ProtocolError("E_ALLOWLIST")
+ items = raw.get("items")
+ if not isinstance(items, list) or not 1 <= len(items) <= 10_000:
+ raise ProtocolError("E_CATALOG")
+ records: dict[str, dict[str, Any]] = {}
+ for value in items:
+ if not isinstance(value, dict) or set(value) != _INGRESS_KEYS:
+ raise ProtocolError("E_CATALOG")
+ job = validate_ingress_record(value, frozenset({config["creator_uid"]}))
+ if job["creator_uid"] != config["creator_uid"]:
+ raise ProtocolError("E_ALLOWLIST")
+ allowlist = config["bvid_allowlist"]
+ if allowlist is not None and job["bvid"] not in allowlist:
+ continue
+ previous = records.get(job["bvid"])
+ if previous is not None and previous != value:
+ raise ProtocolError("E_CATALOG_CONFLICT")
+ records.setdefault(job["bvid"], value)
+ return records
+
+
+def collect_jobs(config: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+ records, terminal, dynamic_hash = load_dynamic_manifest(config)
+ registered = load_registered_catalog(config)
+ for bvid, record in registered.items():
+ previous = records.get(bvid)
+ if previous is not None and previous != record:
+ raise ProtocolError("E_CATALOG_CONFLICT")
+ records.setdefault(bvid, record)
+ allowlist = config["bvid_allowlist"]
+ if allowlist is not None:
+ missing = set(allowlist).difference(records, terminal)
+ if missing:
+ raise ProtocolError("E_CATALOG_MISSING")
+ jobs = [records[bvid] for bvid in sorted(records) if bvid not in terminal]
+ if not jobs:
+ raise ProtocolError("E_NO_JOBS")
+ return jobs, {
+ "dynamic_manifest_sha256": dynamic_hash,
+ "registered_catalog_sha256": config["_registered_catalog_sha256"],
+ "catalog_jobs": len(records),
+ "terminal_skipped": len(set(records).intersection(terminal)),
+ }
+
+
+def run(config_path: Path, *, append: bool, append_successors: bool = False) -> dict[str, Any]:
+ if append and append_successors:
+ raise ProtocolError("E_CONFIG")
+ config = load_producer_configuration(config_path)
+ jobs, evidence = collect_jobs(config)
+ paths = config["queue_paths"]
+ result: dict[str, int] = {"appended": 0, "unchanged": 0}
+ if append or append_successors:
+ runtime_root = paths["queue_path"].parent
+ if append_successors:
+ _check_existing_chain(runtime_root, require_file=False)
+ _check_existing_chain(paths["queue_lock_path"], require_file=True)
+ if not runtime_root.is_dir():
+ raise ProtocolError("E_DEPLOYMENT_NOT_READY")
+ else:
+ runtime_root.mkdir(parents=True, exist_ok=True)
+ _check_existing_chain(runtime_root, require_file=False)
+ store = QueueStore(
+ paths["queue_path"], paths["queue_state_path"], paths["queue_lock_path"],
+ frozenset({config["creator_uid"]}),
+ )
+ if append_successors:
+ result = store.append_authorized_successors(lambda: _load_release_approval(config), jobs)
+ else:
+ result = store.append_ingress_jobs(jobs)
+ return {
+ "schema": 2 if append_successors else 1,
+ "producer_id": PRODUCER_ID,
+ "status": (
+ "SUCCESSOR_APPENDED" if append_successors and result["appended"]
+ else "NO_CHANGE" if append_successors
+ else "APPENDED" if append and result["appended"]
+ else "NO_CHANGE" if append
+ else "VALIDATION_PASS_ONLY"
+ ),
+ "job_count": len(jobs),
+ **result,
+ **evidence,
+ }
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Produce governed Bilibili authenticated queue records.")
+ parser.add_argument("--config", required=True, type=Path)
+ action = parser.add_mutually_exclusive_group()
+ action.add_argument("--append", action="store_true", help="Append initial schema-1 jobs after validation")
+ action.add_argument("--append-successors", action="store_true", help="Append one admin-approved schema-2 successor block")
+ return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+ try:
+ arguments = _parser().parse_args(argv)
+ result = run(
+ arguments.config, append=arguments.append,
+ append_successors=arguments.append_successors,
+ )
+ except ProtocolError as exc:
+ print(json.dumps({"schema": 1, "status": "FAILED", "error_code": exc.code}, sort_keys=True, separators=(",", ":")))
+ return 3
+ except (OSError, KeyError, TypeError, ValueError):
+ print(json.dumps({"schema": 1, "status": "FAILED", "error_code": "E_PRODUCER"}, sort_keys=True, separators=(",", ":")))
+ return 3
+ print(json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":")))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/dev/project-dev/bili_authenticated_extension/queue_state.py b/dev/project-dev/bili_authenticated_extension/queue_state.py
new file mode 100644
index 0000000..7d00937
--- /dev/null
+++ b/dev/project-dev/bili_authenticated_extension/queue_state.py
@@ -0,0 +1,1298 @@
+"""Append-only local queue and reload state for generic authenticated jobs."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import secrets
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable, Iterator
+
+from .constants import (
+ COMPLETION_CLOSURE_REQUIRED,
+ EXTENSION_BUILD,
+ ERROR_CODE_RE,
+ HANDOFF_ID_RE,
+ JOB_ID_RE,
+ MESSAGE_ID_RE,
+ MAX_CLAIM_ATTEMPTS,
+ INTERNAL_ONLY_PAGE_PENDING_CODES,
+ LEGACY_MISSING_DIAGNOSTIC_REPLAY_LINES,
+ LEGACY_PAGE_METADATA_REPLAY_LINES,
+ LEGACY_PAGE_METADATA_REPLAY_PREFIX_BYTES,
+ LEGACY_PAGE_METADATA_REPLAY_PREFIX_SHA256,
+ LEGACY_PAGE_METADATA_REPLAY_STATE_SUFFIX,
+ PREPARELESS_REJECT_CODES,
+ QUEUE_LEASE_SECONDS,
+ QUEUE_SCHEMA_VERSION,
+ SUCCESSOR_QUEUE_SCHEMA_VERSION,
+ UPPER_SHA256_RE,
+ RELOAD_BACKOFF_SECONDS,
+ canonical_url,
+ stable_job_id,
+ stable_successor_job_id,
+ validate_bvid,
+ validate_creator_uid,
+ validate_prepareless_terminal,
+ validate_postprocess_terminal,
+ validate_runtime_diagnostic,
+)
+from .protocol import (
+ ProtocolError, encode_json, strict_json_loads, validate_job,
+ validate_media_complete_identity,
+)
+
+
+@dataclass(frozen=True)
+class AuthorizedSuccessor:
+ creator_uid: str
+ bvid: str
+ predecessor_job_id: str
+ retry_generation: int
+ terminal_error_code: str
+
+
+@dataclass(frozen=True)
+class ReleaseApproval:
+ """Immutable value emitted only by the producer's deployment trust gate."""
+
+ authorization_message_id: str
+ authorization_handoff_id: str
+ authorization_sha256: str
+ repair_review_result_message_id: str
+ repair_audit_id: str
+ repair_audit_bytes: int
+ repair_audit_sha256: str
+ successors: tuple[AuthorizedSuccessor, ...]
+
+
+def _read_jsonl(path: Path, *, missing_ok: bool) -> list[dict[str, Any]]:
+ if not path.exists():
+ if missing_ok:
+ return []
+ raise ProtocolError("E_QUEUE")
+ if not path.is_file() or path.is_symlink():
+ raise ProtocolError("E_QUEUE")
+ payload = path.read_bytes()
+ if len(payload) > 16 * 1024 * 1024:
+ raise ProtocolError("E_QUEUE")
+ if payload and not payload.endswith(b"\n"):
+ raise ProtocolError("E_QUEUE_PARTIAL")
+ result: list[dict[str, Any]] = []
+ for line in payload.splitlines():
+ if not line or len(line) > 4096:
+ raise ProtocolError("E_QUEUE")
+ result.append(strict_json_loads(line))
+ return result
+
+
+def _append_jsonl(path: Path, value: dict[str, Any]) -> None:
+ payload = encode_json(value, 4096) + b"\n"
+ with path.open("ab", buffering=0) as stream:
+ if stream.write(payload) != len(payload):
+ raise ProtocolError("E_QUEUE_WRITE")
+ stream.flush()
+ os.fsync(stream.fileno())
+
+
+def _media_complete_durability_test_seam(_stage: str) -> None:
+ """No-op production seam for crash-window durability counterexamples."""
+
+ return None
+
+
+def _postprocess_recovery_claim_test_seam(_stage: str) -> None:
+ """No-op production seam for recovery-claim crash counterexamples."""
+
+ return None
+
+
+def _append_jsonl_batch(path: Path, values: list[dict[str, Any]]) -> None:
+ if not values:
+ return
+ payload = b"".join(encode_json(value, 4096) + b"\n" for value in values)
+ with path.open("ab", buffering=0) as stream:
+ if stream.write(payload) != len(payload):
+ raise ProtocolError("E_QUEUE_WRITE")
+ stream.flush()
+ os.fsync(stream.fileno())
+
+
+def _append_bytes(path: Path, payload: bytes) -> None:
+ if not payload:
+ return
+ with path.open("ab", buffering=0) as stream:
+ if stream.write(payload) != len(payload):
+ raise ProtocolError("E_QUEUE_WRITE")
+ stream.flush()
+ os.fsync(stream.fileno())
+
+
+def _canonical_line(value: dict[str, Any]) -> bytes:
+ return encode_json(value, 4096) + b"\n"
+
+
+@dataclass(frozen=True)
+class _QueueSnapshot:
+ jobs: dict[str, dict[str, Any]]
+ records: dict[str, dict[str, Any]]
+ children: dict[str, str]
+ blocks: tuple[tuple[str, str, str, tuple[str, ...], bytes], ...]
+ committed_bytes: bytes
+ pending_suffix: bytes
+
+
+_LINEAGE_KEYS = {
+ "predecessor_job_id", "retry_generation", "predecessor_terminal_error_code",
+ "authorization_message_id", "authorization_handoff_id", "authorization_sha256",
+ "repair_review_result_message_id", "repair_audit_id", "repair_audit_bytes",
+ "repair_audit_sha256",
+}
+_SUCCESSOR_RECORD_KEYS = {
+ "schema", "record_type", "job_id", "bvid", "creator_uid",
+ "expected_duration_ms", "discovered_at_unix_ms", "published_at", "title", "lineage",
+}
+_BEGIN_KEYS = {
+ "schema", "record_type", "authorization_message_id", "authorization_handoff_id",
+ "authorization_sha256", "successor_count",
+}
+_COMMIT_KEYS = _BEGIN_KEYS | {"block_sha256"}
+
+
+def _validate_authorization_identity(message_id: Any, handoff_id: Any, sha256: Any) -> None:
+ if (
+ not isinstance(message_id, str) or not MESSAGE_ID_RE.fullmatch(message_id)
+ or not isinstance(handoff_id, str) or not HANDOFF_ID_RE.fullmatch(handoff_id)
+ or not isinstance(sha256, str) or not UPPER_SHA256_RE.fullmatch(sha256)
+ ):
+ raise ProtocolError("E_LINEAGE")
+
+
+def _successor_record(value: dict[str, Any], allowed_creators: frozenset[str]) -> dict[str, Any]:
+ if set(value) != _SUCCESSOR_RECORD_KEYS or value.get("schema") != SUCCESSOR_QUEUE_SCHEMA_VERSION or value.get("record_type") != "SUCCESSOR_JOB":
+ raise ProtocolError("E_LINEAGE")
+ creator = value.get("creator_uid")
+ if creator not in allowed_creators:
+ raise ProtocolError("E_ALLOWLIST")
+ discovered = value.get("discovered_at_unix_ms")
+ if isinstance(discovered, bool) or not isinstance(discovered, int) or discovered <= 0:
+ raise ProtocolError("E_LINEAGE")
+ lineage = value.get("lineage")
+ if not isinstance(lineage, dict) or set(lineage) != _LINEAGE_KEYS:
+ raise ProtocolError("E_LINEAGE")
+ runtime = {
+ "job_id": value["job_id"],
+ "bvid": value["bvid"],
+ "creator_uid": creator,
+ "canonical_url": canonical_url(value["bvid"]),
+ "expected_duration_ms": value["expected_duration_ms"],
+ "published_at": value["published_at"],
+ "title": value["title"],
+ "lineage": lineage,
+ }
+ try:
+ validate_job(runtime)
+ except (ProtocolError, ValueError) as exc:
+ raise ProtocolError("E_LINEAGE") from exc
+ return runtime
+
+
+def _queue_snapshot(payload: bytes, allowed_creators: frozenset[str]) -> _QueueSnapshot:
+ """Parse committed schema-1 records and schema-2 blocks, retaining one tail suffix."""
+ if len(payload) > 16 * 1024 * 1024:
+ raise ProtocolError("E_QUEUE")
+ jobs: dict[str, dict[str, Any]] = {}
+ records: dict[str, dict[str, Any]] = {}
+ ingress: dict[str, dict[str, Any]] = {}
+ children: dict[str, str] = {}
+ blocks: list[tuple[str, str, str, tuple[str, ...], bytes]] = []
+ lines: list[tuple[int, int, bytes, dict[str, Any]]] = []
+ offset = 0
+ for raw_line in payload.splitlines(keepends=True):
+ start = offset
+ offset += len(raw_line)
+ if not raw_line.endswith(b"\n"):
+ break
+ content = raw_line[:-1]
+ if content.endswith(b"\r"):
+ content = content[:-1]
+ if not content or len(content) > 4096:
+ raise ProtocolError("E_QUEUE")
+ value = strict_json_loads(content)
+ if not isinstance(value, dict):
+ raise ProtocolError("E_QUEUE")
+ lines.append((start, offset, raw_line, value))
+
+ index = 0
+ committed_end = 0
+ while index < len(lines):
+ start, end, raw_line, value = lines[index]
+ if value.get("schema") == QUEUE_SCHEMA_VERSION:
+ if value.get("record_type") is not None:
+ raise ProtocolError("E_QUEUE")
+ job = _ingress_job(value, allowed_creators)
+ job_id = job["job_id"]
+ previous = ingress.get(job_id)
+ if previous is not None and previous != value:
+ raise ProtocolError("E_QUEUE_CONFLICT")
+ ingress.setdefault(job_id, value)
+ prior_job = jobs.get(job_id)
+ if prior_job is not None and prior_job != job:
+ raise ProtocolError("E_QUEUE_CONFLICT")
+ jobs.setdefault(job_id, job)
+ records.setdefault(job_id, value)
+ committed_end = end
+ index += 1
+ continue
+
+ if value.get("schema") != SUCCESSOR_QUEUE_SCHEMA_VERSION or value.get("record_type") != "SUCCESSOR_BEGIN":
+ raise ProtocolError("E_LINEAGE_RECOVERY")
+ if set(value) != _BEGIN_KEYS or raw_line != _canonical_line(value):
+ raise ProtocolError("E_LINEAGE")
+ _validate_authorization_identity(
+ value.get("authorization_message_id"), value.get("authorization_handoff_id"),
+ value.get("authorization_sha256"),
+ )
+ count = value.get("successor_count")
+ if isinstance(count, bool) or not isinstance(count, int) or not 1 <= count <= 100:
+ raise ProtocolError("E_LINEAGE")
+ needed = count + 2
+ if index + needed > len(lines):
+ break
+ block_lines = lines[index:index + needed]
+ job_ids: list[str] = []
+ block_records: list[tuple[dict[str, Any], dict[str, Any]]] = []
+ for _, _, job_line, raw_job in block_lines[1:-1]:
+ if job_line != _canonical_line(raw_job):
+ raise ProtocolError("E_LINEAGE")
+ runtime = _successor_record(raw_job, allowed_creators)
+ lineage = runtime["lineage"]
+ if (
+ lineage["authorization_message_id"] != value["authorization_message_id"]
+ or lineage["authorization_handoff_id"] != value["authorization_handoff_id"]
+ or lineage["authorization_sha256"] != value["authorization_sha256"]
+ ):
+ raise ProtocolError("E_LINEAGE")
+ job_ids.append(runtime["job_id"])
+ block_records.append((raw_job, runtime))
+ if job_ids != sorted(job_ids) or len(set(job_ids)) != count:
+ raise ProtocolError("E_LINEAGE")
+ _, block_end, commit_line, commit = block_lines[-1]
+ if set(commit) != _COMMIT_KEYS or commit.get("schema") != SUCCESSOR_QUEUE_SCHEMA_VERSION or commit.get("record_type") != "SUCCESSOR_COMMIT" or commit_line != _canonical_line(commit):
+ raise ProtocolError("E_LINEAGE")
+ for key in _BEGIN_KEYS - {"record_type"}:
+ if commit.get(key) != value.get(key):
+ raise ProtocolError("E_LINEAGE")
+ block_prefix = b"".join(item[2] for item in block_lines[:-1])
+ block_hash = hashlib.sha256(block_prefix).hexdigest().upper()
+ if commit.get("block_sha256") != block_hash:
+ raise ProtocolError("E_LINEAGE")
+ full_block = block_prefix + commit_line
+ for raw_job, runtime in block_records:
+ job_id = runtime["job_id"]
+ parent = runtime["lineage"]["predecessor_job_id"]
+ if job_id in jobs or parent in children:
+ raise ProtocolError("E_LINEAGE_CONFLICT")
+ jobs[job_id] = runtime
+ records[job_id] = raw_job
+ children[parent] = job_id
+ blocks.append((
+ value["authorization_message_id"], value["authorization_handoff_id"],
+ value["authorization_sha256"], tuple(job_ids), full_block,
+ ))
+ committed_end = block_end
+ index += needed
+
+ return _QueueSnapshot(
+ jobs=jobs,
+ records=records,
+ children=children,
+ blocks=tuple(blocks),
+ committed_bytes=payload[:committed_end],
+ pending_suffix=payload[committed_end:],
+ )
+
+
+def _ingress_job(value: dict[str, Any], allowed_creators: frozenset[str]) -> dict[str, Any]:
+ expected = {"schema", "bvid", "creator_uid", "expected_duration_ms", "discovered_at_unix_ms", "published_at", "title"}
+ if set(value) != expected or value.get("schema") != QUEUE_SCHEMA_VERSION:
+ raise ProtocolError("E_QUEUE")
+ try:
+ bvid = validate_bvid(value["bvid"])
+ creator = validate_creator_uid(value["creator_uid"])
+ except ValueError as exc:
+ raise ProtocolError("E_JOB") from exc
+ duration = value["expected_duration_ms"]
+ discovered = value["discovered_at_unix_ms"]
+ if (
+ creator not in allowed_creators
+ or isinstance(duration, bool) or not isinstance(duration, int) or not 1_000 <= duration <= 86_400_000
+ or isinstance(discovered, bool) or not isinstance(discovered, int) or discovered <= 0
+ ):
+ raise ProtocolError("E_ALLOWLIST" if creator not in allowed_creators else "E_JOB")
+ job = {
+ "job_id": stable_job_id(creator, bvid),
+ "bvid": bvid,
+ "creator_uid": creator,
+ "canonical_url": canonical_url(bvid),
+ "expected_duration_ms": duration,
+ "published_at": value["published_at"],
+ "title": value["title"],
+ }
+ try:
+ return validate_job(job)
+ except ProtocolError as exc:
+ raise ProtocolError("E_JOB") from exc
+
+
+def validate_ingress_record(
+ value: dict[str, Any], allowed_creators: frozenset[str]
+) -> dict[str, Any]:
+ """Public producer/consumer boundary for one exact schema-1 record."""
+ return _ingress_job(value, allowed_creators)
+
+
+_EVENTS = {
+ "CLAIMED", "STARTED", "MEDIA_COMPLETE", "POSTPROCESS_CLAIMED",
+ "COMPLETE", "FAILED", "POSTPROCESS_FAILED",
+}
+_TERMINAL_EVENTS = {"COMPLETE", "FAILED", "POSTPROCESS_FAILED"}
+_POSTPROCESS_RECOVERY_KEYS = {
+ "media_complete_event_sha256", "media_complete_lease_id", "media",
+}
+
+
+def _media_complete_event_sha256(value: dict[str, Any]) -> str:
+ if value.get("event") != "MEDIA_COMPLETE":
+ raise ProtocolError("E_QUEUE_STATE")
+ return hashlib.sha256(_canonical_line(value)).hexdigest().upper()
+
+
+def _postprocess_recovery_binding(value: dict[str, Any]) -> dict[str, Any]:
+ if value.get("event") != "MEDIA_COMPLETE" or "media" not in value:
+ raise ProtocolError("E_QUEUE_STATE")
+ return {
+ "media_complete_event_sha256": _media_complete_event_sha256(value),
+ "media_complete_lease_id": value["lease_id"],
+ "media": value["media"],
+ }
+
+
+def _validate_postprocess_recovery_binding(
+ value: object, job: dict[str, Any],
+) -> dict[str, Any]:
+ if not isinstance(value, dict) or set(value) != _POSTPROCESS_RECOVERY_KEYS:
+ raise ProtocolError("E_QUEUE_STATE")
+ digest = value["media_complete_event_sha256"]
+ prior_lease = value["media_complete_lease_id"]
+ if not isinstance(digest, str) or UPPER_SHA256_RE.fullmatch(digest) is None:
+ raise ProtocolError("E_QUEUE_STATE")
+ if (
+ not isinstance(prior_lease, str) or len(prior_lease) != 32
+ or any(ch not in "0123456789abcdef" for ch in prior_lease)
+ ):
+ raise ProtocolError("E_QUEUE_STATE")
+ try:
+ media = validate_media_complete_identity(value["media"], job)
+ except ProtocolError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ return {
+ "media_complete_event_sha256": digest,
+ "media_complete_lease_id": prior_lease,
+ "media": media,
+ }
+
+
+def _event(
+ value: dict[str, Any], jobs: dict[str, dict[str, Any]], *,
+ allow_exact_legacy_replay: bool = False,
+ allow_exact_missing_diagnostic_replay: bool = False,
+) -> dict[str, Any]:
+ expected = {
+ "schema", "event", "job_id", "bvid", "creator_uid", "lease_id",
+ "at_unix_ms", "lease_expires_unix_ms", "error_code",
+ }
+ allowed_shapes = (
+ expected, expected | {"diagnostic"}, expected | {"media"},
+ expected | {"recovery"},
+ )
+ if set(value) not in allowed_shapes or value.get("schema") != QUEUE_SCHEMA_VERSION or value.get("event") not in _EVENTS:
+ raise ProtocolError("E_QUEUE_STATE")
+ if not isinstance(value["job_id"], str) or not JOB_ID_RE.fullmatch(value["job_id"]):
+ raise ProtocolError("E_QUEUE_STATE")
+ job = jobs.get(value["job_id"])
+ if job is None or value["creator_uid"] != job["creator_uid"] or value["bvid"] != job["bvid"]:
+ raise ProtocolError("E_QUEUE_STATE")
+ lease = value["lease_id"]
+ if not isinstance(lease, str) or len(lease) != 32 or any(ch not in "0123456789abcdef" for ch in lease):
+ raise ProtocolError("E_QUEUE_STATE")
+ for name in ("at_unix_ms", "lease_expires_unix_ms"):
+ if isinstance(value[name], bool) or not isinstance(value[name], int) or value[name] <= 0:
+ raise ProtocolError("E_QUEUE_STATE")
+ error = value["error_code"]
+ if error is not None and (not isinstance(error, str) or not error.startswith("E_")):
+ raise ProtocolError("E_QUEUE_STATE")
+ if value["event"] in {"FAILED", "POSTPROCESS_FAILED"} and error is None:
+ raise ProtocolError("E_QUEUE_STATE")
+ if value["event"] not in {"FAILED", "POSTPROCESS_FAILED"} and error is not None:
+ raise ProtocolError("E_QUEUE_STATE")
+ if value["event"] == "MEDIA_COMPLETE":
+ if "media" not in value:
+ raise ProtocolError("E_QUEUE_STATE")
+ try:
+ validate_media_complete_identity(value["media"], job)
+ except ProtocolError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ elif "media" in value:
+ raise ProtocolError("E_QUEUE_STATE")
+ if value["event"] == "POSTPROCESS_CLAIMED":
+ if "recovery" not in value:
+ raise ProtocolError("E_QUEUE_STATE")
+ _validate_postprocess_recovery_binding(value["recovery"], job)
+ elif "recovery" in value:
+ raise ProtocolError("E_QUEUE_STATE")
+ if error in INTERNAL_ONLY_PAGE_PENDING_CODES and not allow_exact_legacy_replay:
+ raise ProtocolError("E_QUEUE_STATE")
+ if "diagnostic" in value:
+ if value["event"] not in {"FAILED", "POSTPROCESS_FAILED"}:
+ raise ProtocolError("E_QUEUE_STATE")
+ try:
+ if value["event"] == "POSTPROCESS_FAILED":
+ validate_postprocess_terminal(error, value["diagnostic"])
+ elif error in PREPARELESS_REJECT_CODES:
+ validate_prepareless_terminal(error, value["diagnostic"])
+ else:
+ validate_runtime_diagnostic(value["diagnostic"])
+ except ValueError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ elif value["event"] == "POSTPROCESS_FAILED":
+ try:
+ validate_postprocess_terminal(error, None)
+ except ValueError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ elif error in PREPARELESS_REJECT_CODES and not allow_exact_missing_diagnostic_replay:
+ raise ProtocolError("E_QUEUE_STATE")
+ return value
+
+
+def _terminal_request(
+ job: dict[str, Any], lease_id: str, now_ms: int, *, complete: bool,
+ error_code: str | None, diagnostic: dict[str, object] | None,
+) -> dict[str, Any]:
+ """Validate one live terminal request before any idempotent decision."""
+
+ validated_job = validate_job(job)
+ if not isinstance(complete, bool):
+ raise ProtocolError("E_QUEUE_STATE")
+ if error_code == COMPLETION_CLOSURE_REQUIRED:
+ raise ProtocolError("E_QUEUE_STATE")
+ payload: dict[str, Any] = {
+ "schema": QUEUE_SCHEMA_VERSION,
+ "event": "COMPLETE" if complete else "FAILED",
+ "job_id": validated_job["job_id"],
+ "bvid": validated_job["bvid"],
+ "creator_uid": validated_job["creator_uid"],
+ "lease_id": lease_id,
+ "at_unix_ms": now_ms,
+ "lease_expires_unix_ms": now_ms + QUEUE_LEASE_SECONDS * 1_000,
+ "error_code": error_code,
+ }
+ if diagnostic is not None:
+ payload["diagnostic"] = diagnostic
+ return _event(payload, {validated_job["job_id"]: validated_job})
+
+
+def _is_governed_legacy_state_path(path: Path) -> bool:
+ if not path.is_absolute():
+ return False
+ suffix = tuple(part.casefold() for part in LEGACY_PAGE_METADATA_REPLAY_STATE_SUFFIX)
+ parts = tuple(part.casefold() for part in path.parts)
+ return len(parts) >= len(suffix) and parts[-len(suffix):] == suffix
+
+
+def _replay_state_events(path: Path, jobs: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
+ if not path.exists():
+ return []
+ if not path.is_file() or path.is_symlink():
+ raise ProtocolError("E_QUEUE")
+ payload = path.read_bytes()
+ if len(payload) > 16 * 1024 * 1024:
+ raise ProtocolError("E_QUEUE")
+ if payload and not payload.endswith(b"\n"):
+ raise ProtocolError("E_QUEUE_PARTIAL")
+
+ governed = (
+ _is_governed_legacy_state_path(path)
+ and len(payload) >= LEGACY_PAGE_METADATA_REPLAY_PREFIX_BYTES
+ )
+ if governed and hashlib.sha256(
+ payload[:LEGACY_PAGE_METADATA_REPLAY_PREFIX_BYTES]
+ ).hexdigest().upper() != LEGACY_PAGE_METADATA_REPLAY_PREFIX_SHA256:
+ raise ProtocolError("E_QUEUE_STATE")
+
+ expected = {
+ line_number: (line_bytes, line_sha256)
+ for line_number, line_bytes, line_sha256 in LEGACY_PAGE_METADATA_REPLAY_LINES
+ }
+ expected_missing = {
+ line_number: (line_bytes, line_sha256, error_code)
+ for line_number, line_bytes, line_sha256, error_code
+ in LEGACY_MISSING_DIAGNOSTIC_REPLAY_LINES
+ }
+ observed: list[tuple[int, int, str]] = []
+ observed_missing: list[tuple[int, int, str, str]] = []
+ events: list[dict[str, Any]] = []
+ latest_by_job: dict[str, dict[str, Any]] = {}
+ media_complete_by_job: dict[str, dict[str, Any]] = {}
+ for line_number, line in enumerate(payload.splitlines(keepends=True), 1):
+ if not line.endswith(b"\n") or len(line) <= 1 or len(line) - 1 > 4096:
+ raise ProtocolError("E_QUEUE")
+ value = strict_json_loads(line[:-1])
+ legacy = value.get("error_code") in INTERNAL_ONLY_PAGE_PENDING_CODES
+ allow = False
+ allow_missing = False
+ if legacy:
+ identity = (len(line), hashlib.sha256(line).hexdigest().upper())
+ allow = governed and expected.get(line_number) == identity
+ if not allow:
+ raise ProtocolError("E_QUEUE_STATE")
+ observed.append((line_number, *identity))
+ elif value.get("error_code") in PREPARELESS_REJECT_CODES and "diagnostic" not in value:
+ identity_with_error = (
+ len(line), hashlib.sha256(line).hexdigest().upper(), value["error_code"],
+ )
+ allow_missing = governed and expected_missing.get(line_number) == identity_with_error
+ if not allow_missing:
+ raise ProtocolError("E_QUEUE_STATE")
+ observed_missing.append((line_number, *identity_with_error))
+ parsed = _event(
+ value, jobs, allow_exact_legacy_replay=allow,
+ allow_exact_missing_diagnostic_replay=allow_missing,
+ )
+ job_id = parsed["job_id"]
+ latest = latest_by_job.get(job_id)
+ if parsed["event"] == "MEDIA_COMPLETE":
+ if job_id in media_complete_by_job:
+ raise ProtocolError("E_QUEUE_STATE")
+ if (
+ latest is None or latest["event"] != "STARTED"
+ or latest["lease_id"] != parsed["lease_id"]
+ ):
+ raise ProtocolError("E_QUEUE_STATE")
+ media_complete_by_job[job_id] = parsed
+ elif parsed["event"] == "POSTPROCESS_CLAIMED":
+ prior_media = media_complete_by_job.get(job_id)
+ if (
+ prior_media is None
+ or latest is None
+ or latest["event"] not in {"MEDIA_COMPLETE", "POSTPROCESS_CLAIMED"}
+ or parsed["recovery"] != _postprocess_recovery_binding(prior_media)
+ ):
+ raise ProtocolError("E_QUEUE_STATE")
+ elif latest is not None and latest["event"] == "MEDIA_COMPLETE":
+ if (
+ parsed["event"] not in {"COMPLETE", "POSTPROCESS_FAILED"}
+ or parsed["lease_id"] != latest["lease_id"]
+ ):
+ raise ProtocolError("E_QUEUE_STATE")
+ elif latest is not None and latest["event"] == "POSTPROCESS_CLAIMED":
+ if (
+ parsed["event"] not in {"COMPLETE", "POSTPROCESS_FAILED"}
+ or parsed["lease_id"] != latest["lease_id"]
+ ):
+ raise ProtocolError("E_QUEUE_STATE")
+ events.append(parsed)
+ latest_by_job[job_id] = parsed
+ if governed and tuple(observed) != LEGACY_PAGE_METADATA_REPLAY_LINES:
+ raise ProtocolError("E_QUEUE_STATE")
+ if governed and tuple(observed_missing) != LEGACY_MISSING_DIAGNOSTIC_REPLAY_LINES:
+ raise ProtocolError("E_QUEUE_STATE")
+ return events
+
+
+class QueueStore:
+ def __init__(
+ self,
+ queue_path: Path,
+ state_path: Path,
+ lock_path: Path,
+ allowed_creators: frozenset[str],
+ ) -> None:
+ self.queue_path = queue_path
+ self.state_path = state_path
+ self.lock_path = lock_path
+ self.allowed_creators = allowed_creators
+
+ @contextmanager
+ def _locked(self) -> Iterator[None]:
+ import msvcrt
+
+ self.lock_path.parent.mkdir(parents=True, exist_ok=True)
+ with self.lock_path.open("a+b") as stream:
+ if stream.seek(0, os.SEEK_END) == 0:
+ stream.write(b"\0")
+ stream.flush()
+ os.fsync(stream.fileno())
+ stream.seek(0)
+ try:
+ msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
+ except OSError as exc:
+ raise ProtocolError("E_QUEUE_BUSY") from exc
+ try:
+ yield
+ finally:
+ stream.seek(0)
+ msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
+
+ def _jobs(self) -> list[dict[str, Any]]:
+ payload = self.queue_path.read_bytes() if self.queue_path.exists() else b""
+ snapshot = _queue_snapshot(payload, self.allowed_creators)
+ if snapshot.pending_suffix:
+ raise ProtocolError("E_QUEUE_PARTIAL")
+ return list(snapshot.jobs.values())
+
+ def append_ingress_jobs(self, records: list[dict[str, Any]]) -> dict[str, int]:
+ """Append a prevalidated producer batch under the consumer's exact lock.
+
+ The full incoming batch and the existing queue are checked before the
+ first append. Replaying byte-equivalent schema-1 records is idempotent;
+ any field drift for a stable job id fails closed.
+ """
+ if not isinstance(records, list) or not 1 <= len(records) <= 10_000:
+ raise ProtocolError("E_QUEUE")
+ incoming: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {}
+ order: list[str] = []
+ for raw in records:
+ if not isinstance(raw, dict):
+ raise ProtocolError("E_QUEUE")
+ job = _ingress_job(raw, self.allowed_creators)
+ job_id = job["job_id"]
+ previous = incoming.get(job_id)
+ if previous is not None and previous[0] != raw:
+ raise ProtocolError("E_QUEUE_CONFLICT")
+ if previous is None:
+ incoming[job_id] = (raw, job)
+ order.append(job_id)
+
+ with self._locked():
+ payload = self.queue_path.read_bytes() if self.queue_path.exists() else b""
+ snapshot = _queue_snapshot(payload, self.allowed_creators)
+ if snapshot.pending_suffix:
+ raise ProtocolError("E_QUEUE_PARTIAL")
+ existing = snapshot.records
+
+ to_append: list[dict[str, Any]] = []
+ unchanged = 0
+ for job_id in order:
+ raw = incoming[job_id][0]
+ previous = existing.get(job_id)
+ if previous is None:
+ to_append.append(raw)
+ elif previous == raw:
+ unchanged += 1
+ else:
+ raise ProtocolError("E_QUEUE_CONFLICT")
+ _append_jsonl_batch(self.queue_path, to_append)
+ return {"appended": len(to_append), "unchanged": unchanged}
+
+ @staticmethod
+ def _validate_release(approval: ReleaseApproval) -> None:
+ if not isinstance(approval, ReleaseApproval):
+ raise ProtocolError("E_AUTH_TRUST")
+ _validate_authorization_identity(
+ approval.authorization_message_id, approval.authorization_handoff_id,
+ approval.authorization_sha256,
+ )
+ if (
+ not MESSAGE_ID_RE.fullmatch(approval.repair_review_result_message_id)
+ or not isinstance(approval.repair_audit_id, str)
+ or not approval.repair_audit_id.startswith("DEV-AUDIT-")
+ or isinstance(approval.repair_audit_bytes, bool)
+ or not isinstance(approval.repair_audit_bytes, int)
+ or approval.repair_audit_bytes <= 0
+ or not UPPER_SHA256_RE.fullmatch(approval.repair_audit_sha256)
+ or not isinstance(approval.successors, tuple)
+ or not 1 <= len(approval.successors) <= 100
+ ):
+ raise ProtocolError("E_AUTH_TRUST")
+ identities: list[tuple[str, str, str]] = []
+ for item in approval.successors:
+ if not isinstance(item, AuthorizedSuccessor):
+ raise ProtocolError("E_AUTH_TRUST")
+ try:
+ validate_creator_uid(item.creator_uid)
+ validate_bvid(item.bvid)
+ except ValueError as exc:
+ raise ProtocolError("E_AUTH_TRUST") from exc
+ if (
+ not JOB_ID_RE.fullmatch(item.predecessor_job_id)
+ or isinstance(item.retry_generation, bool)
+ or not isinstance(item.retry_generation, int)
+ or not 1 <= item.retry_generation <= 1_000_000
+ or not ERROR_CODE_RE.fullmatch(item.terminal_error_code)
+ ):
+ raise ProtocolError("E_AUTH_TRUST")
+ identities.append((item.creator_uid, item.bvid, item.predecessor_job_id))
+ if identities != sorted(identities) or len(set(identities)) != len(identities):
+ raise ProtocolError("E_AUTH_TRUST")
+
+ @staticmethod
+ def _build_successor_block(
+ approval: ReleaseApproval, records: list[dict[str, Any]]
+ ) -> tuple[bytes, tuple[str, ...]]:
+ ordered = sorted(records, key=lambda value: value["job_id"])
+ begin = {
+ "schema": SUCCESSOR_QUEUE_SCHEMA_VERSION,
+ "record_type": "SUCCESSOR_BEGIN",
+ "authorization_message_id": approval.authorization_message_id,
+ "authorization_handoff_id": approval.authorization_handoff_id,
+ "authorization_sha256": approval.authorization_sha256,
+ "successor_count": len(ordered),
+ }
+ prefix = _canonical_line(begin) + b"".join(_canonical_line(value) for value in ordered)
+ commit = {
+ **begin,
+ "record_type": "SUCCESSOR_COMMIT",
+ "block_sha256": hashlib.sha256(prefix).hexdigest().upper(),
+ }
+ block = prefix + _canonical_line(commit)
+ if len(block) > 1024 * 1024:
+ raise ProtocolError("E_LINEAGE")
+ return block, tuple(value["job_id"] for value in ordered)
+
+ def append_authorized_successors(
+ self,
+ approval_loader: Callable[[], ReleaseApproval],
+ catalog_records: list[dict[str, Any]],
+ ) -> dict[str, int]:
+ """Append or recover one deterministic authorized schema-2 block."""
+ catalog: dict[tuple[str, str], dict[str, Any]] = {}
+ if not isinstance(catalog_records, list) or not catalog_records:
+ raise ProtocolError("E_CATALOG")
+ for raw in catalog_records:
+ if not isinstance(raw, dict):
+ raise ProtocolError("E_CATALOG")
+ job = _ingress_job(raw, self.allowed_creators)
+ key = (job["creator_uid"], job["bvid"])
+ previous = catalog.get(key)
+ if previous is not None and previous != raw:
+ raise ProtocolError("E_CATALOG_CONFLICT")
+ catalog.setdefault(key, raw)
+
+ with self._locked():
+ approval = approval_loader()
+ self._validate_release(approval)
+ payload = self.queue_path.read_bytes() if self.queue_path.exists() else b""
+ snapshot = _queue_snapshot(payload, self.allowed_creators)
+ events = self._events(snapshot.jobs)
+ latest: dict[str, dict[str, Any]] = {}
+ for event in events:
+ latest[event["job_id"]] = event
+ successor_records: list[dict[str, Any]] = []
+ for item in approval.successors:
+ predecessor = snapshot.jobs.get(item.predecessor_job_id)
+ source = snapshot.records.get(item.predecessor_job_id)
+ current_catalog = catalog.get((item.creator_uid, item.bvid))
+ if predecessor is None or source is None or current_catalog is None:
+ raise ProtocolError("E_LINEAGE_TERMINAL")
+ if predecessor["creator_uid"] != item.creator_uid or predecessor["bvid"] != item.bvid:
+ raise ProtocolError("E_LINEAGE_CONFLICT")
+ state = latest.get(item.predecessor_job_id)
+ failed_terminal = (
+ state is not None and state["event"] in {"FAILED", "POSTPROCESS_FAILED"}
+ and item.terminal_error_code != COMPLETION_CLOSURE_REQUIRED
+ and state["error_code"] == item.terminal_error_code
+ )
+ completion_closure = (
+ state is not None and state["event"] == "COMPLETE"
+ and state["error_code"] is None
+ and item.terminal_error_code == COMPLETION_CLOSURE_REQUIRED
+ )
+ if not (failed_terminal or completion_closure):
+ raise ProtocolError("E_LINEAGE_TERMINAL")
+ predecessor_lineage = predecessor.get("lineage")
+ expected_generation = 1 if predecessor_lineage is None else predecessor_lineage["retry_generation"] + 1
+ if item.retry_generation != expected_generation:
+ raise ProtocolError("E_LINEAGE_GENERATION")
+ for key in (
+ "bvid", "creator_uid", "expected_duration_ms", "discovered_at_unix_ms",
+ "published_at", "title",
+ ):
+ if source.get(key) != current_catalog.get(key):
+ raise ProtocolError("E_CATALOG_CONFLICT")
+ lineage = {
+ "predecessor_job_id": item.predecessor_job_id,
+ "retry_generation": item.retry_generation,
+ "predecessor_terminal_error_code": item.terminal_error_code,
+ "authorization_message_id": approval.authorization_message_id,
+ "authorization_handoff_id": approval.authorization_handoff_id,
+ "authorization_sha256": approval.authorization_sha256,
+ "repair_review_result_message_id": approval.repair_review_result_message_id,
+ "repair_audit_id": approval.repair_audit_id,
+ "repair_audit_bytes": approval.repair_audit_bytes,
+ "repair_audit_sha256": approval.repair_audit_sha256,
+ }
+ try:
+ job_id = stable_successor_job_id(
+ item.creator_uid, item.bvid, item.predecessor_job_id,
+ item.retry_generation, item.terminal_error_code,
+ approval.authorization_message_id, approval.authorization_handoff_id,
+ approval.authorization_sha256, approval.repair_review_result_message_id,
+ approval.repair_audit_id, approval.repair_audit_bytes,
+ approval.repair_audit_sha256,
+ )
+ except ValueError as exc:
+ raise ProtocolError("E_LINEAGE") from exc
+ successor_records.append({
+ "schema": SUCCESSOR_QUEUE_SCHEMA_VERSION,
+ "record_type": "SUCCESSOR_JOB",
+ "job_id": job_id,
+ "bvid": item.bvid,
+ "creator_uid": item.creator_uid,
+ "expected_duration_ms": source["expected_duration_ms"],
+ "discovered_at_unix_ms": source["discovered_at_unix_ms"],
+ "published_at": source["published_at"],
+ "title": source["title"],
+ "lineage": lineage,
+ })
+
+ block, target_ids = self._build_successor_block(approval, successor_records)
+ matching_blocks = [
+ existing for existing in snapshot.blocks
+ if existing[:3] == (
+ approval.authorization_message_id, approval.authorization_handoff_id,
+ approval.authorization_sha256,
+ )
+ ]
+ if matching_blocks:
+ if len(matching_blocks) != 1 or matching_blocks[0][3] != target_ids or matching_blocks[0][4] != block or snapshot.pending_suffix:
+ raise ProtocolError("E_LINEAGE_CONFLICT")
+ return {"appended": 0, "unchanged": len(target_ids), "recovered": 0}
+ for item, target_id in zip(approval.successors, (
+ value["job_id"] for value in successor_records
+ )):
+ existing_child = snapshot.children.get(item.predecessor_job_id)
+ if existing_child is not None and existing_child != target_id:
+ raise ProtocolError("E_LINEAGE_CONFLICT")
+ suffix = snapshot.pending_suffix
+ if suffix and not block.startswith(suffix):
+ raise ProtocolError("E_LINEAGE_RECOVERY")
+ remaining = block[len(suffix):]
+ _append_bytes(self.queue_path, remaining)
+ final_payload = self.queue_path.read_bytes()
+ if not final_payload.startswith(payload) or final_payload != snapshot.committed_bytes + block:
+ raise ProtocolError("E_LINEAGE_RECOVERY")
+ final_snapshot = _queue_snapshot(final_payload, self.allowed_creators)
+ if final_snapshot.pending_suffix or not any(existing[3] == target_ids and existing[4] == block for existing in final_snapshot.blocks):
+ raise ProtocolError("E_LINEAGE_RECOVERY")
+ return {
+ "appended": len(target_ids),
+ "unchanged": 0,
+ "recovered": len(target_ids) if suffix else 0,
+ }
+
+ def _events(self, jobs: dict[str, dict[str, Any]] | None = None) -> list[dict[str, Any]]:
+ if jobs is None:
+ payload = self.queue_path.read_bytes() if self.queue_path.exists() else b""
+ snapshot = _queue_snapshot(payload, self.allowed_creators)
+ if snapshot.pending_suffix:
+ raise ProtocolError("E_QUEUE_PARTIAL")
+ jobs = snapshot.jobs
+ return _replay_state_events(self.state_path, jobs)
+
+ def claim_next(self, now_ms: int) -> tuple[dict[str, Any], str] | None:
+ with self._locked():
+ payload = self.queue_path.read_bytes() if self.queue_path.exists() else b""
+ snapshot = _queue_snapshot(payload, self.allowed_creators)
+ if snapshot.pending_suffix:
+ raise ProtocolError("E_QUEUE_PARTIAL")
+ jobs = list(snapshot.jobs.values())
+ events = self._events(snapshot.jobs)
+ latest: dict[str, dict[str, Any]] = {}
+ for item in events:
+ latest[item["job_id"]] = item
+ for job in jobs:
+ current = latest.get(job["job_id"])
+ if current and current["event"] in _TERMINAL_EVENTS:
+ continue
+ if current and current["event"] in {"MEDIA_COMPLETE", "POSTPROCESS_CLAIMED"}:
+ if (
+ current["event"] == "POSTPROCESS_CLAIMED"
+ and current["lease_expires_unix_ms"] >= now_ms
+ ):
+ continue
+ prior = [
+ item for item in events
+ if item["job_id"] == job["job_id"]
+ and item["event"] == "MEDIA_COMPLETE"
+ ]
+ if len(prior) != 1:
+ raise ProtocolError("E_QUEUE_STATE")
+ lease = secrets.token_hex(16)
+ recovery = _postprocess_recovery_binding(prior[0])
+ _postprocess_recovery_claim_test_seam("BEFORE_APPEND")
+ self._append_event(
+ job, lease, "POSTPROCESS_CLAIMED", now_ms, None,
+ recovery=recovery,
+ )
+ _postprocess_recovery_claim_test_seam("AFTER_APPEND_BEFORE_READBACK")
+ reread = [
+ item for item in self._events(snapshot.jobs)
+ if item["job_id"] == job["job_id"]
+ ]
+ if (
+ not reread or reread[-1]["event"] != "POSTPROCESS_CLAIMED"
+ or reread[-1]["lease_id"] != lease
+ or reread[-1].get("recovery") != recovery
+ ):
+ raise ProtocolError("E_QUEUE_WRITE")
+ _postprocess_recovery_claim_test_seam("AFTER_READBACK")
+ return job, lease
+ if current and current["event"] == "STARTED":
+ if current["lease_expires_unix_ms"] < now_ms:
+ self._append_event(job, current["lease_id"], "FAILED", now_ms, "E_ORPHANED")
+ continue
+ if current and current["event"] == "CLAIMED":
+ if current["lease_expires_unix_ms"] >= now_ms:
+ continue
+ attempts = sum(
+ item["event"] == "CLAIMED" and item["job_id"] == job["job_id"]
+ for item in events
+ )
+ if attempts >= MAX_CLAIM_ATTEMPTS:
+ self._append_event(job, current["lease_id"], "FAILED", now_ms, "E_CLAIM_EXPIRED")
+ continue
+ lease = secrets.token_hex(16)
+ self._append_event(job, lease, "CLAIMED", now_ms, None)
+ return job, lease
+ return None
+
+ def _append_event(
+ self, job: dict[str, Any], lease_id: str, event: str, now_ms: int, error_code: str | None,
+ diagnostic: dict[str, object] | None = None,
+ media: dict[str, Any] | None = None,
+ recovery: dict[str, Any] | None = None,
+ ) -> None:
+ validate_job(job)
+ if error_code == COMPLETION_CLOSURE_REQUIRED:
+ raise ProtocolError("E_QUEUE_STATE")
+ if error_code in INTERNAL_ONLY_PAGE_PENDING_CODES:
+ raise ProtocolError("E_QUEUE_STATE")
+ if error_code in PREPARELESS_REJECT_CODES:
+ try:
+ diagnostic = validate_prepareless_terminal(error_code, diagnostic)
+ except ValueError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ if event != "FAILED":
+ raise ProtocolError("E_QUEUE_STATE")
+ elif event == "POSTPROCESS_FAILED":
+ try:
+ diagnostic = validate_postprocess_terminal(error_code, diagnostic)
+ except ValueError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ elif diagnostic is not None:
+ try:
+ validate_runtime_diagnostic(diagnostic)
+ except ValueError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ if event not in {"FAILED", "POSTPROCESS_FAILED"}:
+ raise ProtocolError("E_QUEUE_STATE")
+ payload = {
+ "schema": QUEUE_SCHEMA_VERSION,
+ "event": event,
+ "job_id": job["job_id"],
+ "bvid": job["bvid"],
+ "creator_uid": job["creator_uid"],
+ "lease_id": lease_id,
+ "at_unix_ms": now_ms,
+ "lease_expires_unix_ms": now_ms + QUEUE_LEASE_SECONDS * 1_000,
+ "error_code": error_code,
+ }
+ if diagnostic is not None:
+ payload["diagnostic"] = diagnostic
+ if event == "MEDIA_COMPLETE":
+ try:
+ payload["media"] = validate_media_complete_identity(media, job)
+ except ProtocolError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ elif media is not None:
+ raise ProtocolError("E_QUEUE_STATE")
+ if event == "POSTPROCESS_CLAIMED":
+ payload["recovery"] = _validate_postprocess_recovery_binding(recovery, job)
+ elif recovery is not None:
+ raise ProtocolError("E_QUEUE_STATE")
+ _append_jsonl(
+ self.state_path,
+ payload,
+ )
+
+ def assert_claim(self, job: dict[str, Any], lease_id: str, now_ms: int) -> None:
+ validate_job(job)
+ with self._locked():
+ latest = None
+ for item in self._events():
+ if item["job_id"] == job["job_id"]:
+ latest = item
+ if (
+ latest is None or latest["event"] != "CLAIMED" or latest["lease_id"] != lease_id
+ or latest["lease_expires_unix_ms"] < now_ms
+ ):
+ raise ProtocolError("E_LEASE")
+
+ def mark_started(self, job: dict[str, Any], lease_id: str, now_ms: int) -> None:
+ with self._locked():
+ latest = None
+ for item in self._events():
+ if item["job_id"] == job["job_id"]:
+ latest = item
+ if latest is None or latest["event"] != "CLAIMED" or latest["lease_id"] != lease_id:
+ raise ProtocolError("E_LEASE")
+ self._append_event(job, lease_id, "STARTED", now_ms, None)
+
+ def postprocess_recovery_claim(
+ self, job: dict[str, Any], lease_id: str,
+ ) -> dict[str, Any] | None:
+ """Return the exact durable recovery binding, or None for an ordinary claim."""
+
+ validated_job = validate_job(job)
+ with self._locked():
+ matching = [
+ item for item in self._events()
+ if item["job_id"] == validated_job["job_id"]
+ ]
+ latest = matching[-1] if matching else None
+ if latest is None or latest["lease_id"] != lease_id:
+ raise ProtocolError("E_LEASE")
+ if latest["event"] == "CLAIMED":
+ return None
+ if latest["event"] != "POSTPROCESS_CLAIMED":
+ raise ProtocolError("E_LEASE")
+ prior = [item for item in matching if item["event"] == "MEDIA_COMPLETE"]
+ if (
+ len(prior) != 1
+ or latest.get("recovery") != _postprocess_recovery_binding(prior[0])
+ ):
+ raise ProtocolError("E_QUEUE_STATE")
+ return dict(latest["recovery"])
+
+ def mark_media_complete(
+ self, job: dict[str, Any], lease_id: str, now_ms: int,
+ media: dict[str, Any],
+ ) -> dict[str, Any]:
+ """Persist verified user media before any reentrant postprocess outcome."""
+ validated_job = validate_job(job)
+ validated_media = validate_media_complete_identity(media, validated_job)
+ with self._locked():
+ matching = [
+ item for item in self._events()
+ if item["job_id"] == job["job_id"]
+ ]
+ latest = matching[-1] if matching else None
+ prior = [item for item in matching if item["event"] == "MEDIA_COMPLETE"]
+ if prior:
+ if (
+ len(prior) != 1
+ or prior[0].get("media") != validated_media
+ ):
+ raise ProtocolError("E_LEASE")
+ if latest is None or latest["lease_id"] != lease_id:
+ raise ProtocolError("E_LEASE")
+ if prior[0]["lease_id"] == lease_id:
+ if latest["event"] not in {
+ "MEDIA_COMPLETE", "COMPLETE", "POSTPROCESS_FAILED",
+ }:
+ raise ProtocolError("E_LEASE")
+ else:
+ recovery = [
+ item for item in matching
+ if item["event"] == "POSTPROCESS_CLAIMED"
+ and item["lease_id"] == lease_id
+ ]
+ if (
+ len(recovery) != 1
+ or recovery[0].get("recovery") != _postprocess_recovery_binding(prior[0])
+ or latest["event"] not in {
+ "POSTPROCESS_CLAIMED", "COMPLETE", "POSTPROCESS_FAILED",
+ }
+ ):
+ raise ProtocolError("E_LEASE")
+ return prior[0]
+ if latest is None or latest["event"] != "STARTED" or latest["lease_id"] != lease_id:
+ raise ProtocolError("E_LEASE")
+ _media_complete_durability_test_seam("BEFORE_APPEND")
+ self._append_event(
+ validated_job, lease_id, "MEDIA_COMPLETE", now_ms, None,
+ media=validated_media,
+ )
+ _media_complete_durability_test_seam("AFTER_APPEND_BEFORE_READBACK")
+ matching = [
+ item for item in self._events()
+ if item["job_id"] == validated_job["job_id"]
+ ]
+ if not matching or matching[-1].get("media") != validated_media:
+ raise ProtocolError("E_QUEUE_WRITE")
+ _media_complete_durability_test_seam("AFTER_READBACK")
+ return matching[-1]
+
+ def mark_postprocess_failed(
+ self, job: dict[str, Any], lease_id: str, now_ms: int, *,
+ error_code: str, diagnostic: dict[str, object] | None = None,
+ ) -> None:
+ """Terminalize postprocess without erasing the prior MEDIA_COMPLETE fact."""
+ validated_job = validate_job(job)
+ try:
+ diagnostic = validate_postprocess_terminal(error_code, diagnostic)
+ except ValueError as exc:
+ raise ProtocolError("E_QUEUE_STATE") from exc
+ if (
+ not isinstance(error_code, str)
+ or not error_code.startswith("E_")
+ ):
+ raise ProtocolError("E_QUEUE_STATE")
+ requested = {
+ "schema": QUEUE_SCHEMA_VERSION,
+ "event": "POSTPROCESS_FAILED",
+ "job_id": validated_job["job_id"],
+ "bvid": validated_job["bvid"],
+ "creator_uid": validated_job["creator_uid"],
+ "lease_id": lease_id,
+ "at_unix_ms": now_ms,
+ "lease_expires_unix_ms": now_ms + QUEUE_LEASE_SECONDS * 1_000,
+ "error_code": error_code,
+ }
+ if diagnostic is not None:
+ requested["diagnostic"] = diagnostic
+ requested = _event(requested, {validated_job["job_id"]: validated_job})
+ with self._locked():
+ matching = [
+ item for item in self._events()
+ if item["job_id"] == job["job_id"]
+ ]
+ latest = matching[-1] if matching else None
+ if latest and latest["event"] in _TERMINAL_EVENTS:
+ identity_fields = (
+ "event", "job_id", "bvid", "creator_uid", "lease_id", "error_code",
+ )
+ if (
+ any(latest[name] != requested[name] for name in identity_fields)
+ or ("diagnostic" in latest) != ("diagnostic" in requested)
+ or latest.get("diagnostic") != requested.get("diagnostic")
+ ):
+ raise ProtocolError("E_LEASE")
+ return
+ if (
+ latest is None
+ or latest["event"] not in {"MEDIA_COMPLETE", "POSTPROCESS_CLAIMED"}
+ or latest["lease_id"] != lease_id
+ ):
+ raise ProtocolError("E_LEASE")
+ self._append_event(
+ job, lease_id, "POSTPROCESS_FAILED", now_ms,
+ error_code, requested.get("diagnostic"),
+ )
+
+ def reject_claim(
+ self, job: dict[str, Any], lease_id: str, now_ms: int, error_code: str,
+ diagnostic: dict[str, object] | None = None,
+ ) -> None:
+ validate_job(job)
+ try:
+ validated_diagnostic = validate_prepareless_terminal(error_code, diagnostic)
+ except ValueError as exc:
+ raise ProtocolError("E_REJECT")
+ with self._locked():
+ latest = None
+ for item in self._events():
+ if item["job_id"] == job["job_id"]:
+ latest = item
+ if latest and latest["event"] == "FAILED" and latest["lease_id"] == lease_id:
+ if latest["error_code"] != error_code or latest.get("diagnostic") != validated_diagnostic:
+ raise ProtocolError("E_LEASE")
+ return
+ if latest is None or latest["event"] != "CLAIMED" or latest["lease_id"] != lease_id:
+ raise ProtocolError("E_LEASE")
+ self._append_event(job, lease_id, "FAILED", now_ms, error_code, validated_diagnostic)
+
+ def mark_terminal(
+ self, job: dict[str, Any], lease_id: str, now_ms: int, *, complete: bool,
+ error_code: str | None, diagnostic: dict[str, object] | None = None,
+ ) -> None:
+ requested = _terminal_request(
+ job, lease_id, now_ms, complete=complete,
+ error_code=error_code, diagnostic=diagnostic,
+ )
+ with self._locked():
+ latest = None
+ for item in self._events():
+ if item["job_id"] == job["job_id"]:
+ latest = item
+ if latest and latest["event"] in _TERMINAL_EVENTS:
+ identity_fields = (
+ "event", "job_id", "bvid", "creator_uid", "lease_id", "error_code",
+ )
+ if (
+ any(latest[name] != requested[name] for name in identity_fields)
+ or ("diagnostic" in latest) != ("diagnostic" in requested)
+ or latest.get("diagnostic") != requested.get("diagnostic")
+ ):
+ raise ProtocolError("E_LEASE")
+ return
+ if latest is None or latest["lease_id"] != lease_id:
+ raise ProtocolError("E_LEASE")
+ if latest["event"] == "MEDIA_COMPLETE" and requested["event"] != "COMPLETE":
+ raise ProtocolError("E_QUEUE_STATE")
+ if latest["event"] == "POSTPROCESS_CLAIMED" and requested["event"] != "COMPLETE":
+ raise ProtocolError("E_QUEUE_STATE")
+ self._append_event(
+ job, lease_id, requested["event"], now_ms,
+ requested["error_code"], requested.get("diagnostic"),
+ )
+
+
+class ReloadStore:
+ def __init__(self, path: Path, generation: str) -> None:
+ self.path = path
+ self.generation = generation
+
+ def _records(self) -> list[dict[str, Any]]:
+ records = _read_jsonl(self.path, missing_ok=True)
+ for value in records:
+ if set(value) != {"schema", "generation", "event", "token", "from_build", "to_build", "at_unix_ms"}:
+ raise ProtocolError("E_RELOAD_STATE")
+ if value["schema"] != 1 or value["event"] not in {"OFFERED", "BEGIN", "APPLIED"}:
+ raise ProtocolError("E_RELOAD_STATE")
+ if not all(isinstance(value[name], str) for name in ("generation", "token", "from_build", "to_build")):
+ raise ProtocolError("E_RELOAD_STATE")
+ if len(value["token"]) != 32 or any(ch not in "0123456789abcdef" for ch in value["token"]):
+ raise ProtocolError("E_RELOAD_STATE")
+ if isinstance(value["at_unix_ms"], bool) or not isinstance(value["at_unix_ms"], int):
+ raise ProtocolError("E_RELOAD_STATE")
+ return records
+
+ def status(self, current_build: str, now_ms: int) -> dict[str, Any]:
+ records = [item for item in self._records() if item["generation"] == self.generation]
+ if current_build == EXTENSION_BUILD:
+ if not records or records[-1]["event"] != "APPLIED":
+ token = records[-1]["token"] if records else secrets.token_hex(16)
+ _append_jsonl(self.path, {"schema": 1, "generation": self.generation, "event": "APPLIED", "token": token, "from_build": current_build, "to_build": EXTENSION_BUILD, "at_unix_ms": now_ms})
+ return {"required_extension_build": EXTENSION_BUILD, "reload_required": False, "reload_token": None, "retry_after_unix_ms": 0}
+ begun = next((item for item in reversed(records) if item["event"] == "BEGIN"), None)
+ if begun is not None:
+ return {"required_extension_build": EXTENSION_BUILD, "reload_required": False, "reload_token": None, "retry_after_unix_ms": begun["at_unix_ms"] + RELOAD_BACKOFF_SECONDS * 1_000}
+ offered = records[-1] if records and records[-1]["event"] == "OFFERED" else None
+ if offered is None:
+ offered = {"schema": 1, "generation": self.generation, "event": "OFFERED", "token": secrets.token_hex(16), "from_build": current_build, "to_build": EXTENSION_BUILD, "at_unix_ms": now_ms}
+ _append_jsonl(self.path, offered)
+ return {"required_extension_build": EXTENSION_BUILD, "reload_required": True, "reload_token": offered["token"], "retry_after_unix_ms": 0}
+
+ def begin(self, current_build: str, token: str, now_ms: int) -> None:
+ records = [item for item in self._records() if item["generation"] == self.generation]
+ if current_build == EXTENSION_BUILD or not records:
+ raise ProtocolError("E_RELOAD")
+ latest = records[-1]
+ if latest["event"] != "OFFERED" or latest["token"] != token or latest["from_build"] != current_build:
+ raise ProtocolError("E_RELOAD")
+ _append_jsonl(self.path, {**latest, "event": "BEGIN", "at_unix_ms": now_ms})
diff --git a/dev/project-dev/bili_authenticated_extension/sidepanel.html b/dev/project-dev/bili_authenticated_extension/sidepanel.html
index 7db5abe..9dbf678 100644
--- a/dev/project-dev/bili_authenticated_extension/sidepanel.html
+++ b/dev/project-dev/bili_authenticated_extension/sidepanel.html
@@ -9,18 +9,8 @@
<body>
<main>
<h1>完整视频任务</h1>
- <p class="target">仅限 BV1HA3o6oEJJ</p>
- <p>请确认当前页面显示“充电中”，播放器总时长为 52:14，并可正常播放完整内容。</p>
- <label class="confirm"><input id="confirmed" type="checkbox"> 我已在页面中完成上述可见确认</label>
- <button id="validate" type="button">1. 校验当前页面</button>
- <button id="start" type="button" disabled>2. 添加完整视频任务</button>
- <button id="retry" type="button" hidden>失败重试</button>
- <button id="cancel" type="button" hidden>取消任务</button>
- <section aria-live="polite">
- <div id="phase">状态：IDLE</div>
- <progress id="progress" max="100" value="0"></progress>
- <div id="detail"></div>
- </section>
+ <p class="target">任务只由本机受控队列与 creator allowlist 自动派发。</p>
+ <p>此历史静态页不注册为 side panel，也不提供校验、启动、重试或取消入口。</p>
</main>
<script src="sidepanel.js"></script>
</body>
diff --git a/dev/project-dev/bili_authenticated_extension/sidepanel.js b/dev/project-dev/bili_authenticated_extension/sidepanel.js
index cf99b19..a82bd9f 100644
--- a/dev/project-dev/bili_authenticated_extension/sidepanel.js
+++ b/dev/project-dev/bili_authenticated_extension/sidepanel.js
@@ -1,71 +1,3 @@
-const confirmed = document.querySelector("#confirmed");
-const validateButton = document.querySelector("#validate");
-const startButton = document.querySelector("#start");
-const retryButton = document.querySelector("#retry");
-const cancelButton = document.querySelector("#cancel");
-const phase = document.querySelector("#phase");
-const progress = document.querySelector("#progress");
-const detail = document.querySelector("#detail");
-let startUiLease = false;
-
-function render(state) {
- if (!state) return;
- phase.textContent = `状态：${state.phase}`;
- progress.value = Number.isInteger(state.progress) ? state.progress : 0;
- detail.textContent = state.error_code
- ? `错误码：${state.error_code}`
- : state.formal_filename
- ? `已完成：${state.formal_filename}`
- : "";
- const active = ["CHECKING", "DOWNLOADING", "MERGING", "VALIDATING", "PUBLISHING"].includes(state.phase);
- cancelButton.hidden = !active;
- retryButton.hidden = state.phase !== "FAILED";
- retryButton.disabled = startUiLease;
- validateButton.disabled = active || startUiLease;
- startButton.disabled = startUiLease || state.phase !== "READY";
-}
-
-async function call(action) {
- const response = await chrome.runtime.sendMessage({action});
- if (!response?.ok) {
- render({phase: "FAILED", progress: 0, error_code: response?.error_code || "E_EXTENSION"});
- return null;
- }
- render(response.state);
- return response.state;
-}
-
-async function callStart(action) {
- if (startUiLease) return;
- startUiLease = true;
- startButton.disabled = true;
- retryButton.disabled = true;
- validateButton.disabled = true;
- try {
- await call(action);
- } finally {
- startUiLease = false;
- }
-}
-
-validateButton.addEventListener("click", async () => {
- if (!confirmed.checked) {
- render({phase: "FAILED", progress: 0, error_code: "E_VISIBLE_CONFIRMATION"});
- return;
- }
- const state = await call("validate");
- startButton.disabled = state?.phase !== "READY";
-});
-
-startButton.addEventListener("click", () => callStart("start"));
-retryButton.addEventListener("click", async () => {
- if (!confirmed.checked) {
- render({phase: "FAILED", progress: 0, error_code: "E_VISIBLE_CONFIRMATION"});
- return;
- }
- await callStart("retry");
-});
-cancelButton.addEventListener("click", () => call("cancel"));
-
-call("status");
-setInterval(() => call("status"), 1000);
+// Historical projection asset retained for exact-set compatibility.
+// The generic queue build does not register a side panel or manual start action.
+document.documentElement.dataset.automation = "host-queue-only";
diff --git a/dev/project-dev/bili_authenticated_extension/source-artifact-manifest.json b/dev/project-dev/bili_authenticated_extension/source-artifact-manifest.json
index 1f597aa..5fa6650 100644
--- a/dev/project-dev/bili_authenticated_extension/source-artifact-manifest.json
+++ b/dev/project-dev/bili_authenticated_extension/source-artifact-manifest.json
@@ -1,27 +1,16 @@
{
"schema": 1,
- "target": "BV1HA3o6oEJJ",
+ "scope": "generic-bilibili-queue",
"extension_id": "oidmclckpdmpabbfedplkbdplmfcenbb",
- "extension_build": "project-info-bili-auth-ingress/1.0.0+20260805.v002",
- "host_build": "project-info-bili-auth-native-host/1.0.0+20260805.v002",
+ "extension_build": "project-info-bili-auth-ingress/1.2.25+20260829.generic.v027",
+ "host_build": "project-info-bili-auth-native-host/1.2.25+20260829.generic.v027",
"archive_metadata_contract": {
"schema": 1,
"root": "yt_dlp-2026.7.4.dist-info",
- "relative_files": [
- "INSTALLER",
- "METADATA",
- "RECORD",
- "REQUESTED",
- "WHEEL",
- "entry_points.txt",
- "licenses/LICENSE"
- ],
+ "relative_files": ["INSTALLER", "METADATA", "RECORD", "REQUESTED", "WHEEL", "entry_points.txt", "licenses/LICENSE"],
"distribution_name": "yt-dlp",
"distribution_version": "2026.7.4",
- "allowed_type_codes": [
- "b",
- "x"
- ],
+ "allowed_type_codes": ["b", "x"],
"source_date_epoch": 1786207924,
"tree_hash_algorithm": "sha256(path-utf8,nul,decimal-bytes-ascii,nul,payload-sha256-lower-hex-ascii,lf)-upper-hex-v1",
"canonical_tree_sha256": "32F1DC23F6704966AE4511CB173318CD11FCBA031323028D268D9F2488589E70"
@@ -29,90 +18,26 @@
"dependency_artifact_manifest_bytes": 723,
"dependency_artifact_manifest_sha256": "2943A0EF7968523A342C28824CACC16E2E0D8324D36DEEEE2E6FBABB910AC27D",
"files": [
- {
- "path": "__init__.py",
- "bytes": 290,
- "sha256": "AD80D1743ECD344FA10B28F0C3B2702EC18EF4C2136B7BF90E516597C99C5FF1"
- },
- {
- "path": "background.js",
- "bytes": 13201,
- "sha256": "27C0598DD8D70659B1E518C841F2CC5908165458F99E8531D58A6B15F767BDE8"
- },
- {
- "path": "build_host.ps1",
- "bytes": 35414,
- "sha256": "6184EBA087470BB5DBD6FEC29EB22068F8119FBDE2ED640715E8317B5A928C04"
- },
- {
- "path": "config.example.json",
- "bytes": 760,
- "sha256": "FDFACA2935A29027848DEB11B746576957CC72766BA9B6161C7D7AD7C318DC8C"
- },
- {
- "path": "constants.py",
- "bytes": 3868,
- "sha256": "B5FE4157F1747BF7E8928793860AF02F98DA98C992B57FCC43104F8726554096"
- },
- {
- "path": "dependencies/dependency-artifact-manifest.json",
- "bytes": 723,
- "sha256": "2943A0EF7968523A342C28824CACC16E2E0D8324D36DEEEE2E6FBABB910AC27D"
- },
- {
- "path": "dependencies/yt_dlp-2026.7.4-py3-none-any.whl",
- "bytes": 3184705,
- "sha256": "F11F2B11D5A8AC4059F9BDF29FA4407DC7C6BB00C5097E95CA22A7A9DB518266"
- },
- {
- "path": "install_native_host.ps1",
- "bytes": 24079,
- "sha256": "AA9667A3CD765B21058F234FD82DFDB9621161246041EC595E4BCB17DDCFDE86"
- },
- {
- "path": "job.py",
- "bytes": 15586,
- "sha256": "09540CBF4F7693D22C0AB91E9C45D379ABA472156298F84296143485C6E0D42A"
- },
- {
- "path": "manifest.json",
- "bytes": 1011,
- "sha256": "9B5CEB76758D69C3A9F7DEBDE5A55B026E70BA064769180A6F959A04283EA865"
- },
- {
- "path": "native-host-manifest.template.json",
- "bytes": 265,
- "sha256": "2E55FD8915258557964CDC2C00491C83194D4B6F7B9AEAC0FC18C2A331C10918"
- },
- {
- "path": "native_host.py",
- "bytes": 29843,
- "sha256": "41CA31D688F39E53B5C7A11ACE1390C3BEA1D8E08D0FFFF49C4C62D320F42937"
- },
- {
- "path": "protocol.py",
- "bytes": 11393,
- "sha256": "8E8A1BBB742B37358A2848B28D2A7B10ACA9B20167844388455CAAB59DBE422D"
- },
- {
- "path": "sidepanel.css",
- "bytes": 400,
- "sha256": "46AD95FC47D623CBE72A8FB060E238BC6066C152DD2C63DE9E0740D92BE4D01C"
- },
- {
- "path": "sidepanel.html",
- "bytes": 1094,
- "sha256": "9836EE44AAFE4002EAB27AB77AB249BC416908C565138A956A19480F663EB3EB"
- },
- {
- "path": "sidepanel.js",
- "bytes": 2364,
- "sha256": "88794C840C0491E0017E7E8D15A956E4AA28DCEB71950A991F4A40399C2449B4"
- },
- {
- "path": "worker.py",
- "bytes": 33118,
- "sha256": "4A37BB7788D29D095A59812D6596E947094737669B9AD36B66D8088F27B03B78"
- }
+ {"path": "__init__.py", "bytes": 313, "sha256": "BE02EC724F4195E2BB37FF73FE34CF3B2F84EB1DCD5D02DD4901249509373F30"},
+ {"path": "background.js", "bytes": 71667, "sha256": "7A70FB0B6CC604E5D16855EBE876EBFFE6898D192CC4F7237C36BB7BEA0CF9C1"},
+ {"path": "build_host.ps1", "bytes": 36383, "sha256": "4B25281E6EDA0E3ECB1C95DFD7DDE9F9F5D68E9E9A666F8DB53BDAF9BBEAB139"},
+ {"path": "config.example.json", "bytes": 1292, "sha256": "D3E6E080C277492CA1D5FB8C144E01A046F0C386E2E8CDA3EFCF7DD6D032861F"},
+ {"path": "constants.py", "bytes": 17631, "sha256": "CBAECED4B90D936651D646D1EA9BD7BCF8E597DBFCFFE60FC53B03D7A8112D6D"},
+ {"path": "dependencies/dependency-artifact-manifest.json", "bytes": 723, "sha256": "2943A0EF7968523A342C28824CACC16E2E0D8324D36DEEEE2E6FBABB910AC27D"},
+ {"path": "dependencies/yt_dlp-2026.7.4-py3-none-any.whl", "bytes": 3184705, "sha256": "F11F2B11D5A8AC4059F9BDF29FA4407DC7C6BB00C5097E95CA22A7A9DB518266"},
+ {"path": "formal_legacy_identity_manifest.py", "bytes": 7858, "sha256": "2255C9B63DF5297F96F8F140CD0EFAAC3BA2505C03CF2A753B64DE9EDD10EEB4"},
+ {"path": "install_native_host.ps1", "bytes": 45995, "sha256": "251BAAF92672D81F5E1C6BB67E29BDE00B01988082DDBAB0BF88062D2287669C"},
+ {"path": "job.py", "bytes": 15586, "sha256": "09540CBF4F7693D22C0AB91E9C45D379ABA472156298F84296143485C6E0D42A"},
+ {"path": "manifest.json", "bytes": 991, "sha256": "FDA38B8430CA90A628696DF448FC98F04416E008BF400DF8131D826421559883"},
+ {"path": "native-host-manifest.template.json", "bytes": 265, "sha256": "2E55FD8915258557964CDC2C00491C83194D4B6F7B9AEAC0FC18C2A331C10918"},
+ {"path": "native_host.py", "bytes": 85557, "sha256": "FF9FF2E417E4E53E7F6772AA4D6F0B305AD5715D4D862B03CFFC533C7D96B0ED"},
+ {"path": "protocol.py", "bytes": 21803, "sha256": "A081B335652253CE2E12D49E7B294ADE7B563EE78051C0A06067B109BF236845"},
+ {"path": "queue-producer.example.json", "bytes": 570, "sha256": "8C458B2720DE4DD4D638FE2BE96FBCF54F964D7C8887C6CE05DB17D187B8605D"},
+ {"path": "queue_producer.py", "bytes": 70120, "sha256": "50579347FE8B6576FC141E590163A770D855D3456155F97E4158AB6B8D9AF88F"},
+ {"path": "queue_state.py", "bytes": 58440, "sha256": "6E12E516E2AADDE1A9CCDED443C8202BB2C15A7BD0CC49F7C0DD85110733FAEE"},
+ {"path": "sidepanel.css", "bytes": 400, "sha256": "46AD95FC47D623CBE72A8FB060E238BC6066C152DD2C63DE9E0740D92BE4D01C"},
+ {"path": "sidepanel.html", "bytes": 546, "sha256": "D407BE259F273B164493DB2D93D3B55A534EB88B10335CA165629331AF3B56AE"},
+ {"path": "sidepanel.js", "bytes": 216, "sha256": "834AA17BAF0EED75E657698EB159D9C882EF9AAC7EC1E0E54741DE3BB70551D4"},
+ {"path": "worker.py", "bytes": 89337, "sha256": "5D14DE2B2846F0F30D8F1CC741A5850417B4240319E0E38CB570CC66F2FA52F8"}
]
}
diff --git a/dev/project-dev/bili_authenticated_extension/worker.py b/dev/project-dev/bili_authenticated_extension/worker.py
index 6d1a2ca..3516a7c 100644
--- a/dev/project-dev/bili_authenticated_extension/worker.py
+++ b/dev/project-dev/bili_authenticated_extension/worker.py
@@ -7,17 +7,22 @@
from __future__ import annotations
+import ctypes
import hashlib
import importlib.util
import io
import json
import math
import os
+import re
import shutil
+import stat
import subprocess
import sys
import time
import uuid
+from collections.abc import Mapping
+from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Iterable, Sequence
@@ -25,27 +30,63 @@
from .constants import (
BRIDGE_TIMEOUT_SECONDS,
- CANONICAL_URL,
- DURATION_TOLERANCE_MS,
- EXPECTED_DURATION_MS,
+ EXTENSION_BUILD,
EXTRACTOR_RETRIES,
FILE_ACCESS_RETRIES,
FRAGMENT_RETRIES,
HTTP_RETRIES,
+ RELOAD_GENERATION,
SOCKET_TIMEOUT_SECONDS,
- TARGET_BVID,
YTDLP_MODULE_SHA256,
YTDLP_VERSION,
+ duration_tolerance_ms,
+ validate_bvid,
+ validate_creator_uid,
)
-from .protocol import ProtocolError, strict_json_loads, validate_start
+from .formal_legacy_identity_manifest import (
+ FORMAL_LEGACY_CREATOR_UID,
+ FORMAL_LEGACY_INTEGER_UID_ROWS,
+ FORMAL_LEGACY_ROWS,
+ FORMAL_PREFIX_BYTES,
+ FORMAL_PREFIX_LINES,
+ FORMAL_PREFIX_SHA256,
+)
+from .protocol import (
+ ProtocolError, strict_json_loads, validate_media_complete_identity, validate_start,
+)
-FROZEN_BRIDGE_SHA256 = "749FC486B0F42315BD463F11771FE2A7C71CAB53DD9AC2E411CE82E1175DFF13"
+FROZEN_BRIDGE_SHA256 = "00F11DAF8387160DB863C89F0B33AB8480422233FF42199189222C989C7ED07E"
+SUBPROCESS_POLICY_ERROR_CODES = frozenset({
+ "E_SUBPROCESS_POLICY_ARGUMENTS",
+ "E_SUBPROCESS_POLICY_ENVIRONMENT",
+ "E_SUBPROCESS_POLICY_EVENT_SHAPE",
+ "E_SUBPROCESS_POLICY_EXECUTABLE",
+ "E_SUBPROCESS_POLICY_LOCAL_PATH",
+ "E_SUBPROCESS_POLICY_SECRET",
+})
+_BRIDGE_MAPPING_KEYS = frozenset({
+ "schema_version", "bvid", "source", "published_at", "title", "local_file",
+ "bytes", "sha256", "duration_seconds", "remote_duration_seconds",
+ "local_duration_seconds", "duration_delta_seconds", "duration_tolerance_seconds",
+ "format_name", "video_codec", "audio_codec", "completed_at", "acquisition_mode",
+ "handoff_source_sha256",
+})
+_BRIDGE_ITEM_KEYS = _BRIDGE_MAPPING_KEYS | {"status"}
+_BRIDGE_ITEM_WARNING_KEYS = _BRIDGE_ITEM_KEYS | {"warning"}
+_LOWER_SHA256_RE = re.compile(r"[0-9a-f]{64}")
+_BRIDGE_CLEANUP_WARNING_RE = re.compile(
+ r"staging cleanup requires attention: [A-Za-z][A-Za-z0-9_]{0,63}"
+)
+_UTC_ISO_RE = re.compile(
+ r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?\+00:00"
+)
class WorkerError(Exception):
- def __init__(self, code: str) -> None:
+ def __init__(self, code: str, diagnostic: dict[str, object] | None = None) -> None:
super().__init__(code)
self.code = code
+ self.diagnostic = diagnostic
class CancelRequested(BaseException):
@@ -116,13 +157,17 @@
@dataclass(frozen=True)
class HostConfig:
+ creator_allowlist: frozenset[str]
ffmpeg: Path
ffprobe: Path
bridge_python: Path
bridge_script: Path
- batch_json: Path
yt_dlp_executable: Path
destination: Path
+ queue_lock_path: Path | None = None
+ formal_manifest_path: Path | None = None
+ processing_handoff_path: Path | None = None
+ creator_name: str = ""
@staticmethod
def _safe_absolute_file(value: Any, expected_hash: Any) -> Path:
@@ -148,8 +193,13 @@
raise WorkerError("E_CONFIG") from exc
expected = {
"schema",
- "target",
- "canonical_url",
+ "creator_allowlist",
+ "queue_path",
+ "queue_state_path",
+ "queue_lock_path",
+ "reload_state_path",
+ "reload_generation",
+ "required_extension_build",
"ffmpeg",
"ffmpeg_sha256",
"ffprobe",
@@ -158,16 +208,86 @@
"bridge_python_sha256",
"bridge_script",
"bridge_script_sha256",
- "batch_json",
- "batch_json_sha256",
"yt_dlp_executable",
"yt_dlp_executable_sha256",
"destination",
+ "formal_manifest_path",
+ "processing_handoff_path",
+ "creator_name",
}
if not isinstance(raw, dict) or set(raw) != expected:
raise WorkerError("E_CONFIG")
- if raw["schema"] != 1 or raw["target"] != TARGET_BVID or raw["canonical_url"] != CANONICAL_URL:
+ if raw["schema"] != 2:
raise WorkerError("E_CONFIG")
+ if raw["required_extension_build"] != EXTENSION_BUILD or raw["reload_generation"] != RELOAD_GENERATION:
+ raise WorkerError("E_CONFIG")
+ creators = raw["creator_allowlist"]
+ try:
+ if (
+ not isinstance(creators, list) or not creators or len(creators) > 64
+ or [validate_creator_uid(item) for item in creators] != sorted(set(creators))
+ ):
+ raise WorkerError("E_CONFIG")
+ except ValueError as exc:
+ raise WorkerError("E_CONFIG") from exc
+ for name in (
+ "queue_path", "queue_state_path", "queue_lock_path", "reload_state_path",
+ "formal_manifest_path", "processing_handoff_path",
+ ):
+ value = raw[name]
+ if not isinstance(value, str):
+ raise WorkerError("E_CONFIG")
+ creator_name = raw["creator_name"]
+ if (
+ not isinstance(creator_name, str) or not creator_name.strip()
+ or len(creator_name.encode("utf-8")) > 240
+ or any(ord(ch) < 32 or ord(ch) == 127 for ch in creator_name)
+ ):
+ raise WorkerError("E_CONFIG")
+ queue_lock_path = Path(raw["queue_lock_path"])
+ formal_manifest_path = Path(raw["formal_manifest_path"])
+ processing_handoff_path = Path(raw["processing_handoff_path"])
+ governed_paths: list[Path] = []
+ for governed_path, must_exist in (
+ (queue_lock_path, False),
+ (formal_manifest_path, True),
+ (processing_handoff_path, False),
+ ):
+ if not governed_path.is_absolute() or str(governed_path).startswith("\\\\"):
+ raise WorkerError("E_CONFIG")
+ try:
+ lexical_parent = governed_path.parent
+ parent_stat = lexical_parent.lstat()
+ resolved_parent = lexical_parent.resolve(strict=True)
+ if (
+ not stat.S_ISDIR(parent_stat.st_mode)
+ or lexical_parent.is_symlink()
+ or _is_reparse(lexical_parent)
+ or resolved_parent != lexical_parent
+ ):
+ raise WorkerError("E_CONFIG")
+ resolved_path = resolved_parent / governed_path.name
+ try:
+ path_stat = governed_path.lstat()
+ except FileNotFoundError:
+ if must_exist:
+ raise WorkerError("E_CONFIG")
+ else:
+ if (
+ not stat.S_ISREG(path_stat.st_mode)
+ or governed_path.is_symlink()
+ or _is_reparse(governed_path)
+ or governed_path.resolve(strict=True) != resolved_path
+ ):
+ raise WorkerError("E_CONFIG")
+ except WorkerError:
+ raise
+ except OSError as exc:
+ raise WorkerError("E_CONFIG") from exc
+ governed_paths.append(resolved_path)
+ if len(set(governed_paths)) != len(governed_paths):
+ raise WorkerError("E_CONFIG")
+ queue_lock_path, formal_manifest_path, processing_handoff_path = governed_paths
bridge_script = cls._safe_absolute_file(raw["bridge_script"], raw["bridge_script_sha256"])
if raw["bridge_script_sha256"].upper() != FROZEN_BRIDGE_SHA256:
raise WorkerError("E_CONFIG_HASH")
@@ -178,17 +298,21 @@
if not destination.is_dir() or destination.is_symlink():
raise WorkerError("E_CONFIG")
return cls(
+ creator_allowlist=frozenset(creators),
ffmpeg=cls._safe_absolute_file(raw["ffmpeg"], raw["ffmpeg_sha256"]),
ffprobe=cls._safe_absolute_file(raw["ffprobe"], raw["ffprobe_sha256"]),
bridge_python=cls._safe_absolute_file(
raw["bridge_python"], raw["bridge_python_sha256"]
),
bridge_script=bridge_script,
- batch_json=cls._safe_absolute_file(raw["batch_json"], raw["batch_json_sha256"]),
yt_dlp_executable=cls._safe_absolute_file(
raw["yt_dlp_executable"], raw["yt_dlp_executable_sha256"]
),
destination=destination,
+ queue_lock_path=queue_lock_path,
+ formal_manifest_path=formal_manifest_path,
+ processing_handoff_path=processing_handoff_path,
+ creator_name=creator_name,
)
@@ -229,13 +353,17 @@
return resolved
-def fixed_stage_root() -> Path:
+def fixed_stage_root(bvid: str) -> Path:
local_app_data = validated_local_app_data()
+ try:
+ bvid = validate_bvid(bvid)
+ except ValueError as exc:
+ raise WorkerError("E_STAGE") from exc
logical_root = (
local_app_data
/ "project-info"
/ "bili-auth-ingress"
- / TARGET_BVID
+ / bvid
)
_reject_reparse_path(logical_root, local_app_data)
resolved = logical_root.resolve(strict=False)
@@ -258,7 +386,7 @@
def cleanup_stale_runs(root: Path, *, boundary: Path | None = None) -> None:
"""Remove only uncommitted run-* directories below the fixed stage root."""
- allowed_root = fixed_stage_root() if boundary is None else boundary.resolve()
+ allowed_root = root.resolve() if boundary is None else boundary.resolve()
_ensure_within(root, allowed_root)
if not root.exists():
return
@@ -270,8 +398,8 @@
shutil.rmtree(child)
-def create_run_directory(root: Path | None = None) -> Path:
- stage_root = fixed_stage_root() if root is None else root.resolve()
+def create_run_directory(root: Path) -> Path:
+ stage_root = root.resolve()
_ensure_within(stage_root, stage_root)
stage_root.mkdir(parents=True, exist_ok=True)
_reject_reparse_chain(stage_root, stage_root)
@@ -285,10 +413,10 @@
raise WorkerError("E_STAGE")
-def prepare_run_directory(root: Path | None = None) -> Path:
+def prepare_run_directory(root: Path) -> Path:
"""Clean stale runs and create the secret-free task lease."""
- stage_root = fixed_stage_root() if root is None else root.resolve()
- cleanup_stale_runs(stage_root, boundary=stage_root if root is not None else None)
+ stage_root = root.resolve()
+ cleanup_stale_runs(stage_root, boundary=stage_root)
return create_run_directory(stage_root)
@@ -345,8 +473,8 @@
return converted
-def validate_processed_info(info: Any) -> dict[str, Any]:
- if not isinstance(info, dict) or info.get("id") != TARGET_BVID:
+def validate_processed_info(info: Any, job: dict[str, Any]) -> dict[str, Any]:
+ if not isinstance(info, dict) or info.get("id") != job["bvid"]:
raise WorkerError("E_METADATA")
if info.get("entries") not in (None, []) or info.get("_type") not in (None, "video"):
raise WorkerError("E_MULTI_PART")
@@ -358,8 +486,11 @@
raise WorkerError("E_DRM")
if info.get("availability") not in (None, "public", "unlisted"):
raise WorkerError("E_ENTITLEMENT")
+ owner_ids = {str(value) for value in (info.get("uploader_id"), info.get("channel_id")) if value is not None}
+ if job["creator_uid"] not in owner_ids:
+ raise WorkerError("E_OWNER")
duration_ms = round(_finite_number(info.get("duration")) * 1000)
- if abs(duration_ms - EXPECTED_DURATION_MS) > DURATION_TOLERANCE_MS:
+ if abs(duration_ms - job["expected_duration_ms"]) > duration_tolerance_ms(job["expected_duration_ms"]):
raise WorkerError("E_DURATION")
formats = info.get("formats")
if not isinstance(formats, list) or not formats:
@@ -385,10 +516,11 @@
def validate_download_info(
download_info: Any,
params: dict[str, Any],
+ job: dict[str, Any],
*,
downloader_resolver: Callable[..., Any] | None = None,
) -> tuple[list[dict[str, Any]], bool]:
- if not isinstance(download_info, dict) or download_info.get("id") != TARGET_BVID:
+ if not isinstance(download_info, dict) or download_info.get("id") != job["bvid"]:
raise WorkerError("E_FORMAT")
leaves, single = _format_leaves(download_info)
if single:
@@ -426,6 +558,7 @@
def prepare_download_info(
ydl: Any,
+ job: dict[str, Any],
*,
downloader_resolver: Callable[..., Any] | None = None,
) -> tuple[dict[str, Any], bool, tuple[str, ...]]:
@@ -440,8 +573,8 @@
return original_extract(*args, **kwargs)
ydl.extract_info = one_extract
- processed = ydl.extract_info(CANONICAL_URL, download=False, process=True)
- validate_processed_info(processed)
+ processed = ydl.extract_info(job["canonical_url"], download=False, process=True)
+ validate_processed_info(processed, job)
selector = ydl.build_format_selector("bestvideo+bestaudio/best")
selected = list(ydl._select_formats(ydl._get_formats(processed), selector))
if len(selected) != 1:
@@ -451,6 +584,7 @@
leaves, single = validate_download_info(
download_info,
ydl.params,
+ job,
downloader_resolver=downloader_resolver,
)
signed_urls = tuple(str(leaf["url"]) for leaf in leaves)
@@ -467,6 +601,23 @@
class SubprocessPolicy:
"""Pre-CreateProcess audit for local-only child command lines."""
+ _FORBIDDEN = ("://", "-headers", "-cookies", "authorization", "cookie:", "referer:", "user-agent:")
+ _FFMPEG_FLAGS = frozenset({"-nostdin", "-y"})
+ _FFMPEG_SCALAR_OPTIONS = frozenset(
+ {"-v", "-loglevel", "-map", "-c", "-map_metadata", "-f", "-movflags"}
+ )
+ _FFMPEG_REPEATABLE_OPTIONS = frozenset({"-map"})
+ _FFPROBE_FLAGS = frozenset({"-hide_banner", "-show_format", "-show_streams"})
+ _FFPROBE_SCALAR_OPTIONS = frozenset({"-v", "-show_entries", "-of", "-print_format"})
+ _WINDOWS_RESERVED_NAMES = frozenset(
+ {"CON", "PRN", "AUX", "NUL", *(f"COM{index}" for index in range(1, 10)),
+ *(f"LPT{index}" for index in range(1, 10))}
+ )
+ _MAX_ARGUMENTS = 256
+ _MAX_COMMAND_LINE = 32_767
+ _MAX_ENVIRONMENT_ITEMS = 256
+ _MAX_LOCAL_FILE_OPERANDS = 16
+
def __init__(
self,
run_root: Path,
@@ -477,46 +628,269 @@
self.executables = {os.path.normcase(str(item.resolve())) for item in executables}
self.secrets = {item.casefold() for item in secrets if item}
+ @staticmethod
+ def _fail(code: str) -> None:
+ if code not in SUBPROCESS_POLICY_ERROR_CODES:
+ code = "E_SUBPROCESS_POLICY_EVENT_SHAPE"
+ raise WorkerError(code)
+
+ @classmethod
+ def _windows_arguments(cls, command_line: Any) -> list[str]:
+ if (
+ not isinstance(command_line, str)
+ or not command_line
+ or len(command_line) > cls._MAX_COMMAND_LINE
+ or "\x00" in command_line
+ ):
+ cls._fail("E_SUBPROCESS_POLICY_EVENT_SHAPE")
+ argc = ctypes.c_int()
+ command_line_to_argv = ctypes.windll.shell32.CommandLineToArgvW
+ command_line_to_argv.argtypes = (ctypes.c_wchar_p, ctypes.POINTER(ctypes.c_int))
+ command_line_to_argv.restype = ctypes.POINTER(ctypes.c_wchar_p)
+ pointer = command_line_to_argv(command_line, ctypes.byref(argc))
+ if not pointer:
+ cls._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ try:
+ result = [pointer[index] for index in range(argc.value)]
+ finally:
+ local_free = ctypes.windll.kernel32.LocalFree
+ local_free.argtypes = (ctypes.c_void_p,)
+ local_free.restype = ctypes.c_void_p
+ local_free(ctypes.cast(pointer, ctypes.c_void_p))
+ if (
+ not result
+ or len(result) > cls._MAX_ARGUMENTS
+ or any(not isinstance(item, str) or "\x00" in item for item in result)
+ or subprocess.list2cmdline(result) != command_line
+ ):
+ cls._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ return result
+
+ @classmethod
+ def _portable_arguments(cls, raw: Any) -> list[str]:
+ if not isinstance(raw, (list, tuple)) or not raw or len(raw) > cls._MAX_ARGUMENTS:
+ cls._fail("E_SUBPROCESS_POLICY_EVENT_SHAPE")
+ if any(type(item) is not str or "\x00" in item for item in raw):
+ cls._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ return list(raw)
+
+ @staticmethod
+ def _absolute_executable(value: Any) -> tuple[str, Path]:
+ if not isinstance(value, (str, os.PathLike)):
+ SubprocessPolicy._fail("E_SUBPROCESS_POLICY_EVENT_SHAPE")
+ filesystem_value = os.fspath(value)
+ if not isinstance(filesystem_value, str) or not filesystem_value or "\x00" in filesystem_value:
+ SubprocessPolicy._fail("E_SUBPROCESS_POLICY_EVENT_SHAPE")
+ path = Path(filesystem_value)
+ if not path.is_absolute() or str(path).startswith("\\\\"):
+ SubprocessPolicy._fail("E_SUBPROCESS_POLICY_EXECUTABLE")
+ try:
+ resolved = path.resolve(strict=True)
+ except OSError as exc:
+ raise WorkerError("E_SUBPROCESS_POLICY_EXECUTABLE") from exc
+ if not resolved.is_file() or resolved.is_symlink() or _is_reparse(resolved):
+ SubprocessPolicy._fail("E_SUBPROCESS_POLICY_EXECUTABLE")
+ return os.path.normcase(str(resolved)), resolved
+
+ def _validate_environment(self, environment: Any) -> None:
+ if environment is None:
+ return
+ if not isinstance(environment, Mapping) or len(environment) > self._MAX_ENVIRONMENT_ITEMS:
+ self._fail("E_SUBPROCESS_POLICY_ENVIRONMENT")
+ encoded: list[str] = []
+ for key, value in environment.items():
+ if (
+ type(key) is not str
+ or type(value) is not str
+ or not key
+ or "\x00" in key
+ or "\x00" in value
+ ):
+ self._fail("E_SUBPROCESS_POLICY_ENVIRONMENT")
+ encoded.append(f"{key}={value}")
+ folded = "\x00".join(encoded).casefold()
+ if any(item in folded for item in self._FORBIDDEN) or any(item in folded for item in self.secrets):
+ self._fail("E_SUBPROCESS_POLICY_ENVIRONMENT")
+
def check(self, event: str, arguments: tuple[Any, ...]) -> None:
if event != "subprocess.Popen":
return
- executable, argv, _cwd, environment = arguments
- if not isinstance(executable, (str, os.PathLike)) or not isinstance(argv, (list, tuple)):
- raise WorkerError("E_SUBPROCESS_POLICY")
- executable_key = os.path.normcase(str(Path(executable).resolve()))
+ if not isinstance(arguments, tuple) or len(arguments) != 4:
+ self._fail("E_SUBPROCESS_POLICY_EVENT_SHAPE")
+ executable, raw_arguments, cwd, environment = arguments
+ if cwd is not None:
+ self._fail("E_SUBPROCESS_POLICY_EVENT_SHAPE")
+ text_args = (
+ self._windows_arguments(raw_arguments)
+ if os.name == "nt"
+ else self._portable_arguments(raw_arguments)
+ )
+ argv_executable_key, _ = self._absolute_executable(text_args[0])
+ if executable is None:
+ executable_key = argv_executable_key
+ else:
+ executable_key, _ = self._absolute_executable(executable)
+ if executable_key != argv_executable_key:
+ self._fail("E_SUBPROCESS_POLICY_EXECUTABLE")
if executable_key not in self.executables:
- raise WorkerError("E_SUBPROCESS_POLICY")
- text_args = [str(item) for item in argv]
+ self._fail("E_SUBPROCESS_POLICY_EXECUTABLE")
folded = "\x00".join(text_args).casefold()
- forbidden = ("://", "-headers", "-cookies", "authorization", "cookie:", "referer:", "user-agent:")
- if any(item in folded for item in forbidden) or any(item in folded for item in self.secrets):
- raise WorkerError("E_SUBPROCESS_POLICY")
- if environment is not None:
- encoded_env = "\x00".join(f"{key}={value}" for key, value in environment.items()).casefold()
- if any(item in encoded_env for item in self.secrets):
- raise WorkerError("E_SUBPROCESS_POLICY")
- if executable_key.endswith("ffmpeg.exe") or executable_key.endswith("ffprobe.exe"):
- for index, argument in enumerate(text_args[:-1]):
- if argument == "-i":
- self._local_path(text_args[index + 1], must_exist=True)
- output = text_args[-1]
- if output not in {"-", "NUL"} and not output.startswith("-"):
- self._local_path(output, must_exist=False)
+ if any(item in folded for item in self._FORBIDDEN) or any(item in folded for item in self.secrets):
+ self._fail("E_SUBPROCESS_POLICY_SECRET")
+ self._validate_environment(environment)
+ executable_name = Path(executable_key).name.casefold()
+ if executable_name == "ffmpeg.exe":
+ self._validate_ffmpeg_arguments(text_args)
+ elif executable_name == "ffprobe.exe":
+ self._validate_ffprobe_arguments(text_args)
+
+ @staticmethod
+ def _valid_ffmpeg_scalar(option: str, value: str) -> bool:
+ if option == "-v":
+ return value == "error"
+ if option == "-loglevel":
+ return value == "repeat+info"
+ if option == "-map":
+ return re.fullmatch(r"\d+(?::[av](?::\d+)?)?", value) is not None
+ if option == "-c":
+ return value == "copy"
+ if option == "-map_metadata":
+ return value == "-1"
+ if option == "-f":
+ return value == "matroska"
+ if option == "-movflags":
+ return value == "+faststart"
+ return False
+
+ @staticmethod
+ def _valid_ffprobe_scalar(option: str, value: str) -> bool:
+ if option == "-v":
+ return value == "error"
+ if option == "-show_entries":
+ return value == "format=format_name,duration:stream=codec_type"
+ if option in {"-of", "-print_format"}:
+ return value == "json"
+ return False
+
+ def _validate_ffmpeg_arguments(self, arguments: Sequence[str]) -> None:
+ if tuple(arguments[1:]) == ("-bsfs",):
+ return
+ index = 1
+ input_count = 0
+ file_operand_count = 0
+ output_seen = False
+ seen_options: set[str] = set()
+ while index < len(arguments):
+ option = arguments[index]
+ if option in self._FFMPEG_FLAGS:
+ if option in seen_options:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ seen_options.add(option)
+ index += 1
+ continue
+ if option in {"-i", "-attach"}:
+ if index + 1 >= len(arguments) or arguments[index + 1].startswith("-"):
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ self._local_path(arguments[index + 1], must_exist=True)
+ file_operand_count += 1
+ input_count += option == "-i"
+ if file_operand_count > self._MAX_LOCAL_FILE_OPERANDS:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ index += 2
+ continue
+ if option in self._FFMPEG_SCALAR_OPTIONS:
+ if index + 1 >= len(arguments):
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ value = arguments[index + 1]
+ if not self._valid_ffmpeg_scalar(option, value):
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ if option not in self._FFMPEG_REPEATABLE_OPTIONS:
+ if option in seen_options:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ seen_options.add(option)
+ index += 2
+ continue
+ if re.fullmatch(r"-bsf:a:\d+", option):
+ if index + 1 >= len(arguments) or arguments[index + 1] != "aac_adtstoasc":
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ if option in seen_options:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ seen_options.add(option)
+ index += 2
+ continue
+ if option.startswith("-") or output_seen or index != len(arguments) - 1:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ self._local_path(option, must_exist=False)
+ output_seen = True
+ file_operand_count += 1
+ index += 1
+ if input_count < 1 or not output_seen or file_operand_count > self._MAX_LOCAL_FILE_OPERANDS:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+
+ def _validate_ffprobe_arguments(self, arguments: Sequence[str]) -> None:
+ if tuple(arguments[1:]) == ("-bsfs",):
+ return
+ index = 1
+ input_seen = False
+ seen_options: set[str] = set()
+ while index < len(arguments):
+ option = arguments[index]
+ if option in self._FFPROBE_FLAGS:
+ if option in seen_options:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ seen_options.add(option)
+ index += 1
+ continue
+ if option in self._FFPROBE_SCALAR_OPTIONS:
+ if index + 1 >= len(arguments) or not self._valid_ffprobe_scalar(
+ option, arguments[index + 1]
+ ):
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ if option in seen_options:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ seen_options.add(option)
+ index += 2
+ continue
+ if option.startswith("-") or input_seen or index != len(arguments) - 1:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
+ self._local_path(option, must_exist=True)
+ input_seen = True
+ index += 1
+ if not input_seen:
+ self._fail("E_SUBPROCESS_POLICY_ARGUMENTS")
def _local_path(self, value: str, *, must_exist: bool) -> Path:
+ if value.startswith("file:"):
+ value = value[5:]
path = Path(value)
- if not path.is_absolute() or str(path).startswith("\\\\"):
- raise WorkerError("E_SUBPROCESS_POLICY")
+ if (
+ not value
+ or not path.is_absolute()
+ or str(path).startswith("\\\\")
+ or ":" in value[2:]
+ or any(part == ".." or part.endswith((" ", ".")) for part in path.parts)
+ or path.name.split(".", 1)[0].upper() in self._WINDOWS_RESERVED_NAMES
+ ):
+ self._fail("E_SUBPROCESS_POLICY_LOCAL_PATH")
try:
- resolved = path.resolve(strict=must_exist)
- except OSError as exc:
- raise WorkerError("E_SUBPROCESS_POLICY") from exc
- try:
- resolved.relative_to(self.run_root)
- except ValueError as exc:
- raise WorkerError("E_SUBPROCESS_POLICY") from exc
- if must_exist and (not resolved.is_file() or resolved.is_symlink()):
- raise WorkerError("E_SUBPROCESS_POLICY")
+ path.relative_to(self.run_root)
+ _reject_reparse_chain(path, self.run_root)
+ if must_exist:
+ resolved = path.resolve(strict=True)
+ resolved.relative_to(self.run_root)
+ if not resolved.is_file() or resolved.is_symlink() or _is_reparse(resolved):
+ self._fail("E_SUBPROCESS_POLICY_LOCAL_PATH")
+ else:
+ if path.exists() or path.is_symlink():
+ self._fail("E_SUBPROCESS_POLICY_LOCAL_PATH")
+ parent = path.parent.resolve(strict=True)
+ parent.relative_to(self.run_root)
+ _reject_reparse_chain(path.parent, self.run_root)
+ if not parent.is_dir() or parent.is_symlink() or _is_reparse(parent):
+ self._fail("E_SUBPROCESS_POLICY_LOCAL_PATH")
+ resolved = parent / path.name
+ except (OSError, ValueError, WorkerError) as exc:
+ raise WorkerError("E_SUBPROCESS_POLICY_LOCAL_PATH") from exc
return resolved
def install(self) -> None:
@@ -619,7 +993,7 @@
raise WorkerError("E_MERGE")
-def probe_mkv(ffprobe: Path, candidate: Path) -> None:
+def probe_mkv(ffprobe: Path, candidate: Path, job: dict[str, Any] | None = None) -> None:
try:
result = _run_local(
[
@@ -627,7 +1001,7 @@
"-v",
"error",
"-show_entries",
- "format=format_name:stream=codec_type",
+ "format=format_name,duration:stream=codec_type",
"-of",
"json",
str(candidate),
@@ -645,6 +1019,13 @@
format_name = payload.get("format", {}).get("format_name", "")
if result.returncode != 0 or {"video", "audio"} - stream_types or "matroska" not in format_name:
raise WorkerError("E_MEDIA_VALIDATION")
+ if job is not None:
+ raw_duration = payload.get("format", {}).get("duration")
+ if isinstance(raw_duration, str) and re.fullmatch(r"[0-9]+(?:\.[0-9]+)?", raw_duration):
+ raw_duration = float(raw_duration)
+ duration_ms = round(_finite_number(raw_duration) * 1000)
+ if abs(duration_ms - job["expected_duration_ms"]) > duration_tolerance_ms(job["expected_duration_ms"]):
+ raise WorkerError("E_DURATION")
def validate_unique_candidate(run_directory: Path) -> Path:
@@ -658,96 +1039,981 @@
return candidates[0]
-def _bridge_command(config: HostConfig, candidate: Path) -> list[str]:
+def _ordinary_exact_file(path: Path, parent: Path) -> os.stat_result:
+ try:
+ if path.parent.resolve(strict=True) != parent.resolve(strict=True):
+ raise WorkerError("E_COLLISION")
+ _reject_reparse_path(path, parent)
+ value = path.lstat()
+ except OSError as exc:
+ raise WorkerError("E_COLLISION") from exc
+ if not stat.S_ISREG(value.st_mode) or _is_reparse(path):
+ raise WorkerError("E_COLLISION")
+ return value
+
+
+def _stable_file_identity(value: os.stat_result) -> tuple[int, ...]:
+ """Return cross-API file identity fields; content is bound separately by SHA."""
+ return (
+ value.st_dev,
+ value.st_ino,
+ value.st_mode,
+ value.st_nlink,
+ value.st_size,
+ )
+
+
+def _stable_pair_test_seam(_: str) -> None:
+ """Named no-op seams used only by production-shaped race regressions."""
+ return None
+
+
+def _completion_test_seam(_: str) -> None:
+ """Named no-op seams for durable completion transaction regressions."""
+ return None
+
+
+def _canonical_json_line(value: Mapping[str, Any]) -> bytes:
+ return json.dumps(
+ dict(value), ensure_ascii=False, allow_nan=False, separators=(",", ":")
+ ).encode("utf-8") + b"\n"
+
+
+_FORMAL_PRIOR_REQUIRED_KEYS = frozenset({
+ "stable_id", "creator_uid", "source_url", "published_at", "item_type", "status",
+})
+_FORMAL_PRIOR_STATUS_RE = re.compile(r"VIDEO_[A-Z0-9_]{1,127}\Z")
+_FORMAL_LEGACY_BY_LINE = {row[0]: row for row in FORMAL_LEGACY_ROWS}
+_FORMAL_LEGACY_INTEGER_UID_BY_LINE = {
+ row[0]: row for row in FORMAL_LEGACY_INTEGER_UID_ROWS
+}
+
+
+def _formal_raw_lines(payload: bytes) -> list[bytes]:
+ complete = payload if not payload or payload.endswith(b"\n") else payload[:payload.rfind(b"\n") + 1]
+ return complete.splitlines()
+
+
+def _validate_formal_legacy_manifest() -> None:
+ if (
+ FORMAL_PREFIX_BYTES != 103_766
+ or FORMAL_PREFIX_LINES != 119
+ or not re.fullmatch(r"[A-F0-9]{64}", FORMAL_PREFIX_SHA256)
+ or not re.fullmatch(r"[1-9][0-9]{1,19}", FORMAL_LEGACY_CREATOR_UID)
+ or len(FORMAL_LEGACY_ROWS) != 26
+ or len(_FORMAL_LEGACY_BY_LINE) != len(FORMAL_LEGACY_ROWS)
+ or len(FORMAL_LEGACY_INTEGER_UID_ROWS) != 1
+ or len(_FORMAL_LEGACY_INTEGER_UID_BY_LINE)
+ != len(FORMAL_LEGACY_INTEGER_UID_ROWS)
+ or set(_FORMAL_LEGACY_BY_LINE) & set(_FORMAL_LEGACY_INTEGER_UID_BY_LINE)
+ ):
+ raise WorkerError("E_COMPLETION_FORMAL")
+ for row in FORMAL_LEGACY_ROWS:
+ if (
+ not isinstance(row, tuple) or len(row) != 8
+ or not isinstance(row[0], int) or not 1 <= row[0] <= FORMAL_PREFIX_LINES
+ or not isinstance(row[1], int) or row[1] < 2
+ or not isinstance(row[2], str) or not re.fullmatch(r"[A-F0-9]{64}", row[2])
+ or not all(isinstance(value, str) and value for value in row[3:])
+ or row[6] != "video" or _FORMAL_PRIOR_STATUS_RE.fullmatch(row[7]) is None
+ ):
+ raise WorkerError("E_COMPLETION_FORMAL")
+ for row in FORMAL_LEGACY_INTEGER_UID_ROWS:
+ if (
+ not isinstance(row, tuple) or len(row) != 9
+ or type(row[0]) is not int or not 1 <= row[0] <= FORMAL_PREFIX_LINES
+ or type(row[1]) is not int or row[1] < 2
+ or not isinstance(row[2], str) or not re.fullmatch(r"[A-F0-9]{64}", row[2])
+ or type(row[3]) is not int or row[3] <= 0
+ or str(row[3]) != FORMAL_LEGACY_CREATOR_UID
+ or not all(isinstance(value, str) and value for value in row[4:])
+ or row[7] != "video" or _FORMAL_PRIOR_STATUS_RE.fullmatch(row[8]) is None
+ ):
+ raise WorkerError("E_COMPLETION_FORMAL")
+
+
+def _validate_legacy_formal_prior(
+ payload: bytes,
+ raw_lines: Sequence[bytes],
+ line_ordinal: int,
+ value: dict[str, Any],
+ config: HostConfig,
+ job: dict[str, Any],
+) -> str:
+ _validate_formal_legacy_manifest()
+ if (
+ len(payload) < FORMAL_PREFIX_BYTES
+ or len(raw_lines) < FORMAL_PREFIX_LINES
+ or hashlib.sha256(payload[:FORMAL_PREFIX_BYTES]).hexdigest().upper() != FORMAL_PREFIX_SHA256
+ or payload[:FORMAL_PREFIX_BYTES].count(b"\n") != FORMAL_PREFIX_LINES
+ or not payload[:FORMAL_PREFIX_BYTES].endswith(b"\n")
+ or job["creator_uid"] != FORMAL_LEGACY_CREATOR_UID
+ ):
+ raise WorkerError("E_COMPLETION_FORMAL")
+ expected = _FORMAL_LEGACY_BY_LINE.get(line_ordinal)
+ if expected is None:
+ raise WorkerError("E_COMPLETION_FORMAL")
+ raw_line = raw_lines[line_ordinal - 1]
+ if (
+ len(raw_line) != expected[1]
+ or hashlib.sha256(raw_line).hexdigest().upper() != expected[2]
+ or set(_FORMAL_PRIOR_REQUIRED_KEYS) - set(value) != {"creator_uid"}
+ or value.get("stable_id") != expected[3]
+ or value.get("source_url") != expected[4]
+ or value.get("published_at") != expected[5]
+ or value.get("item_type") != expected[6]
+ or value.get("status") != expected[7]
+ or value.get("schema_version") != 1
+ or value.get("creator") != config.creator_name
+ or expected[3] != job["bvid"]
+ or expected[4] != job["canonical_url"]
+ or expected[5] != job["published_at"]
+ ):
+ raise WorkerError("E_COMPLETION_FORMAL")
+ return expected[7]
+
+
+def _validate_current_formal_prior(
+ value: dict[str, Any], config: HostConfig, job: dict[str, Any]
+) -> str:
+ if (
+ not _FORMAL_PRIOR_REQUIRED_KEYS.issubset(value)
+ or type(value.get("schema_version")) is not int
+ or value["schema_version"] != 1
+ or value.get("creator") != config.creator_name
+ or not isinstance(value.get("stable_id"), str)
+ or not isinstance(value.get("creator_uid"), str)
+ or not isinstance(value.get("source_url"), str)
+ or not isinstance(value.get("published_at"), str)
+ or not isinstance(value.get("item_type"), str)
+ or not isinstance(value.get("status"), str)
+ or value["stable_id"] != job["bvid"]
+ or value["creator_uid"] != job["creator_uid"]
+ or value["source_url"] != job["canonical_url"]
+ or value["published_at"] != job["published_at"]
+ or value["item_type"] != "video"
+ or _FORMAL_PRIOR_STATUS_RE.fullmatch(value["status"]) is None
+ ):
+ raise WorkerError("E_COMPLETION_FORMAL")
+ return value["status"]
+
+
+def _validate_legacy_integer_uid_formal_prior(
+ payload: bytes,
+ raw_lines: Sequence[bytes],
+ line_ordinal: int,
+ value: dict[str, Any],
+ config: HostConfig,
+ job: dict[str, Any],
+) -> str:
+ _validate_formal_legacy_manifest()
+ if (
+ len(payload) < FORMAL_PREFIX_BYTES
+ or len(raw_lines) < FORMAL_PREFIX_LINES
+ or hashlib.sha256(payload[:FORMAL_PREFIX_BYTES]).hexdigest().upper()
+ != FORMAL_PREFIX_SHA256
+ or payload[:FORMAL_PREFIX_BYTES].count(b"\n") != FORMAL_PREFIX_LINES
+ or not payload[:FORMAL_PREFIX_BYTES].endswith(b"\n")
+ ):
+ raise WorkerError("E_COMPLETION_FORMAL")
+ expected = _FORMAL_LEGACY_INTEGER_UID_BY_LINE.get(line_ordinal)
+ if expected is None:
+ raise WorkerError("E_COMPLETION_FORMAL")
+ raw_line = raw_lines[line_ordinal - 1]
+ if (
+ len(raw_line) != expected[1]
+ or hashlib.sha256(raw_line).hexdigest().upper() != expected[2]
+ or not _FORMAL_PRIOR_REQUIRED_KEYS.issubset(value)
+ or type(value.get("schema_version")) is not int
+ or value["schema_version"] != 1
+ or value.get("creator") != config.creator_name
+ or type(value.get("creator_uid")) is not int
+ or value["creator_uid"] != expected[3]
+ or value.get("stable_id") != expected[4]
+ or value.get("source_url") != expected[5]
+ or value.get("published_at") != expected[6]
+ or value.get("item_type") != expected[7]
+ or value.get("status") != expected[8]
+ or str(expected[3]) != job["creator_uid"]
+ or expected[4] != job["bvid"]
+ or expected[5] != job["canonical_url"]
+ or expected[6] != job["published_at"]
+ ):
+ raise WorkerError("E_COMPLETION_FORMAL")
+ return expected[8]
+
+
+def _read_jsonl_objects(path: Path, *, limit: int, error_code: str) -> tuple[bytes, list[dict[str, Any]]]:
+ try:
+ if path.exists():
+ parent = path.parent.resolve(strict=True)
+ _ordinary_exact_file(path, parent)
+ payload = path.read_bytes()
+ else:
+ payload = b""
+ if len(payload) > limit:
+ raise WorkerError(error_code)
+ complete = payload if not payload or payload.endswith(b"\n") else payload[:payload.rfind(b"\n") + 1]
+ records: list[dict[str, Any]] = []
+ for raw_line in complete.splitlines():
+ if not raw_line:
+ raise WorkerError(error_code)
+ value = strict_json_loads(raw_line)
+ records.append(value)
+ return payload, records
+ except WorkerError:
+ raise
+ except (OSError, ProtocolError, UnicodeError, json.JSONDecodeError) as exc:
+ raise WorkerError(error_code) from exc
+
+
+def _append_jsonl_idempotent(
+ path: Path,
+ record: dict[str, Any],
+ *,
+ identity_key: str,
+ identity_value: str,
+ limit: int,
+ error_code: str,
+) -> None:
+ expected = _canonical_json_line(record)
+ payload, records = _read_jsonl_objects(path, limit=limit, error_code=error_code)
+ matches = [value for value in records if value.get(identity_key) == identity_value]
+ if len(matches) > 1 or (matches and matches[0] != record):
+ raise WorkerError("E_COMPLETION_REPLAY")
+ if matches:
+ return
+ suffix = b"" if not payload or payload.endswith(b"\n") else payload[payload.rfind(b"\n") + 1:]
+ if suffix and (len(suffix) >= len(expected) or expected[:len(suffix)] != suffix):
+ raise WorkerError(error_code)
+ parent = path.parent.resolve(strict=True)
+ try:
+ if not path.exists():
+ with path.open("xb") as created:
+ created.flush()
+ os.fsync(created.fileno())
+ _ordinary_exact_file(path, parent)
+ with path.open("r+b", buffering=0) as stream:
+ current = stream.read()
+ if current != payload:
+ raise WorkerError(error_code)
+ stream.seek(0, os.SEEK_END)
+ stream.write(expected[len(suffix):])
+ stream.flush()
+ os.fsync(stream.fileno())
+ _completion_test_seam(f"AFTER_{identity_key.upper()}_APPEND")
+ final_payload, final_records = _read_jsonl_objects(path, limit=limit, error_code=error_code)
+ if not final_payload.endswith(b"\n") or sum(
+ value.get(identity_key) == identity_value and value == record for value in final_records
+ ) != 1:
+ raise WorkerError(error_code)
+ except WorkerError:
+ raise
+ except OSError as exc:
+ raise WorkerError(error_code) from exc
+
+
+@contextmanager
+def _completion_lock(path: Path) -> Iterable[None]:
+ try:
+ path.parent.resolve(strict=True)
+ with path.open("a+b") as stream:
+ if stream.seek(0, os.SEEK_END) == 0:
+ stream.write(b"\0")
+ stream.flush()
+ os.fsync(stream.fileno())
+ _ordinary_exact_file(path, path.parent)
+ stream.seek(0)
+ if os.name == "nt":
+ import msvcrt # noqa: PLC0415
+
+ msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
+ try:
+ yield
+ finally:
+ stream.seek(0)
+ msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl # noqa: PLC0415
+
+ fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ try:
+ yield
+ finally:
+ fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
+ except WorkerError:
+ raise
+ except OSError as exc:
+ raise WorkerError("E_COMPLETION_BUSY") from exc
+
+
+def _commit_formal_and_handoff(
+ config: HostConfig,
+ job: dict[str, Any],
+ formal_name: str,
+ mapping_name: str,
+ persisted: dict[str, Any],
+ *,
+ media_complete_acknowledged: bool = False,
+) -> None:
+ """Idempotently close formal publication and processing handoff before COMPLETE."""
+ if (
+ not isinstance(config.queue_lock_path, Path)
+ or not isinstance(config.formal_manifest_path, Path)
+ or not isinstance(config.processing_handoff_path, Path)
+ or not isinstance(config.creator_name, str) or not config.creator_name
+ ):
+ raise WorkerError("E_COMPLETION_CONFIG")
+ handoff_id = f"HANDOFF-BILI-MEDIA-{job['job_id'][:32].upper()}"
+ with _completion_lock(config.queue_lock_path):
+ formal_payload, formal_records = _read_jsonl_objects(
+ config.formal_manifest_path, limit=32 * 1024 * 1024, error_code="E_COMPLETION_FORMAL"
+ )
+ raw_lines = _formal_raw_lines(formal_payload)
+ if len(raw_lines) != len(formal_records):
+ raise WorkerError("E_COMPLETION_FORMAL")
+ prior_status = None
+ seen_prior_identities: set[tuple[str, str, str, str, str, str]] = set()
+ for line_ordinal, value in enumerate(formal_records, 1):
+ if value.get("queue_job_id") == job["job_id"]:
+ continue
+ stable_matches = value.get("stable_id") == job["bvid"]
+ source_matches = value.get("source_url") == job["canonical_url"]
+ if not stable_matches and not source_matches:
+ continue
+ item_type = value.get("item_type")
+ if item_type == "video_transcript" and stable_matches and source_matches:
+ continue
+ if not stable_matches or not source_matches:
+ raise WorkerError("E_COMPLETION_FORMAL")
+ missing = _FORMAL_PRIOR_REQUIRED_KEYS - set(value)
+ if line_ordinal in _FORMAL_LEGACY_INTEGER_UID_BY_LINE:
+ prior_status = _validate_legacy_integer_uid_formal_prior(
+ formal_payload, raw_lines, line_ordinal, value, config, job
+ )
+ creator_uid = job["creator_uid"]
+ elif missing == {"creator_uid"}:
+ prior_status = _validate_legacy_formal_prior(
+ formal_payload, raw_lines, line_ordinal, value, config, job
+ )
+ creator_uid = job["creator_uid"]
+ else:
+ prior_status = _validate_current_formal_prior(value, config, job)
+ creator_uid = value["creator_uid"]
+ prior_identity = (
+ value["stable_id"], creator_uid, value["source_url"],
+ value["published_at"], value["item_type"], prior_status,
+ )
+ if prior_identity in seen_prior_identities:
+ raise WorkerError("E_COMPLETION_FORMAL")
+ seen_prior_identities.add(prior_identity)
+ if prior_status is None and media_complete_acknowledged is not True:
+ # A first formal row has no historical status to supersede. It is
+ # permitted only after the typed Host ACK proves the governed queue
+ # job's exact MEDIA_COMPLETE identity is already durable.
+ raise WorkerError("E_COMPLETION_FORMAL")
+ formal_record = {
+ "schema_version": 1,
+ "creator": config.creator_name,
+ "creator_uid": job["creator_uid"],
+ "item_type": "video",
+ "stable_id": job["bvid"],
+ "title": job["title"],
+ "source_url": job["canonical_url"],
+ "published_at": job["published_at"],
+ "collected_at": persisted["completed_at"],
+ "status": "VIDEO_DOWNLOADED_COMPLETE_HANDOFF_SENT",
+ "video_path": str(config.destination / formal_name),
+ "mapping_path": str(config.destination / mapping_name),
+ "bytes": persisted["bytes"],
+ "sha256": persisted["sha256"],
+ "duration_seconds": persisted["duration_seconds"],
+ "video_codec": persisted["video_codec"],
+ "audio_codec": persisted["audio_codec"],
+ "processing_handoff_id": handoff_id,
+ "queue_job_id": job["job_id"],
+ }
+ if prior_status is not None:
+ formal_record["supersedes_status"] = prior_status
+ handoff_record = {
+ "schema": 1,
+ "type": "media-processing-handoff",
+ "status": "READY",
+ "handoff_id": handoff_id,
+ "queue_job_id": job["job_id"],
+ "creator_uid": job["creator_uid"],
+ "bvid": job["bvid"],
+ "source_url": job["canonical_url"],
+ "media_path": str(config.destination / formal_name),
+ "mapping_path": str(config.destination / mapping_name),
+ "bytes": persisted["bytes"],
+ "sha256": persisted["sha256"],
+ "duration_seconds": persisted["duration_seconds"],
+ "video_codec": persisted["video_codec"],
+ "audio_codec": persisted["audio_codec"],
+ "created_at": persisted["completed_at"],
+ }
+ _append_jsonl_idempotent(
+ config.formal_manifest_path, formal_record,
+ identity_key="queue_job_id", identity_value=job["job_id"],
+ limit=32 * 1024 * 1024, error_code="E_COMPLETION_FORMAL",
+ )
+ _completion_test_seam("BETWEEN_FORMAL_AND_HANDOFF")
+ _append_jsonl_idempotent(
+ config.processing_handoff_path, handoff_record,
+ identity_key="queue_job_id", identity_value=job["job_id"],
+ limit=8 * 1024 * 1024, error_code="E_COMPLETION_HANDOFF",
+ )
+ _completion_test_seam("BEFORE_COMPLETION_RETURN")
+
+
+@dataclass
+class _LockedPublishedFile:
+ """Read-only handle whose sharing mode denies writers, deletion and replacement."""
+
+ path: Path
+ parent: Path
+ error_code: str
+ stream: Any
+ identity: tuple[int, ...]
+
+ @classmethod
+ def open(cls, path: Path, parent: Path, error_code: str) -> "_LockedPublishedFile":
+ stream: Any | None = None
+ try:
+ before = _ordinary_exact_file(path, parent)
+ if os.name != "nt":
+ # The deployed Host is Windows-only. Keep non-Windows imports
+ # fail-closed while retaining a no-follow advisory read lock.
+ import fcntl # noqa: PLC0415
+
+ flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
+ descriptor = os.open(path, flags)
+ try:
+ fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
+ stream = os.fdopen(descriptor, "rb", closefd=True)
+ except BaseException:
+ os.close(descriptor)
+ raise
+ else:
+ import msvcrt # noqa: PLC0415
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ create_file = kernel32.CreateFileW
+ create_file.argtypes = (
+ ctypes.c_wchar_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p,
+ ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p,
+ )
+ create_file.restype = ctypes.c_void_p
+ close_handle = kernel32.CloseHandle
+ close_handle.argtypes = (ctypes.c_void_p,)
+ close_handle.restype = ctypes.c_int
+ handle = create_file(
+ str(path),
+ 0x80000000, # GENERIC_READ
+ 0x00000001, # FILE_SHARE_READ: deny write/delete/path replacement
+ None,
+ 3, # OPEN_EXISTING
+ 0x00000080 | 0x00200000 | 0x08000000,
+ # FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OPEN_REPARSE_POINT |
+ # FILE_FLAG_SEQUENTIAL_SCAN
+ None,
+ )
+ if handle in (None, ctypes.c_void_p(-1).value):
+ code = ctypes.get_last_error()
+ raise OSError(code, ctypes.FormatError(code), str(path))
+ descriptor: int | None = None
+ try:
+ descriptor = msvcrt.open_osfhandle(
+ int(handle), os.O_RDONLY | getattr(os, "O_BINARY", 0)
+ )
+ handle = None
+ stream = os.fdopen(descriptor, "rb", closefd=True)
+ descriptor = None
+ finally:
+ if descriptor is not None:
+ os.close(descriptor)
+ if handle is not None:
+ close_handle(handle)
+ after = _ordinary_exact_file(path, parent)
+ identity = _stable_file_identity(os.fstat(stream.fileno()))
+ if identity != _stable_file_identity(before) or identity != _stable_file_identity(after):
+ raise WorkerError(error_code)
+ return cls(path=path, parent=parent, error_code=error_code, stream=stream, identity=identity)
+ except BaseException as exc:
+ if stream is not None:
+ stream.close()
+ if isinstance(exc, WorkerError) and exc.code == error_code:
+ raise
+ if isinstance(exc, (KeyboardInterrupt, SystemExit)):
+ raise
+ raise WorkerError(error_code) from exc
+
+ def close(self) -> None:
+ self.stream.close()
+
+ def assert_path_identity(self) -> os.stat_result:
+ try:
+ path_stat = _ordinary_exact_file(self.path, self.parent)
+ handle_stat = os.fstat(self.stream.fileno())
+ except (OSError, WorkerError) as exc:
+ raise WorkerError(self.error_code) from exc
+ if (
+ _stable_file_identity(path_stat) != self.identity
+ or _stable_file_identity(handle_stat) != self.identity
+ ):
+ raise WorkerError(self.error_code)
+ return handle_stat
+
+ def read_all(self, maximum: int) -> bytes:
+ try:
+ self.stream.seek(0)
+ payload = self.stream.read(maximum + 1)
+ if len(payload) > maximum or self.stream.read(1) != b"":
+ raise WorkerError(self.error_code)
+ self.assert_path_identity()
+ return payload
+ except WorkerError:
+ raise
+ except OSError as exc:
+ raise WorkerError(self.error_code) from exc
+
+ def sha256(self) -> str:
+ digest = hashlib.sha256()
+ try:
+ self.stream.seek(0)
+ for chunk in iter(lambda: self.stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ self.assert_path_identity()
+ return digest.hexdigest()
+ except WorkerError:
+ raise
+ except OSError as exc:
+ raise WorkerError(self.error_code) from exc
+
+
+def _bridge_number(value: Any) -> float:
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ raise WorkerError("E_BRIDGE_OUTPUT_SCHEMA")
+ number = float(value)
+ if not math.isfinite(number) or number < 0:
+ raise WorkerError("E_BRIDGE_OUTPUT_SCHEMA")
+ return number
+
+
+def _bridge_remote_matches_expected(expected_duration_ms: int, remote: float) -> bool:
+ expected = expected_duration_ms / 1000
+ if abs(remote - expected) <= 0.001:
+ return True
+ return (
+ expected_duration_ms % 1000 == 0
+ and 0 < expected - remote < 1
+ and math.ceil(remote) == int(expected)
+ )
+
+
+def _validate_complete_bridge_item(
+ item: Any,
+ job: dict[str, Any],
+ *,
+ persisted: bool,
+) -> dict[str, Any]:
+ if not isinstance(item, dict):
+ raise WorkerError("E_BRIDGE_OUTPUT_SCHEMA")
+ expected_keys = _BRIDGE_MAPPING_KEYS if persisted else _BRIDGE_ITEM_KEYS
+ if set(item) != expected_keys:
+ if persisted or set(item) != _BRIDGE_ITEM_WARNING_KEYS:
+ raise WorkerError("E_BRIDGE_OUTPUT_SCHEMA")
+ warning = item["warning"]
+ if (
+ not isinstance(warning, str)
+ or _BRIDGE_CLEANUP_WARNING_RE.fullmatch(warning) is None
+ ):
+ raise WorkerError("E_BRIDGE_OUTPUT_SCHEMA")
+ if not persisted and item["status"] != "COMPLETE":
+ raise WorkerError("E_BRIDGE_OUTPUT_SCHEMA")
+ if (
+ item["schema_version"] != "1.0"
+ or item["bvid"] != job["bvid"]
+ or item["source"] != job["canonical_url"]
+ or item["published_at"] != job["published_at"]
+ or item["local_file"] != f"{job['bvid']}.mkv"
+ or item["acquisition_mode"] != "authorized_browser_file_handoff"
+ or not isinstance(item["title"], str)
+ or not 1 <= len(item["title"]) <= 1024
+ or any(ord(character) < 0x20 for character in item["title"])
+ or not isinstance(item["bytes"], int)
+ or isinstance(item["bytes"], bool)
+ or item["bytes"] <= 0
+ or not isinstance(item["sha256"], str)
+ or _LOWER_SHA256_RE.fullmatch(item["sha256"]) is None
+ or item["handoff_source_sha256"] != item["sha256"]
+ or not isinstance(item["format_name"], str)
+ or "matroska" not in item["format_name"].split(",")
+ or not isinstance(item["video_codec"], str)
+ or not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", item["video_codec"])
+ or not isinstance(item["audio_codec"], str)
+ or not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", item["audio_codec"])
+ or not isinstance(item["completed_at"], str)
+ or _UTC_ISO_RE.fullmatch(item["completed_at"]) is None
+ ):
+ raise WorkerError("E_BRIDGE_OUTPUT_SCHEMA")
+ duration = _bridge_number(item["duration_seconds"])
+ remote = _bridge_number(item["remote_duration_seconds"])
+ local = _bridge_number(item["local_duration_seconds"])
+ delta = _bridge_number(item["duration_delta_seconds"])
+ tolerance = _bridge_number(item["duration_tolerance_seconds"])
+ expected_tolerance = max(3.0, remote * 0.001)
+ if (
+ not _bridge_remote_matches_expected(job["expected_duration_ms"], remote)
+ or abs(duration - local) > 1e-9
+ or abs(delta - abs(local - remote)) > 1e-9
+ or abs(tolerance - expected_tolerance) > 1e-9
+ or delta > tolerance
+ ):
+ raise WorkerError("E_BRIDGE_DURATION_SHA")
+ return item
+
+
+def _read_exact_published_bridge_result(
+ config: HostConfig,
+ job: dict[str, Any],
+ *,
+ expected_item: dict[str, Any] | None = None,
+ on_media_verified: Callable[[str, str, dict[str, Any], dict[str, Any]], None] | None = None,
+ on_verified: Callable[[str, str, dict[str, Any]], None] | None = None,
+) -> tuple[str, str] | None:
+ destination = config.destination.resolve(strict=True)
+ formal_name = f"{job['bvid']}.mkv"
+ mapping_name = f"{job['bvid']}.download.json"
+ formal_path = destination / formal_name
+ mapping_path = destination / mapping_name
+ present = (formal_path.exists(), mapping_path.exists())
+ if present == (False, False):
+ return None
+ if present != (True, True):
+ raise WorkerError("E_BRIDGE_MEDIA_MAPPING")
+ formal_lock = _LockedPublishedFile.open(formal_path, destination, "E_BRIDGE_DURATION_SHA")
+ try:
+ mapping_lock = _LockedPublishedFile.open(
+ mapping_path, destination, "E_BRIDGE_MAPPING_READBACK"
+ )
+ try:
+ _stable_pair_test_seam("LOCKS_ACQUIRED")
+ mapping_stat = mapping_lock.assert_path_identity()
+ if mapping_stat.st_size <= 0 or mapping_stat.st_size > 64 * 1024:
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK")
+ mapping_bytes = mapping_lock.read_all(64 * 1024)
+ try:
+ persisted = _validate_complete_bridge_item(
+ strict_json_loads(mapping_bytes), job, persisted=True
+ )
+ except WorkerError as exc:
+ if exc.code == "E_BRIDGE_DURATION_SHA":
+ raise
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK") from exc
+ except (ProtocolError, UnicodeError, json.JSONDecodeError) as exc:
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK") from exc
+ if expected_item is not None:
+ if {key: expected_item[key] for key in _BRIDGE_MAPPING_KEYS} != persisted:
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK")
+ formal_stat = formal_lock.assert_path_identity()
+ formal_sha = formal_lock.sha256().casefold()
+ if persisted["bytes"] != formal_stat.st_size or persisted["sha256"] != formal_sha:
+ raise WorkerError("E_BRIDGE_DURATION_SHA")
+ _stable_pair_test_seam("AFTER_INITIAL_PAIR")
+
+ # The two Windows handles deny write/delete/path replacement while
+ # FFprobe opens its read-only view. Hashing and JSON parsing use these
+ # same handles, and the consumer commit executes before handle release.
+ probe_mkv(config.ffprobe, formal_path, job)
+ formal_lock.assert_path_identity()
+ mapping_lock.assert_path_identity()
+ final_formal_sha = formal_lock.sha256().casefold()
+ _stable_pair_test_seam("AFTER_FINAL_MEDIA_HASH")
+ final_mapping_bytes = mapping_lock.read_all(64 * 1024)
+ _stable_pair_test_seam("AFTER_FINAL_MAPPING_READ")
+ if final_formal_sha != persisted["sha256"]:
+ raise WorkerError("E_BRIDGE_DURATION_SHA")
+ if final_mapping_bytes != mapping_bytes:
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK")
+ try:
+ final_persisted = _validate_complete_bridge_item(
+ strict_json_loads(final_mapping_bytes), job, persisted=True
+ )
+ except (WorkerError, ProtocolError, UnicodeError, json.JSONDecodeError) as exc:
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK") from exc
+ if final_persisted != persisted:
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK")
+ formal_lock.assert_path_identity()
+ mapping_lock.assert_path_identity()
+ _stable_pair_test_seam("BEFORE_COMMIT")
+ media_identity = validate_media_complete_identity(
+ {
+ "formal_filename": formal_name,
+ "mapping_filename": mapping_name,
+ "media_bytes": persisted["bytes"],
+ "media_sha256": persisted["sha256"].upper(),
+ "mapping_bytes": len(final_mapping_bytes),
+ "mapping_sha256": hashlib.sha256(final_mapping_bytes).hexdigest().upper(),
+ "duration_milliseconds": int(round(
+ float(persisted["local_duration_seconds"]) * 1_000
+ )),
+ "video_codec": persisted["video_codec"],
+ "audio_codec": persisted["audio_codec"],
+ },
+ job,
+ )
+ if on_media_verified is not None:
+ on_media_verified(formal_name, mapping_name, persisted, media_identity)
+ if on_verified is not None:
+ on_verified(formal_name, mapping_name, persisted)
+ return formal_name, mapping_name
+ finally:
+ mapping_lock.close()
+ finally:
+ formal_lock.close()
+
+
+def recover_published_task(
+ config: HostConfig,
+ job: dict[str, Any],
+ *,
+ cancel_check: Callable[[], bool],
+ report: Callable[..., None],
+ commit_begin: Callable[[], None],
+) -> tuple[str, str]:
+ """Verify and consume one exact published pair before secret transfer."""
+ if job["creator_uid"] not in config.creator_allowlist:
+ raise WorkerError("E_ALLOWLIST")
+ committed: tuple[str, str] | None = None
+
+ def media_verified(
+ _formal_name: str, _mapping_name: str, _persisted: dict[str, Any],
+ media_identity: dict[str, Any],
+ ) -> None:
+ report("MEDIA_COMPLETE", 100, media_identity)
+
+ def consume(formal_name: str, mapping_name: str, persisted: dict[str, Any]) -> None:
+ nonlocal committed
+ if cancel_check():
+ raise CancelRequested()
+ report("POSTPROCESS_PENDING", 100)
+ commit_begin()
+ _commit_formal_and_handoff(
+ config, job, formal_name, mapping_name, persisted,
+ media_complete_acknowledged=True,
+ )
+ committed = (formal_name, mapping_name)
+
+ recovered = _read_exact_published_bridge_result(
+ config, job, on_media_verified=media_verified, on_verified=consume
+ )
+ if recovered is None:
+ raise WorkerError("E_BRIDGE_MEDIA_MAPPING")
+ if committed != recovered:
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK")
+ return recovered
+
+
+def _bridge_command(
+ config: HostConfig, candidate: Path, job: dict[str, Any], batch_json: Path
+) -> list[str]:
return [
str(config.bridge_python),
str(config.bridge_script),
"--input",
- str(config.batch_json),
- "--yt-dlp",
- str(config.yt_dlp_executable),
+ str(batch_json),
"accept-browser-file",
"--bvid",
- TARGET_BVID,
+ job["bvid"],
"--media-file",
str(candidate),
"--destination",
str(config.destination),
"--ffprobe",
str(config.ffprobe),
+ "--expected-duration-ms",
+ str(job["expected_duration_ms"]),
]
-def run_frozen_bridge(config: HostConfig, candidate: Path) -> tuple[str, str]:
+def run_frozen_bridge(
+ config: HostConfig,
+ candidate: Path,
+ job: dict[str, Any],
+ *,
+ on_media_verified: Callable[[str, str, dict[str, Any], dict[str, Any]], None] | None = None,
+ on_verified: Callable[[str, str, dict[str, Any]], None] | None = None,
+) -> tuple[str, str]:
+ started = time.monotonic()
+
+ def failure(code: str, state: str, reason: str) -> WorkerError:
+ return WorkerError(code, {
+ "attempts": 1,
+ "elapsed_ms": max(0, min(7_200_000, int((time.monotonic() - started) * 1000))),
+ "state": state,
+ "reason": reason,
+ })
+
+ batch_json = candidate.parent / "bridge-input.json"
+ batch_payload = {
+ "schema_version": "1.0",
+ "batch_id": f"generic-{job['job_id'][:16]}",
+ "items": [{
+ "bvid": job["bvid"],
+ "source_url": job["canonical_url"],
+ "published_at": job["published_at"],
+ "title": job["title"],
+ "expected_duration_ms": job["expected_duration_ms"],
+ }],
+ }
+ try:
+ with batch_json.open("xb") as stream:
+ stream.write((json.dumps(batch_payload, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8"))
+ stream.flush()
+ os.fsync(stream.fileno())
+ except BaseException as exc:
+ batch_json.unlink(missing_ok=True)
+ raise failure(
+ "E_BRIDGE_INVOCATION", "BRIDGE_INVOCATION", "BATCH_RECEIPT_CREATE_FAILED"
+ ) from exc
try:
result = _run_local(
- _bridge_command(config, candidate),
+ _bridge_command(config, candidate, job, batch_json),
BRIDGE_TIMEOUT_SECONDS,
capture_stdout=True,
)
except subprocess.TimeoutExpired as exc:
- raise WorkerError("E_BACKHALF") from exc
+ raise failure(
+ "E_BRIDGE_INVOCATION", "BRIDGE_INVOCATION", "SUBPROCESS_TIMEOUT"
+ ) from exc
+ except OSError as exc:
+ raise failure(
+ "E_BRIDGE_INVOCATION", "BRIDGE_INVOCATION", "SUBPROCESS_INVOCATION_FAILED"
+ ) from exc
+ finally:
+ batch_json.unlink(missing_ok=True)
if result.returncode != 0:
- raise WorkerError("E_BACKHALF")
+ try:
+ stopped = strict_json_loads(result.stdout)
+ except (ProtocolError, UnicodeError, json.JSONDecodeError):
+ stopped = None
+ bridge_stops = {
+ "E_BRIDGE_SOURCE_STABILITY": (
+ "BRIDGE_SOURCE_STABILITY", "SOURCE_FILE_INVALID"
+ ),
+ "E_BRIDGE_METADATA_BINDING": (
+ "BRIDGE_METADATA_BINDING", "EXPECTED_METADATA_MISMATCH"
+ ),
+ "E_BRIDGE_FFPROBE": (
+ "BRIDGE_FFPROBE", "LOCAL_MEDIA_PROBE_FAILED"
+ ),
+ "E_BRIDGE_DURATION_SHA": (
+ "DURATION_SHA_VERIFICATION", "DURATION_OR_SHA_MISMATCH"
+ ),
+ "E_BRIDGE_PUBLISH": (
+ "CREATE_NEW_PUBLISH", "BRIDGE_REPORTED_PUBLISH_FAILURE"
+ ),
+ }
+ if (
+ isinstance(stopped, dict)
+ and set(stopped) == {"result", "error_code"}
+ and stopped.get("result") == "SAFETY_STOP"
+ and stopped.get("error_code") in bridge_stops
+ ):
+ code = stopped["error_code"]
+ state, reason = bridge_stops[code]
+ raise failure(code, state, reason)
+ raise failure("E_BRIDGE_EXIT", "BRIDGE_EXIT", "NONZERO_EXIT")
+
+ def read_published(expected_item: dict[str, Any] | None) -> tuple[str, str]:
+ try:
+ published_value = _read_exact_published_bridge_result(
+ config, job, expected_item=expected_item,
+ on_media_verified=on_media_verified, on_verified=on_verified,
+ )
+ except WorkerError as exc:
+ if exc.code.startswith("E_COMPLETION_"):
+ raise
+ code = exc.code if exc.code.startswith("E_BRIDGE_") else "E_BRIDGE_MAPPING_READBACK"
+ state, reason = {
+ "E_BRIDGE_MEDIA_MAPPING": ("MEDIA_MAPPING_PRESENCE", "MEDIA_OR_MAPPING_MISSING"),
+ "E_BRIDGE_DURATION_SHA": ("DURATION_SHA_VERIFICATION", "DURATION_OR_SHA_MISMATCH"),
+ }.get(code, ("MAPPING_READBACK", "PERSISTED_MAPPING_INVALID"))
+ raise failure(code, state, reason) from exc
+ if published_value is None:
+ raise failure(
+ "E_BRIDGE_MEDIA_MAPPING", "MEDIA_MAPPING_PRESENCE", "MEDIA_OR_MAPPING_MISSING"
+ )
+ return published_value
+
+ def preserve_verified_media_before_output_failure(_cause: BaseException | None = None) -> bool:
+ """Persist media truth, but do not commit postprocess through an invalid wire result."""
+ try:
+ if os.path.lexists(config.destination / ".bili-download-staging"):
+ raise failure(
+ "E_BRIDGE_OUTPUT_SCHEMA", "BRIDGE_OUTPUT_SCHEMA", "OUTPUT_SCHEMA_INVALID"
+ )
+ published_value = _read_exact_published_bridge_result(
+ config, job, on_media_verified=on_media_verified
+ )
+ if published_value is None:
+ raise WorkerError("E_BRIDGE_MEDIA_MAPPING")
+ return True
+ except WorkerError as exc:
+ if exc.code.startswith("E_COMPLETION_"):
+ raise
+ return False
+
try:
- payload = json.loads(result.stdout.decode("utf-8"))
- except (UnicodeError, json.JSONDecodeError) as exc:
- raise WorkerError("E_BACKHALF") from exc
- if not isinstance(payload, dict) or payload.get("result") != "PASS":
- raise WorkerError("E_BACKHALF")
+ payload = strict_json_loads(result.stdout)
+ except (ProtocolError, UnicodeError, json.JSONDecodeError) as exc:
+ preserve_verified_media_before_output_failure(exc)
+ raise failure(
+ "E_BRIDGE_OUTPUT_SCHEMA", "BRIDGE_OUTPUT_SCHEMA", "OUTPUT_SCHEMA_INVALID"
+ ) from exc
+ if (
+ not isinstance(payload, dict)
+ or set(payload) != {"batch_id", "command", "result", "success_count", "failure_count", "items"}
+ or payload.get("batch_id") != f"generic-{job['job_id'][:16]}"
+ or payload.get("result") != "PASS"
+ or payload.get("command") != "accept-browser-file"
+ or payload.get("success_count") != 1
+ or payload.get("failure_count") != 0
+ ):
+ preserve_verified_media_before_output_failure()
+ raise failure(
+ "E_BRIDGE_OUTPUT_SCHEMA", "BRIDGE_OUTPUT_SCHEMA", "OUTPUT_SCHEMA_INVALID"
+ )
items = payload.get("items")
if not isinstance(items, list) or len(items) != 1 or not isinstance(items[0], dict):
- raise WorkerError("E_BACKHALF")
- item = items[0]
- required = {"bvid", "local_file", "bytes", "sha256", "duration_seconds", "acquisition_mode"}
- if not required.issubset(item) or item["bvid"] != TARGET_BVID:
- raise WorkerError("E_BACKHALF")
- if item["acquisition_mode"] != "authorized_browser_file_handoff":
- raise WorkerError("E_BACKHALF")
- formal = item["local_file"]
- mapping = f"{TARGET_BVID}.download.json"
- if formal != f"{TARGET_BVID}.mkv":
- raise WorkerError("E_BACKHALF")
- formal_path = config.destination / formal
- mapping_path = config.destination / mapping
- if not formal_path.is_file() or not mapping_path.is_file():
- raise WorkerError("E_BACKHALF")
+ preserve_verified_media_before_output_failure()
+ raise failure(
+ "E_BRIDGE_OUTPUT_SCHEMA", "BRIDGE_OUTPUT_SCHEMA", "OUTPUT_SCHEMA_INVALID"
+ )
try:
- duration_ms = round(_finite_number(item["duration_seconds"]) * 1000)
+ item = _validate_complete_bridge_item(items[0], job, persisted=False)
except WorkerError as exc:
- raise WorkerError("E_BACKHALF") from exc
- if abs(duration_ms - EXPECTED_DURATION_MS) > DURATION_TOLERANCE_MS:
- raise WorkerError("E_BACKHALF")
- formal_sha = sha256_file(formal_path)
- if (
- isinstance(item["bytes"], bool)
- or not isinstance(item["bytes"], int)
- or item["bytes"] != formal_path.stat().st_size
- or not isinstance(item["sha256"], str)
- or item["sha256"] != formal_sha
- ):
- raise WorkerError("E_BACKHALF")
- try:
- persisted = strict_json_loads(mapping_path.read_bytes())
- except (OSError, ProtocolError) as exc:
- raise WorkerError("E_BACKHALF") from exc
- matched_fields = {
- "bvid": TARGET_BVID,
- "source": CANONICAL_URL,
- "local_file": formal,
- "bytes": item["bytes"],
- "sha256": formal_sha,
- "acquisition_mode": "authorized_browser_file_handoff",
- }
- if any(persisted.get(key) != expected for key, expected in matched_fields.items()):
- raise WorkerError("E_BACKHALF")
- try:
- persisted_duration_ms = round(_finite_number(persisted.get("duration_seconds")) * 1000)
- except WorkerError as exc:
- raise WorkerError("E_BACKHALF") from exc
- if persisted_duration_ms != duration_ms:
- raise WorkerError("E_BACKHALF")
- return formal, mapping
+ preserve_verified_media_before_output_failure(exc)
+ code = exc.code if exc.code == "E_BRIDGE_DURATION_SHA" else "E_BRIDGE_OUTPUT_SCHEMA"
+ state, reason = (
+ ("DURATION_SHA_VERIFICATION", "DURATION_OR_SHA_MISMATCH")
+ if code == "E_BRIDGE_DURATION_SHA"
+ else ("BRIDGE_OUTPUT_SCHEMA", "OUTPUT_SCHEMA_INVALID")
+ )
+ raise failure(code, state, reason) from exc
+ if "warning" in item:
+ try:
+ if os.path.lexists(config.destination / ".bili-download-staging"):
+ raise failure(
+ "E_BRIDGE_MAPPING_READBACK", "MAPPING_READBACK", "PERSISTED_MAPPING_INVALID"
+ )
+ except OSError as exc:
+ raise failure(
+ "E_BRIDGE_MAPPING_READBACK", "MAPPING_READBACK", "PERSISTED_MAPPING_INVALID"
+ ) from exc
+ return read_published(item)
def ytdlp_options(
@@ -799,15 +2065,19 @@
config: HostConfig,
*,
cancel_check: Callable[[], bool],
- report: Callable[[str, int], None],
+ report: Callable[..., None],
stage_root: Path | None = None,
prepared_run_directory: Path | None = None,
commit_begin: Callable[[], None] | None = None,
closure_report: Callable[[bool], None] | None = None,
+ recovery_required: bool = False,
) -> tuple[str, str, bool]:
"""Run the exact task. The caller must already own this worker in a job."""
validate_start(start)
- root = fixed_stage_root() if stage_root is None else stage_root.resolve()
+ job = start["job"]
+ if job["creator_uid"] not in config.creator_allowlist:
+ raise WorkerError("E_ALLOWLIST")
+ root = fixed_stage_root(job["bvid"]) if stage_root is None else stage_root.resolve()
if prepared_run_directory is None:
run_directory = prepare_run_directory(root)
else:
@@ -825,6 +2095,38 @@
raise CancelRequested()
checkpoint()
+ def consume_recovered(
+ formal_name: str, mapping_name: str, persisted: dict[str, Any]
+ ) -> None:
+ nonlocal cookie_closed, committed, outcome
+ checkpoint()
+ report("POSTPROCESS_PENDING", 100)
+ if commit_begin is not None:
+ commit_begin()
+ _commit_formal_and_handoff(
+ config, job, formal_name, mapping_name, persisted,
+ media_complete_acknowledged=True,
+ )
+ cookie_closed = True
+ committed = True
+ outcome = (formal_name, mapping_name, True)
+
+ def recovered_media_verified(
+ _formal_name: str, _mapping_name: str, _persisted: dict[str, Any],
+ media_identity: dict[str, Any],
+ ) -> None:
+ report("MEDIA_COMPLETE", 100, media_identity)
+
+ recovered = _read_exact_published_bridge_result(
+ config, job, on_media_verified=recovered_media_verified,
+ on_verified=consume_recovered,
+ )
+ if recovered is not None:
+ if outcome != (recovered[0], recovered[1], True):
+ raise WorkerError("E_BRIDGE_MAPPING_READBACK")
+ return outcome
+ if recovery_required:
+ raise WorkerError("E_BRIDGE_MEDIA_MAPPING")
cookie_stream = build_cookie_stream(start)
yt_dlp, _ = bootstrap_ytdlp()
secret_values = [
@@ -852,7 +2154,7 @@
opts = ytdlp_options(config, run_directory, cookie_stream, progress_hook)
with yt_dlp.YoutubeDL(opts) as ydl:
report("CHECKING", 0)
- download_info, single, signed_urls = prepare_download_info(ydl)
+ download_info, single, signed_urls = prepare_download_info(ydl, job)
policy.secrets.update(url.casefold() for url in signed_urls)
checkpoint()
ydl.process_info(download_info)
@@ -882,7 +2184,7 @@
checkpoint()
candidate = validate_unique_candidate(run_directory)
checkpoint()
- probe_mkv(config.ffprobe, candidate)
+ probe_mkv(config.ffprobe, candidate, job)
checkpoint()
cookie_closed = close_cookie_stream(cookie_stream)
cookie_stream = None
@@ -892,7 +2194,29 @@
if commit_begin is not None:
commit_begin()
checkpoint()
- formal, mapping = run_frozen_bridge(config, candidate)
+ completion_committed = False
+
+ def consume_published(
+ formal_name: str, mapping_name: str, persisted: dict[str, Any]
+ ) -> None:
+ nonlocal completion_committed
+ checkpoint()
+ report("POSTPROCESS_PENDING", 100)
+ _commit_formal_and_handoff(
+ config, job, formal_name, mapping_name, persisted,
+ media_complete_acknowledged=True,
+ )
+ completion_committed = True
+
+ formal, mapping = run_frozen_bridge(
+ config, candidate, job,
+ on_media_verified=lambda _formal, _mapping, _persisted, media: report(
+ "MEDIA_COMPLETE", 100, media
+ ),
+ on_verified=consume_published,
+ )
+ if not completion_committed:
+ raise WorkerError("E_COMPLETION_HANDOFF")
committed = True
outcome = (formal, mapping, cookie_closed)
finally:
diff --git a/dev/project-dev/bili_authenticated_extension_unpacked_validator.py b/dev/project-dev/bili_authenticated_extension_unpacked_validator.py
new file mode 100644
index 0000000..ace6d69
--- /dev/null
+++ b/dev/project-dev/bili_authenticated_extension_unpacked_validator.py
@@ -0,0 +1,605 @@
+#!/usr/bin/env python3
+"""Offline validator for the local-unpacked Bilibili extension projection."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import ctypes
+from ctypes import wintypes
+from dataclasses import asdict, dataclass
+import hashlib
+import json
+import os
+from pathlib import Path, PurePosixPath
+import re
+import stat
+import sys
+from typing import Callable, Iterable
+
+
+SCHEMA = 1
+PROJECT_ID = "project-info"
+TASK_ID = "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001"
+SOURCE_ROOT_REL = PurePosixPath("dev/project-dev/bili_authenticated_extension")
+PROJECTION_ROOT_REL = PurePosixPath("dev/project-dev/bili_authenticated_extension_unpacked")
+CONTRACT_REL = PurePosixPath("dev/project-dev/bili_authenticated_extension_unpacked_contract.json")
+SOURCE_ARTIFACT_MANIFEST_REL = SOURCE_ROOT_REL / "source-artifact-manifest.json"
+PROJECT_MARKER_REL = PurePosixPath("mbx.project.yaml")
+EXPECTED_FILES = (
+ "background.js",
+ "manifest.json",
+ "sidepanel.css",
+ "sidepanel.html",
+ "sidepanel.js",
+)
+EXPECTED_EXTENSION_ID = "oidmclckpdmpabbfedplkbdplmfcenbb"
+EXPECTED_PUBLIC_DER_SHA256 = "E83C2B2AF3CF011543FBA13FBC524D1122EEA68548F9F27B9F7A82B5D594666C"
+TREE_HASH_ALGORITHM = "path-nul-bytes-nul-sha256-upper-lf-v1"
+SOURCE_ARTIFACT_MANIFEST_BYTES = 3770
+SOURCE_ARTIFACT_MANIFEST_SHA256 = "132B637634550A43B0EDD5E8E74C439205DE4BFECCE0A9276D8EBEF78445ECFC"
+FILE_ATTRIBUTE_DIRECTORY = 0x10
+FILE_ATTRIBUTE_REPARSE_POINT = 0x400
+FILE_READ_ATTRIBUTES = 0x80
+GENERIC_READ = 0x80000000
+FILE_SHARE_READ = 0x1
+FILE_SHARE_WRITE = 0x2
+OPEN_EXISTING = 3
+FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
+
+
+class ValidationError(RuntimeError):
+ def __init__(self, code: str, message: str) -> None:
+ super().__init__(message)
+ self.code = code
+
+
+@dataclass(frozen=True)
+class PathIdentity:
+ relative_path: str
+ kind: str
+ volume_serial: int
+ file_id: int
+ attributes: int
+ reparse_tag: int
+ final_path: str
+
+
+@dataclass(frozen=True)
+class FileEvidence:
+ path: str
+ bytes: int
+ sha256: str
+
+
+class _ByHandleFileInformation(ctypes.Structure):
+ _fields_ = [
+ ("dwFileAttributes", wintypes.DWORD),
+ ("ftCreationTime", wintypes.FILETIME),
+ ("ftLastAccessTime", wintypes.FILETIME),
+ ("ftLastWriteTime", wintypes.FILETIME),
+ ("dwVolumeSerialNumber", wintypes.DWORD),
+ ("nFileSizeHigh", wintypes.DWORD),
+ ("nFileSizeLow", wintypes.DWORD),
+ ("nNumberOfLinks", wintypes.DWORD),
+ ("nFileIndexHigh", wintypes.DWORD),
+ ("nFileIndexLow", wintypes.DWORD),
+ ]
+
+
+if os.name == "nt":
+ _kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ _CreateFileW = _kernel32.CreateFileW
+ _CreateFileW.argtypes = [
+ wintypes.LPCWSTR,
+ wintypes.DWORD,
+ wintypes.DWORD,
+ wintypes.LPVOID,
+ wintypes.DWORD,
+ wintypes.DWORD,
+ wintypes.HANDLE,
+ ]
+ _CreateFileW.restype = wintypes.HANDLE
+ _GetFileInformationByHandle = _kernel32.GetFileInformationByHandle
+ _GetFileInformationByHandle.argtypes = [
+ wintypes.HANDLE,
+ ctypes.POINTER(_ByHandleFileInformation),
+ ]
+ _GetFileInformationByHandle.restype = wintypes.BOOL
+ _GetFinalPathNameByHandleW = _kernel32.GetFinalPathNameByHandleW
+ _GetFinalPathNameByHandleW.argtypes = [
+ wintypes.HANDLE,
+ wintypes.LPWSTR,
+ wintypes.DWORD,
+ wintypes.DWORD,
+ ]
+ _GetFinalPathNameByHandleW.restype = wintypes.DWORD
+ _CloseHandle = _kernel32.CloseHandle
+ _CloseHandle.argtypes = [wintypes.HANDLE]
+ _CloseHandle.restype = wintypes.BOOL
+
+
+class _HandleSet:
+ def __init__(self) -> None:
+ self.handles: list[int] = []
+
+ def add(self, handle: int) -> None:
+ self.handles.append(handle)
+
+ def close(self) -> None:
+ while self.handles:
+ _CloseHandle(self.handles.pop())
+
+ def __enter__(self) -> "_HandleSet":
+ return self
+
+ def __exit__(self, *_: object) -> None:
+ self.close()
+
+
+def _strict_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
+ result: dict[str, object] = {}
+ for key, value in pairs:
+ if key in result:
+ raise ValidationError("E_JSON", "Duplicate JSON key.")
+ result[key] = value
+ return result
+
+
+def _strict_json(data: bytes) -> dict[str, object]:
+ try:
+ value = json.loads(data.decode("utf-8"), object_pairs_hook=_strict_object)
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+ raise ValidationError("E_JSON", "Invalid strict UTF-8 JSON.") from exc
+ if not isinstance(value, dict):
+ raise ValidationError("E_JSON", "JSON root must be an object.")
+ return value
+
+
+def _validate_project_root_text(value: str) -> Path:
+ if os.name != "nt":
+ raise ValidationError("E_WINDOWS_REQUIRED", "Windows is required.")
+ if not re.fullmatch(r"[A-Za-z]:\\[^:*?\"<>|]+(?:\\[^:*?\"<>|]+)*", value):
+ raise ValidationError("E_PROJECT_ROOT", "Project root must be an unambiguous absolute drive path.")
+ if value.startswith(("\\\\", "\\\\?\\", "\\\\.\\")):
+ raise ValidationError("E_PROJECT_ROOT", "UNC and device paths are forbidden.")
+ components = value[3:].split("\\") if len(value) > 3 else []
+ if not components or any(
+ part in {"", ".", ".."} or part.endswith((" ", ".")) or ":" in part
+ for part in components
+ ):
+ raise ValidationError("E_PROJECT_ROOT", "Ambiguous project-root components are forbidden.")
+ root = Path(value)
+ if root.name != PROJECT_ID:
+ raise ValidationError("E_PROJECT_ROOT", "Project-root identity mismatch.")
+ return root
+
+
+def _native(relative: PurePosixPath) -> tuple[str, ...]:
+ parts = tuple(relative.parts)
+ if not parts or any(part in {"", ".", ".."} or part.endswith((" ", ".")) or ":" in part for part in parts):
+ raise ValidationError("E_PATH", "Invalid fixed relative path.")
+ return parts
+
+
+def _join(root: Path, relative: PurePosixPath) -> Path:
+ return root.joinpath(*_native(relative))
+
+
+def _lstat(path: Path) -> os.stat_result:
+ try:
+ return os.lstat(path)
+ except OSError as exc:
+ raise ValidationError("E_PATH_MISSING", "Required lexical path is unavailable.") from exc
+
+
+def _reparse_tag(st: os.stat_result) -> int:
+ return int(getattr(st, "st_reparse_tag", 0) or 0)
+
+
+def _attributes(st: os.stat_result) -> int:
+ return int(getattr(st, "st_file_attributes", 0) or 0)
+
+
+def _open_identity(
+ path: Path,
+ *,
+ relative: str,
+ expect_dir: bool,
+ handles: _HandleSet,
+) -> PathIdentity:
+ st = _lstat(path)
+ attrs = _attributes(st)
+ tag = _reparse_tag(st)
+ if attrs & FILE_ATTRIBUTE_REPARSE_POINT or tag or stat.S_ISLNK(st.st_mode):
+ raise ValidationError("E_REPARSE", "Reparse paths are forbidden.")
+ if expect_dir:
+ if not stat.S_ISDIR(st.st_mode):
+ raise ValidationError("E_PATH_KIND", "Expected directory.")
+ access = FILE_READ_ATTRIBUTES
+ share = FILE_SHARE_READ | FILE_SHARE_WRITE
+ else:
+ if not stat.S_ISREG(st.st_mode):
+ raise ValidationError("E_PATH_KIND", "Expected regular file.")
+ access = GENERIC_READ
+ share = FILE_SHARE_READ
+ handle = _CreateFileW(
+ str(path),
+ access,
+ share,
+ None,
+ OPEN_EXISTING,
+ FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
+ None,
+ )
+ if handle == INVALID_HANDLE_VALUE:
+ raise ValidationError("E_PATH_HANDLE", "Cannot open no-follow path handle.")
+ handle_value = int(handle)
+ handles.add(handle_value)
+ info = _ByHandleFileInformation()
+ if not _GetFileInformationByHandle(handle_value, ctypes.byref(info)):
+ raise ValidationError("E_PATH_HANDLE", "Cannot read path identity.")
+ if info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT:
+ raise ValidationError("E_REPARSE", "Reparse handle is forbidden.")
+ is_directory = bool(info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+ if is_directory != expect_dir:
+ raise ValidationError("E_PATH_KIND", "Handle kind mismatch.")
+ size = 512
+ while True:
+ buffer = ctypes.create_unicode_buffer(size)
+ result = _GetFinalPathNameByHandleW(handle_value, buffer, size, 0)
+ if result == 0:
+ raise ValidationError("E_PATH_HANDLE", "Cannot read final path.")
+ if result < size:
+ final_path = buffer.value
+ break
+ size = result + 1
+ if final_path.startswith("\\\\?\\UNC\\"):
+ raise ValidationError("E_PATH_ESCAPE", "UNC final paths are forbidden.")
+ if final_path.startswith("\\\\?\\"):
+ final_path = final_path[4:]
+ file_id = (int(info.nFileIndexHigh) << 32) | int(info.nFileIndexLow)
+ return PathIdentity(
+ relative_path=relative,
+ kind="directory" if expect_dir else "file",
+ volume_serial=int(info.dwVolumeSerialNumber),
+ file_id=file_id,
+ attributes=int(info.dwFileAttributes),
+ reparse_tag=tag,
+ final_path=os.path.normpath(final_path),
+ )
+
+
+def _inside(root_final: str, child_final: str) -> bool:
+ root_norm = os.path.normcase(os.path.normpath(root_final))
+ child_norm = os.path.normcase(os.path.normpath(child_final))
+ try:
+ return os.path.commonpath((root_norm, child_norm)) == root_norm
+ except ValueError:
+ return False
+
+
+def _expected_paths(root: Path) -> list[tuple[Path, str, bool]]:
+ paths: dict[str, tuple[Path, str, bool]] = {}
+
+ def add(path: Path, relative: str, expect_dir: bool) -> None:
+ key = os.path.normcase(os.path.normpath(str(path)))
+ paths[key] = (path, relative, expect_dir)
+
+ add(root, ".", True)
+ add(_join(root, PROJECT_MARKER_REL), PROJECT_MARKER_REL.as_posix(), False)
+ for relative in (
+ SOURCE_ROOT_REL,
+ PROJECTION_ROOT_REL,
+ CONTRACT_REL,
+ SOURCE_ARTIFACT_MANIFEST_REL,
+ ):
+ current = root
+ current_parts: list[str] = []
+ for part in _native(relative):
+ current = current / part
+ current_parts.append(part)
+ relative_text = PurePosixPath(*current_parts).as_posix()
+ is_leaf = len(current_parts) == len(relative.parts)
+ expect_dir = not is_leaf or relative in {SOURCE_ROOT_REL, PROJECTION_ROOT_REL}
+ add(current, relative_text, expect_dir)
+ for parent in (SOURCE_ROOT_REL, PROJECTION_ROOT_REL):
+ for name in EXPECTED_FILES:
+ add(_join(root, parent) / name, (parent / name).as_posix(), False)
+ return sorted(paths.values(), key=lambda item: (len(Path(item[0]).parts), item[1]))
+
+
+def _capture(root: Path) -> tuple[dict[str, PathIdentity], dict[str, FileEvidence], dict[str, object]]:
+ with _HandleSet() as handles:
+ identities: dict[str, PathIdentity] = {}
+ for path, relative, expect_dir in _expected_paths(root):
+ identity = _open_identity(
+ path,
+ relative=relative,
+ expect_dir=expect_dir,
+ handles=handles,
+ )
+ identities[relative] = identity
+ root_identity = identities["."]
+ if root_identity.final_path != os.path.normpath(str(root)):
+ raise ValidationError("E_PATH_CASE_DRIFT", "Project-root physical casing mismatch.")
+ if os.path.normcase(root_identity.final_path) != os.path.normcase(os.path.normpath(str(root))):
+ raise ValidationError("E_PATH_ESCAPE", "Project-root physical identity mismatch.")
+ for identity in identities.values():
+ if identity.volume_serial != root_identity.volume_serial or not _inside(
+ root_identity.final_path, identity.final_path
+ ):
+ raise ValidationError("E_PATH_ESCAPE", "Path escapes the trusted physical project root.")
+ expected_final = root_identity.final_path
+ if identity.relative_path != ".":
+ expected_final = os.path.normpath(
+ os.path.join(root_identity.final_path, *PurePosixPath(identity.relative_path).parts)
+ )
+ if identity.final_path != expected_final:
+ raise ValidationError("E_PATH_CASE_DRIFT", "Actual NTFS path casing does not match the frozen path.")
+ marker_path = _join(root, PROJECT_MARKER_REL)
+ marker = marker_path.read_text(encoding="utf-8")
+ if not re.search(r"(?m)^project:\s*$", marker) or not re.search(
+ rf"(?m)^ id: {re.escape(PROJECT_ID)}\s*$", marker
+ ):
+ raise ValidationError("E_PROJECT_ROOT", "Project marker mismatch.")
+ contract_path = _join(root, CONTRACT_REL)
+ contract = _strict_json(contract_path.read_bytes())
+ _validate_contract_shape(contract)
+ source_artifact_manifest = _join(root, SOURCE_ARTIFACT_MANIFEST_REL).read_bytes()
+ _validate_source_artifact_manifest(source_artifact_manifest, contract)
+ source_root = _join(root, SOURCE_ROOT_REL)
+ projection_root = _join(root, PROJECTION_ROOT_REL)
+ _validate_projection_set(projection_root)
+ source_evidence = _read_files(source_root)
+ projection_evidence = _read_files(projection_root)
+ _validate_evidence(contract, source_evidence, projection_evidence)
+ _validate_manifest((projection_root / "manifest.json").read_bytes())
+ return identities, projection_evidence, contract
+
+
+def _validate_projection_set(root: Path) -> None:
+ actual: list[str] = []
+ try:
+ with os.scandir(root) as entries:
+ for entry in entries:
+ if entry.name.startswith("_"):
+ raise ValidationError("E_RESERVED_NAME", "Reserved underscore path is forbidden.")
+ if entry.is_symlink() or not entry.is_file(follow_symlinks=False):
+ raise ValidationError("E_PROJECTION_SET", "Projection must contain only regular files.")
+ actual.append(entry.name)
+ except OSError as exc:
+ raise ValidationError("E_PROJECTION_SET", "Cannot enumerate projection.") from exc
+ if tuple(sorted(actual)) != EXPECTED_FILES:
+ raise ValidationError("E_PROJECTION_SET", "Projection exact file set mismatch.")
+
+
+def _read_files(root: Path) -> dict[str, FileEvidence]:
+ result: dict[str, FileEvidence] = {}
+ for name in EXPECTED_FILES:
+ data = (root / name).read_bytes()
+ result[name] = FileEvidence(name, len(data), hashlib.sha256(data).hexdigest().upper())
+ return result
+
+
+def _validate_contract_shape(contract: dict[str, object]) -> None:
+ expected_keys = {
+ "schema",
+ "task_id",
+ "project_id",
+ "source_root",
+ "projection_root",
+ "expected_extension_id",
+ "public_key_der_sha256",
+ "tree_hash_algorithm",
+ "tree_sha256",
+ "files",
+ }
+ if set(contract) != expected_keys:
+ raise ValidationError("E_CONTRACT", "Contract key set mismatch.")
+ fixed = {
+ "schema": SCHEMA,
+ "task_id": TASK_ID,
+ "project_id": PROJECT_ID,
+ "source_root": SOURCE_ROOT_REL.as_posix(),
+ "projection_root": PROJECTION_ROOT_REL.as_posix(),
+ "expected_extension_id": EXPECTED_EXTENSION_ID,
+ "public_key_der_sha256": EXPECTED_PUBLIC_DER_SHA256,
+ "tree_hash_algorithm": TREE_HASH_ALGORITHM,
+ }
+ if any(contract.get(key) != value for key, value in fixed.items()):
+ raise ValidationError("E_CONTRACT", "Contract fixed identity mismatch.")
+ files = contract.get("files")
+ if not isinstance(files, list) or len(files) != len(EXPECTED_FILES):
+ raise ValidationError("E_CONTRACT", "Contract file set mismatch.")
+
+
+def _tree_hash(evidence: Iterable[FileEvidence]) -> str:
+ payload = bytearray()
+ for item in sorted(evidence, key=lambda value: value.path):
+ payload.extend(item.path.encode("utf-8"))
+ payload.append(0)
+ payload.extend(str(item.bytes).encode("ascii"))
+ payload.append(0)
+ payload.extend(item.sha256.upper().encode("ascii"))
+ payload.append(0x0A)
+ return hashlib.sha256(payload).hexdigest().upper()
+
+
+def _contract_file_evidence(contract: dict[str, object]) -> dict[str, FileEvidence]:
+ files = contract["files"]
+ assert isinstance(files, list)
+ expected: dict[str, FileEvidence] = {}
+ for raw in files:
+ if not isinstance(raw, dict) or set(raw) != {"path", "bytes", "sha256"}:
+ raise ValidationError("E_CONTRACT", "Contract file entry mismatch.")
+ path = raw.get("path")
+ size = raw.get("bytes")
+ digest = raw.get("sha256")
+ if (
+ not isinstance(path, str)
+ or path not in EXPECTED_FILES
+ or isinstance(size, bool)
+ or not isinstance(size, int)
+ or size <= 0
+ or not isinstance(digest, str)
+ or not re.fullmatch(r"[0-9A-F]{64}", digest)
+ or path in expected
+ ):
+ raise ValidationError("E_CONTRACT", "Invalid contract file entry.")
+ expected[path] = FileEvidence(path, size, digest)
+ if tuple(sorted(expected)) != EXPECTED_FILES:
+ raise ValidationError("E_CONTRACT", "Contract file paths mismatch.")
+ return expected
+
+
+def _validate_source_artifact_manifest(data: bytes, contract: dict[str, object]) -> None:
+ if (
+ len(data) != SOURCE_ARTIFACT_MANIFEST_BYTES
+ or hashlib.sha256(data).hexdigest().upper() != SOURCE_ARTIFACT_MANIFEST_SHA256
+ ):
+ raise ValidationError("E_SOURCE_MANIFEST", "Frozen source artifact manifest bytes mismatch.")
+ source_manifest = _strict_json(data)
+ files = source_manifest.get("files")
+ if not isinstance(files, list):
+ raise ValidationError("E_SOURCE_MANIFEST", "Source artifact manifest files are invalid.")
+ expected = _contract_file_evidence(contract)
+ observed: dict[str, FileEvidence] = {}
+ for raw in files:
+ if not isinstance(raw, dict):
+ raise ValidationError("E_SOURCE_MANIFEST", "Source artifact manifest entry is invalid.")
+ path = raw.get("path")
+ if path not in EXPECTED_FILES:
+ continue
+ if set(raw) != {"path", "bytes", "sha256"} or path in observed:
+ raise ValidationError("E_SOURCE_MANIFEST", "Source artifact manifest exact-five entry is invalid.")
+ size = raw.get("bytes")
+ digest = raw.get("sha256")
+ if isinstance(size, bool) or not isinstance(size, int) or not isinstance(digest, str):
+ raise ValidationError("E_SOURCE_MANIFEST", "Source artifact manifest exact-five value is invalid.")
+ observed[path] = FileEvidence(path, size, digest)
+ if observed != expected:
+ raise ValidationError("E_SOURCE_MANIFEST", "Source artifact manifest exact-five binding mismatch.")
+
+
+def _validate_evidence(
+ contract: dict[str, object],
+ source: dict[str, FileEvidence],
+ projection: dict[str, FileEvidence],
+) -> None:
+ expected = _contract_file_evidence(contract)
+ if source != expected or projection != expected or source != projection:
+ raise ValidationError("E_BYTE_DRIFT", "Source/projection bytes are not exactly locked.")
+ tree = _tree_hash(projection.values())
+ if contract.get("tree_sha256") != tree:
+ raise ValidationError("E_TREE_HASH", "Projection tree hash mismatch.")
+
+
+def _validate_manifest(data: bytes) -> None:
+ manifest = _strict_json(data)
+ if (
+ manifest.get("manifest_version") != 3
+ or manifest.get("name") != "project-info Bilibili 完整视频入口"
+ or manifest.get("version") != "1.2.25"
+ or manifest.get("version_name") != "1.2.25+20260829.generic.v027"
+ or manifest.get("background") != {"service_worker": "background.js", "type": "module"}
+ or manifest.get("permissions") != ["alarms", "cookies", "nativeMessaging", "scripting", "storage", "tabs"]
+ or manifest.get("host_permissions") != [
+ "https://www.bilibili.com/*",
+ "https://*.bilibili.com/*",
+ ]
+ or "side_panel" in manifest
+ or "action" in manifest
+ ):
+ raise ValidationError("E_MANIFEST", "Manifest identity mismatch.")
+ key = manifest.get("key")
+ if not isinstance(key, str):
+ raise ValidationError("E_MANIFEST", "Manifest public key is missing.")
+ try:
+ der = base64.b64decode(key, validate=True)
+ except ValueError as exc:
+ raise ValidationError("E_MANIFEST", "Manifest public key is invalid.") from exc
+ digest = hashlib.sha256(der).hexdigest().upper()
+ if digest != EXPECTED_PUBLIC_DER_SHA256:
+ raise ValidationError("E_MANIFEST", "Manifest public key hash mismatch.")
+ extension_id = "".join(chr(ord("a") + int(nibble, 16)) for nibble in digest[:32].lower())
+ if extension_id != EXPECTED_EXTENSION_ID:
+ raise ValidationError("E_EXTENSION_ID", "Derived extension ID mismatch.")
+
+
+def _identity_key(value: PathIdentity) -> tuple[object, ...]:
+ return (
+ value.relative_path,
+ value.kind,
+ value.volume_serial,
+ value.file_id,
+ value.attributes,
+ value.reparse_tag,
+ value.final_path,
+ )
+
+
+def validate_project(
+ project_root_text: str,
+ *,
+ between_passes: Callable[[], None] | None = None,
+) -> dict[str, object]:
+ root = _validate_project_root_text(project_root_text)
+ first_identities, first_files, first_contract = _capture(root)
+ if between_passes is not None:
+ between_passes()
+ second_identities, second_files, second_contract = _capture(root)
+ if set(first_identities) != set(second_identities) or any(
+ _identity_key(first_identities[key]) != _identity_key(second_identities[key])
+ for key in first_identities
+ ):
+ raise ValidationError("E_PATH_IDENTITY_DRIFT", "Path identity changed between validation passes.")
+ if first_files != second_files or first_contract != second_contract:
+ raise ValidationError("E_PATH_IDENTITY_DRIFT", "Content identity changed between validation passes.")
+ root_identity = second_identities["."]
+ return {
+ "schema": 1,
+ "status": "VALIDATION_PASS_ONLY",
+ "project_id": PROJECT_ID,
+ "task_id": TASK_ID,
+ "project_root": str(root),
+ "project_root_final": root_identity.final_path,
+ "project_volume_serial": root_identity.volume_serial,
+ "project_file_id": root_identity.file_id,
+ "source_root": SOURCE_ROOT_REL.as_posix(),
+ "projection_root": PROJECTION_ROOT_REL.as_posix(),
+ "file_count": len(EXPECTED_FILES),
+ "tree_sha256": second_contract["tree_sha256"],
+ "extension_id": EXPECTED_EXTENSION_ID,
+ "source_artifact_manifest": {
+ "bytes": SOURCE_ARTIFACT_MANIFEST_BYTES,
+ "sha256": SOURCE_ARTIFACT_MANIFEST_SHA256,
+ },
+ "files": [asdict(second_files[name]) for name in EXPECTED_FILES],
+ "path_identities": [asdict(second_identities[key]) for key in sorted(second_identities)],
+ }
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ description="Validate the exact local-unpacked Bilibili extension projection without launching Chrome.",
+ )
+ parser.add_argument("--project-root", required=True, help="Exact absolute trusted project root.")
+ return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = _parser().parse_args(argv)
+ try:
+ result = validate_project(args.project_root)
+ except ValidationError as exc:
+ print(json.dumps({"schema": 1, "status": "SAFETY_STOP", "error_code": exc.code}, separators=(",", ":")))
+ return 3
+ print(json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":")))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/dev/project-dev/bili_dynamic_collector.py b/dev/project-dev/bili_dynamic_collector.py
index 003f4b2..fd96e5e 100644
--- a/dev/project-dev/bili_dynamic_collector.py
+++ b/dev/project-dev/bili_dynamic_collector.py
@@ -593,12 +593,17 @@
if refresh_raw is not None:
if not isinstance(refresh_raw, Mapping):
raise CollectorError("E_CONFIG", "config.refresh must be an object.")
+ if creator_uid is None:
+ raise CollectorError("E_CONFIG", "refresh requires creator.uid.", safety=True)
+ refresh_page = urlsplit(dynamic_url)
if (
- creator_name.strip() != "青枫浦上Q"
- or creator_uid != "1420210197"
- or dynamic_url != "https://space.bilibili.com/1420210197/dynamic"
+ refresh_page.scheme != "https"
+ or refresh_page.hostname != "space.bilibili.com"
+ or refresh_page.query
+ or refresh_page.fragment
+ or refresh_page.path.rstrip("/") != f"/{creator_uid}/dynamic"
):
- raise CollectorError("E_CONFIG", "refresh supports only the registered creator identity.", safety=True)
+ raise CollectorError("E_CONFIG", "refresh page URL must bind creator.uid.", safety=True)
def bounded_int(field: str, default: int, lower: int, upper: int) -> int:
raw = refresh_raw.get(field, default)
@@ -1617,7 +1622,14 @@
handoff = subparsers.add_parser("handoff", help="Generate an unsent canonical Codex-native video handoff")
handoff.add_argument("--output", type=Path, help="Optional no-overwrite Markdown output path")
handoff.add_argument("--now", help="Optional offset-aware ISO-8601 clock for deterministic runs")
- begin = subparsers.add_parser("refresh-begin", help="Create one durable hourly browser-refresh run")
+ run_refresh = subparsers.add_parser(
+ "refresh-run",
+ help="Fail closed: trusted local-unpacked extension/Host is required",
+ )
+ run_refresh.add_argument("--now", help="Optional offset-aware ISO-8601 clock for deterministic runs")
+ begin = subparsers.add_parser(
+ "refresh-begin", help="Recover an existing refresh run (new schema3 runs require refresh-run)"
+ )
begin.add_argument("--now", help="Optional offset-aware ISO-8601 clock for deterministic runs")
commit = subparsers.add_parser("refresh-commit", help="Validate and commit one local browser evidence file")
commit.add_argument("--input", required=True, type=Path, help="Final browser evidence JSON from refresh-begin")
@@ -1641,12 +1653,16 @@
if args.output:
output = args.output if args.output.is_absolute() else Path.cwd() / args.output
result = generate_handoff(config, output, now)
- elif args.command in {"refresh-begin", "refresh-commit"}:
+ elif args.command in {"refresh-run", "refresh-begin", "refresh-commit"}:
from bili_dynamic_refresh import refresh_begin, refresh_commit
if config.refresh is None:
raise CollectorError("E_CONFIG", "config.refresh is required for refresh commands.")
- if args.command == "refresh-begin":
+ if args.command == "refresh-run":
+ from bili_dynamic_refresh_controller import run_product
+
+ result = run_product(config, config_path_value, now)
+ elif args.command == "refresh-begin":
result = refresh_begin(config, config_path_value, now)
else:
result = refresh_commit(config, config_path_value, absolute_lexical(args.input), now)
diff --git a/dev/project-dev/bili_dynamic_refresh.py b/dev/project-dev/bili_dynamic_refresh.py
index d7aeb3c..ab94822 100644
--- a/dev/project-dev/bili_dynamic_refresh.py
+++ b/dev/project-dev/bili_dynamic_refresh.py
@@ -14,12 +14,12 @@
import math
import os
import re
-import secrets
import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path, PurePosixPath
from typing import Any, Iterable, Mapping, Sequence
+from urllib.parse import urlsplit
import bili_dynamic_collector as core
@@ -171,48 +171,6 @@
return core.canonical_json_bytes(signable, newline=False)
-def _attest_controller_evidence(
- pending: Mapping[str, Any],
- evidence: dict[str, Any],
- *,
- action_dispatched: bool,
- monotonic_run_started_ms: int,
- monotonic_action_started_ms: int | None,
- monotonic_action_finished_ms: int | None,
- monotonic_observation_started_ms: int | None,
- monotonic_observation_finished_ms: int | None,
- monotonic_evidence_write_started_ms: int,
-) -> dict[str, Any]:
- """Bind evidence to the source-controlled controller's actual call envelope.
-
- The per-run capability is durable state that is never returned by
- ``refresh-begin``. It is not browser/session material; it only prevents a
- caller-authored JSON document from selecting a successful runtime state.
- """
- capability = pending.get("controller_capability")
- if not isinstance(capability, str) or re.fullmatch(r"[0-9a-f]{64}", capability) is None:
- raise core.CollectorError("E_RECOVERY_AMBIGUOUS", "Controller capability is invalid.", safety=True)
- runtime = pending["runtime_contract"]
- evidence["controller_attestation"] = {
- "controller_id": runtime["controller_id"],
- "controller_sha256": runtime["controller_sha256"],
- "binding_algorithm": runtime["binding_algorithm"],
- "action_dispatched": action_dispatched,
- "monotonic_run_started_ms": monotonic_run_started_ms,
- "monotonic_action_started_ms": monotonic_action_started_ms,
- "monotonic_action_finished_ms": monotonic_action_finished_ms,
- "monotonic_observation_started_ms": monotonic_observation_started_ms,
- "monotonic_observation_finished_ms": monotonic_observation_finished_ms,
- "monotonic_evidence_write_started_ms": monotonic_evidence_write_started_ms,
- "binding_sha256": None,
- }
- digest = hmac.new(
- bytes.fromhex(capability), _controller_attestation_payload(evidence), hashlib.sha256
- ).hexdigest()
- evidence["controller_attestation"]["binding_sha256"] = digest
- return evidence
-
-
def _strict_json_bytes(payload: bytes, description: str) -> Any:
if payload.startswith(b"\xef\xbb\xbf"):
raise core.CollectorError("E_EVIDENCE_SCHEMA", f"{description} must not contain a BOM.")
@@ -316,11 +274,11 @@
"intake_root", "evidence_identity", "planned_terminal", "transaction_identity",
"last_transition_at",
"runtime_contract",
- "controller_capability",
+ "controller_key_commitment", "controller_binding_sha256",
}
schema = value.get("schema_version")
if schema == LEGACY_PENDING_SCHEMA:
- legacy_required = required - {"runtime_contract", "controller_capability"}
+ legacy_required = required - {"runtime_contract", "controller_key_commitment", "controller_binding_sha256"}
if set(value) != legacy_required:
raise core.CollectorError("E_RECOVERY_AMBIGUOUS", "Legacy refresh pending schema is invalid.", safety=True)
elif schema != PENDING_SCHEMA or set(value) != required:
@@ -449,13 +407,28 @@
raise core.CollectorError("E_RUN_HOUR_OCCUPIED", "Run slot is not safely reusable.", safety=True)
-def refresh_begin(config: core.CollectorConfig, config_path: Path, now: datetime) -> dict[str, Any]:
+def refresh_begin(
+ config: core.CollectorConfig,
+ config_path: Path,
+ now: datetime,
+ *,
+ _controller_key_commitment: str | None = None,
+) -> dict[str, Any]:
refresh = config.refresh
assert refresh is not None
_validate_refresh_roots(config, create_state=True)
recovered = _recover_or_replay(config, config_path, now)
if recovered is not None:
return recovered
+ if (
+ not isinstance(_controller_key_commitment, str)
+ or core.LOWER_SHA256_PATTERN.fullmatch(_controller_key_commitment) is None
+ ):
+ raise core.CollectorError(
+ "E_CONTROLLER_ENTRY_REQUIRED",
+ "New runtime-v2 runs must be created by the source-controlled refresh-run entry.",
+ safety=True,
+ )
hour_epoch = math.floor(now.timestamp() / 3600)
_validate_slot_available(config, hour_epoch)
run_id = hashlib.sha256(
@@ -487,7 +460,8 @@
"transaction_identity": None,
"last_transition_at": core.canonical_datetime(now),
"runtime_contract": runtime_identity,
- "controller_capability": secrets.token_hex(32),
+ "controller_key_commitment": _controller_key_commitment,
+ "controller_binding_sha256": None,
}
_write_pending(config, pending, create=True)
_ensure_started_slot(config, pending)
@@ -719,6 +693,8 @@
config: core.CollectorConfig,
pending: Mapping[str, Any],
path: Path,
+ *,
+ _controller_key: bytes | None = None,
) -> tuple[dict[str, Any], bytes, dict[str, Any] | None, bool]:
global _OBSERVATION_CONFIG
if str(path) != pending["evidence_path"]:
@@ -752,12 +728,24 @@
binding = attestation["binding_sha256"]
if not isinstance(binding, str) or core.LOWER_SHA256_PATTERN.fullmatch(binding) is None:
raise core.CollectorError("E_CONTROLLER_ATTESTATION", "Controller binding is invalid.", safety=True)
- capability = pending["controller_capability"]
- expected_binding = hmac.new(
- bytes.fromhex(capability), _controller_attestation_payload(value), hashlib.sha256
- ).hexdigest()
- if not hmac.compare_digest(binding, expected_binding):
- raise core.CollectorError("E_CONTROLLER_ATTESTATION", "Controller envelope binding mismatch.", safety=True)
+ if _controller_key is not None:
+ commitment = hashlib.sha256(_controller_key).hexdigest()
+ if not hmac.compare_digest(commitment, pending["controller_key_commitment"]):
+ raise core.CollectorError("E_CONTROLLER_ATTESTATION", "Controller authority commitment mismatch.", safety=True)
+ expected_binding = hmac.new(
+ _controller_key, _controller_attestation_payload(value), hashlib.sha256
+ ).hexdigest()
+ if not hmac.compare_digest(binding, expected_binding):
+ raise core.CollectorError("E_CONTROLLER_ATTESTATION", "Controller envelope binding mismatch.", safety=True)
+ elif (
+ pending.get("phase") != "EVIDENCE_BOUND"
+ or pending.get("controller_binding_sha256") != binding
+ ):
+ raise core.CollectorError(
+ "E_CONTROLLER_REQUIRED",
+ "Caller-authored runtime-v2 evidence cannot enter the commit path.",
+ safety=True,
+ )
runtime_contract = _exact_keys(
value["runtime_contract"], {"contract_id", "contract_bytes", "contract_sha256"}, "runtime_contract"
)
@@ -944,45 +932,39 @@
return value, payload, observation_result, identity_match
-def bind_controller_evidence(
- config: core.CollectorConfig,
- pending: dict[str, Any],
- path: Path,
- *,
- transitioned_at: datetime,
-) -> None:
- """Validate and durably bind the controller-created evidence exactly once."""
- if pending.get("schema_version") != PENDING_SCHEMA or pending.get("phase") != "AWAITING_EVIDENCE":
- raise core.CollectorError("E_CONTROLLER_STATE", "Controller evidence can only bind the active runtime-v2 run.", safety=True)
- deadline = core.parse_datetime(pending["deadline_at"], "pending.deadline_at")
- if transitioned_at > deadline:
- raise core.CollectorError(
- "E_OVERALL_DEADLINE",
- "Controller evidence cannot be bound after the total deadline.",
- safety=True,
- )
- _, payload, _, _ = _validate_evidence(config, pending, path)
- pending["evidence_identity"] = {"bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest()}
- pending["phase"] = "EVIDENCE_BOUND"
- pending["last_transition_at"] = core.canonical_datetime(transitioned_at)
- _write_pending(config, pending)
- if _load_pending(config) != pending:
- raise core.CollectorError("E_CONTROLLER_STATE", "Controller evidence binding readback mismatch.", safety=True)
-
-
def _formal_tokens(event: Mapping[str, Any], config: core.CollectorConfig) -> list[str]:
stable = event.get("stable_id")
tokens: list[str] = []
+ legacy_image: re.Match[str] | None = None
if isinstance(stable, str) and stable:
if re.fullmatch(r"BV[0-9A-Za-z]{10}", stable, re.IGNORECASE):
tokens.append(f"bvid:{stable.lower()}")
elif re.fullmatch(r"[0-9]{1,32}", stable):
tokens.append(f"opus:{stable}")
else:
+ legacy_image = re.fullmatch(r"([0-9]{1,32}):(image|cover):([1-9][0-9]*)", stable)
+ if legacy_image is not None and (
+ event.get("item_type") == legacy_image.group(2)
+ and event.get("source_parent_stable_id") == legacy_image.group(1)
+ ):
+ # Exact legacy image rows are independently deduped by their
+ # canonical image URL. Do not merge them into the parent opus
+ # component because one parent can legitimately own many URLs.
+ pass
+ elif not tokens:
raise core.CollectorError("E_CATALOG_IDENTITY_CONFLICT", "Formal stable_id is invalid.", safety=True)
source = event.get("source_url")
if isinstance(source, str):
- canonical = core.validate_url(source, "formal.source_url", config.allowed_source_hosts)
+ if legacy_image is not None:
+ parsed = urlsplit(source)
+ if (
+ parsed.scheme != "https" or parsed.hostname not in {"i0.hdslb.com", "i1.hdslb.com", "i2.hdslb.com"}
+ or parsed.query or parsed.fragment or not parsed.path.startswith("/bfs/")
+ ):
+ raise core.CollectorError("E_CATALOG_IDENTITY_CONFLICT", "Formal image URL identity conflicts.", safety=True)
+ canonical = source
+ else:
+ canonical = core.validate_url(source, "formal.source_url", config.allowed_source_hosts)
tokens.append(f"url:{canonical}")
path_parts = PurePosixPath(canonical.split("?", 1)[0].split("#", 1)[0]).parts
if len(path_parts) >= 3 and path_parts[-2] == "opus" and path_parts[-1].isdecimal():
@@ -2302,10 +2284,14 @@
evidence_hash=None, coverage=None, input_count=0, new_count=0,
created=[], formal_changed=False,
)
+ if pending.get("phase") != "EVIDENCE_BOUND":
+ raise core.CollectorError(
+ "E_CONTROLLER_REQUIRED",
+ "Schema 3 evidence is accepted only after the trusted refresh-run controller binds it.",
+ safety=True,
+ )
evidence, payload, observation, identity_match = _validate_evidence(config, pending, evidence_path)
evidence_hash = hashlib.sha256(payload).hexdigest()
- if pending.get("phase") != "EVIDENCE_BOUND":
- raise core.CollectorError("E_CONTROLLER_REQUIRED", "Evidence must be durably bound by the trusted controller.", safety=True)
if pending.get("evidence_identity") != {"bytes": len(payload), "sha256": evidence_hash}:
raise core.CollectorError("E_RECOVERY_AMBIGUOUS", "Bound evidence identity drifted.", safety=True)
action_outcome = evidence["runtime_observation"]["refresh_action_outcome"]
diff --git a/dev/project-dev/bili_dynamic_refresh_extension/manifest.json b/dev/project-dev/bili_dynamic_refresh_extension/manifest.json
new file mode 100644
index 0000000..38e6fca
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_extension/manifest.json
@@ -0,0 +1,21 @@
+{
+ "manifest_version": 3,
+ "name": "project-info Bilibili dynamic refresh trusted adapter",
+ "version": "1.0.0",
+ "minimum_chrome_version": "120",
+ "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2ZpvAHz7tziXdSB6A74a1nl1k9h9y6pbxR8xyu3rZTuFT5c5MZ9GRB0AVSmcCgwz94v184CNOtXgq7q8iGAzIfkEljLMlbn8hsMgLNXLCXg0Sx9LFhGHj0VyVD/fqpC2zYAoPIbOBF4WsHYgqimptGiCZ949zcleouCLAr1QDXfQBA3y09q/TTk9GRWtgEcYCuobEzhZMqj+10KmALLfR3r/NkqTCPN0uM6iZijE94XaxRSuFgdxuWM5qavryPrWovV6suK0nH/EJs6nhr/mKU0pUTVEVOJB8qEvhmeg+sVefZj0kX+FjtWWqD6tg0jBmO+FQ1nSQA/OYzwGxrP2awIDAQAB",
+ "permissions": [
+ "alarms",
+ "nativeMessaging",
+ "scripting",
+ "storage",
+ "tabs"
+ ],
+ "host_permissions": [
+ "https://space.bilibili.com/*/dynamic*"
+ ],
+ "background": {
+ "service_worker": "service_worker.js",
+ "type": "module"
+ }
+}
diff --git a/dev/project-dev/bili_dynamic_refresh_extension/package.json b/dev/project-dev/bili_dynamic_refresh_extension/package.json
new file mode 100644
index 0000000..3dbc1ca
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_extension/package.json
@@ -0,0 +1,3 @@
+{
+ "type": "module"
+}
diff --git a/dev/project-dev/bili_dynamic_refresh_extension/page_extract.js b/dev/project-dev/bili_dynamic_refresh_extension/page_extract.js
new file mode 100644
index 0000000..1303374
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_extension/page_extract.js
@@ -0,0 +1,88 @@
+function targetProof(value) {
+ const parsed = new URL(String(value));
+ const parts = parsed.pathname.split("/").filter(Boolean);
+ if (parsed.protocol !== "https:" || parsed.hostname !== "space.bilibili.com" || parsed.search || parsed.hash || parts.length !== 2 || !/^[1-9][0-9]{0,19}$/u.test(parts[0]) || parts[1] !== "dynamic") throw new Error("E_PAGE_URL");
+ return {uid: parts[0], dynamic_url: `https://space.bilibili.com/${parts[0]}/dynamic`, profile_url: `https://space.bilibili.com/${parts[0]}`};
+}
+
+function text(value) {
+ return String(value ?? "").replace(/\s+/gu, " ").trim();
+}
+
+function canonicalUrl(value, target) {
+ const proof = targetProof(target);
+ const parsed = new URL(String(value), proof.dynamic_url);
+ if (parsed.protocol !== "https:" || parsed.hostname !== "space.bilibili.com") throw new Error("E_PAGE_URL");
+ parsed.hash = "";
+ return parsed.toString();
+}
+
+export function canonicalizeSnapshot(raw, target) {
+ const proof = targetProof(target);
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error("E_PAGE_SHAPE");
+ const cards = Array.isArray(raw.cards) ? raw.cards : [];
+ const normalized = cards.map((card, position) => {
+ if (!card || typeof card !== "object" || Array.isArray(card)) throw new Error("E_CARD_SHAPE");
+ const dynamicId = text(card.dynamic_id);
+ const publishedAt = text(card.published_at);
+ const body = text(card.text);
+ const url = canonicalUrl(card.url, target);
+ if (!/^\d{1,32}$/u.test(dynamicId) || !publishedAt || !body) throw new Error("E_CARD_IDENTITY");
+ return {position, dynamic_id: dynamicId, published_at: publishedAt, text: body, url};
+ });
+ const seen = new Set();
+ for (const card of normalized) {
+ if (seen.has(card.dynamic_id)) throw new Error("E_CARD_DUPLICATE");
+ seen.add(card.dynamic_id);
+ }
+ const marker = text(raw.terminal_marker_text);
+ if (text(raw.creator?.uid) !== proof.uid || canonicalUrl(raw.creator?.profile_url, target).replace(/\/$/u, "") !== proof.profile_url) throw new Error("E_CREATOR_IDENTITY");
+ return {
+ schema_version: 1,
+ final_url: canonicalUrl(raw.final_url, target),
+ page_title: text(raw.page_title),
+ ready_state: text(raw.ready_state),
+ visibility_state: text(raw.visibility_state),
+ creator: {uid: proof.uid, name: text(raw.creator?.name), profile_url: proof.profile_url},
+ cards: normalized,
+ unparsed_nodes: Number(raw.unparsed_nodes ?? 0),
+ terminal_marker_text: marker,
+ coverage_complete: marker === "已经到底了" && Number(raw.unparsed_nodes ?? 0) === 0,
+ };
+}
+
+export function collectVisiblePage(target) {
+ const proof = targetProof(target);
+ const normalizeText = (value) => String(value ?? "").replace(/\s+/gu, " ").trim();
+ const normalizeUrl = (value) => {
+ const parsed = new URL(String(value), proof.dynamic_url);
+ if (parsed.protocol !== "https:" || parsed.hostname !== "space.bilibili.com") throw new Error("E_PAGE_URL");
+ parsed.hash = "";
+ return parsed.toString();
+ };
+ if (normalizeUrl(location.href).replace(/\/$/u, "") !== proof.dynamic_url) throw new Error("E_PAGE_IDENTITY");
+ const cards = [...document.querySelectorAll("[data-dynamic-id]")].map((node) => ({
+ dynamic_id: normalizeText(node.getAttribute("data-dynamic-id")),
+ published_at: normalizeText(node.getAttribute("data-published-at") || node.querySelector("time")?.getAttribute("datetime")),
+ text: normalizeText(node.querySelector("[data-dynamic-text]")?.textContent || node.textContent),
+ url: normalizeUrl(node.querySelector("a[href]")?.href || location.href),
+ }));
+ const terminal = [...document.querySelectorAll("body *")].map((node) => String(node.textContent || "").trim()).find((value) => value === "已经到底了") || "";
+ const seen = new Set();
+ for (const card of cards) {
+ if (!/^\d{1,32}$/u.test(card.dynamic_id) || !card.published_at || !card.text || seen.has(card.dynamic_id)) throw new Error("E_CARD_IDENTITY");
+ seen.add(card.dynamic_id);
+ }
+ return {
+ schema_version: 1,
+ final_url: proof.dynamic_url,
+ page_title: normalizeText(document.title),
+ ready_state: normalizeText(document.readyState),
+ visibility_state: normalizeText(document.visibilityState),
+ creator: {uid: proof.uid, name: normalizeText(document.querySelector("h1")?.textContent || ""), profile_url: proof.profile_url},
+ cards: cards.map((card, position) => ({position, ...card})),
+ unparsed_nodes: 0,
+ terminal_marker_text: terminal,
+ coverage_complete: terminal === "已经到底了",
+ };
+}
diff --git a/dev/project-dev/bili_dynamic_refresh_extension/protocol.js b/dev/project-dev/bili_dynamic_refresh_extension/protocol.js
new file mode 100644
index 0000000..c7aff46
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_extension/protocol.js
@@ -0,0 +1,101 @@
+const encoder = new TextEncoder();
+
+function stable(value) {
+ if (Array.isArray(value)) return value.map(stable);
+ if (value && typeof value === "object") {
+ return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])]));
+ }
+ return value;
+}
+
+export function canonicalJson(value) {
+ return JSON.stringify(stable(value));
+}
+
+async function hmacHex(secretHex, value) {
+ const secret = Uint8Array.from(secretHex.match(/../gu) || [], (item) => Number.parseInt(item, 16));
+ const key = await crypto.subtle.importKey("raw", secret, {name: "HMAC", hash: "SHA-256"}, false, ["sign"]);
+ const result = await crypto.subtle.sign("HMAC", key, encoder.encode(canonicalJson(value)));
+ return [...new Uint8Array(result)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+async function verify(secret, message) {
+ if (!message || typeof message !== "object" || typeof message.hmac !== "string") throw new Error("E_PROTOCOL");
+ const unsigned = {...message};
+ const supplied = unsigned.hmac;
+ delete unsigned.hmac;
+ if (!/^[0-9a-f]{64}$/u.test(supplied)) throw new Error("E_HMAC");
+ const rawSecret = Uint8Array.from(secret.match(/../gu) || [], (item) => Number.parseInt(item, 16));
+ const signature = Uint8Array.from(supplied.match(/../gu) || [], (item) => Number.parseInt(item, 16));
+ const key = await crypto.subtle.importKey("raw", rawSecret, {name: "HMAC", hash: "SHA-256"}, false, ["verify"]);
+ const valid = await crypto.subtle.verify("HMAC", key, signature, encoder.encode(canonicalJson(unsigned)));
+ if (!valid) throw new Error("E_HMAC");
+}
+
+export class TrustedRuntimeSession {
+ constructor(api, hello) {
+ this.api = api;
+ this.hello = Object.freeze({...hello});
+ this.secret = null;
+ this.runId = null;
+ this.requestId = null;
+ this.sequence = 0;
+ this.prepared = null;
+ this.dispatched = false;
+ this.actionResult = null;
+ }
+
+ helloFrame() {
+ return {schema_version: 1, type: "EXTENSION_HELLO", sequence: 1, ...this.hello};
+ }
+
+ async accept(message) {
+ if (message?.type === "HOST_CHALLENGE") {
+ if (this.secret !== null || !/^[0-9a-f]{64}$/u.test(String(message.secret || ""))) throw new Error("E_CHALLENGE");
+ this.secret = message.secret;
+ await verify(this.secret, message);
+ this.runId = message.run_id;
+ this.requestId = message.request_id;
+ this.sequence = 2;
+ return this.sign("EXTENSION_CHALLENGE_ACCEPTED", 3, {challenge_id: message.challenge_id});
+ }
+ if (!this.secret) throw new Error("E_CHALLENGE_REQUIRED");
+ await verify(this.secret, message);
+ if (message.run_id !== this.runId || message.request_id !== this.requestId || message.sequence !== this.sequence + 2) {
+ throw new Error("E_SEQUENCE");
+ }
+ this.sequence = message.sequence;
+ if (message.type === "HOST_ACTION_PREPARE") {
+ if (this.prepared) throw new Error("E_REPLAY");
+ this.prepared = await this.api.prepare(message.action);
+ return this.sign("EXTENSION_READY_TO_DISPATCH", message.sequence + 1, {
+ action_id: message.action.action_id,
+ prepared: this.prepared
+ });
+ }
+ if (message.type === "HOST_DISPATCH_PERMIT") {
+ if (!this.prepared || this.dispatched || message.permit.action_id !== this.prepared.action_id) throw new Error("E_PERMIT");
+ if (Date.now() > Date.parse(message.permit.deadline_at)) throw new Error("E_DEADLINE");
+ this.dispatched = true;
+ this.actionResult = await this.api.dispatch(this.prepared);
+ return this.sign("EXTENSION_ACTION_RESULT", message.sequence + 1, {
+ permit_id: message.permit.permit_id,
+ result: this.actionResult
+ });
+ }
+ if (message.type === "HOST_OBSERVATION_REQUEST") {
+ if (!this.dispatched || !this.actionResult) throw new Error("E_ACTION_RESULT_REQUIRED");
+ const observation = await this.api.observe(this.actionResult);
+ return this.sign("EXTENSION_OBSERVATION", message.sequence + 1, {observation});
+ }
+ if (message.type === "HOST_COMMIT_RESULT") {
+ return null;
+ }
+ throw new Error("E_MESSAGE_TYPE");
+ }
+
+ async sign(type, sequence, fields) {
+ const unsigned = {schema_version: 1, type, run_id: this.runId, request_id: this.requestId, sequence, ...fields};
+ return {...unsigned, hmac: await hmacHex(this.secret, unsigned)};
+ }
+}
diff --git a/dev/project-dev/bili_dynamic_refresh_extension/runtime.js b/dev/project-dev/bili_dynamic_refresh_extension/runtime.js
new file mode 100644
index 0000000..7983ef6
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_extension/runtime.js
@@ -0,0 +1,519 @@
+const RECORD_SCHEMA = 1;
+export const OWNED_TAB_STORAGE_KEY = "project_info_dynamic_owned_tab_v1";
+export const SLOT_STATE_STORAGE_KEY = "project_info_dynamic_slot_state_v1";
+export const FIXED_LIFECYCLE = Object.freeze({
+ CLOSED: "OWNED_TAB_CLOSED",
+ ALREADY_CLOSED: "OWNED_TAB_ALREADY_CLOSED",
+ IDENTITY_DRIFT: "OWNED_TAB_IDENTITY_DRIFT",
+ CLOSE_FAILED: "OWNED_TAB_CLOSE_FAILED",
+ PEER_CLOSED: "E_NATIVE_PEER_CLOSED",
+ TIMEOUT: "E_NATIVE_TIMEOUT"
+});
+
+const HEX32 = /^[0-9a-f]{32}$/u;
+const HEX64 = /^[0-9a-f]{64}$/u;
+const SLOT_MATERIAL = /^dynamic-slot:(0|[1-9][0-9]{0,15})$/u;
+const SLOT_STATES = new Set(["STARTED", "COMPLETE", "FAILED"]);
+const SLOT_HISTORY_LIMIT = 336;
+
+function plain(value) {
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+ const prototype = Object.getPrototypeOf(value);
+ return prototype === Object.prototype || prototype === null;
+}
+
+function exactKeys(value, keys) {
+ return plain(value) && Object.keys(value).sort().join("\0") === [...keys].sort().join("\0");
+}
+
+export function canonicalDynamicUrl(value) {
+ const parsed = new URL(String(value));
+ const parts = parsed.pathname.split("/").filter(Boolean);
+ if (parsed.protocol !== "https:" || parsed.hostname !== "space.bilibili.com" ||
+ parsed.search !== "" || parsed.hash !== "" || parts.length !== 2 ||
+ !/^[1-9][0-9]{0,19}$/u.test(parts[0]) || parts[1] !== "dynamic") {
+ throw new Error("E_ACTION");
+ }
+ return `https://space.bilibili.com/${parts[0]}/dynamic`;
+}
+
+function tabLocationMatches(tab, expected) {
+ for (const raw of [tab?.url, tab?.pendingUrl]) {
+ if (typeof raw !== "string") continue;
+ try {
+ const parsed = new URL(raw);
+ const canonical = canonicalDynamicUrl(`${parsed.origin}${parsed.pathname}`);
+ if (canonical === expected && parsed.search === "" && parsed.hash === "") return true;
+ } catch {
+ // A non-canonical location never proves ownership.
+ }
+ }
+ return false;
+}
+
+function exactRecord(value) {
+ const keys = ["schema", "slot_id", "lease_id", "tab_id", "window_id", "target_url"];
+ if (!exactKeys(value, keys) || value.schema !== RECORD_SCHEMA ||
+ !HEX64.test(value.slot_id) || !HEX32.test(value.lease_id) ||
+ !Number.isInteger(value.tab_id) || value.tab_id < 0 ||
+ !Number.isInteger(value.window_id) || value.window_id < 0) return null;
+ let target;
+ try { target = canonicalDynamicUrl(value.target_url); } catch { return null; }
+ if (target !== value.target_url) return null;
+ return Object.freeze({...value});
+}
+
+function markerValue(record) {
+ return `${record.slot_id}:${record.lease_id}`;
+}
+
+function setOwnershipMarker(expected) {
+ document.documentElement.dataset.projectInfoDynamicOwner = expected;
+ return document.documentElement.dataset.projectInfoDynamicOwner;
+}
+
+function readOwnershipMarker() {
+ return document.documentElement.dataset.projectInfoDynamicOwner || null;
+}
+
+async function sleep(milliseconds) {
+ await new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+
+export class OwnedTabLifecycle {
+ constructor(chromeApi, {randomHex, delay = sleep, pollMilliseconds = 100, maxPolls = 300} = {}) {
+ this.chrome = chromeApi;
+ this.randomHex = randomHex || ((bytes) => {
+ const value = new Uint8Array(bytes);
+ crypto.getRandomValues(value);
+ return [...value].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+ });
+ this.delay = delay;
+ this.pollMilliseconds = pollMilliseconds;
+ this.maxPolls = maxPolls;
+ this.lastDiagnostic = null;
+ }
+
+ async load() {
+ const container = await this.chrome.storage.session.get(OWNED_TAB_STORAGE_KEY);
+ const raw = plain(container) ? container[OWNED_TAB_STORAGE_KEY] : null;
+ if (raw === undefined || raw === null) return null;
+ const record = exactRecord(raw);
+ if (record !== null) return record;
+ await this.chrome.storage.session.remove(OWNED_TAB_STORAGE_KEY);
+ this.lastDiagnostic = FIXED_LIFECYCLE.IDENTITY_DRIFT;
+ return null;
+ }
+
+ async cleanupFresh(record, tab) {
+ // This capability never escapes create(). The tab id comes directly from
+ // tabs.create, so a pre-existing user tab is never an eligible target.
+ try { await this.chrome.storage.session.remove(OWNED_TAB_STORAGE_KEY); } catch { /* best effort */ }
+ const tabId = record?.tab_id ?? tab?.id;
+ const windowId = record?.window_id ?? tab?.windowId;
+ if (!Number.isInteger(tabId) || !Number.isInteger(windowId) ||
+ tab?.id !== tabId || tab?.windowId !== windowId) return;
+ try {
+ await this.chrome.tabs.remove(tabId);
+ this.lastDiagnostic = FIXED_LIFECYCLE.CLOSED;
+ } catch {
+ this.lastDiagnostic = FIXED_LIFECYCLE.CLOSE_FAILED;
+ }
+ }
+
+ async create(slotId, targetUrl, leaseId = null) {
+ if (!HEX64.test(slotId)) throw new Error("E_SLOT_ID");
+ const target = canonicalDynamicUrl(targetUrl);
+ const tab = await this.chrome.tabs.create({url: target, active: false});
+ if (!Number.isInteger(tab?.id) || !Number.isInteger(tab?.windowId)) {
+ await this.cleanupFresh(null, tab);
+ throw new Error("E_TAB_CREATE");
+ }
+ const record = exactRecord({
+ schema: RECORD_SCHEMA,
+ slot_id: slotId,
+ lease_id: leaseId === null ? this.randomHex(16) : leaseId,
+ tab_id: tab.id,
+ window_id: tab.windowId,
+ target_url: target
+ });
+ if (record === null) {
+ await this.cleanupFresh(null, tab);
+ throw new Error("E_TAB_IDENTITY");
+ }
+ try {
+ await this.chrome.storage.session.set({[OWNED_TAB_STORAGE_KEY]: record});
+ const ready = await this.waitUntilComplete(record);
+ const injected = await this.chrome.scripting.executeScript({
+ target: {tabId: ready.id},
+ func: setOwnershipMarker,
+ args: [markerValue(record)]
+ });
+ if (!Array.isArray(injected) || injected.length !== 1 || injected[0]?.result !== markerValue(record)) {
+ throw new Error("E_TAB_MARKER");
+ }
+ return {record, tab: ready};
+ } catch (error) {
+ await this.cleanupFresh(record, tab);
+ throw error;
+ }
+ }
+
+ async waitUntilComplete(record) {
+ for (let index = 0; index < this.maxPolls; index += 1) {
+ const tab = await this.chrome.tabs.get(record.tab_id);
+ if (tab?.id !== record.tab_id || tab?.windowId !== record.window_id || !tabLocationMatches(tab, record.target_url)) {
+ throw new Error("E_TAB_IDENTITY");
+ }
+ if (tab.status === "complete") return tab;
+ await this.delay(this.pollMilliseconds);
+ }
+ throw new Error("E_TAB_TIMEOUT");
+ }
+
+ async cleanup(recordValue = null) {
+ const record = exactRecord(recordValue) || await this.load();
+ if (record === null) return FIXED_LIFECYCLE.ALREADY_CLOSED;
+ let tab;
+ try {
+ tab = await this.chrome.tabs.get(record.tab_id);
+ } catch {
+ await this.chrome.storage.session.remove(OWNED_TAB_STORAGE_KEY);
+ this.lastDiagnostic = FIXED_LIFECYCLE.ALREADY_CLOSED;
+ return this.lastDiagnostic;
+ }
+ let marker = null;
+ try {
+ const observed = await this.chrome.scripting.executeScript({
+ target: {tabId: record.tab_id},
+ func: readOwnershipMarker
+ });
+ if (Array.isArray(observed) && observed.length === 1) marker = observed[0]?.result ?? null;
+ } catch {
+ marker = null;
+ }
+ if (tab?.id !== record.tab_id || tab?.windowId !== record.window_id ||
+ !tabLocationMatches(tab, record.target_url) || marker !== markerValue(record)) {
+ await this.chrome.storage.session.remove(OWNED_TAB_STORAGE_KEY);
+ this.lastDiagnostic = FIXED_LIFECYCLE.IDENTITY_DRIFT;
+ return this.lastDiagnostic;
+ }
+ // Retire the durable capability before the sole non-atomic tab mutation.
+ // A failed removal is terminal: never retry or navigate the numeric tab id.
+ await this.chrome.storage.session.remove(OWNED_TAB_STORAGE_KEY);
+ try {
+ await this.chrome.tabs.remove(record.tab_id);
+ this.lastDiagnostic = FIXED_LIFECYCLE.CLOSED;
+ } catch {
+ this.lastDiagnostic = FIXED_LIFECYCLE.CLOSE_FAILED;
+ }
+ return this.lastDiagnostic;
+ }
+}
+
+function exactSlotEntry(value) {
+ const keys = ["schema", "slot_material", "slot_number", "slot_id", "lease_id", "state", "result_code"];
+ if (!exactKeys(value, keys) || value.schema !== 1 || !SLOT_MATERIAL.test(value.slot_material) ||
+ !Number.isSafeInteger(value.slot_number) || value.slot_number < 0 ||
+ value.slot_material !== `dynamic-slot:${value.slot_number}` || !HEX64.test(value.slot_id) ||
+ !HEX32.test(value.lease_id) || !SLOT_STATES.has(value.state)) return null;
+ if (value.state === "STARTED" && value.result_code !== null) return null;
+ if (value.state === "COMPLETE" && value.result_code !== "SLOT_COMPLETE") return null;
+ if (value.state === "FAILED" &&
+ (typeof value.result_code !== "string" || !/^E_[A-Z0-9_]+$/u.test(value.result_code))) return null;
+ return Object.freeze({...value});
+}
+
+function exactSlotRoot(value) {
+ if (!exactKeys(value, ["schema", "entries"]) || value.schema !== 1 || !Array.isArray(value.entries) ||
+ value.entries.length > SLOT_HISTORY_LIMIT) return null;
+ const entries = [];
+ const seen = new Set();
+ for (const raw of value.entries) {
+ const entry = exactSlotEntry(raw);
+ if (entry === null || seen.has(entry.slot_material)) return null;
+ seen.add(entry.slot_material);
+ entries.push(entry);
+ }
+ if (entries.filter((entry) => entry.state === "STARTED").length > 1) return null;
+ return Object.freeze({schema: 1, entries: Object.freeze(entries)});
+}
+
+export class SlotStateStore {
+ constructor(chromeApi, {randomHex, historyLimit = SLOT_HISTORY_LIMIT} = {}) {
+ this.chrome = chromeApi;
+ this.historyLimit = historyLimit;
+ this.randomHex = randomHex || ((bytes) => {
+ const value = new Uint8Array(bytes);
+ crypto.getRandomValues(value);
+ return [...value].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+ });
+ if (!Number.isSafeInteger(historyLimit) || historyLimit < 2 || historyLimit > SLOT_HISTORY_LIMIT) {
+ throw new Error("E_SLOT_STATE");
+ }
+ }
+
+ async read() {
+ const container = await this.chrome.storage.local.get(SLOT_STATE_STORAGE_KEY);
+ const raw = plain(container) ? container[SLOT_STATE_STORAGE_KEY] : undefined;
+ if (raw === undefined) return Object.freeze({schema: 1, entries: Object.freeze([])});
+ const root = exactSlotRoot(raw);
+ if (root === null) throw new Error("E_SLOT_STATE");
+ return root;
+ }
+
+ async claim(slotMaterial, slotId) {
+ if (!SLOT_MATERIAL.test(slotMaterial) || !HEX64.test(slotId)) throw new Error("E_SLOT_STATE");
+ const slotNumber = Number(slotMaterial.slice("dynamic-slot:".length));
+ if (!Number.isSafeInteger(slotNumber) || slotMaterial !== `dynamic-slot:${slotNumber}`) {
+ throw new Error("E_SLOT_STATE");
+ }
+ const root = await this.read();
+ const existing = root.entries.find((entry) => entry.slot_material === slotMaterial);
+ if (existing !== undefined) {
+ if (existing.slot_id !== slotId) throw new Error("E_SLOT_STATE");
+ return {disposition: existing.state === "STARTED" ? "RESUME" : "TERMINAL", entry: existing};
+ }
+ if (root.entries.some((entry) => entry.state === "STARTED")) return {disposition: "BUSY", entry: null};
+ const entry = exactSlotEntry({
+ schema: 1,
+ slot_material: slotMaterial,
+ slot_number: slotNumber,
+ slot_id: slotId,
+ lease_id: this.randomHex(16),
+ state: "STARTED",
+ result_code: null
+ });
+ if (entry === null) throw new Error("E_SLOT_STATE");
+ const terminal = root.entries.filter((item) => item.state !== "STARTED");
+ const kept = terminal.slice(Math.max(0, terminal.length - (this.historyLimit - 1)));
+ const next = {schema: 1, entries: [...kept, entry]};
+ await this.chrome.storage.local.set({[SLOT_STATE_STORAGE_KEY]: next});
+ const rebound = await this.read();
+ const stored = rebound.entries.find((item) => item.slot_material === slotMaterial);
+ if (stored === undefined || JSON.stringify(stored) !== JSON.stringify(entry)) throw new Error("E_SLOT_STATE");
+ return {disposition: "CLAIMED", entry: stored};
+ }
+
+ async started() {
+ const root = await this.read();
+ return root.entries.find((entry) => entry.state === "STARTED") ?? null;
+ }
+
+ async finish(entryValue, state, resultCode) {
+ const expected = exactSlotEntry(entryValue);
+ if (expected === null || expected.state !== "STARTED" || !["COMPLETE", "FAILED"].includes(state)) {
+ throw new Error("E_SLOT_STATE");
+ }
+ const candidate = exactSlotEntry({...expected, state, result_code: resultCode});
+ if (candidate === null) throw new Error("E_SLOT_STATE");
+ const root = await this.read();
+ const index = root.entries.findIndex((item) => item.slot_material === expected.slot_material);
+ if (index < 0 || JSON.stringify(root.entries[index]) !== JSON.stringify(expected)) throw new Error("E_SLOT_STATE");
+ const entries = root.entries.map((item, itemIndex) => itemIndex === index ? candidate : item);
+ await this.chrome.storage.local.set({[SLOT_STATE_STORAGE_KEY]: {schema: 1, entries}});
+ const rebound = await this.read();
+ const stored = rebound.entries.find((item) => item.slot_material === expected.slot_material);
+ if (stored === undefined || JSON.stringify(stored) !== JSON.stringify(candidate)) throw new Error("E_SLOT_STATE");
+ return stored;
+ }
+}
+
+export class NativePortTransport {
+ constructor(port, {timeoutMilliseconds = 120000} = {}) {
+ this.port = port;
+ this.timeoutMilliseconds = timeoutMilliseconds;
+ this.closed = false;
+ this.writeChain = Promise.resolve();
+ this.listeners = new Set();
+ this.disconnectListeners = new Set();
+ // Install handlers before the caller can perform its first write.
+ port.onMessage.addListener((message) => {
+ for (const listener of [...this.listeners]) listener(message);
+ });
+ port.onDisconnect.addListener(() => {
+ this.closed = true;
+ for (const listener of [...this.disconnectListeners]) listener(FIXED_LIFECYCLE.PEER_CLOSED);
+ });
+ }
+
+ onMessage(listener) { this.listeners.add(listener); }
+ onDisconnect(listener) { this.disconnectListeners.add(listener); }
+
+ send(message) {
+ const operation = this.writeChain.then(() => {
+ if (this.closed) throw new Error(FIXED_LIFECYCLE.PEER_CLOSED);
+ try {
+ this.port.postMessage(message);
+ } catch {
+ this.closed = true;
+ throw new Error(FIXED_LIFECYCLE.PEER_CLOSED);
+ }
+ if (this.closed) throw new Error(FIXED_LIFECYCLE.PEER_CLOSED);
+ });
+ this.writeChain = operation.catch(() => undefined);
+ return operation;
+ }
+
+ disconnect() {
+ if (this.closed) return;
+ this.closed = true;
+ try { this.port.disconnect(); } catch { /* peer already closed */ }
+ }
+}
+
+export async function runNativeSession({transport, session}) {
+ return await new Promise((resolve, reject) => {
+ let settled = false;
+ const timer = setTimeout(() => finish(new Error(FIXED_LIFECYCLE.TIMEOUT)), transport.timeoutMilliseconds);
+ const finish = (error, value) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ transport.disconnect();
+ if (error) reject(error); else resolve(value);
+ };
+ transport.onDisconnect((code) => finish(new Error(code)));
+ transport.onMessage((message) => {
+ Promise.resolve(session.accept(message)).then(async (reply) => {
+ if (reply !== null) await transport.send(reply);
+ if (message?.type === "HOST_COMMIT_RESULT") finish(null, message.result ?? null);
+ }).catch((error) => finish(error));
+ });
+ transport.send(session.helloFrame()).catch((error) => finish(error));
+ });
+}
+
+export class SlotCoordinator {
+ constructor({lifecycle, slotStore, connect, createSession, hashSlot, collectPage}) {
+ this.lifecycle = lifecycle;
+ this.slotStore = slotStore;
+ this.connect = connect;
+ this.createSession = createSession;
+ this.hashSlot = hashSlot;
+ this.collectPage = collectPage;
+ this.active = false;
+ this.operationTail = Promise.resolve();
+ }
+
+ async serialize(operation) {
+ const prior = this.operationTail;
+ let release;
+ this.operationTail = new Promise((resolve) => { release = resolve; });
+ await prior.catch(() => undefined);
+ try {
+ return await operation();
+ } finally {
+ release();
+ }
+ }
+
+ async recoverUnlocked() {
+ const started = await this.slotStore.started();
+ if (started === null) {
+ await this.lifecycle.cleanup();
+ return null;
+ }
+ const prior = await this.lifecycle.load();
+ if (prior !== null && (prior.slot_id !== started.slot_id || prior.lease_id !== started.lease_id)) {
+ throw new Error("E_SLOT_STATE");
+ }
+ const lifecycle_code = prior === null
+ ? FIXED_LIFECYCLE.ALREADY_CLOSED
+ : await this.lifecycle.cleanup(prior);
+ await this.slotStore.finish(started, "FAILED", "E_SLOT_INTERRUPTED");
+ return Object.freeze({
+ slot_material: started.slot_material,
+ lease_id: started.lease_id,
+ result_code: "E_SLOT_INTERRUPTED",
+ lifecycle_code
+ });
+ }
+
+ async recover() {
+ return await this.serialize(() => this.recoverUnlocked());
+ }
+
+ async runUnlocked(slotMaterial) {
+ let owned = null;
+ let claim = null;
+ let outcome = null;
+ try {
+ const recovered = await this.recoverUnlocked();
+ if (recovered?.slot_material === slotMaterial) {
+ return {status: "FAILED", error_code: recovered.result_code};
+ }
+ const slotId = await this.hashSlot(slotMaterial);
+ if (!HEX64.test(slotId)) throw new Error("E_SLOT_STATE");
+ claim = await this.slotStore.claim(slotMaterial, slotId);
+ if (claim.disposition === "BUSY") return {status: "SKIPPED_OVERLAP"};
+ if (claim.disposition === "TERMINAL") {
+ return {status: "SKIPPED_TERMINAL", terminal_state: claim.entry.state, result_code: claim.entry.result_code};
+ }
+ if (claim.disposition === "RESUME") {
+ const prior = await this.lifecycle.load();
+ if (prior !== null && (prior.slot_id !== claim.entry.slot_id || prior.lease_id !== claim.entry.lease_id)) {
+ throw new Error("E_SLOT_STATE");
+ }
+ if (prior !== null) await this.lifecycle.cleanup(prior);
+ await this.slotStore.finish(claim.entry, "FAILED", "E_SLOT_INTERRUPTED");
+ return {status: "FAILED", error_code: "E_SLOT_INTERRUPTED"};
+ }
+ await this.lifecycle.cleanup();
+ const lifecycle = this.lifecycle;
+ const collectPage = this.collectPage;
+ const api = {
+ async prepare(action) {
+ if (!plain(action) || !["reload", "goto"].includes(action.kind)) throw new Error("E_ACTION");
+ return {action_id: action.action_id, kind: "goto", url: canonicalDynamicUrl(action.url)};
+ },
+ async dispatch(prepared) {
+ owned = await lifecycle.create(slotId, prepared.url, claim.entry.lease_id);
+ return {action_id: prepared.action_id, tab_id: owned.record.tab_id, url: prepared.url};
+ },
+ async observe(result) {
+ return await lifecycle.chrome.scripting.executeScript({
+ target: {tabId: result.tab_id},
+ func: collectPage,
+ args: [result.url]
+ }).then((items) => {
+ if (!Array.isArray(items) || items.length !== 1) throw new Error("E_OBSERVATION_COUNT");
+ return items[0].result;
+ });
+ }
+ };
+ const transport = this.connect();
+ const result = await runNativeSession({transport, session: this.createSession(api)});
+ outcome = {status: "COMPLETE", result};
+ } catch (error) {
+ const code = typeof error?.message === "string" && /^E_[A-Z0-9_]+$/u.test(error.message)
+ ? error.message : "E_SLOT_FAILED";
+ outcome = {status: "FAILED", error_code: code};
+ } finally {
+ if (owned !== null) await this.lifecycle.cleanup(owned.record);
+ }
+ if (claim?.disposition === "CLAIMED") {
+ try {
+ await this.slotStore.finish(
+ claim.entry,
+ outcome?.status === "COMPLETE" ? "COMPLETE" : "FAILED",
+ outcome?.status === "COMPLETE" ? "SLOT_COMPLETE" : (outcome?.error_code || "E_SLOT_FAILED")
+ );
+ } catch {
+ return {status: "FAILED", error_code: "E_SLOT_STATE"};
+ }
+ }
+ return outcome || {status: "FAILED", error_code: "E_SLOT_FAILED"};
+ }
+
+ async run(slotMaterial) {
+ if (this.active) return {status: "SKIPPED_OVERLAP"};
+ this.active = true;
+ try {
+ return await this.serialize(() => this.runUnlocked(slotMaterial));
+ } finally {
+ // The serialized operation includes terminal append and readback.
+ this.active = false;
+ }
+ }
+}
diff --git a/dev/project-dev/bili_dynamic_refresh_extension/service_worker.js b/dev/project-dev/bili_dynamic_refresh_extension/service_worker.js
new file mode 100644
index 0000000..b9ffdad
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_extension/service_worker.js
@@ -0,0 +1,65 @@
+import {TrustedRuntimeSession} from "./protocol.js";
+import {collectVisiblePage} from "./page_extract.js";
+import {NativePortTransport, OwnedTabLifecycle, SlotCoordinator, SlotStateStore} from "./runtime.js";
+
+const HOST = "com.project_info.bili_dynamic_refresh";
+const ALARM = "project-info-dynamic-refresh-half-hour";
+const PERIOD_MINUTES = 30;
+
+async function hashSlot(value) {
+ const material = new TextEncoder().encode(String(value));
+ const digest = await crypto.subtle.digest("SHA-256", material);
+ return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+function nextBoundary(now = Date.now()) {
+ const interval = PERIOD_MINUTES * 60 * 1000;
+ return Math.floor(now / interval) * interval + interval;
+}
+
+function createSession(api) {
+ return new TrustedRuntimeSession(api, {
+ extension_id: chrome.runtime.id,
+ version: chrome.runtime.getManifest().version,
+ manifest_name: chrome.runtime.getManifest().name
+ });
+}
+
+const lifecycle = new OwnedTabLifecycle(chrome);
+const slotStore = new SlotStateStore(chrome);
+const coordinator = new SlotCoordinator({
+ lifecycle,
+ slotStore,
+ connect: () => new NativePortTransport(chrome.runtime.connectNative(HOST)),
+ createSession,
+ hashSlot,
+ collectPage: collectVisiblePage
+});
+
+function schedule() {
+ chrome.alarms.create(ALARM, {when: nextBoundary(), periodInMinutes: PERIOD_MINUTES});
+}
+
+function recoverAtStartup() {
+ // Never leak a raw storage/runtime exception from a startup event. The next
+ // scheduled run will surface the same strict fixed code without acting.
+ void coordinator.recover().catch(() => console.error("E_SLOT_STATE"));
+}
+
+chrome.runtime.onInstalled.addListener(() => {
+ schedule();
+ recoverAtStartup();
+});
+
+chrome.runtime.onStartup.addListener(() => {
+ schedule();
+ recoverAtStartup();
+});
+
+chrome.alarms.onAlarm.addListener((alarm) => {
+ if (alarm?.name !== ALARM) return;
+ // The slot material is deterministic at the half-hour boundary and carries
+ // no session, cookie, profile, URL query, or page payload data.
+ const slot = Math.floor(Number(alarm.scheduledTime) / (PERIOD_MINUTES * 60 * 1000));
+ void coordinator.run(`dynamic-slot:${slot}`);
+});
diff --git a/dev/project-dev/bili_dynamic_refresh_extension/source-artifact-manifest.json b/dev/project-dev/bili_dynamic_refresh_extension/source-artifact-manifest.json
new file mode 100644
index 0000000..d33598f
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_extension/source-artifact-manifest.json
@@ -0,0 +1,39 @@
+{
+ "schema": 1,
+ "extension_id": "gllihoanalkiollgpeggamfhajmmnmml",
+ "root": "dev/project-dev/bili_dynamic_refresh_extension",
+ "entries": [
+ {
+ "path": "manifest.json",
+ "bytes": 821,
+ "sha256": "0dd4704e93a9989e1c44748888b0f779ad9a5030420f25c9dedfb2eec0b7fe86"
+ },
+ {
+ "path": "package.json",
+ "bytes": 23,
+ "sha256": "3ca9d4afd21425087cf31893b8f9f63c81b0b8408db5e343ca76e5f8aa26ab9a"
+ },
+ {
+ "path": "page_extract.js",
+ "bytes": 4541,
+ "sha256": "9a2fa6ca23e41bbe74a25429e36413e52eb330671354a493a22d3c11be989b80"
+ },
+ {
+ "path": "protocol.js",
+ "bytes": 4345,
+ "sha256": "f15d2aa3834e1212932f1e91c33fb1d3f7be760ae6ecfec1ef1230dac21b6e0d"
+ },
+ {
+ "path": "runtime.js",
+ "bytes": 20803,
+ "sha256": "8f83133202a93dfdc018cc59fb27c60a256a065faf5f28732c3dff5c78f65faa"
+ },
+ {
+ "path": "service_worker.js",
+ "bytes": 2204,
+ "sha256": "ae026f233e1c0d58a4d669508142806f03d7bc4396918bee65fe824f30d56d8b"
+ }
+ ],
+ "payload_tree_sha256": "20e8a83089a0f6a656577efa34a1beb87165d803a24c9f11252e8611ada814f6",
+ "manifest_semantic_sha256": "52c30fbe97b6e4890bd9d0bcd9f018c3f0e1a9d5e52d61f51b21d947df76abc2"
+}
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/__init__.py b/dev/project-dev/bili_dynamic_refresh_native_host/__init__.py
new file mode 100644
index 0000000..c8ce887
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/__init__.py
@@ -0,0 +1,5 @@
+"""Offline-reviewed native host core for the dynamic refresh trusted adapter."""
+
+from .constants import EXTENSION_ID, EXTENSION_ORIGIN, HOST_NAME
+
+__all__ = ["EXTENSION_ID", "EXTENSION_ORIGIN", "HOST_NAME"]
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/constants.py b/dev/project-dev/bili_dynamic_refresh_native_host/constants.py
new file mode 100644
index 0000000..3ae47ee
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/constants.py
@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+EXTENSION_ID = "gllihoanalkiollgpeggamfhajmmnmml"
+EXTENSION_ORIGIN = f"chrome-extension://{EXTENSION_ID}/"
+EXTENSION_NAME = "project-info Bilibili dynamic refresh trusted adapter"
+EXTENSION_VERSION = "1.0.0"
+HOST_NAME = "com.project_info.bili_dynamic_refresh"
+SOURCE_MANIFEST_NAME = "source-artifact-manifest.json"
+MAX_FRAME_BYTES = 1_048_576
+PROTOCOL_SCHEMA = 1
+PENDING_SCHEMA = 1
+TOTAL_DEADLINE_SECONDS = 120
+TASK_ID = "DEV-PROJECT-INFO-BILI-DYNAMIC-REFRESH-COLLECTOR-20260813-001"
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/durable.py b/dev/project-dev/bili_dynamic_refresh_native_host/durable.py
new file mode 100644
index 0000000..58c9081
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/durable.py
@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import ctypes
+import hashlib
+import os
+import secrets
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+from .constants import PENDING_SCHEMA
+from .strict_json import canonical_bytes, loads
+
+
+class DurabilityError(RuntimeError):
+ def __init__(self, code: str, message: str) -> None:
+ super().__init__(message)
+ self.code = code
+
+
+def fsync_directory(path: Path) -> None:
+ if os.name != "nt":
+ fd = os.open(path, os.O_RDONLY)
+ try:
+ os.fsync(fd)
+ finally:
+ os.close(fd)
+ return
+ create_file = ctypes.windll.kernel32.CreateFileW
+ handle = create_file(str(path), 0x40000000, 0x7, None, 3, 0x02000000, None)
+ if handle == ctypes.c_void_p(-1).value:
+ raise OSError(ctypes.get_last_error(), "CreateFileW directory failed")
+ try:
+ if not ctypes.windll.kernel32.FlushFileBuffers(handle):
+ raise OSError(ctypes.get_last_error(), "FlushFileBuffers directory failed")
+ finally:
+ ctypes.windll.kernel32.CloseHandle(handle)
+
+
+class PendingStore:
+ def __init__(self, path: Path, boundary_hook: Callable[[str], None] | None = None) -> None:
+ self.path = path
+ self._hook = boundary_hook or (lambda _phase: None)
+
+ def _boundary(self, name: str) -> None:
+ self._hook(name)
+
+ def load(self) -> dict[str, Any] | None:
+ if not self.path.exists():
+ return None
+ try:
+ value = loads(self.path.read_bytes())
+ except Exception as exc:
+ raise DurabilityError("E_DISPATCH_DURABILITY_AMBIGUOUS", "pending state is unreadable") from exc
+ required = {
+ "schema", "run_id", "request_id", "phase", "action_budget_consumed", "may_have_dispatched",
+ "refresh_count", "retry_count", "permit_id", "permit_payload_sha256", "deadline_at",
+ }
+ if not isinstance(value, dict) or set(value) != required or value["schema"] != PENDING_SCHEMA:
+ raise DurabilityError("E_DISPATCH_DURABILITY_AMBIGUOUS", "pending state schema differs")
+ return value
+
+ def write(self, value: Mapping[str, Any]) -> None:
+ payload = canonical_bytes(dict(value), newline=True)
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ partial = self.path.parent / f".{self.path.name}.{secrets.token_hex(8)}.partial"
+ self._boundary("before_open")
+ binary = getattr(os, "O_BINARY", 0)
+ fd = os.open(partial, os.O_CREAT | os.O_EXCL | os.O_WRONLY | binary, 0o600)
+ try:
+ self._boundary("after_open")
+ view = memoryview(payload)
+ while view:
+ written = os.write(fd, view)
+ if written <= 0:
+ raise OSError("short pending write")
+ view = view[written:]
+ self._boundary("after_write")
+ os.fsync(fd)
+ self._boundary("after_file_fsync")
+ finally:
+ os.close(fd)
+ os.replace(partial, self.path)
+ self._boundary("after_replace")
+ if self.path.read_bytes() != payload:
+ raise DurabilityError("E_DISPATCH_DURABILITY_AMBIGUOUS", "pending readback differs")
+ self._boundary("after_readback")
+ fsync_directory(self.path.parent)
+ self._boundary("after_directory_fsync")
+ if self.path.read_bytes() != payload:
+ raise DurabilityError("E_DISPATCH_DURABILITY_AMBIGUOUS", "pending second readback differs")
+ self._boundary("after_second_readback")
+
+ def initialize(self, *, run_id: str, request_id: str, deadline_at: str) -> dict[str, Any]:
+ value = {
+ "schema": PENDING_SCHEMA,
+ "run_id": run_id,
+ "request_id": request_id,
+ "phase": "PREPARED",
+ "action_budget_consumed": False,
+ "may_have_dispatched": False,
+ "refresh_count": 0,
+ "retry_count": 0,
+ "permit_id": None,
+ "permit_payload_sha256": None,
+ "deadline_at": deadline_at,
+ }
+ self.write(value)
+ return value
+
+ def consume(self, value: Mapping[str, Any], *, permit_id: str, permit_payload: bytes) -> dict[str, Any]:
+ if value.get("phase") != "PREPARED" or value.get("action_budget_consumed") is not False:
+ raise DurabilityError("E_DISPATCH_REPLAY", "action budget is not available")
+ updated = dict(value)
+ updated.update(
+ phase="ACTION_BUDGET_CONSUMED",
+ action_budget_consumed=True,
+ may_have_dispatched=True,
+ refresh_count=1,
+ retry_count=0,
+ permit_id=permit_id,
+ permit_payload_sha256=hashlib.sha256(permit_payload).hexdigest(),
+ )
+ self.write(updated)
+ return updated
+
+ def recovery_projection(self) -> dict[str, Any]:
+ try:
+ value = self.load()
+ except DurabilityError:
+ return {"error_code": "E_DISPATCH_DURABILITY_AMBIGUOUS", "refresh_count": 1, "retry_count": 0, "may_have_dispatched": True}
+ if value is None or value["action_budget_consumed"] is False:
+ return {"error_code": "E_PRE_PERMIT_STOP", "refresh_count": 0, "retry_count": 0, "may_have_dispatched": False}
+ return {"error_code": "E_POST_PERMIT_UNCERTAIN", "refresh_count": 1, "retry_count": 0, "may_have_dispatched": True}
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/identity.py b/dev/project-dev/bili_dynamic_refresh_native_host/identity.py
new file mode 100644
index 0000000..a519d09
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/identity.py
@@ -0,0 +1,219 @@
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Mapping
+
+from .constants import (
+ EXTENSION_ID,
+ EXTENSION_NAME,
+ EXTENSION_ORIGIN,
+ EXTENSION_VERSION,
+ HOST_NAME,
+ SOURCE_MANIFEST_NAME,
+ TASK_ID,
+)
+from .strict_json import canonical_bytes, loads
+
+LOWER_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+
+
+class IdentityError(RuntimeError):
+ def __init__(self, code: str, message: str) -> None:
+ super().__init__(message)
+ self.code = code
+
+
+@dataclass(frozen=True)
+class VerifiedLocalIdentity:
+ source_root: Path
+ source_manifest_bytes: int
+ source_manifest_sha256: str
+ payload_tree_sha256: str
+ extension_id: str
+ host_install_receipt_sha256: str
+ chrome_parent_pid: int
+
+
+def _exact_keys(value: Any, expected: set[str], field: str) -> Mapping[str, Any]:
+ if not isinstance(value, dict) or set(value) != expected:
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", f"{field} keys differ")
+ return value
+
+
+def _is_reparse(path: Path) -> bool:
+ info = path.lstat()
+ attrs = getattr(info, "st_file_attributes", 0)
+ return stat.S_ISLNK(info.st_mode) or bool(attrs & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400))
+
+
+def _safe_root(path: Path) -> Path:
+ absolute = Path(os.path.abspath(path))
+ current = Path(absolute.anchor)
+ for part in absolute.parts[1:]:
+ current /= part
+ if not current.exists() or _is_reparse(current):
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source root is missing or reparse-backed")
+ if not absolute.is_dir():
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source root is not a directory")
+ return absolute
+
+
+def _tree_hash(entries: list[Mapping[str, Any]]) -> str:
+ material = bytearray()
+ for entry in sorted(entries, key=lambda item: str(item["path"])):
+ material.extend(str(entry["path"]).encode("utf-8"))
+ material.extend(b"\0")
+ material.extend(str(entry["bytes"]).encode("ascii"))
+ material.extend(b"\0")
+ material.extend(str(entry["sha256"]).encode("ascii"))
+ material.extend(b"\n")
+ return hashlib.sha256(bytes(material)).hexdigest()
+
+
+def verify_source_tree(source_root: Path) -> dict[str, Any]:
+ root = _safe_root(source_root)
+ manifest_path = root / SOURCE_MANIFEST_NAME
+ if not manifest_path.is_file() or _is_reparse(manifest_path):
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source manifest is missing or unsafe")
+ raw = manifest_path.read_bytes()
+ value = loads(raw)
+ manifest = _exact_keys(
+ value,
+ {"schema", "extension_id", "root", "entries", "payload_tree_sha256", "manifest_semantic_sha256"},
+ "source manifest",
+ )
+ if manifest["schema"] != 1 or manifest["extension_id"] != EXTENSION_ID:
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source manifest identity differs")
+ entries = manifest["entries"]
+ if not isinstance(entries, list) or not entries:
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source manifest entries are invalid")
+ expected_paths: set[str] = set()
+ for item in entries:
+ entry = _exact_keys(item, {"path", "bytes", "sha256"}, "source entry")
+ path = entry["path"]
+ if not isinstance(path, str) or not path or "\\" in path or path.startswith("/") or ".." in Path(path).parts:
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source manifest path is unsafe")
+ if path in expected_paths or not isinstance(entry["bytes"], int) or not LOWER_SHA256.fullmatch(str(entry["sha256"])):
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source manifest entry is invalid")
+ expected_paths.add(path)
+ actual_paths: set[str] = set()
+ actual_entries: list[dict[str, Any]] = []
+ for candidate in root.rglob("*"):
+ relative = candidate.relative_to(root).as_posix()
+ if relative == SOURCE_MANIFEST_NAME:
+ continue
+ if _is_reparse(candidate) or not candidate.is_file():
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source tree contains a non-regular entry")
+ actual_paths.add(relative)
+ payload = candidate.read_bytes()
+ actual_entries.append({"path": relative, "bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest()})
+ if actual_paths != expected_paths:
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source tree exact set differs")
+ expected_by_path = {str(item["path"]): item for item in entries}
+ for actual in actual_entries:
+ if actual != expected_by_path[actual["path"]]:
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "source file identity differs")
+ if _tree_hash(actual_entries) != manifest["payload_tree_sha256"]:
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "payload tree identity differs")
+ semantic = dict(manifest)
+ semantic["manifest_semantic_sha256"] = None
+ if hashlib.sha256(canonical_bytes(semantic)).hexdigest() != manifest["manifest_semantic_sha256"]:
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "manifest semantic identity differs")
+ return {
+ "root": root,
+ "manifest_bytes": len(raw),
+ "manifest_sha256": hashlib.sha256(raw).hexdigest(),
+ "payload_tree_sha256": manifest["payload_tree_sha256"],
+ }
+
+
+def verify_local_identity(source_root: Path, facts: Mapping[str, Any]) -> VerifiedLocalIdentity:
+ facts = _exact_keys(facts, {"source_approval", "load_approval", "host"}, "identity facts")
+ source = verify_source_tree(source_root)
+ source_approval = _exact_keys(
+ facts["source_approval"],
+ {
+ "schema", "task_id", "scope", "source_manifest_path", "source_manifest_bytes",
+ "source_manifest_sha256", "payload_tree_sha256", "manifest_bytes", "manifest_sha256",
+ "extension_id", "approved_by_role", "review_audit_id", "created_at",
+ },
+ "source approval",
+ )
+ if (
+ source_approval["schema"] != 1 or source_approval["task_id"] != TASK_ID
+ or source_approval["scope"] != "controlled-local-unpacked-source-manifest"
+ or source_approval["approved_by_role"] != "dev.reviewer.project"
+ or Path(str(source_approval["source_manifest_path"])) != source["root"] / SOURCE_MANIFEST_NAME
+ ):
+ raise IdentityError("E_TRUSTED_ADAPTER_UNAVAILABLE", "reviewer source approval is unavailable")
+ manifest_payload = (source["root"] / "manifest.json").read_bytes()
+ manifest_sha = hashlib.sha256(manifest_payload).hexdigest()
+ required_source = {
+ "source_manifest_bytes": source["manifest_bytes"],
+ "source_manifest_sha256": source["manifest_sha256"],
+ "payload_tree_sha256": source["payload_tree_sha256"],
+ "manifest_bytes": len(manifest_payload),
+ "manifest_sha256": manifest_sha,
+ "extension_id": EXTENSION_ID,
+ }
+ if any(source_approval[key] != value for key, value in required_source.items()):
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "reviewer source approval binding differs")
+ load = _exact_keys(
+ facts["load_approval"],
+ {
+ "schema", "task_id", "scope", "account_holder_confirmed", "observed_extension_id",
+ "observed_name", "observed_version", "observed_enabled", "observed_error_count",
+ "source_root_absolute", "source_manifest_bytes", "source_manifest_sha256", "payload_tree_sha256",
+ "host_install_receipt_bytes", "host_install_receipt_sha256", "host_manifest_sha256",
+ "chrome_parent_pid", "chrome_parent_path_sha256", "chrome_parent_signature_verified",
+ "chrome_parent_started_at", "observed_at", "approved_by_role", "approval_reason",
+ },
+ "load approval",
+ )
+ if (
+ load["schema"] != 1 or load["task_id"] != TASK_ID or load["scope"] != "local-unpacked-load"
+ or load["approved_by_role"] not in {"project.admin", "case_analysis.video_downloader"}
+ or load["account_holder_confirmed"] is not True or load["observed_enabled"] is not True
+ or load["observed_error_count"] != 0 or load["observed_extension_id"] != EXTENSION_ID
+ or load["observed_name"] != EXTENSION_NAME or load["observed_version"] != EXTENSION_VERSION
+ or Path(str(load["source_root_absolute"])) != source["root"]
+ or load["source_manifest_bytes"] != source["manifest_bytes"]
+ or load["source_manifest_sha256"] != source["manifest_sha256"]
+ or load["payload_tree_sha256"] != source["payload_tree_sha256"]
+ or load["chrome_parent_signature_verified"] is not True
+ ):
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "local unpacked load approval differs")
+ host = _exact_keys(
+ facts["host"],
+ {
+ "host_name", "allowed_origins", "install_receipt_bytes", "install_receipt_sha256",
+ "host_manifest_sha256", "chrome_parent_pid", "chrome_parent_path_sha256",
+ "chrome_parent_signature_verified", "chrome_parent_started_at",
+ },
+ "host identity",
+ )
+ if (
+ host["host_name"] != HOST_NAME or host["allowed_origins"] != [EXTENSION_ORIGIN]
+ or host["install_receipt_bytes"] != load["host_install_receipt_bytes"]
+ or host["install_receipt_sha256"] != load["host_install_receipt_sha256"]
+ or host["host_manifest_sha256"] != load["host_manifest_sha256"]
+ or host["chrome_parent_pid"] != load["chrome_parent_pid"]
+ or host["chrome_parent_path_sha256"] != load["chrome_parent_path_sha256"]
+ or host["chrome_parent_signature_verified"] is not True
+ or host["chrome_parent_started_at"] != load["chrome_parent_started_at"]
+ ):
+ raise IdentityError("E_LOCAL_EXTENSION_IDENTITY", "host or current Chrome identity differs")
+ return VerifiedLocalIdentity(
+ source_root=source["root"],
+ source_manifest_bytes=source["manifest_bytes"],
+ source_manifest_sha256=source["manifest_sha256"],
+ payload_tree_sha256=source["payload_tree_sha256"],
+ extension_id=EXTENSION_ID,
+ host_install_receipt_sha256=str(load["host_install_receipt_sha256"]),
+ chrome_parent_pid=int(load["chrome_parent_pid"]),
+ )
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/native-host-manifest.template.json b/dev/project-dev/bili_dynamic_refresh_native_host/native-host-manifest.template.json
new file mode 100644
index 0000000..a09f8f2
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/native-host-manifest.template.json
@@ -0,0 +1,9 @@
+{
+ "name": "com.project_info.bili_dynamic_refresh",
+ "description": "project-info Bilibili dynamic refresh trusted adapter",
+ "path": "__REVIEWED_ABSOLUTE_EXE_PATH__",
+ "type": "stdio",
+ "allowed_origins": [
+ "chrome-extension://gllihoanalkiollgpeggamfhajmmnmml/"
+ ]
+}
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/native_host.py b/dev/project-dev/bili_dynamic_refresh_native_host/native_host.py
new file mode 100644
index 0000000..8b865fb
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/native_host.py
@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import json
+import errno
+import struct
+import sys
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, BinaryIO
+
+from .constants import MAX_FRAME_BYTES
+from .strict_json import canonical_bytes
+
+
+PEER_CLOSED_CODE = "E_NATIVE_PEER_CLOSED"
+_WINDOWS_PEER_CLOSED = {38, 109, 232, 233}
+
+
+@dataclass(frozen=True)
+class WriteResult:
+ written: bool
+ error_code: str | None
+
+
+def _is_peer_closed(error: BaseException) -> bool:
+ if isinstance(error, (BrokenPipeError, EOFError)):
+ return True
+ return isinstance(error, OSError) and (
+ error.errno == errno.EPIPE or getattr(error, "winerror", None) in _WINDOWS_PEER_CLOSED
+ )
+
+
+class NativeFrameWriter:
+ """One-owner serialized native-messaging writer with bounded peer-close semantics."""
+
+ def __init__(self, stream: BinaryIO) -> None:
+ self._stream = stream
+ self._lock = threading.Lock()
+ self._closed = False
+
+ def write_frame(self, value: dict[str, Any]) -> WriteResult:
+ payload = canonical_bytes(value)
+ frame = struct.pack("<I", len(payload)) + payload
+ with self._lock:
+ if self._closed:
+ return WriteResult(False, PEER_CLOSED_CODE)
+ try:
+ # A single write narrows the Windows WriteFile/peer-exit race.
+ self._stream.write(frame)
+ self._stream.flush()
+ except BaseException as error:
+ if not _is_peer_closed(error):
+ raise
+ self._closed = True
+ return WriteResult(False, PEER_CLOSED_CODE)
+ return WriteResult(True, None)
+
+
+def _write_frame(value: dict[str, Any]) -> WriteResult:
+ return NativeFrameWriter(sys.stdout.buffer).write_frame(value)
+
+
+def main() -> int:
+ """Fail closed until reviewed build/install and visible local-load approvals exist.
+
+ No argv, environment variable, stdin JSONL, fixture or transcript can supply
+ those identities. A later reviewed installer writes the adjacent immutable
+ runtime config consumed by the packaged host.
+ """
+ config_path = Path(sys.executable).with_name("runtime-config.json")
+ if not config_path.is_file():
+ written = _write_frame({
+ "schema_version": 1,
+ "ok": False,
+ "status": "REFRESH_FAILED_PAGE_UNREADABLE",
+ "error_code": "E_TRUSTED_ADAPTER_UNAVAILABLE",
+ "authoritative": False,
+ "saved": False,
+ "no_new": False,
+ })
+ return 4 if written.written else 0
+ written = _write_frame({
+ "schema_version": 1,
+ "ok": False,
+ "status": "REFRESH_FAILED_PAGE_UNREADABLE",
+ "error_code": "E_TRUSTED_ADAPTER_UNAVAILABLE",
+ "authoritative": False,
+ "saved": False,
+ "no_new": False,
+ })
+ return 4 if written.written else 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/protocol.py b/dev/project-dev/bili_dynamic_refresh_native_host/protocol.py
new file mode 100644
index 0000000..ca83574
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/protocol.py
@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import secrets
+from datetime import datetime, timezone
+from typing import Any, Callable, Mapping
+from urllib.parse import urlsplit, urlunsplit
+
+from .constants import EXTENSION_ID, EXTENSION_NAME, EXTENSION_VERSION
+from .durable import DurabilityError, PendingStore
+from .identity import VerifiedLocalIdentity
+from .strict_json import canonical_bytes
+
+
+class ProtocolError(RuntimeError):
+ def __init__(self, code: str, message: str) -> None:
+ super().__init__(message)
+ self.code = code
+
+
+def _timestamp(value: str) -> datetime:
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise ProtocolError("E_DEADLINE", "deadline is invalid") from exc
+ if parsed.tzinfo is None:
+ raise ProtocolError("E_DEADLINE", "deadline must be offset-aware")
+ return parsed.astimezone(timezone.utc)
+
+
+def _sign(secret: bytes, frame: Mapping[str, Any]) -> str:
+ return hmac.new(secret, canonical_bytes(dict(frame)), hashlib.sha256).hexdigest()
+
+
+def _target_url(value: str) -> str:
+ parsed = urlsplit(value)
+ parts = [part for part in parsed.path.split("/") if part]
+ if (
+ parsed.scheme != "https"
+ or parsed.hostname != "space.bilibili.com"
+ or parsed.query
+ or parsed.fragment
+ or len(parts) != 2
+ or not parts[0].isdigit()
+ or parts[0].startswith("0")
+ or parts[1] != "dynamic"
+ ):
+ raise ProtocolError("E_PAGE_IDENTITY", "target URL is not a canonical creator dynamic page")
+ return urlunsplit(("https", "space.bilibili.com", f"/{parts[0]}/dynamic", "", ""))
+
+
+class HostSession:
+ def __init__(
+ self,
+ identity: VerifiedLocalIdentity,
+ pending: PendingStore,
+ commit_observation: Callable[[Mapping[str, Any]], Mapping[str, Any]],
+ target_url: str,
+ *,
+ now: Callable[[], datetime] | None = None,
+ ) -> None:
+ if type(identity) is not VerifiedLocalIdentity:
+ raise ProtocolError("E_TRUSTED_ADAPTER_UNAVAILABLE", "verified local identity is required")
+ self.identity = identity
+ self.pending = pending
+ self.commit_observation = commit_observation
+ self.target_url = _target_url(target_url)
+ self.now = now or (lambda: datetime.now(timezone.utc))
+ self.secret = secrets.token_bytes(32)
+ self.run_id: str | None = None
+ self.request_id: str | None = None
+ self.deadline_at: str | None = None
+ self.sequence = 0
+ self.action: dict[str, Any] | None = None
+ self.permit_id: str | None = None
+ self.action_result: Mapping[str, Any] | None = None
+
+ def start(self, hello: Mapping[str, Any], *, run_id: str, request_id: str, deadline_at: str) -> dict[str, Any]:
+ if set(hello) != {"schema_version", "type", "sequence", "extension_id", "version", "manifest_name"}:
+ raise ProtocolError("E_LOCAL_EXTENSION_IDENTITY", "extension hello shape differs")
+ if (
+ hello["schema_version"] != 1 or hello["type"] != "EXTENSION_HELLO" or hello["sequence"] != 1
+ or hello["extension_id"] != EXTENSION_ID or hello["version"] != EXTENSION_VERSION
+ or hello["manifest_name"] != EXTENSION_NAME
+ ):
+ raise ProtocolError("E_LOCAL_EXTENSION_IDENTITY", "extension hello identity differs")
+ deadline = _timestamp(deadline_at)
+ if self.now().astimezone(timezone.utc) >= deadline:
+ raise ProtocolError("E_OVERALL_DEADLINE", "original deadline expired before challenge")
+ self.run_id, self.request_id, self.deadline_at, self.sequence = run_id, request_id, deadline_at, 2
+ self.pending.initialize(run_id=run_id, request_id=request_id, deadline_at=deadline_at)
+ return self._host_frame("HOST_CHALLENGE", 2, {"challenge_id": secrets.token_hex(16), "secret": self.secret.hex()})
+
+ def accept(self, frame: Mapping[str, Any]) -> dict[str, Any]:
+ self._verify_extension(frame)
+ kind = frame["type"]
+ if kind == "EXTENSION_CHALLENGE_ACCEPTED" and frame["sequence"] == 3:
+ self.sequence = 4
+ self.action = {"action_id": secrets.token_hex(16), "kind": "reload", "url": self.target_url}
+ return self._host_frame("HOST_ACTION_PREPARE", 4, {"action": self.action})
+ if kind == "EXTENSION_READY_TO_DISPATCH" and frame["sequence"] == 5:
+ if self.action is None or frame.get("action_id") != self.action["action_id"]:
+ raise ProtocolError("E_PROTOCOL", "prepared action identity differs")
+ if self.now().astimezone(timezone.utc) >= _timestamp(str(self.deadline_at)):
+ raise ProtocolError("E_OVERALL_DEADLINE", "original deadline expired before dispatch")
+ candidate_permit_id = secrets.token_hex(16)
+ permit = {"permit_id": candidate_permit_id, "action_id": self.action["action_id"], "deadline_at": self.deadline_at}
+ permit_payload = canonical_bytes(permit)
+ current = self.pending.load()
+ if current is None:
+ raise DurabilityError("E_DISPATCH_DURABILITY_AMBIGUOUS", "pending disappeared")
+ self.pending.consume(current, permit_id=candidate_permit_id, permit_payload=permit_payload)
+ if self.now().astimezone(timezone.utc) >= _timestamp(str(self.deadline_at)):
+ raise ProtocolError("E_OVERALL_DEADLINE", "original deadline expired after durable budget")
+ self.permit_id = candidate_permit_id
+ self.sequence = 6
+ return self._host_frame("HOST_DISPATCH_PERMIT", 6, {"permit": permit})
+ if kind == "EXTENSION_ACTION_RESULT" and frame["sequence"] == 7:
+ if self.permit_id is None or frame.get("permit_id") != self.permit_id:
+ raise ProtocolError("E_PROTOCOL", "action result permit differs")
+ self.action_result = frame.get("result")
+ self.sequence = 8
+ return self._host_frame("HOST_OBSERVATION_REQUEST", 8, {"action_id": self.action["action_id"]})
+ if kind in {"EXTENSION_OBSERVATION", "EXTENSION_TERMINAL_ERROR"} and frame["sequence"] == 9:
+ if self.action_result is None:
+ raise ProtocolError("E_ACTION_RESULT_REQUIRED", "observation preceded action result")
+ observation = frame.get("observation") if kind == "EXTENSION_OBSERVATION" else {"terminal_error": frame.get("error_code")}
+ result = dict(self.commit_observation({
+ "run_id": self.run_id,
+ "request_id": self.request_id,
+ "deadline_at": self.deadline_at,
+ "action": self.action,
+ "action_result": self.action_result,
+ "observation": observation,
+ "trusted_identity": {
+ "extension_id": self.identity.extension_id,
+ "source_manifest_sha256": self.identity.source_manifest_sha256,
+ "payload_tree_sha256": self.identity.payload_tree_sha256,
+ "host_install_receipt_sha256": self.identity.host_install_receipt_sha256,
+ },
+ }))
+ self.sequence = 10
+ return self._host_frame("HOST_COMMIT_RESULT", 10, {"result": result})
+ raise ProtocolError("E_SEQUENCE", "message type or sequence differs")
+
+ def _verify_extension(self, frame: Mapping[str, Any]) -> None:
+ required = {"schema_version", "type", "run_id", "request_id", "sequence", "hmac"}
+ if not isinstance(frame, dict) or not required.issubset(frame):
+ raise ProtocolError("E_PROTOCOL", "extension frame is incomplete")
+ if frame["run_id"] != self.run_id or frame["request_id"] != self.request_id:
+ raise ProtocolError("E_PROTOCOL", "run or request identity differs")
+ unsigned = dict(frame)
+ supplied = unsigned.pop("hmac")
+ if not isinstance(supplied, str) or not hmac.compare_digest(supplied, _sign(self.secret, unsigned)):
+ raise ProtocolError("E_HMAC", "extension frame HMAC differs")
+
+ def _host_frame(self, kind: str, sequence: int, fields: Mapping[str, Any]) -> dict[str, Any]:
+ frame = {
+ "schema_version": 1,
+ "type": kind,
+ "run_id": self.run_id,
+ "request_id": self.request_id,
+ "sequence": sequence,
+ **dict(fields),
+ }
+ frame["hmac"] = _sign(self.secret, frame)
+ return frame
+
+
+def sign_extension_frame(secret_hex: str, frame: Mapping[str, Any]) -> dict[str, Any]:
+ """Test/extension parity helper; production main never exposes the session secret."""
+ unsigned = dict(frame)
+ unsigned["hmac"] = _sign(bytes.fromhex(secret_hex), unsigned)
+ return unsigned
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/source-artifact-manifest.json b/dev/project-dev/bili_dynamic_refresh_native_host/source-artifact-manifest.json
new file mode 100644
index 0000000..37d6a9b
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/source-artifact-manifest.json
@@ -0,0 +1,49 @@
+{
+ "schema": 1,
+ "extension_id": "gllihoanalkiollgpeggamfhajmmnmml",
+ "root": "dev/project-dev/bili_dynamic_refresh_native_host",
+ "entries": [
+ {
+ "path": "__init__.py",
+ "bytes": 208,
+ "sha256": "7a0e1e2f2df7bb2332c9a9d351526892102e329baadc4554326fdb1acad2de65"
+ },
+ {
+ "path": "constants.py",
+ "bytes": 520,
+ "sha256": "072efa0b7d555366b68c358dd3dcc5e4e2f9dbd3a9d26c0eed2722dd87cd271c"
+ },
+ {
+ "path": "durable.py",
+ "bytes": 5409,
+ "sha256": "15f06128594de674d94d30312d87df9296061f6697daf0fedab0709728825972"
+ },
+ {
+ "path": "identity.py",
+ "bytes": 10348,
+ "sha256": "2406dfa2324751f3fe153e7b0224eb33062d31a63b8c910fdd075efcc29ac257"
+ },
+ {
+ "path": "native-host-manifest.template.json",
+ "bytes": 278,
+ "sha256": "0233077552c752f73f0b84e147d1ba4e3fa7a21f24074f960e7492fb1b6a81f9"
+ },
+ {
+ "path": "native_host.py",
+ "bytes": 2983,
+ "sha256": "507595bd40ea5f760aba29d1522efb2658d3797ff30cbc44519334d8f8165989"
+ },
+ {
+ "path": "protocol.py",
+ "bytes": 8840,
+ "sha256": "c8cdb4ba5fce931cb33a8628c80f9ecb8e8100daf2fcbaec9c3e31a05692bbc2"
+ },
+ {
+ "path": "strict_json.py",
+ "bytes": 964,
+ "sha256": "3e0faf5ff334821182f520c8ec40c8ded2ba240ab9b3cf704a75b8edef4c8b48"
+ }
+ ],
+ "payload_tree_sha256": "253b8af6c3f748a546cfd9763b313206334b39d32911576dfc639c0d78841d16",
+ "manifest_semantic_sha256": "15809321096e46c12848cd66b755c36af3e7e9b087838fa50a398cd927b61d1d"
+}
diff --git a/dev/project-dev/bili_dynamic_refresh_native_host/strict_json.py b/dev/project-dev/bili_dynamic_refresh_native_host/strict_json.py
new file mode 100644
index 0000000..6e3d257
--- /dev/null
+++ b/dev/project-dev/bili_dynamic_refresh_native_host/strict_json.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+class StrictJsonError(ValueError):
+ pass
+
+
+def _pairs(values: list[tuple[str, Any]]) -> dict[str, Any]:
+ result: dict[str, Any] = {}
+ for key, value in values:
+ if key in result:
+ raise StrictJsonError(f"duplicate key: {key}")
+ result[key] = value
+ return result
+
+
+def loads(payload: bytes) -> Any:
+ try:
+ text = payload.decode("utf-8", errors="strict")
+ return json.loads(text, object_pairs_hook=_pairs, parse_constant=lambda value: (_ for _ in ()).throw(StrictJsonError(value)))
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+ raise StrictJsonError("invalid strict UTF-8 JSON") from exc
+
+
+def canonical_bytes(value: Any, *, newline: bool = False) -> bytes:
+ payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ return payload + (b"\n" if newline else b"")
diff --git a/dev/project-dev/bili_half_hour_pipeline.config.json b/dev/project-dev/bili_half_hour_pipeline.config.json
new file mode 100644
index 0000000..a2c7082
--- /dev/null
+++ b/dev/project-dev/bili_half_hour_pipeline.config.json
@@ -0,0 +1,46 @@
+{
+ "schema_version": 1,
+ "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "interval_minutes": 30,
+ "creator": {
+ "uid": "1420210197",
+ "name": "青枫浦上Q",
+ "dynamic_url": "https://space.bilibili.com/1420210197/dynamic"
+ },
+ "paths": {
+ "project_root": "../..",
+ "archive_root": "ana-data/news-青枫浦上Q",
+ "formal_manifest": "ana-data/news-青枫浦上Q/manifest.jsonl",
+ "processing_handoffs": "ana-data/news-青枫浦上Q/video-processing-handoffs.jsonl",
+ "state_dir": "dev/tmp/bili-half-hour-pipeline-state",
+ "video_root": "F:/video/青枫浦上Q"
+ },
+ "downstream": {
+ "video_downloader_thread_id": "019fcc5d-798f-7ea1-8325-3a4d1f2dc5a5",
+ "media_processor_thread_id": "019fb7a4-bdfd-79f2-bd6b-e67e2b7d8efd",
+ "minutes_thread_id": "019fae88-ef98-7f83-b964-dcd4c5f842ef",
+ "reply_thread_id": "019fbcbb-bed7-7c90-83ab-f50610f80d3a"
+ },
+ "git": {
+ "remote": "origin",
+ "branch": "master",
+ "allowed_extensions": [
+ ".txt",
+ ".md",
+ ".json",
+ ".jsonl",
+ ".srt",
+ ".pdf",
+ ".png",
+ ".jpg",
+ ".jpeg",
+ ".webp"
+ ],
+ "allowed_docs": [
+ "ana-data/目录导读.md",
+ "ana-data/news-青枫浦上Q/目录导读.md",
+ "dev-doc/project-doc/B站博主动态采集辅助工具.md",
+ "dev-doc/project-doc/目录导读.md"
+ ]
+ }
+}
diff --git a/dev/project-dev/bili_half_hour_pipeline.py b/dev/project-dev/bili_half_hour_pipeline.py
new file mode 100644
index 0000000..28814cd
--- /dev/null
+++ b/dev/project-dev/bili_half_hour_pipeline.py
@@ -0,0 +1,3016 @@
+#!/usr/bin/env python3
+"""Durable half-hour coordinator for one configured Bilibili creator.
+
+The coordinator never reads browser credentials and never downloads media. It
+turns already validated collector/formal records into exact-once role outboxes,
+tracks non-overlapping half-hour runs, accepts exact downstream receipts, and
+performs narrowly allowlisted Git delivery. Browser capture and native media
+work remain in their reviewed components.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import hashlib
+import json
+import os
+import re
+import stat
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable, Iterable, Iterator, Mapping, Sequence
+from urllib.parse import urlsplit
+
+
+SCHEMA = 1
+INTERVAL_MINUTES = 30
+TASK_ID = "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001"
+UID = re.compile(r"^[1-9][0-9]{0,19}$")
+BVID = re.compile(r"^BV1[1-9A-HJ-NP-Za-km-z]{9}$")
+SHA256 = re.compile(r"^[0-9A-F]{64}$")
+SHA256_MIXED_ASCII = re.compile(r"^[0-9A-Fa-f]{64}$", re.ASCII)
+THREAD_ID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
+SAFE_ID = re.compile(r"^[A-Za-z0-9_.:-]{1,240}$")
+SECRET_KEY = re.compile(r"(?i)(cookie|sessdata|token|credential|authorization|localstorage|profile|signed_url)")
+SECRET_VALUE = re.compile(r"(?i)(SYNTHETIC_SECRET|sessdata=|cookie\s*[:=]|authorization\s*[:=]|bearer\s+|token=|signed_url=|localstorage)")
+FORBIDDEN_GIT_SUFFIXES = (
+ ".mkv", ".mp4", ".mov", ".webm", ".download.json", ".flac", ".partial", ".crdownload"
+)
+RECEIPT_GIT_KIND_SUFFIX = {
+ "transcript": ".txt",
+ "transcript_txt": ".txt",
+ "transcript_srt": ".srt",
+ "transcript_json": ".json",
+ "minutes": ".md",
+ "minutes_md": ".md",
+ "minutes_pdf": ".pdf",
+ "relocation_manifest": ".json",
+ "documentation": ".md",
+}
+CONTENT_TYPES = frozenset({"article", "text", "image"})
+VIDEO_COMPLETE = "VIDEO_DOWNLOADED_COMPLETE_HANDOFF_SENT"
+CANONICAL_TITLE_MAX_LENGTH = 64
+WINDOWS_INVALID_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
+WINDOWS_RESERVED_NAMES = {
+ "CON", "PRN", "AUX", "NUL",
+ *(f"COM{value}" for value in range(1, 10)),
+ *(f"LPT{value}" for value in range(1, 10)),
+}
+RELOCATION_REPORT_TYPE = "VIDEO_ARTIFACT_RELOCATION_BATCH"
+PUBLIC_VIDEO_KINDS = {
+ "transcript_txt": ".txt",
+ "transcript_srt": ".srt",
+ "transcript_json": ".json",
+ "minutes_md": ".md",
+ "minutes_pdf": ".pdf",
+}
+
+
+class PipelineError(RuntimeError):
+ def __init__(self, code: str, message: str) -> None:
+ super().__init__(message)
+ self.code = code
+
+
+@dataclass(frozen=True)
+class Config:
+ path: Path
+ project_root: Path
+ creator_uid: str
+ creator_name: str
+ dynamic_url: str
+ archive_root: Path
+ formal_manifest: Path
+ processing_handoffs: Path
+ state_dir: Path
+ video_root: Path
+ video_downloader_thread_id: str
+ media_thread_id: str
+ minutes_thread_id: str
+ reply_thread_id: str
+ git_remote: str
+ git_branch: str
+ git_extensions: frozenset[str]
+ git_doc_paths: frozenset[str]
+
+ @property
+ def state_path(self) -> Path:
+ return self.state_dir / "state.json"
+
+ @property
+ def runs_path(self) -> Path:
+ return self.state_dir / "runs.jsonl"
+
+ @property
+ def outbox_path(self) -> Path:
+ return self.state_dir / "outbox.jsonl"
+
+ @property
+ def terminals_path(self) -> Path:
+ return self.state_dir / "terminals.jsonl"
+
+ @property
+ def lock_path(self) -> Path:
+ return self.state_dir / "coordinator.lock"
+
+ def git_index_guard_path(self, batch_id: str) -> Path:
+ return self.state_dir / f"git-shared-index-guard-{batch_id}.json"
+
+ @property
+ def relocation_root(self) -> Path:
+ return self.archive_root / "artifact-relocations"
+
+
+def _exact(value: Any, keys: Iterable[str], field: str) -> Mapping[str, Any]:
+ expected = set(keys)
+ if not isinstance(value, dict) or set(value) != expected:
+ raise PipelineError("E_SCHEMA", f"{field} keys differ")
+ return value
+
+
+def _reject_secrets(value: Any, path: str = "$") -> None:
+ if isinstance(value, dict):
+ for key, item in value.items():
+ if not isinstance(key, str) or SECRET_KEY.search(key):
+ raise PipelineError("E_SECRET_FIELD", f"secret-like field at {path}")
+ _reject_secrets(item, f"{path}.{key}")
+ elif isinstance(value, list):
+ for index, item in enumerate(value):
+ _reject_secrets(item, f"{path}[{index}]")
+ elif isinstance(value, str):
+ if SECRET_VALUE.search(value):
+ raise PipelineError("E_SECRET_FIELD", f"secret-like value at {path}")
+ parsed = urlsplit(value)
+ if parsed.scheme in {"http", "https"} and (parsed.username is not None or parsed.password is not None):
+ raise PipelineError("E_SECRET_FIELD", f"credential-bearing URL at {path}")
+
+
+def _canonical(value: Mapping[str, Any]) -> bytes:
+ _reject_secrets(value)
+ return (json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii")
+
+
+def _strict_json(path: Path, field: str) -> tuple[Any, bytes]:
+ try:
+ payload = path.read_bytes()
+ except OSError as exc:
+ raise PipelineError("E_INPUT", f"{field} is unavailable") from exc
+ if not payload or payload.startswith(b"\xef\xbb\xbf") or b"\r" in payload or not payload.endswith(b"\n"):
+ raise PipelineError("E_INPUT", f"{field} is not strict UTF-8 JSON")
+ try:
+ text = payload.decode("utf-8")
+ value = json.loads(text)
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+ raise PipelineError("E_INPUT", f"{field} is invalid JSON") from exc
+ _reject_secrets(value)
+ return value, payload
+
+
+def _absolute(base: Path, value: Any, field: str) -> Path:
+ if not isinstance(value, str) or not value:
+ raise PipelineError("E_CONFIG", f"{field} must be a path")
+ candidate = Path(value)
+ if not candidate.is_absolute():
+ candidate = base / candidate
+ return Path(os.path.abspath(candidate))
+
+
+def _within(child: Path, parent: Path) -> bool:
+ try:
+ child.relative_to(parent)
+ return True
+ except ValueError:
+ return False
+
+
+def _canonical_dynamic_url(value: Any, uid: str) -> str:
+ if not isinstance(value, str):
+ raise PipelineError("E_CONFIG", "dynamic_url must be a string")
+ parsed = urlsplit(value)
+ if parsed.scheme != "https" or parsed.hostname != "space.bilibili.com" or parsed.query or parsed.fragment or parsed.path.rstrip("/") != f"/{uid}/dynamic":
+ raise PipelineError("E_CONFIG", "dynamic_url is not canonical")
+ return f"https://space.bilibili.com/{uid}/dynamic"
+
+
+def _canonical_video_url(value: Any, bvid: str) -> str:
+ if not isinstance(value, str) or not BVID.fullmatch(bvid):
+ raise PipelineError("E_SOURCE_BINDING", "video source identity is incomplete")
+ _reject_secrets(value)
+ parsed = urlsplit(value)
+ if (
+ parsed.scheme != "https"
+ or parsed.hostname != "www.bilibili.com"
+ or parsed.username is not None
+ or parsed.password is not None
+ or parsed.port is not None
+ or parsed.query
+ or parsed.fragment
+ or parsed.path.rstrip("/") != f"/video/{bvid}"
+ ):
+ raise PipelineError("E_SOURCE_BINDING", "video source URL is not canonical")
+ return f"https://www.bilibili.com/video/{bvid}"
+
+
+def load_config(path: Path) -> Config:
+ value, _ = _strict_json(path, "config")
+ root = _exact(value, {"schema_version", "task_id", "interval_minutes", "creator", "paths", "downstream", "git"}, "config")
+ if (
+ type(root["schema_version"]) is not int
+ or root["schema_version"] != SCHEMA
+ or root["task_id"] != TASK_ID
+ or type(root["interval_minutes"]) is not int
+ or root["interval_minutes"] != INTERVAL_MINUTES
+ ):
+ raise PipelineError("E_CONFIG", "config identity differs")
+ creator = _exact(root["creator"], {"uid", "name", "dynamic_url"}, "creator")
+ uid = creator["uid"]
+ if not isinstance(uid, str) or not UID.fullmatch(uid) or not isinstance(creator["name"], str) or not creator["name"].strip():
+ raise PipelineError("E_CONFIG", "creator identity differs")
+ paths = _exact(root["paths"], {"project_root", "archive_root", "formal_manifest", "processing_handoffs", "state_dir", "video_root"}, "paths")
+ base = path.parent
+ project_root = _absolute(base, paths["project_root"], "project_root")
+ archive_root = _absolute(project_root, paths["archive_root"], "archive_root")
+ formal = _absolute(project_root, paths["formal_manifest"], "formal_manifest")
+ handoffs = _absolute(project_root, paths["processing_handoffs"], "processing_handoffs")
+ state_dir = _absolute(project_root, paths["state_dir"], "state_dir")
+ video_root = _absolute(project_root, paths["video_root"], "video_root")
+ if not _within(archive_root, project_root) or not _within(formal, archive_root) or not _within(handoffs, archive_root) or not _within(state_dir, project_root):
+ raise PipelineError("E_CONFIG", "project output path escaped its governed root")
+ downstream = _exact(
+ root["downstream"],
+ {"video_downloader_thread_id", "media_processor_thread_id", "minutes_thread_id", "reply_thread_id"},
+ "downstream",
+ )
+ for field, thread_id in downstream.items():
+ if not isinstance(thread_id, str) or not THREAD_ID.fullmatch(thread_id):
+ raise PipelineError("E_CONFIG", f"{field} is not a thread id")
+ git = _exact(root["git"], {"remote", "branch", "allowed_extensions", "allowed_docs"}, "git")
+ if (
+ git["remote"] != "origin"
+ or not isinstance(git["branch"], str)
+ or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}", git["branch"])
+ or git["branch"].endswith(("/", "."))
+ or any(marker in git["branch"] for marker in ("..", "//", "@{"))
+ ):
+ raise PipelineError("E_CONFIG", "git destination differs")
+ extensions = git["allowed_extensions"]
+ if not isinstance(extensions, list) or not extensions or any(not isinstance(item, str) or not item.startswith(".") or item.lower() in FORBIDDEN_GIT_SUFFIXES for item in extensions):
+ raise PipelineError("E_CONFIG", "git extension allowlist is invalid")
+ docs = git["allowed_docs"]
+ if not isinstance(docs, list) or not docs:
+ raise PipelineError("E_CONFIG", "git document allowlist is invalid")
+ normalized_docs: list[str] = []
+ for value in docs:
+ if not isinstance(value, str) or not value or Path(value).is_absolute():
+ raise PipelineError("E_CONFIG", "git document allowlist is invalid")
+ target = Path(os.path.abspath(project_root / Path(value)))
+ if not _within(target, project_root) or target.suffix.lower() != ".md":
+ raise PipelineError("E_CONFIG", "git document allowlist escaped the project")
+ normalized_docs.append(target.relative_to(project_root).as_posix())
+ if len(set(normalized_docs)) != len(normalized_docs):
+ raise PipelineError("E_CONFIG", "git document allowlist is duplicated")
+ return Config(
+ path=path, project_root=project_root, creator_uid=uid, creator_name=creator["name"].strip(),
+ dynamic_url=_canonical_dynamic_url(creator["dynamic_url"], uid), archive_root=archive_root,
+ formal_manifest=formal, processing_handoffs=handoffs, state_dir=state_dir, video_root=video_root,
+ video_downloader_thread_id=downstream["video_downloader_thread_id"],
+ media_thread_id=downstream["media_processor_thread_id"], minutes_thread_id=downstream["minutes_thread_id"],
+ reply_thread_id=downstream["reply_thread_id"], git_remote=git["remote"], git_branch=git["branch"],
+ git_extensions=frozenset(item.lower() for item in extensions), git_doc_paths=frozenset(normalized_docs),
+ )
+
+
+def _file_identity(path: Path) -> dict[str, Any]:
+ payload = path.read_bytes() if path.exists() else b""
+ if payload and not payload.endswith(b"\n"):
+ raise PipelineError("E_JOURNAL", f"{path.name} lacks final LF")
+ return {"bytes": len(payload), "lines": payload.count(b"\n"), "sha256": hashlib.sha256(payload).hexdigest().upper()}
+
+
+def _atomic(path: Path, payload: bytes) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ fd, raw = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".partial", dir=path.parent)
+ partial = Path(raw)
+ try:
+ with os.fdopen(fd, "wb") as stream:
+ stream.write(payload)
+ stream.flush()
+ os.fsync(stream.fileno())
+ os.replace(partial, path)
+ if path.read_bytes() != payload:
+ raise PipelineError("E_DURABILITY", f"{path.name} readback differs")
+ finally:
+ with contextlib.suppress(FileNotFoundError):
+ partial.unlink()
+
+
+def _append(path: Path, value: Mapping[str, Any]) -> dict[str, Any]:
+ payload = _canonical(value)
+ pre = path.read_bytes() if path.exists() else b""
+ if pre and not pre.endswith(b"\n"):
+ raise PipelineError("E_JOURNAL", f"{path.name} is malformed")
+ _atomic(path, pre + payload)
+ if not path.read_bytes().startswith(pre):
+ raise PipelineError("E_DURABILITY", f"{path.name} prefix changed")
+ return _file_identity(path)
+
+
+def _create_new(path: Path, payload: bytes) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)
+ try:
+ descriptor = os.open(path, flags, 0o600)
+ except FileExistsError as exc:
+ raise PipelineError("E_DURABILITY", f"{path.name} already exists") from exc
+ try:
+ with os.fdopen(descriptor, "wb", closefd=False) as stream:
+ stream.write(payload)
+ stream.flush()
+ os.fsync(stream.fileno())
+ finally:
+ os.close(descriptor)
+ if path.read_bytes() != payload:
+ raise PipelineError("E_DURABILITY", f"{path.name} readback differs")
+
+
+@contextlib.contextmanager
+def _lock(config: Config) -> Iterator[None]:
+ config.state_dir.mkdir(parents=True, exist_ok=True)
+ stream = config.lock_path.open("a+b")
+ try:
+ if os.name == "nt":
+ import msvcrt
+ stream.seek(0)
+ if stream.tell() == stream.seek(0, os.SEEK_END) == 0:
+ stream.write(b"0")
+ stream.flush()
+ stream.seek(0)
+ try:
+ msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
+ except OSError as exc:
+ raise PipelineError("E_RUN_ACTIVE", "another coordinator owns the run lock") from exc
+ else:
+ import fcntl
+ try:
+ fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except OSError as exc:
+ raise PipelineError("E_RUN_ACTIVE", "another coordinator owns the run lock") from exc
+ yield
+ finally:
+ if os.name == "nt":
+ with contextlib.suppress(OSError):
+ stream.seek(0)
+ msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ with contextlib.suppress(OSError):
+ fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
+ stream.close()
+
+
+def _read_lines(
+ path: Path,
+ field: str,
+ *,
+ reject_secrets: bool = True,
+) -> tuple[list[dict[str, Any]], bytes]:
+ if not path.exists():
+ return [], b""
+ payload = path.read_bytes()
+ if payload and (b"\r" in payload or not payload.endswith(b"\n")):
+ raise PipelineError("E_INPUT", f"{field} is not strict JSONL")
+ rows: list[dict[str, Any]] = []
+ for index, line in enumerate(payload.splitlines(), 1):
+ try:
+ row = json.loads(line.decode("utf-8"))
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+ raise PipelineError("E_INPUT", f"{field} line {index} is invalid") from exc
+ if not isinstance(row, dict):
+ raise PipelineError("E_INPUT", f"{field} line {index} is not an object")
+ if reject_secrets:
+ _reject_secrets(row)
+ rows.append(row)
+ return rows, payload
+
+
+def _load_state(config: Config) -> dict[str, Any]:
+ value, _ = _strict_json(config.state_path, "state")
+ state = _exact(value, {"schema_version", "task_id", "creator_uid", "initialized_at", "cursors", "active_run"}, "state")
+ if (
+ type(state["schema_version"]) is not int
+ or state["schema_version"] != SCHEMA
+ or state["task_id"] != TASK_ID
+ or state["creator_uid"] != config.creator_uid
+ ):
+ raise PipelineError("E_STATE", "state identity differs")
+ cursors = _exact(state["cursors"], {"formal_lines", "formal_sha256", "handoff_lines", "handoff_sha256"}, "cursors")
+ if (
+ type(cursors["formal_lines"]) is not int
+ or cursors["formal_lines"] < 0
+ or type(cursors["handoff_lines"]) is not int
+ or cursors["handoff_lines"] < 0
+ or not isinstance(cursors["formal_sha256"], str)
+ or not SHA256.fullmatch(cursors["formal_sha256"])
+ or not isinstance(cursors["handoff_sha256"], str)
+ or not SHA256.fullmatch(cursors["handoff_sha256"])
+ ):
+ raise PipelineError("E_STATE", "state cursor identity differs")
+ return dict(state)
+
+
+def _write_state(config: Config, state: Mapping[str, Any]) -> None:
+ _atomic(config.state_path, _canonical(state))
+
+
+def _now(value: str | None) -> datetime:
+ if value is None:
+ return datetime.now(timezone.utc)
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise PipelineError("E_TIME", "now is invalid") from exc
+ if parsed.tzinfo is None:
+ raise PipelineError("E_TIME", "now must be offset-aware")
+ return parsed.astimezone(timezone.utc)
+
+
+def _published_at(value: Any, field: str = "published_at") -> datetime:
+ if not isinstance(value, str) or not value:
+ raise PipelineError("E_SOURCE_BINDING", f"{field} differs")
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise PipelineError("E_SOURCE_BINDING", f"{field} differs") from exc
+ if parsed.tzinfo is None:
+ raise PipelineError("E_SOURCE_BINDING", f"{field} differs")
+ return parsed
+
+
+def _canonical_title(value: Any) -> str:
+ if not isinstance(value, str) or not value or "\x00" in value:
+ raise PipelineError("E_SOURCE_BINDING", "video title differs")
+ cleaned = WINDOWS_INVALID_CHARS.sub("_", value)
+ cleaned = re.sub(r"\s+", " ", cleaned).strip(" .")
+ if not cleaned:
+ cleaned = "untitled"
+ if cleaned.upper().split(".", 1)[0] in WINDOWS_RESERVED_NAMES:
+ cleaned = "_" + cleaned
+ cleaned = cleaned[:CANONICAL_TITLE_MAX_LENGTH].rstrip(" .")
+ return cleaned or "untitled"
+
+
+def _canonical_video_base(stable_id: Any, title: Any, published_at: Any) -> str:
+ if not isinstance(stable_id, str) or not BVID.fullmatch(stable_id):
+ raise PipelineError("E_SOURCE_BINDING", "video stable identity differs")
+ published = _published_at(published_at)
+ return f"{published:%Y%m%d-%H%M%S}_video_{_canonical_title(title)}_{stable_id}"
+
+
+def _project_relative(config: Config, path: Path, code: str = "E_ARTIFACT") -> str:
+ target = Path(os.path.abspath(path))
+ if not _within(target, config.project_root):
+ raise PipelineError(code, "path escaped the project root")
+ return target.relative_to(config.project_root).as_posix()
+
+
+def initialize(config: Config, now: datetime) -> dict[str, Any]:
+ with _lock(config):
+ if config.state_path.exists():
+ state = _load_state(config)
+ return {"status": "ALREADY_INITIALIZED", "state": state, "state_identity": _file_identity(config.state_path)}
+ formal = _file_identity(config.formal_manifest)
+ handoff = _file_identity(config.processing_handoffs)
+ state = {
+ "schema_version": SCHEMA, "task_id": TASK_ID, "creator_uid": config.creator_uid,
+ "initialized_at": now.isoformat(),
+ "cursors": {
+ "formal_lines": formal["lines"], "formal_sha256": formal["sha256"],
+ "handoff_lines": handoff["lines"], "handoff_sha256": handoff["sha256"],
+ },
+ "active_run": None,
+ }
+ _write_state(config, state)
+ return {"status": "INITIALIZED", "baseline": {"formal": formal, "handoff": handoff}, "state_identity": _file_identity(config.state_path)}
+
+
+def _run_id(config: Config, slot: int) -> str:
+ return hashlib.sha256(f"bili-half-hour-v1\0{config.creator_uid}\0{slot}".encode("ascii")).hexdigest()
+
+
+def begin(config: Config, now: datetime) -> dict[str, Any]:
+ with _lock(config):
+ state = _load_state(config)
+ slot = int(now.timestamp()) // (INTERVAL_MINUTES * 60)
+ run_id = _run_id(config, slot)
+ active = state["active_run"]
+ if active is not None:
+ if active.get("run_id") == run_id:
+ return {"status": "RUN_RESUMED", "run": active}
+ raise PipelineError("E_RUN_ACTIVE", "a prior half-hour run is still active")
+ event = {
+ "schema_version": SCHEMA, "event": "RUN_STARTED", "run_id": run_id, "slot": slot,
+ "creator_uid": config.creator_uid, "started_at": now.isoformat(),
+ }
+ _append(config.runs_path, event)
+ state["active_run"] = {"run_id": run_id, "slot": slot, "started_at": now.isoformat()}
+ _write_state(config, state)
+ return {"status": "RUN_STARTED", "run": state["active_run"], "dynamic_url": config.dynamic_url}
+
+
+def _journal_source_row(config: Config, source: Mapping[str, Any]) -> tuple[str, Mapping[str, Any]]:
+ if set(source) != {"journal", "line", "sha256"}:
+ raise PipelineError("E_SOURCE_BINDING", "journal source shape differs")
+ journal = source.get("journal")
+ line = source.get("line")
+ digest = source.get("sha256")
+ if journal not in {"formal", "processing_handoff"} or type(line) is not int or line <= 0 or not isinstance(digest, str) or not SHA256.fullmatch(digest):
+ raise PipelineError("E_SOURCE_BINDING", "journal source identity differs")
+ path = config.formal_manifest if journal == "formal" else config.processing_handoffs
+ # Formal and processing journals contain immutable historical audit fields.
+ # Do not treat a legacy key name (for example a handoff identity containing
+ # "authorization") as a credential. Every consumable row is rebound to a
+ # narrow source-controlled projection below, and the projected outbox is
+ # still subject to the normal key/value secret rejection.
+ rows, raw = _read_lines(path, journal, reject_secrets=False)
+ raw_lines = raw.splitlines()
+ if line > len(rows) or _row_digest(raw_lines[line - 1]) != digest:
+ raise PipelineError("E_SOURCE_BINDING", "journal source bytes differ")
+ row = rows[line - 1]
+ if _creator_uid(row) != config.creator_uid:
+ raise PipelineError("E_SOURCE_BINDING", "journal creator differs")
+ return journal, row
+
+
+def _safe_text(value: Any, field: str, *, pattern: re.Pattern[str] | None = None) -> str:
+ if not isinstance(value, str) or not value or "\x00" in value or SECRET_KEY.search(value):
+ raise PipelineError("E_SOURCE_BINDING", f"{field} differs")
+ if pattern is not None and not pattern.fullmatch(value):
+ raise PipelineError("E_SOURCE_BINDING", f"{field} differs")
+ return value
+
+
+def _project_formal_payload(config: Config, kind: str, row: Mapping[str, Any]) -> dict[str, Any]:
+ item_type = row.get("item_type")
+ status_value = row.get("status")
+ stable_id = row.get("stable_id")
+ if kind == "GIT_DELIVERY_READY":
+ if item_type not in CONTENT_TYPES or status_value != "SAVED" or not isinstance(stable_id, str) or not stable_id:
+ raise PipelineError("E_SOURCE_BINDING", "formal content source differs")
+ return {"stable_id": stable_id, "files": _content_artifacts(row, config), "reason": "CONTENT_ARCHIVED"}
+ if kind == "VIDEO_DOWNLOAD_READY":
+ if item_type != "video" or status_value == VIDEO_COMPLETE or not isinstance(stable_id, str) or not BVID.fullmatch(stable_id):
+ raise PipelineError("E_SOURCE_BINDING", "formal video source differs")
+ duration = row.get("expected_duration_seconds")
+ if isinstance(duration, bool) or not isinstance(duration, (int, float)) or not (0 < float(duration) < 86400):
+ raise PipelineError("E_SOURCE_BINDING", "video duration differs")
+ return {
+ "bvid": stable_id,
+ "source_url": _canonical_video_url(row.get("source_url"), stable_id),
+ "title": _safe_text(row.get("title"), "video title"),
+ "published_at": _safe_text(row.get("published_at"), "video publication"),
+ "expected_duration_seconds": duration,
+ }
+ raise PipelineError("E_SOURCE_BINDING", "formal source cannot produce this kind")
+
+
+def _project_handoff_payload(config: Config, row: Mapping[str, Any]) -> dict[str, Any]:
+ common_keys = {
+ "type", "status", "handoff_id", "queue_job_id", "creator_uid", "bvid",
+ "source_url", "media_path", "mapping_path", "bytes", "sha256", "duration_seconds",
+ "video_codec", "audio_codec", "created_at",
+ }
+ schema_keys = set(row) - common_keys
+ if (
+ schema_keys not in ({"schema"}, {"schema_version"})
+ or set(row) != common_keys | schema_keys
+ or row.get("type") != "media-processing-handoff"
+ or row.get("status") != "READY"
+ ):
+ raise PipelineError("E_SOURCE_BINDING", "processing handoff shape differs")
+ schema_value = row[next(iter(schema_keys))]
+ if type(schema_value) is not int or schema_value != SCHEMA or row.get("creator_uid") != config.creator_uid:
+ raise PipelineError("E_SOURCE_BINDING", "processing handoff identity differs")
+ bvid = row.get("bvid")
+ byte_count = row.get("bytes")
+ duration = row.get("duration_seconds")
+ if (
+ not isinstance(bvid, str) or not BVID.fullmatch(bvid)
+ or type(byte_count) is not int or byte_count <= 0
+ or isinstance(duration, bool) or not isinstance(duration, (int, float)) or float(duration) <= 0
+ or not isinstance(row.get("sha256"), str) or not SHA256_MIXED_ASCII.fullmatch(row["sha256"])
+ or not isinstance(row.get("queue_job_id"), str) or not re.fullmatch(r"[0-9a-f]{64}", row["queue_job_id"])
+ ):
+ raise PipelineError("E_SOURCE_BINDING", "processing handoff media identity differs")
+ projected = {
+ "type": "media-processing-handoff", "status": "READY",
+ "handoff_id": _safe_text(row.get("handoff_id"), "handoff id", pattern=SAFE_ID),
+ "queue_job_id": row["queue_job_id"], "creator_uid": config.creator_uid, "bvid": bvid,
+ "source_url": _canonical_video_url(row.get("source_url"), bvid),
+ "media_path": _safe_text(row.get("media_path"), "media path"),
+ "mapping_path": _safe_text(row.get("mapping_path"), "mapping path"),
+ "bytes": byte_count, "sha256": row["sha256"].upper(), "duration_seconds": duration,
+ "video_codec": _safe_text(row.get("video_codec"), "video codec", pattern=SAFE_ID),
+ "audio_codec": _safe_text(row.get("audio_codec"), "audio codec", pattern=SAFE_ID),
+ "created_at": _safe_text(row.get("created_at"), "handoff creation time"),
+ }
+ return projected
+
+
+def _project_relocation_payload(config: Config, source: Mapping[str, Any]) -> dict[str, Any]:
+ if set(source) != {"journal", "path", "bytes", "sha256"} or source.get("journal") != "relocation_report":
+ raise PipelineError("E_SOURCE_BINDING", "relocation source shape differs")
+ relative = source.get("path")
+ size = source.get("bytes")
+ digest = source.get("sha256")
+ if (
+ not isinstance(relative, str)
+ or Path(relative).is_absolute()
+ or type(size) is not int
+ or size <= 0
+ or not isinstance(digest, str)
+ or not SHA256.fullmatch(digest)
+ ):
+ raise PipelineError("E_SOURCE_BINDING", "relocation source identity differs")
+ path = Path(os.path.abspath(config.project_root / Path(relative)))
+ if not _within(path, config.relocation_root):
+ raise PipelineError("E_SOURCE_BINDING", "relocation source escaped its root")
+ payload = _stable_artifact_bytes(path, config.relocation_root)
+ if len(payload) != size or hashlib.sha256(payload).hexdigest().upper() != digest:
+ raise PipelineError("E_SOURCE_BINDING", "relocation source bytes differ")
+ report = _read_relocation_report(config, path)
+ files: list[dict[str, Any]] = []
+ for item in report["items"]:
+ files.extend(
+ {
+ "path": alias["new_path"],
+ "bytes": alias["bytes"],
+ "sha256": alias["sha256"],
+ "kind": alias["kind"],
+ }
+ for alias in item["aliases"]
+ )
+ files.extend(report["docs"])
+ files.append({
+ "path": relative,
+ "bytes": size,
+ "sha256": digest,
+ "kind": "relocation_manifest",
+ })
+ return {
+ "stable_id": report["batch_id"],
+ "files": files,
+ "remove_paths": report["remove_paths"],
+ "reason": "CANONICAL_VIDEO_ARTIFACT_MIGRATION",
+ }
+
+
+def _expected_outbox_payload(config: Config, kind: str, source: Mapping[str, Any]) -> dict[str, Any]:
+ if source.get("journal") == "relocation_report":
+ if kind != "GIT_DELIVERY_READY":
+ raise PipelineError("E_SOURCE_BINDING", "relocation source kind differs")
+ return _project_relocation_payload(config, source)
+ if source.get("journal") in {"formal", "processing_handoff"}:
+ journal, row = _journal_source_row(config, source)
+ if journal == "formal":
+ return _project_formal_payload(config, kind, row)
+ if kind != "VIDEO_TRANSCRIPTION_READY":
+ raise PipelineError("E_SOURCE_BINDING", "handoff source kind differs")
+ return _project_handoff_payload(config, row)
+ allowed = {"journal", "receipt_sha256"}
+ if source.get("projection") is not None:
+ allowed.add("projection")
+ if set(source) != allowed or source.get("journal") != "terminals" or not isinstance(source.get("receipt_sha256"), str) or not SHA256.fullmatch(source["receipt_sha256"]):
+ raise PipelineError("E_SOURCE_BINDING", "terminal source identity differs")
+ terminals = [row for row in _terminal_rows(config) if row.get("receipt_sha256") == source["receipt_sha256"]]
+ if len(terminals) != 1:
+ raise PipelineError("E_SOURCE_BINDING", "terminal source is absent or ambiguous")
+ terminal = terminals[0]
+ files = terminal["files"]
+ stable_id = terminal["stable_id"]
+ projection = source.get("projection")
+ if kind == "MINUTES_READY" and terminal["event"] == "TRANSCRIPTION_COMPLETE" and projection is None:
+ return {"stable_id": stable_id, "transcript_terminal_id": terminal["terminal_id"], "files": files}
+ if kind == "GIT_DELIVERY_READY" and terminal["event"] == "TRANSCRIPTION_COMPLETE" and projection == "transcript":
+ return {"stable_id": stable_id, "files": files, "reason": "TRANSCRIPT_COMPLETE"}
+ if kind == "GIT_DELIVERY_READY" and terminal["event"] == "MINUTES_COMPLETE" and projection == "minutes":
+ return {"stable_id": stable_id, "files": files, "reason": "MINUTES_COMPLETE"}
+ raise PipelineError("E_SOURCE_BINDING", "terminal source projection differs")
+
+
+def _outbox_id(kind: str, source: Mapping[str, Any], payload: Mapping[str, Any]) -> str:
+ material = _canonical({"kind": kind, "source": dict(source), "payload": dict(payload)})
+ return hashlib.sha256(material).hexdigest()
+
+
+def _outbox_rows(config: Config) -> list[dict[str, Any]]:
+ rows, _ = _read_lines(config.outbox_path, "outbox")
+ allowed_kinds = {"GIT_DELIVERY_READY", "VIDEO_DOWNLOAD_READY", "VIDEO_TRANSCRIPTION_READY", "MINUTES_READY"}
+ grouped: dict[str, list[dict[str, Any]]] = {}
+ for row in rows:
+ if type(row.get("schema_version")) is not int or row.get("schema_version") != SCHEMA:
+ raise PipelineError("E_OUTBOX", "outbox schema identity differs")
+ outbox_id = row.get("outbox_id")
+ event = row.get("event")
+ if not isinstance(outbox_id, str) or not re.fullmatch(r"[0-9a-f]{64}", outbox_id):
+ raise PipelineError("E_OUTBOX", "outbox identity differs")
+ if event not in {"CREATED", "DISPATCH_INTENT", "OBSERVED", "GIT_COMMIT_INTENT", "COMMIT_CREATED", "COMPLETE"}:
+ raise PipelineError("E_OUTBOX", "outbox event differs")
+ grouped.setdefault(outbox_id, []).append(row)
+ for outbox_id, events in grouped.items():
+ created = [row for row in events if row.get("event") == "CREATED"]
+ if len(created) != 1 or events[0].get("event") != "CREATED":
+ raise PipelineError("E_OUTBOX", "outbox creation history differs")
+ origin = created[0]
+ if set(origin) != {"schema_version", "event", "outbox_id", "kind", "creator_uid", "source", "payload", "created_at"}:
+ raise PipelineError("E_OUTBOX", "outbox creation shape differs")
+ kind = origin.get("kind")
+ source = origin.get("source")
+ if (
+ kind not in allowed_kinds
+ or origin.get("creator_uid") != config.creator_uid
+ or not isinstance(source, dict)
+ or not isinstance(origin.get("payload"), dict)
+ or _outbox_id(kind, source, origin["payload"]) != outbox_id
+ or not isinstance(origin.get("created_at"), str)
+ ):
+ raise PipelineError("E_OUTBOX", "outbox creation binding differs")
+ expected_payload = _expected_outbox_payload(config, kind, source)
+ if origin["payload"] != expected_payload:
+ raise PipelineError("E_OUTBOX", "outbox payload differs from its immutable source")
+ counts = {name: sum(row.get("event") == name for row in events) for name in {"DISPATCH_INTENT", "OBSERVED", "GIT_COMMIT_INTENT", "COMMIT_CREATED", "COMPLETE"}}
+ if any(value > 1 for value in counts.values()):
+ raise PipelineError("E_OUTBOX", "outbox event is duplicated")
+ intent = next((row for row in events if row.get("event") == "DISPATCH_INTENT"), None)
+ observed = next((row for row in events if row.get("event") == "OBSERVED"), None)
+ git_intent = next((row for row in events if row.get("event") == "GIT_COMMIT_INTENT"), None)
+ commit = next((row for row in events if row.get("event") == "COMMIT_CREATED"), None)
+ complete = next((row for row in events if row.get("event") == "COMPLETE"), None)
+ if intent is not None:
+ if (
+ kind == "GIT_DELIVERY_READY"
+ or set(intent) != {"schema_version", "event", "outbox_id", "kind", "target_thread_id", "created_at"}
+ or intent.get("kind") != kind
+ or intent.get("target_thread_id") != _dispatch_target(config, kind)
+ or not isinstance(intent.get("created_at"), str)
+ ):
+ raise PipelineError("E_OUTBOX", "dispatch intent binding differs")
+ if observed is not None:
+ if (
+ intent is None
+ or set(observed) != {"schema_version", "event", "outbox_id", "delivery_id", "observed_at"}
+ or not isinstance(observed.get("delivery_id"), str)
+ or not SAFE_ID.fullmatch(observed["delivery_id"])
+ or not isinstance(observed.get("observed_at"), str)
+ ):
+ raise PipelineError("E_OUTBOX", "dispatch observation binding differs")
+ if git_intent is not None:
+ if (
+ kind != "GIT_DELIVERY_READY"
+ or set(git_intent) != {
+ "schema_version", "event", "outbox_id", "parent_sha", "tree_sha", "files",
+ "message", "author_name", "author_email", "authored_at", "created_at",
+ }
+ or not isinstance(git_intent.get("files"), list)
+ or not isinstance(git_intent.get("message"), str)
+ or not isinstance(git_intent.get("author_name"), str)
+ or not isinstance(git_intent.get("author_email"), str)
+ or not isinstance(git_intent.get("authored_at"), str)
+ or not isinstance(git_intent.get("created_at"), str)
+ or git_intent.get("files") != _git_expected_paths(origin.get("payload", {}))
+ ):
+ raise PipelineError("E_OUTBOX", "Git commit intent binding differs")
+ for field in ("parent_sha", "tree_sha"):
+ if not isinstance(git_intent.get(field), str) or not re.fullmatch(r"[0-9a-f]{40,64}", git_intent[field]):
+ raise PipelineError("E_OUTBOX", "Git commit intent identity differs")
+ if commit is not None:
+ if (
+ kind != "GIT_DELIVERY_READY"
+ or git_intent is None
+ or set(commit) != {"schema_version", "event", "outbox_id", "parent_sha", "tree_sha", "commit_sha", "intent_sha256", "files", "created_at"}
+ or not isinstance(commit.get("files"), list)
+ or not isinstance(commit.get("created_at"), str)
+ or commit.get("parent_sha") != git_intent.get("parent_sha")
+ or commit.get("tree_sha") != git_intent.get("tree_sha")
+ or commit.get("files") != git_intent.get("files")
+ or commit.get("intent_sha256") != hashlib.sha256(_canonical(git_intent)).hexdigest().upper()
+ ):
+ raise PipelineError("E_OUTBOX", "Git commit binding differs")
+ for field in ("parent_sha", "tree_sha", "commit_sha"):
+ if not isinstance(commit.get(field), str) or not re.fullmatch(r"[0-9a-f]{40,64}", commit[field]):
+ raise PipelineError("E_OUTBOX", "Git commit identity differs")
+ if complete is not None:
+ result = complete.get("result")
+ if not isinstance(complete.get("completed_at"), str):
+ raise PipelineError("E_OUTBOX", "outbox completion time differs")
+ if kind == "GIT_DELIVERY_READY":
+ expected = {"schema_version", "event", "outbox_id", "result", "completed_at"}
+ if result == "PUSHED":
+ expected.add("commit_sha")
+ if set(complete) != expected or result not in {"PUSHED", "NO_CHANGES"}:
+ raise PipelineError("E_OUTBOX", "Git completion binding differs")
+ if result == "PUSHED" and (commit is None or complete.get("commit_sha") != commit.get("commit_sha")):
+ raise PipelineError("E_OUTBOX", "Git completion commit differs")
+ else:
+ if (
+ intent is None or observed is None
+ or set(complete) != {"schema_version", "event", "outbox_id", "result", "terminal_id", "receipt_sha256", "completed_at"}
+ or result != ({"VIDEO_TRANSCRIPTION_READY": "TRANSCRIPTION_COMPLETE", "MINUTES_READY": "MINUTES_COMPLETE"}.get(kind))
+ or not isinstance(complete.get("terminal_id"), str)
+ or not SAFE_ID.fullmatch(complete["terminal_id"])
+ or not isinstance(complete.get("receipt_sha256"), str)
+ or not SHA256.fullmatch(complete["receipt_sha256"])
+ ):
+ raise PipelineError("E_OUTBOX", "role completion binding differs")
+ sequence = [row["event"] for row in events]
+ if kind == "GIT_DELIVERY_READY":
+ if intent is not None or observed is not None:
+ raise PipelineError("E_OUTBOX", "Git outbox contains role events")
+ if git_intent is not None and sequence.index("GIT_COMMIT_INTENT") <= 0:
+ raise PipelineError("E_OUTBOX", "Git intent order differs")
+ if commit is not None and sequence.index("COMMIT_CREATED") <= sequence.index("GIT_COMMIT_INTENT"):
+ raise PipelineError("E_OUTBOX", "Git commit order differs")
+ if complete is not None and sequence.index("COMPLETE") != len(sequence) - 1:
+ raise PipelineError("E_OUTBOX", "Git completion order differs")
+ else:
+ if git_intent is not None or commit is not None:
+ raise PipelineError("E_OUTBOX", "role outbox contains Git events")
+ if intent is not None and sequence.index("DISPATCH_INTENT") <= 0:
+ raise PipelineError("E_OUTBOX", "dispatch intent order differs")
+ if observed is not None and (intent is None or sequence.index("OBSERVED") <= sequence.index("DISPATCH_INTENT")):
+ raise PipelineError("E_OUTBOX", "dispatch observation order differs")
+ if complete is not None and (observed is None or sequence.index("COMPLETE") <= sequence.index("OBSERVED") or sequence.index("COMPLETE") != len(sequence) - 1):
+ raise PipelineError("E_OUTBOX", "role completion order differs")
+ return rows
+
+
+def _append_outbox(config: Config, kind: str, source: Mapping[str, Any], payload: Mapping[str, Any], created_at: str) -> str:
+ expected_payload = _expected_outbox_payload(config, kind, source)
+ if dict(payload) != expected_payload:
+ raise PipelineError("E_OUTBOX", "new outbox payload differs from its immutable source")
+ outbox_id = _outbox_id(kind, source, payload)
+ rows = _outbox_rows(config)
+ same_source = [
+ row for row in rows
+ if row.get("event") == "CREATED" and row.get("kind") == kind and row.get("source") == dict(source)
+ ]
+ if same_source and not (
+ len(same_source) == 1
+ and same_source[0].get("outbox_id") == outbox_id
+ and same_source[0].get("payload") == dict(payload)
+ ):
+ raise PipelineError("E_OUTBOX", "source identity is already bound to another payload")
+ prior = next((row for row in rows if row.get("event") == "CREATED" and row.get("outbox_id") == outbox_id), None)
+ if prior is not None:
+ if prior.get("kind") != kind or prior.get("source") != dict(source) or prior.get("payload") != dict(payload) or prior.get("creator_uid") != config.creator_uid:
+ raise PipelineError("E_OUTBOX", "existing outbox identity differs")
+ return outbox_id
+ event = {
+ "schema_version": SCHEMA, "event": "CREATED", "outbox_id": outbox_id,
+ "kind": kind, "creator_uid": config.creator_uid, "source": dict(source),
+ "payload": dict(payload), "created_at": created_at,
+ }
+ _append(config.outbox_path, event)
+ return outbox_id
+
+
+def _row_digest(raw_line: bytes) -> str:
+ return hashlib.sha256(raw_line).hexdigest().upper()
+
+
+def _prefix_digest(payload: bytes, lines: int, field: str) -> str:
+ if type(lines) is not int or lines < 0:
+ raise PipelineError("E_STATE", f"{field} cursor line count differs")
+ chunks = payload.splitlines(keepends=True)
+ if len(chunks) < lines:
+ raise PipelineError("E_HISTORY_REWRITE", f"{field} lost rows")
+ return hashlib.sha256(b"".join(chunks[:lines])).hexdigest().upper()
+
+
+def _creator_uid(row: Mapping[str, Any]) -> str | None:
+ value = row.get("creator_uid")
+ if type(value) is int:
+ return str(value)
+ if isinstance(value, str):
+ return value
+ creator = row.get("creator")
+ if isinstance(creator, dict) and isinstance(creator.get("uid"), str):
+ return creator["uid"]
+ return None
+
+
+def _is_reparse(info: os.stat_result) -> bool:
+ return bool(getattr(info, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400))
+
+
+def _stat_identity(info: os.stat_result) -> tuple[int, ...]:
+ return (
+ int(info.st_dev), int(info.st_ino), int(info.st_mode), int(info.st_nlink), int(info.st_size),
+ int(info.st_mtime_ns), int(info.st_ctime_ns), int(getattr(info, "st_file_attributes", 0)),
+ )
+
+
+def _path_handle_identity(info: os.stat_result) -> tuple[int, ...]:
+ identity = _stat_identity(info)
+ return identity[:6] + identity[7:]
+
+
+def _chain_identity(info: os.stat_result, *, final_file: bool) -> tuple[int, ...]:
+ if final_file:
+ return _stat_identity(info)
+ return (int(info.st_dev), int(info.st_ino), int(info.st_mode), int(getattr(info, "st_file_attributes", 0)))
+
+
+def _strict_chain(root: Path, target: Path, *, final_file: bool) -> tuple[tuple[str, tuple[int, ...]], ...]:
+ root = Path(os.path.abspath(root))
+ target = Path(os.path.abspath(target))
+ if not _within(target, root):
+ raise PipelineError("E_ARTIFACT", "path escaped its governed root")
+ root_real = Path(os.path.realpath(root))
+ target_real = Path(os.path.realpath(target))
+ if not _within(target_real, root_real) or os.path.normcase(str(target_real)) != os.path.normcase(str(target)):
+ raise PipelineError("E_ARTIFACT", "path resolution escaped or drifted")
+ anchor = Path(target.anchor)
+ paths: list[Path] = []
+ current = anchor
+ if str(anchor):
+ paths.append(anchor)
+ for part in target.parts[1:] if str(anchor) else target.parts:
+ current = current / part
+ paths.append(current)
+ snapshots: list[tuple[str, tuple[int, ...]]] = []
+ for index, item in enumerate(paths):
+ try:
+ info = os.lstat(item)
+ except OSError as exc:
+ raise PipelineError("E_ARTIFACT", "governed path is unavailable") from exc
+ if stat.S_ISLNK(info.st_mode) or _is_reparse(info):
+ raise PipelineError("E_ARTIFACT", "governed path contains a reparse object")
+ is_final = index == len(paths) - 1
+ if (is_final and final_file and not stat.S_ISREG(info.st_mode)) or ((not is_final or not final_file) and not stat.S_ISDIR(info.st_mode)):
+ raise PipelineError("E_ARTIFACT", "governed path object type differs")
+ snapshots.append((os.path.normcase(str(item)), _chain_identity(info, final_file=is_final and final_file)))
+ return tuple(snapshots)
+
+
+def _stable_artifact_bytes(target: Path, root: Path) -> bytes:
+ before = _strict_chain(root, target, final_file=True)
+ flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
+ try:
+ descriptor = os.open(target, flags)
+ except OSError as exc:
+ raise PipelineError("E_ARTIFACT", "artifact cannot be opened safely") from exc
+ try:
+ opened_before = os.fstat(descriptor)
+ if _path_handle_identity(opened_before) != before[-1][1][:6] + before[-1][1][7:] or not stat.S_ISREG(opened_before.st_mode) or _is_reparse(opened_before):
+ raise PipelineError("E_ARTIFACT", "artifact path and handle differ")
+ with os.fdopen(descriptor, "rb", closefd=False) as stream:
+ payload = stream.read()
+ opened_after = os.fstat(descriptor)
+ after = _strict_chain(root, target, final_file=True)
+ if _stat_identity(opened_before) != _stat_identity(opened_after) or before != after:
+ raise PipelineError("E_ARTIFACT", "artifact identity drifted during read")
+ return payload
+ finally:
+ os.close(descriptor)
+
+
+def _relocation_batch_id(report: Mapping[str, Any]) -> str:
+ material = {
+ "schema_version": report.get("schema_version"),
+ "type": report.get("type"),
+ "task_id": report.get("task_id"),
+ "creator_uid": report.get("creator_uid"),
+ "baseline_head": report.get("baseline_head"),
+ "items": report.get("items"),
+ "docs": report.get("docs"),
+ "remove_paths": report.get("remove_paths"),
+ }
+ return hashlib.sha256(_canonical(material)).hexdigest().upper()
+
+
+def _read_relocation_report(config: Config, path: Path) -> dict[str, Any]:
+ _strict_chain(config.relocation_root, path, final_file=True)
+ value, _ = _strict_json(path, "artifact relocation report")
+ report = _exact(
+ value,
+ {
+ "schema_version", "type", "task_id", "creator_uid", "batch_id", "created_at",
+ "baseline_head", "items", "docs", "remove_paths",
+ },
+ "artifact relocation report",
+ )
+ if (
+ type(report["schema_version"]) is not int
+ or report["schema_version"] != SCHEMA
+ or report["type"] != RELOCATION_REPORT_TYPE
+ or report["task_id"] != TASK_ID
+ or report["creator_uid"] != config.creator_uid
+ or not isinstance(report["baseline_head"], str)
+ or not re.fullmatch(r"[0-9a-f]{40,64}", report["baseline_head"])
+ or not isinstance(report["batch_id"], str)
+ or not SHA256.fullmatch(report["batch_id"])
+ or report["batch_id"] != _relocation_batch_id(report)
+ or path.name != f"{report['batch_id']}.json"
+ or not isinstance(report["created_at"], str)
+ or not isinstance(report["items"], list)
+ or not report["items"]
+ or not isinstance(report["docs"], list)
+ or not isinstance(report["remove_paths"], list)
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation report identity differs")
+ aliases: list[dict[str, Any]] = []
+ stable_ids: set[str] = set()
+ item_ids: list[str] = []
+ for item_value in report["items"]:
+ item = _exact(
+ item_value,
+ {"stable_id", "title", "published_at", "canonical_base", "aliases", "intermediates"},
+ "artifact relocation item",
+ )
+ stable_id = item["stable_id"]
+ if (
+ not isinstance(stable_id, str)
+ or not BVID.fullmatch(stable_id)
+ or stable_id in stable_ids
+ or item["canonical_base"] != _canonical_video_base(stable_id, item["title"], item["published_at"])
+ or not isinstance(item["aliases"], list)
+ or not item["aliases"]
+ or not isinstance(item["intermediates"], list)
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation item identity differs")
+ stable_ids.add(stable_id)
+ item_ids.append(stable_id)
+ canonical_base = item["canonical_base"]
+ expected_alias_paths = {
+ "transcript_txt": (
+ config.archive_root / f"{stable_id}.transcript" / f"{stable_id}.txt",
+ config.archive_root / f"{canonical_base}.transcript" / f"{canonical_base}.txt",
+ ),
+ "transcript_srt": (
+ config.archive_root / f"{stable_id}.transcript" / f"{stable_id}.srt",
+ config.archive_root / f"{canonical_base}.transcript" / f"{canonical_base}.srt",
+ ),
+ "transcript_json": (
+ config.archive_root / f"{stable_id}.transcript" / f"{stable_id}.json",
+ config.archive_root / f"{canonical_base}.transcript" / f"{canonical_base}.json",
+ ),
+ "minutes_md": (
+ config.archive_root / f"{stable_id}.minutes" / f"{stable_id}.md",
+ config.archive_root / f"{canonical_base}.minutes" / f"{canonical_base}.md",
+ ),
+ "minutes_pdf": (
+ config.archive_root / f"{stable_id}.minutes" / f"{stable_id}.pdf",
+ config.archive_root / f"{canonical_base}.minutes" / f"{canonical_base}.pdf",
+ ),
+ }
+ item_kinds: set[str] = set()
+ for alias_value in item["aliases"]:
+ alias = _exact(alias_value, {"kind", "old_path", "new_path", "bytes", "sha256"}, "artifact relocation alias")
+ kind = alias["kind"]
+ if (
+ kind not in PUBLIC_VIDEO_KINDS
+ or not isinstance(alias["old_path"], str)
+ or not isinstance(alias["new_path"], str)
+ or alias["old_path"] == alias["new_path"]
+ or type(alias["bytes"]) is not int
+ or alias["bytes"] < 0
+ or not isinstance(alias["sha256"], str)
+ or not SHA256.fullmatch(alias["sha256"])
+ or Path(alias["new_path"]).suffix.lower() != PUBLIC_VIDEO_KINDS[kind]
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation alias differs")
+ if kind in item_kinds:
+ raise PipelineError("E_RELOCATION", "artifact relocation kind is duplicated")
+ item_kinds.add(kind)
+ expected_old, expected_new = expected_alias_paths[kind]
+ if (
+ alias["old_path"] != _project_relative(config, expected_old)
+ or alias["new_path"] != _project_relative(config, expected_new)
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation path grammar differs")
+ for field in ("old_path", "new_path"):
+ target = Path(os.path.abspath(config.project_root / Path(alias[field])))
+ if Path(alias[field]).is_absolute() or not _within(target, config.archive_root):
+ raise PipelineError("E_RELOCATION", "artifact relocation path escaped the archive")
+ aliases.append(dict(alias))
+ transcript_kinds = {kind for kind in item_kinds if kind.startswith("transcript_")}
+ minutes_kinds = {kind for kind in item_kinds if kind.startswith("minutes_")}
+ if transcript_kinds != {"transcript_txt", "transcript_srt", "transcript_json"} or minutes_kinds not in (
+ set(), {"minutes_md", "minutes_pdf"},
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation kind set differs")
+ if len(item["intermediates"]) > 1:
+ raise PipelineError("E_RELOCATION", "artifact relocation intermediate set differs")
+ for intermediate_value in item["intermediates"]:
+ intermediate = _exact(
+ intermediate_value,
+ {"kind", "old_path", "new_path", "bytes", "sha256"},
+ "artifact relocation intermediate",
+ )
+ if (
+ intermediate["kind"] != "audio_flac"
+ or not isinstance(intermediate["old_path"], str)
+ or not isinstance(intermediate["new_path"], str)
+ or type(intermediate["bytes"]) is not int
+ or intermediate["bytes"] <= 0
+ or not isinstance(intermediate["sha256"], str)
+ or not SHA256.fullmatch(intermediate["sha256"])
+ or not str(intermediate["new_path"]).lower().endswith(".audio.flac")
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation intermediate differs")
+ expected_old = config.archive_root / f"{stable_id}.transcript" / f"{stable_id}.audio.flac"
+ expected_new = config.video_root / "intermediate" / "transcription" / f"{stable_id}.audio.flac"
+ if (
+ intermediate["old_path"] != _project_relative(config, expected_old)
+ or os.path.normcase(intermediate["new_path"]) != os.path.normcase(str(Path(os.path.abspath(expected_new))))
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation intermediate grammar differs")
+ old_target = Path(os.path.abspath(config.project_root / Path(intermediate["old_path"])))
+ new_target = Path(os.path.abspath(Path(intermediate["new_path"])))
+ if Path(intermediate["old_path"]).is_absolute() or not _within(old_target, config.archive_root) or not _within(new_target, config.video_root):
+ raise PipelineError("E_RELOCATION", "artifact relocation intermediate escaped its boundary")
+ if item_ids != sorted(item_ids):
+ raise PipelineError("E_RELOCATION", "artifact relocation item order differs")
+ old_paths = [alias["old_path"] for alias in aliases]
+ new_paths = [alias["new_path"] for alias in aliases]
+ if len(set(old_paths)) != len(old_paths) or len(set(new_paths)) != len(new_paths):
+ raise PipelineError("E_RELOCATION", "artifact relocation aliases are duplicated")
+ if (
+ report["remove_paths"] != sorted(set(report["remove_paths"]))
+ or any(value not in old_paths for value in report["remove_paths"])
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation removal scope differs")
+ docs: list[dict[str, Any]] = []
+ for value in report["docs"]:
+ doc = _exact(value, {"path", "bytes", "sha256", "kind"}, "artifact relocation document")
+ if (
+ doc["kind"] != "documentation"
+ or doc["path"] not in config.git_doc_paths
+ or type(doc["bytes"]) is not int
+ or doc["bytes"] <= 0
+ or not isinstance(doc["sha256"], str)
+ or not SHA256.fullmatch(doc["sha256"])
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation document differs")
+ docs.append(dict(doc))
+ if (
+ len({value["path"] for value in docs}) != len(docs)
+ or [value["path"] for value in docs] != sorted(config.git_doc_paths)
+ ):
+ raise PipelineError("E_RELOCATION", "artifact relocation document set differs")
+ return dict(report)
+
+
+def _relocation_reports(config: Config) -> list[tuple[Path, dict[str, Any]]]:
+ if not config.relocation_root.exists():
+ return []
+ _strict_chain(config.archive_root, config.relocation_root, final_file=False)
+ values: list[tuple[Path, dict[str, Any]]] = []
+ for path in sorted(config.relocation_root.glob("*.json"), key=lambda value: value.name):
+ values.append((path, _read_relocation_report(config, path)))
+ return values
+
+
+def _relocated_artifact_target(
+ config: Config,
+ old_relative: str,
+ expected_bytes: int,
+ expected_sha256: str,
+) -> Path | None:
+ matches: list[dict[str, Any]] = []
+ for _, report in _relocation_reports(config):
+ for item in report["items"]:
+ matches.extend(
+ alias for alias in item["aliases"]
+ if alias["old_path"] == old_relative
+ and alias["bytes"] == expected_bytes
+ and alias["sha256"] == expected_sha256.upper()
+ )
+ if not matches:
+ return None
+ if len(matches) != 1:
+ raise PipelineError("E_RELOCATION", "artifact relocation alias is ambiguous")
+ return Path(os.path.abspath(config.project_root / Path(matches[0]["new_path"])))
+
+
+def _artifact(path_value: Any, bytes_value: Any, sha_value: Any, config: Config) -> dict[str, Any]:
+ if not isinstance(path_value, str) or not path_value or type(bytes_value) is not int or bytes_value < 0 or not isinstance(sha_value, str) or not SHA256.fullmatch(sha_value.upper()):
+ raise PipelineError("E_ARTIFACT", "artifact identity is incomplete")
+ path = Path(path_value)
+ if path.is_absolute():
+ requested = Path(os.path.abspath(path))
+ targets = [requested] if requested.exists() or requested.is_symlink() else []
+ if not targets and _within(requested, config.archive_root):
+ relocated = _relocated_artifact_target(
+ config,
+ requested.relative_to(config.project_root).as_posix(),
+ bytes_value,
+ sha_value.upper(),
+ )
+ if relocated is not None:
+ targets.append(relocated)
+ else:
+ candidates = [Path(os.path.abspath(config.archive_root / path)), Path(os.path.abspath(config.project_root / path))]
+ requested_candidates = [candidate for candidate in candidates if _within(candidate, config.archive_root)]
+ targets = [candidate for candidate in requested_candidates if candidate.exists() or candidate.is_symlink()]
+ if not targets:
+ for candidate in requested_candidates:
+ relocated = _relocated_artifact_target(
+ config,
+ candidate.relative_to(config.project_root).as_posix(),
+ bytes_value,
+ sha_value.upper(),
+ )
+ if relocated is not None:
+ targets.append(relocated)
+ requested = requested_candidates[0] if len(requested_candidates) == 1 else None
+ distinct = {os.path.normcase(str(candidate)): candidate for candidate in targets}
+ if len(distinct) != 1:
+ raise PipelineError("E_ARTIFACT", "artifact path is absent or ambiguous")
+ target = next(iter(distinct.values()))
+ if not _within(target, config.archive_root) or any(str(target).lower().endswith(suffix) for suffix in FORBIDDEN_GIT_SUFFIXES):
+ raise PipelineError("E_ARTIFACT", "artifact path is outside the Git boundary")
+ payload = _stable_artifact_bytes(target, config.archive_root)
+ digest = hashlib.sha256(payload).hexdigest().upper()
+ if len(payload) != bytes_value or digest != sha_value.upper():
+ raise PipelineError("E_ARTIFACT", "artifact readback differs")
+ if path.is_absolute():
+ relative = requested.relative_to(config.project_root).as_posix()
+ else:
+ logical_candidates = [
+ candidate.relative_to(config.project_root).as_posix()
+ for candidate in requested_candidates
+ if _relocated_artifact_target(config, candidate.relative_to(config.project_root).as_posix(), bytes_value, sha_value.upper()) == target
+ or candidate == target
+ ]
+ if len(set(logical_candidates)) != 1:
+ raise PipelineError("E_ARTIFACT", "artifact logical path is ambiguous")
+ relative = logical_candidates[0]
+ if target.suffix.lower() not in config.git_extensions:
+ raise PipelineError("E_ARTIFACT", "artifact extension is not allowlisted")
+ return {"path": relative, "bytes": len(payload), "sha256": digest}
+
+
+def _content_artifacts(row: Mapping[str, Any], config: Config) -> list[dict[str, Any]]:
+ artifacts = [_artifact(row.get("path"), row.get("bytes"), row.get("sha256"), config)]
+ if row.get("image_path") is not None:
+ artifacts.append(_artifact(row.get("image_path"), row.get("image_bytes"), row.get("image_sha256"), config))
+ images = row.get("images")
+ if images is not None:
+ if not isinstance(images, list):
+ raise PipelineError("E_ARTIFACT", "images is not a list")
+ for image in images:
+ if not isinstance(image, dict):
+ raise PipelineError("E_ARTIFACT", "image identity is invalid")
+ candidate = _artifact(image.get("path"), image.get("bytes"), image.get("sha256"), config)
+ if candidate not in artifacts:
+ artifacts.append(candidate)
+ return artifacts
+
+
+def reconcile(config: Config, now: datetime) -> dict[str, Any]:
+ with _lock(config):
+ state = _load_state(config)
+ active = state["active_run"]
+ if active is None:
+ raise PipelineError("E_NO_ACTIVE_RUN", "begin is required before reconcile")
+ formal_rows, formal_payload = _read_lines(
+ config.formal_manifest, "formal manifest", reject_secrets=False
+ )
+ handoff_rows, handoff_payload = _read_lines(
+ config.processing_handoffs, "processing handoff", reject_secrets=False
+ )
+ cursors = state["cursors"]
+ if (
+ _prefix_digest(formal_payload, cursors["formal_lines"], "formal manifest") != cursors["formal_sha256"]
+ or _prefix_digest(handoff_payload, cursors["handoff_lines"], "processing handoff") != cursors["handoff_sha256"]
+ ):
+ raise PipelineError("E_HISTORY_REWRITE", "append-only input prefix changed")
+ created: list[str] = []
+ formal_raw = formal_payload.splitlines()
+ for index in range(cursors["formal_lines"], len(formal_rows)):
+ row = formal_rows[index]
+ if _creator_uid(row) != config.creator_uid:
+ raise PipelineError("E_CREATOR", "new formal row creator differs")
+ source = {"journal": "formal", "line": index + 1, "sha256": _row_digest(formal_raw[index])}
+ item_type = row.get("item_type")
+ if item_type in CONTENT_TYPES and row.get("status") == "SAVED":
+ payload = _expected_outbox_payload(config, "GIT_DELIVERY_READY", source)
+ created.append(_append_outbox(config, "GIT_DELIVERY_READY", source, payload, now.isoformat()))
+ elif item_type == "video" and row.get("status") != VIDEO_COMPLETE:
+ payload = _expected_outbox_payload(config, "VIDEO_DOWNLOAD_READY", source)
+ created.append(_append_outbox(config, "VIDEO_DOWNLOAD_READY", source, payload, now.isoformat()))
+ handoff_raw = handoff_payload.splitlines()
+ for index in range(cursors["handoff_lines"], len(handoff_rows)):
+ row = handoff_rows[index]
+ if _creator_uid(row) != config.creator_uid or row.get("status") != "READY":
+ raise PipelineError("E_HANDOFF", "processing handoff identity differs")
+ stable_id = row.get("bvid")
+ if not isinstance(stable_id, str) or not BVID.fullmatch(stable_id):
+ raise PipelineError("E_HANDOFF", "processing handoff BVID differs")
+ source = {"journal": "processing_handoff", "line": index + 1, "sha256": _row_digest(handoff_raw[index])}
+ payload = _expected_outbox_payload(config, "VIDEO_TRANSCRIPTION_READY", source)
+ created.append(_append_outbox(config, "VIDEO_TRANSCRIPTION_READY", source, payload, now.isoformat()))
+ state["cursors"] = {
+ "formal_lines": len(formal_rows), "formal_sha256": hashlib.sha256(formal_payload).hexdigest().upper(),
+ "handoff_lines": len(handoff_rows), "handoff_sha256": hashlib.sha256(handoff_payload).hexdigest().upper(),
+ }
+ _write_state(config, state)
+ return {"status": "RECONCILED", "run_id": active["run_id"], "created_outbox_ids": sorted(set(created)), "cursors": state["cursors"]}
+
+
+def pending(config: Config) -> dict[str, Any]:
+ with _lock(config):
+ rows = _outbox_rows(config)
+ latest: dict[str, str] = {}
+ created: dict[str, dict[str, Any]] = {}
+ for row in rows:
+ outbox_id = row.get("outbox_id")
+ if isinstance(outbox_id, str):
+ latest[outbox_id] = str(row.get("event"))
+ if row.get("event") == "CREATED":
+ created[outbox_id] = row
+ values = []
+ for key in sorted(created):
+ state = latest.get(key)
+ if state not in {"CREATED", "DISPATCH_INTENT"}:
+ continue
+ values.append({**created[key], "delivery_state": state})
+ return {"status": "PENDING", "count": len(values), "items": values}
+
+
+def _dispatch_target(config: Config, kind: str) -> str:
+ targets = {
+ "VIDEO_DOWNLOAD_READY": config.video_downloader_thread_id,
+ "VIDEO_TRANSCRIPTION_READY": config.media_thread_id,
+ "MINUTES_READY": config.minutes_thread_id,
+ }
+ target = targets.get(kind)
+ if target is None:
+ raise PipelineError("E_DISPATCH_KIND", "outbox item is not a role handoff")
+ return target
+
+
+def _dispatch_envelope(config: Config, created: Mapping[str, Any]) -> dict[str, Any]:
+ return {
+ "schema_version": SCHEMA,
+ "type": created["kind"],
+ "outbox_id": created["outbox_id"],
+ "creator_uid": config.creator_uid,
+ "payload": created["payload"],
+ }
+
+
+def dispatch_intent(config: Config, outbox_id: str, now: datetime) -> dict[str, Any]:
+ with _lock(config):
+ rows = _outbox_rows(config)
+ matches = [row for row in rows if row.get("outbox_id") == outbox_id]
+ created = next((row for row in matches if row.get("event") == "CREATED"), None)
+ if created is None:
+ raise PipelineError("E_OUTBOX", "outbox item is unknown")
+ kind = created["kind"]
+ target = _dispatch_target(config, kind)
+ envelope = _dispatch_envelope(config, created)
+ observed = next((row for row in matches if row.get("event") == "OBSERVED"), None)
+ completed = next((row for row in matches if row.get("event") == "COMPLETE"), None)
+ if completed is not None:
+ return {
+ "status": "DISPATCH_ALREADY_COMPLETE", "outbox_id": outbox_id,
+ "target_thread_id": target, "envelope": envelope,
+ }
+ if observed is not None:
+ return {
+ "status": "DISPATCH_ALREADY_OBSERVED", "outbox_id": outbox_id,
+ "target_thread_id": target, "delivery_id": observed["delivery_id"], "envelope": envelope,
+ }
+ prior = next((row for row in matches if row.get("event") == "DISPATCH_INTENT"), None)
+ if prior is not None:
+ if prior.get("kind") != kind or prior.get("target_thread_id") != target:
+ raise PipelineError("E_OUTBOX", "dispatch intent identity drifted")
+ return {
+ "status": "DISPATCH_INTENT_RESUMED", "outbox_id": outbox_id,
+ "target_thread_id": target, "envelope": envelope,
+ }
+ event = {"schema_version": SCHEMA, "event": "DISPATCH_INTENT", "outbox_id": outbox_id, "kind": kind, "target_thread_id": target, "created_at": now.isoformat()}
+ _append(config.outbox_path, event)
+ return {
+ "status": "DISPATCH_INTENT_DURABLE", "outbox_id": outbox_id,
+ "target_thread_id": target, "envelope": envelope,
+ }
+
+
+def observe_dispatch(config: Config, outbox_id: str, delivery_id: str, now: datetime) -> dict[str, Any]:
+ if not isinstance(delivery_id, str) or not SAFE_ID.fullmatch(delivery_id):
+ raise PipelineError("E_DISPATCH_RECEIPT", "delivery identity differs")
+ with _lock(config):
+ rows = _outbox_rows(config)
+ matches = [row for row in rows if row.get("outbox_id") == outbox_id]
+ if not any(row.get("event") == "DISPATCH_INTENT" for row in matches):
+ raise PipelineError("E_DISPATCH_RECEIPT", "dispatch intent is absent")
+ if any(row.get("event") == "COMPLETE" for row in matches):
+ raise PipelineError("E_DISPATCH_RECEIPT", "completed dispatch cannot accept a late observation")
+ observed = [row for row in matches if row.get("event") == "OBSERVED"]
+ if observed:
+ if len(observed) == 1 and observed[0].get("delivery_id") == delivery_id:
+ return {"status": "DISPATCH_ALREADY_OBSERVED", "outbox_id": outbox_id, "delivery_id": delivery_id}
+ raise PipelineError("E_DISPATCH_RECEIPT", "dispatch receipt conflicts")
+ event = {
+ "schema_version": SCHEMA, "event": "OBSERVED", "outbox_id": outbox_id,
+ "delivery_id": delivery_id, "observed_at": now.isoformat(),
+ }
+ _append(config.outbox_path, event)
+ return {"status": "DISPATCH_OBSERVED", "outbox_id": outbox_id, "delivery_id": delivery_id}
+
+
+def _receipt_files(value: Any, config: Config) -> list[dict[str, Any]]:
+ if not isinstance(value, list) or not value:
+ raise PipelineError("E_RECEIPT", "receipt files are empty")
+ files: list[dict[str, Any]] = []
+ for item in value:
+ item = _exact(item, {"path", "bytes", "sha256", "kind"}, "receipt file")
+ if not isinstance(item["kind"], str) or not SAFE_ID.fullmatch(item["kind"]):
+ raise PipelineError("E_RECEIPT", "receipt file kind differs")
+ files.append({**_artifact(item["path"], item["bytes"], item["sha256"], config), "kind": item["kind"]})
+ if len({item["path"] for item in files}) != len(files):
+ raise PipelineError("E_RECEIPT", "receipt files are duplicated")
+ return files
+
+
+def _terminal_rows(config: Config) -> list[dict[str, Any]]:
+ rows, _ = _read_lines(config.terminals_path, "terminals")
+ seen_sources: set[str] = set()
+ seen_terminals: set[str] = set()
+ for row in rows:
+ if set(row) != {
+ "schema_version", "event", "source_outbox_id", "stable_id", "terminal_id",
+ "receipt_bytes", "receipt_sha256", "files", "committed_at",
+ }:
+ raise PipelineError("E_RECEIPT_CONFLICT", "terminal shape differs")
+ source = row.get("source_outbox_id")
+ terminal = row.get("terminal_id")
+ if (
+ type(row.get("schema_version")) is not int
+ or row.get("schema_version") != SCHEMA
+ or row.get("event") not in {"TRANSCRIPTION_COMPLETE", "MINUTES_COMPLETE"}
+ or not isinstance(source, str)
+ or not re.fullmatch(r"[0-9a-f]{64}", source)
+ or not isinstance(row.get("stable_id"), str)
+ or not isinstance(terminal, str)
+ or not SAFE_ID.fullmatch(terminal)
+ or type(row.get("receipt_bytes")) is not int
+ or row["receipt_bytes"] <= 0
+ or not isinstance(row.get("receipt_sha256"), str)
+ or not SHA256.fullmatch(row["receipt_sha256"])
+ or not isinstance(row.get("files"), list)
+ or not row["files"]
+ or not isinstance(row.get("committed_at"), str)
+ ):
+ raise PipelineError("E_RECEIPT_CONFLICT", "terminal identity differs")
+ if not BVID.fullmatch(row["stable_id"]):
+ raise PipelineError("E_RECEIPT_CONFLICT", "terminal stable identity differs")
+ rebound: list[dict[str, Any]] = []
+ for item in row["files"]:
+ item = _exact(item, {"path", "bytes", "sha256", "kind"}, "terminal file")
+ if not isinstance(item["kind"], str) or not SAFE_ID.fullmatch(item["kind"]):
+ raise PipelineError("E_RECEIPT_CONFLICT", "terminal file kind differs")
+ rebound.append({**_artifact(item["path"], item["bytes"], item["sha256"], config), "kind": item["kind"]})
+ if rebound != row["files"]:
+ raise PipelineError("E_RECEIPT_CONFLICT", "terminal files drifted")
+ if source in seen_sources or terminal in seen_terminals:
+ raise PipelineError("E_RECEIPT_CONFLICT", "terminal identity is duplicated")
+ seen_sources.add(source)
+ seen_terminals.add(terminal)
+ return rows
+
+
+def ingest_receipt(config: Config, receipt_path: Path, now: datetime) -> dict[str, Any]:
+ with _lock(config):
+ value, raw = _strict_json(receipt_path, "downstream receipt")
+ receipt = _exact(value, {"schema_version", "type", "outbox_id", "stable_id", "terminal_id", "status", "files", "created_at"}, "receipt")
+ if (
+ type(receipt["schema_version"]) is not int
+ or receipt["schema_version"] != SCHEMA
+ or receipt["status"] != "COMPLETE"
+ or not isinstance(receipt["outbox_id"], str)
+ or not isinstance(receipt["stable_id"], str)
+ or not isinstance(receipt["terminal_id"], str)
+ or not SAFE_ID.fullmatch(receipt["terminal_id"])
+ ):
+ raise PipelineError("E_RECEIPT", "receipt identity differs")
+ kind = receipt["type"]
+ if kind not in {"TRANSCRIPTION_COMPLETE", "MINUTES_COMPLETE"}:
+ raise PipelineError("E_RECEIPT", "receipt type differs")
+ rows = _outbox_rows(config)
+ source = next((row for row in rows if row.get("event") == "CREATED" and row.get("outbox_id") == receipt["outbox_id"]), None)
+ expected_kind = "VIDEO_TRANSCRIPTION_READY" if kind == "TRANSCRIPTION_COMPLETE" else "MINUTES_READY"
+ if source is None or source.get("kind") != expected_kind:
+ raise PipelineError("E_RECEIPT", "receipt source outbox differs")
+ source_events = [row for row in rows if row.get("outbox_id") == receipt["outbox_id"]]
+ if not any(row.get("event") == "DISPATCH_INTENT" for row in source_events):
+ raise PipelineError("E_RECEIPT", "receipt has no durable dispatch intent")
+ if not any(row.get("event") == "OBSERVED" for row in source_events):
+ raise PipelineError("E_RECEIPT", "receipt arrived before durable dispatch observation")
+ expected_stable_id = source.get("payload", {}).get("bvid" if kind == "TRANSCRIPTION_COMPLETE" else "stable_id")
+ if receipt["stable_id"] != expected_stable_id:
+ raise PipelineError("E_RECEIPT", "receipt stable identity differs")
+ files = _receipt_files(receipt["files"], config)
+ receipt_sha = hashlib.sha256(raw).hexdigest().upper()
+ terminals = _terminal_rows(config)
+ by_source = [item for item in terminals if item.get("source_outbox_id") == receipt["outbox_id"]]
+ if by_source and not (len(by_source) == 1 and by_source[0].get("receipt_sha256") == receipt_sha):
+ raise PipelineError("E_RECEIPT_CONFLICT", "source outbox already has a different terminal")
+ if any(item.get("terminal_id") == receipt["terminal_id"] and item.get("source_outbox_id") != receipt["outbox_id"] for item in terminals):
+ raise PipelineError("E_RECEIPT_CONFLICT", "terminal identity is already bound elsewhere")
+ already_committed = bool(by_source)
+ terminal = {
+ "schema_version": SCHEMA, "event": kind, "source_outbox_id": receipt["outbox_id"],
+ "stable_id": receipt["stable_id"], "terminal_id": receipt["terminal_id"],
+ "receipt_bytes": len(raw), "receipt_sha256": receipt_sha, "files": files,
+ "committed_at": now.isoformat(),
+ }
+ if not already_committed:
+ _append(config.terminals_path, terminal)
+ source_identity = {"journal": "terminals", "receipt_sha256": receipt_sha}
+ created: list[str] = []
+ if kind == "TRANSCRIPTION_COMPLETE":
+ created.append(_append_outbox(config, "MINUTES_READY", source_identity, {
+ "stable_id": receipt["stable_id"], "transcript_terminal_id": receipt["terminal_id"], "files": files
+ }, now.isoformat()))
+ created.append(_append_outbox(config, "GIT_DELIVERY_READY", {**source_identity, "projection": "transcript"}, {
+ "stable_id": receipt["stable_id"], "files": files, "reason": "TRANSCRIPT_COMPLETE"
+ }, now.isoformat()))
+ else:
+ created.append(_append_outbox(config, "GIT_DELIVERY_READY", {**source_identity, "projection": "minutes"}, {
+ "stable_id": receipt["stable_id"], "files": files, "reason": "MINUTES_COMPLETE"
+ }, now.isoformat()))
+ if not any(row.get("event") == "COMPLETE" and row.get("outbox_id") == receipt["outbox_id"] for row in rows):
+ _append(config.outbox_path, {
+ "schema_version": SCHEMA, "event": "COMPLETE", "outbox_id": receipt["outbox_id"],
+ "result": kind, "terminal_id": receipt["terminal_id"],
+ "receipt_sha256": receipt_sha, "completed_at": now.isoformat(),
+ })
+ return {
+ "status": "RECEIPT_ALREADY_COMMITTED" if already_committed else "RECEIPT_COMMITTED",
+ "receipt_sha256": receipt_sha,
+ "created_outbox_ids": [] if already_committed else created,
+ }
+
+
+def finish(config: Config, status: str, now: datetime) -> dict[str, Any]:
+ if status not in {"COMPLETE", "FAILED"}:
+ raise PipelineError("E_SCHEMA", "finish status differs")
+ with _lock(config):
+ state = _load_state(config)
+ active = state["active_run"]
+ if active is None:
+ raise PipelineError("E_NO_ACTIVE_RUN", "there is no active run")
+ event = {
+ "schema_version": SCHEMA, "event": f"RUN_{status}", "run_id": active["run_id"],
+ "slot": active["slot"], "creator_uid": config.creator_uid, "finished_at": now.isoformat(),
+ }
+ _append(config.runs_path, event)
+ state["active_run"] = None
+ _write_state(config, state)
+ return {"status": f"RUN_{status}", "run_id": active["run_id"]}
+
+
+def _run(command: Sequence[str], cwd: Path, runner: Callable[..., subprocess.CompletedProcess[str]]) -> subprocess.CompletedProcess[str]:
+ return runner(list(command), cwd=cwd, text=True, capture_output=True, check=False)
+
+
+def _run_env(
+ command: Sequence[str], cwd: Path, runner: Callable[..., subprocess.CompletedProcess[str]], env: Mapping[str, str],
+) -> subprocess.CompletedProcess[str]:
+ return runner(list(command), cwd=cwd, text=True, capture_output=True, check=False, env=dict(env))
+
+
+def _run_env_bytes(
+ command: Sequence[str], cwd: Path, runner: Callable[..., subprocess.CompletedProcess[Any]], env: Mapping[str, str],
+) -> subprocess.CompletedProcess[bytes]:
+ return runner(list(command), cwd=cwd, text=False, capture_output=True, check=False, env=dict(env))
+
+
+def _run_env_input(
+ command: Sequence[str], cwd: Path, runner: Callable[..., subprocess.CompletedProcess[str]], env: Mapping[str, str], payload: bytes,
+) -> subprocess.CompletedProcess[str]:
+ return runner(list(command), cwd=cwd, input=payload, capture_output=True, check=False, env=dict(env))
+
+
+def _git_artifacts(config: Config, files: Any) -> tuple[list[str], list[dict[str, Any]], list[bytes]]:
+ if not isinstance(files, list) or not files:
+ raise PipelineError("E_GIT_SCOPE", "Git file list is empty")
+ allowed: list[str] = []
+ rebound: list[dict[str, Any]] = []
+ blobs: list[bytes] = []
+ for artifact in files:
+ identity_keys = {"path", "bytes", "sha256"}
+ if not isinstance(artifact, dict) or frozenset(artifact) not in {frozenset(identity_keys), frozenset(identity_keys | {"kind"})}:
+ raise PipelineError("E_GIT_SCOPE", "Git artifact shape differs")
+ path = artifact.get("path")
+ size = artifact.get("bytes")
+ digest = artifact.get("sha256")
+ if not isinstance(path, str) or type(size) is not int or size < 0 or not isinstance(digest, str) or not SHA256.fullmatch(digest):
+ raise PipelineError("E_GIT_SCOPE", "Git artifact identity differs")
+ if Path(path).is_absolute():
+ raise PipelineError("E_GIT_SCOPE", "Git path must be project-relative")
+ requested = Path(os.path.abspath(config.project_root / Path(path)))
+ target = requested
+ if not target.exists() and not target.is_symlink():
+ relocated = _relocated_artifact_target(config, Path(path).as_posix(), size, digest.upper())
+ if relocated is not None:
+ target = relocated
+ relative = target.relative_to(config.project_root).as_posix() if _within(target, config.project_root) else ""
+ if (
+ not (_within(target, config.archive_root) or relative in config.git_doc_paths)
+ or target.suffix.lower() not in config.git_extensions
+ or any(str(target).lower().endswith(suffix) for suffix in FORBIDDEN_GIT_SUFFIXES)
+ ):
+ raise PipelineError("E_GIT_SCOPE", "Git path escaped its allowlist")
+ if "kind" in artifact and RECEIPT_GIT_KIND_SUFFIX.get(artifact["kind"]) != target.suffix.lower():
+ raise PipelineError("E_GIT_SCOPE", "Git receipt artifact kind differs")
+ payload = _stable_artifact_bytes(target, config.archive_root if _within(target, config.archive_root) else config.project_root)
+ current = {"path": relative, "bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest().upper()}
+ expected_identity = {"bytes": artifact["bytes"], "sha256": artifact["sha256"]}
+ if {"bytes": current["bytes"], "sha256": current["sha256"]} != expected_identity:
+ raise PipelineError("E_GIT_SCOPE", "Git artifact identity drifted")
+ allowed.append(current["path"])
+ rebound.append(current)
+ blobs.append(payload)
+ if len(set(allowed)) != len(allowed):
+ raise PipelineError("E_GIT_SCOPE", "Git paths are duplicated")
+ return allowed, rebound, blobs
+
+
+def _git_remove_paths(config: Config, value: Any) -> list[str]:
+ if value is None:
+ return []
+ if not isinstance(value, list) or not value or any(not isinstance(item, str) or not item for item in value):
+ raise PipelineError("E_GIT_SCOPE", "Git removal scope differs")
+ normalized: list[str] = []
+ for item in value:
+ if Path(item).is_absolute():
+ raise PipelineError("E_GIT_SCOPE", "Git removal path must be project-relative")
+ target = Path(os.path.abspath(config.project_root / Path(item)))
+ if not _within(target, config.archive_root) or any(str(target).lower().endswith(suffix) for suffix in FORBIDDEN_GIT_SUFFIXES):
+ raise PipelineError("E_GIT_SCOPE", "Git removal path escaped the archive")
+ if target.exists() or target.is_symlink():
+ raise PipelineError("E_GIT_SCOPE", "relocated Git source still exists in the worktree")
+ ancestor = target.parent
+ while not ancestor.exists() and ancestor != config.archive_root:
+ ancestor = ancestor.parent
+ _strict_chain(config.archive_root, ancestor, final_file=False)
+ normalized.append(target.relative_to(config.project_root).as_posix())
+ if normalized != sorted(set(normalized)):
+ raise PipelineError("E_GIT_SCOPE", "Git removal paths are not canonical")
+ return normalized
+
+
+def _git_expected_paths(payload: Mapping[str, Any]) -> list[str]:
+ files = payload.get("files")
+ add_paths = [item.get("path") for item in files] if isinstance(files, list) else []
+ removals = payload.get("remove_paths")
+ if removals is None:
+ return add_paths
+ if not isinstance(removals, list):
+ return []
+ return sorted([*add_paths, *removals])
+
+
+def _remove_task_index(path: Path, state_dir: Path) -> None:
+ if not path.exists() and not path.is_symlink():
+ return
+ _strict_chain(state_dir, path, final_file=True)
+ path.unlink()
+
+
+def _git_batch(rows: Sequence[Mapping[str, Any]], item: Mapping[str, Any]) -> tuple[str, list[str]]:
+ created_at = item.get("created_at")
+ if not isinstance(created_at, str):
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git batch creation identity differs")
+ outbox_ids = sorted(
+ row["outbox_id"] for row in rows
+ if row.get("event") == "CREATED"
+ and row.get("kind") == "GIT_DELIVERY_READY"
+ and row.get("created_at") == created_at
+ )
+ if not outbox_ids or item.get("outbox_id") not in outbox_ids:
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git batch membership differs")
+ binding = {"created_at": created_at, "outbox_ids": outbox_ids}
+ batch_id = hashlib.sha256(_canonical(binding)).hexdigest().upper()
+ return batch_id, outbox_ids
+
+
+def _decode_git_paths(result: subprocess.CompletedProcess[Any], error_code: str) -> list[str]:
+ if result.returncode != 0 or not isinstance(result.stdout, bytes):
+ raise PipelineError(error_code, "Git staged paths are unavailable")
+ try:
+ return [part.decode("utf-8").replace("\\", "/") for part in result.stdout.split(b"\0") if part]
+ except UnicodeDecodeError as exc:
+ raise PipelineError(error_code, "Git staged path encoding differs") from exc
+
+
+def _shared_index_snapshot(
+ config: Config,
+ baseline_head: str,
+ runner: Callable[..., subprocess.CompletedProcess[Any]],
+) -> dict[str, Any]:
+ if not isinstance(baseline_head, str) or not re.fullmatch(r"[0-9a-f]{40,64}", baseline_head):
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git baseline HEAD identity differs")
+ index_path = Path(os.path.abspath(config.project_root / ".git" / "index"))
+ payload = _stable_artifact_bytes(index_path, config.project_root)
+ chain = _strict_chain(config.project_root, index_path, final_file=True)
+ staged = _run_env_bytes(
+ ["git", "diff", "--cached", "--name-only", "-z", baseline_head, "--"],
+ config.project_root,
+ runner,
+ os.environ,
+ )
+ return {
+ "index_bytes": len(payload),
+ "index_sha256": hashlib.sha256(payload).hexdigest().upper(),
+ "index_identity": list(chain[-1][1]),
+ "staged_paths": _decode_git_paths(staged, "E_GIT_INDEX_DIRTY"),
+ }
+
+
+def _git_guard_value(
+ config: Config,
+ batch_id: str,
+ outbox_ids: Sequence[str],
+ baseline_head: str,
+ snapshot: Mapping[str, Any],
+) -> dict[str, Any]:
+ return {
+ "schema_version": SCHEMA,
+ "task_id": TASK_ID,
+ "creator_uid": config.creator_uid,
+ "batch_id": batch_id,
+ "outbox_ids": list(outbox_ids),
+ "baseline_head": baseline_head,
+ "index_bytes": snapshot["index_bytes"],
+ "index_sha256": snapshot["index_sha256"],
+ "index_identity": snapshot["index_identity"],
+ "staged_paths": snapshot["staged_paths"],
+ }
+
+
+@contextlib.contextmanager
+def _open_no_write_handle(path: Path, error_code: str) -> Iterator[Any]:
+ descriptor: int | None = None
+ stream = None
+ try:
+ if os.name == "nt":
+ import ctypes # noqa: PLC0415
+ import msvcrt # noqa: PLC0415
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ create_file = kernel32.CreateFileW
+ create_file.argtypes = (
+ ctypes.c_wchar_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p,
+ ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p,
+ )
+ create_file.restype = ctypes.c_void_p
+ close_handle = kernel32.CloseHandle
+ close_handle.argtypes = (ctypes.c_void_p,)
+ close_handle.restype = ctypes.c_int
+ handle = create_file(
+ str(path), 0x80000000, 0x00000001, None, 3,
+ 0x00000080 | 0x00200000 | 0x08000000, None,
+ )
+ if handle in (None, ctypes.c_void_p(-1).value):
+ raise OSError(ctypes.get_last_error(), "CreateFileW failed", str(path))
+ try:
+ descriptor = msvcrt.open_osfhandle(int(handle), os.O_RDONLY | getattr(os, "O_BINARY", 0))
+ handle = None
+ finally:
+ if handle is not None:
+ close_handle(handle)
+ else:
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ import fcntl # noqa: PLC0415
+ fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
+ stream = os.fdopen(descriptor, "rb", closefd=True)
+ descriptor = None
+ yield stream
+ except PipelineError:
+ raise
+ except OSError as exc:
+ raise PipelineError(error_code, "stable file handle is unavailable") from exc
+ finally:
+ if stream is not None:
+ stream.close()
+ elif descriptor is not None:
+ os.close(descriptor)
+
+
+def _read_exact_held_file(
+ stream: Any,
+ path: Path,
+ root: Path,
+ expected_chain: tuple[tuple[str, tuple[int, ...]], ...],
+ error_code: str,
+) -> bytes:
+ opened_before = os.fstat(stream.fileno())
+ stream.seek(0)
+ payload = stream.read()
+ opened_after = os.fstat(stream.fileno())
+ chain = _strict_chain(root, path, final_file=True)
+ if (
+ not stat.S_ISREG(opened_before.st_mode)
+ or _is_reparse(opened_before)
+ or _path_handle_identity(opened_before) != expected_chain[-1][1][:6] + expected_chain[-1][1][7:]
+ or _stat_identity(opened_before) != _stat_identity(opened_after)
+ or chain != expected_chain
+ ):
+ raise PipelineError(error_code, "held file identity drifted")
+ return payload
+
+
+@contextlib.contextmanager
+def _open_no_write_delete_handle(path: Path, error_code: str) -> Iterator[Any]:
+ descriptor: int | None = None
+ stream = None
+ try:
+ if os.name == "nt":
+ import ctypes # noqa: PLC0415
+ import msvcrt # noqa: PLC0415
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ create_file = kernel32.CreateFileW
+ create_file.argtypes = (
+ ctypes.c_wchar_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p,
+ ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p,
+ )
+ create_file.restype = ctypes.c_void_p
+ close_handle = kernel32.CloseHandle
+ close_handle.argtypes = (ctypes.c_void_p,)
+ close_handle.restype = ctypes.c_int
+ handle = create_file(
+ str(path), 0x80000000 | 0x00010000, 0x00000001, None, 3,
+ 0x00000080 | 0x00200000 | 0x08000000, None,
+ )
+ if handle in (None, ctypes.c_void_p(-1).value):
+ raise OSError(ctypes.get_last_error(), "CreateFileW failed", str(path))
+ try:
+ descriptor = msvcrt.open_osfhandle(int(handle), os.O_RDONLY | getattr(os, "O_BINARY", 0))
+ handle = None
+ finally:
+ if handle is not None:
+ close_handle(handle)
+ else:
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ import fcntl # noqa: PLC0415
+ fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ stream = os.fdopen(descriptor, "rb", closefd=True)
+ descriptor = None
+ yield stream
+ except PipelineError:
+ raise
+ except OSError as exc:
+ raise PipelineError(error_code, "stable deletable file handle is unavailable") from exc
+ finally:
+ if stream is not None:
+ stream.close()
+ elif descriptor is not None:
+ os.close(descriptor)
+
+
+def _mark_held_file_for_delete(stream: Any, path: Path) -> None:
+ if os.name == "nt":
+ import ctypes # noqa: PLC0415
+ import msvcrt # noqa: PLC0415
+
+ class FileDispositionInfo(ctypes.Structure):
+ _fields_ = [("DeleteFile", ctypes.c_int)]
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ setter = kernel32.SetFileInformationByHandle
+ setter.argtypes = (ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_uint32)
+ setter.restype = ctypes.c_int
+ info = FileDispositionInfo(1)
+ handle = ctypes.c_void_p(msvcrt.get_osfhandle(stream.fileno()))
+ if not setter(handle, 4, ctypes.byref(info), ctypes.sizeof(info)):
+ raise OSError(ctypes.get_last_error(), "SetFileInformationByHandle failed", str(path))
+ return
+ os.unlink(path)
+
+
+@contextlib.contextmanager
+def _held_relocation_source(
+ source: Path,
+ source_root: Path,
+ size: int,
+ digest: str,
+) -> Iterator[tuple[Any, bytes, tuple[tuple[str, tuple[int, ...]], ...]]]:
+ before = _strict_chain(source_root, source, final_file=True)
+ with _open_no_write_delete_handle(source, "E_RELOCATION") as stream:
+ opened_before = os.fstat(stream.fileno())
+ if (
+ _path_handle_identity(opened_before) != before[-1][1][:6] + before[-1][1][7:]
+ or not stat.S_ISREG(opened_before.st_mode)
+ or _is_reparse(opened_before)
+ ):
+ raise PipelineError("E_RELOCATION", "migration source path and handle differ")
+ stream.seek(0)
+ payload = stream.read()
+ opened_after = os.fstat(stream.fileno())
+ after = _strict_chain(source_root, source, final_file=True)
+ if (
+ len(payload) != size
+ or hashlib.sha256(payload).hexdigest().upper() != digest
+ or _stat_identity(opened_before) != _stat_identity(opened_after)
+ or before != after
+ ):
+ raise PipelineError("E_RELOCATION", "migration source identity differs")
+ yield stream, payload, before
+
+
+def _delete_held_relocation_source(
+ stream: Any,
+ source: Path,
+ source_root: Path,
+ payload: bytes,
+ before: tuple[tuple[str, tuple[int, ...]], ...],
+) -> None:
+ stream.seek(0)
+ rebound = stream.read()
+ opened = os.fstat(stream.fileno())
+ after = _strict_chain(source_root, source, final_file=True)
+ if (
+ rebound != payload
+ or before != after
+ or _path_handle_identity(opened) != after[-1][1][:6] + after[-1][1][7:]
+ ):
+ raise PipelineError("E_RELOCATION", "migration source drifted before handle-bound delete")
+ try:
+ _mark_held_file_for_delete(stream, source)
+ except OSError as exc:
+ raise PipelineError("E_RELOCATION", "migration source handle delete failed") from exc
+
+
+@contextlib.contextmanager
+def _held_exact_git_artifacts(config: Config, allowed: Sequence[str], blobs: Sequence[bytes]) -> Iterator[None]:
+ with contextlib.ExitStack() as stack:
+ for relative, expected in zip(allowed, blobs, strict=True):
+ target = Path(os.path.abspath(config.project_root / relative))
+ governed_root = config.archive_root if _within(target, config.archive_root) else config.project_root
+ before = _strict_chain(governed_root, target, final_file=True)
+ stream = stack.enter_context(_open_no_write_handle(target, "E_GIT_SCOPE"))
+ opened_before = os.fstat(stream.fileno())
+ if (
+ _path_handle_identity(opened_before) != before[-1][1][:6] + before[-1][1][7:]
+ or not stat.S_ISREG(opened_before.st_mode)
+ or _is_reparse(opened_before)
+ ):
+ raise PipelineError("E_GIT_SCOPE", "published artifact path and handle differ")
+ stream.seek(0)
+ payload = stream.read()
+ opened_after = os.fstat(stream.fileno())
+ after = _strict_chain(governed_root, target, final_file=True)
+ if payload != expected or _stat_identity(opened_before) != _stat_identity(opened_after) or before != after:
+ raise PipelineError("E_GIT_SCOPE", "published artifact changed before terminal append")
+ yield
+
+
+@contextlib.contextmanager
+def _held_git_ref_locks(
+ config: Config,
+ runner: Callable[..., subprocess.CompletedProcess[Any]],
+) -> Iterator[None]:
+ symbolic = _run(["git", "symbolic-ref", "-q", "HEAD"], config.project_root, runner)
+ expected_local = f"refs/heads/{config.git_branch}"
+ local_ref = symbolic.stdout.strip() if isinstance(symbolic.stdout, str) else ""
+ git_dir_result = _run(["git", "rev-parse", "--absolute-git-dir"], config.project_root, runner)
+ git_dir_text = git_dir_result.stdout.strip() if isinstance(git_dir_result.stdout, str) else ""
+ if symbolic.returncode != 0 or local_ref != expected_local or git_dir_result.returncode != 0 or not git_dir_text:
+ raise PipelineError("E_GIT_PUSH", "published branch ref identity differs")
+ git_dir = Path(os.path.abspath(git_dir_text))
+ _strict_chain(config.project_root, git_dir, final_file=False)
+ lock_paths = sorted({
+ git_dir / "HEAD.lock",
+ git_dir / f"{expected_local}.lock",
+ git_dir / f"refs/remotes/{config.git_remote}/{config.git_branch}.lock",
+ }, key=lambda value: os.path.normcase(str(value)))
+ with contextlib.ExitStack() as stack:
+ for lock_path in lock_paths:
+ _strict_chain(git_dir, lock_path.parent, final_file=False)
+ try:
+ _create_new(lock_path, b"mbx-published-no-changes-lock\n")
+ except PipelineError as exc:
+ raise PipelineError("E_GIT_PUSH", "published Git ref lock is unavailable") from exc
+
+ def cleanup(path: Path = lock_path) -> None:
+ with contextlib.suppress(OSError, PipelineError):
+ _strict_chain(git_dir, path, final_file=True)
+ path.unlink()
+
+ stack.callback(cleanup)
+ stream = stack.enter_context(_open_no_write_handle(lock_path, "E_GIT_PUSH"))
+ if stream.read() != b"mbx-published-no-changes-lock\n":
+ raise PipelineError("E_GIT_PUSH", "published Git ref lock identity differs")
+ yield
+
+
+@contextlib.contextmanager
+def _published_head_with_exact_artifacts(
+ config: Config,
+ allowed: Sequence[str],
+ blobs: Sequence[bytes],
+ remove_paths: Sequence[str],
+ runner: Callable[..., subprocess.CompletedProcess[Any]],
+) -> Iterator[str | None]:
+ head = _run(["git", "rev-parse", "HEAD"], config.project_root, runner)
+ head_sha = head.stdout.strip() if isinstance(head.stdout, str) else ""
+ if head.returncode != 0 or not re.fullmatch(r"[0-9a-f]{40,64}", head_sha):
+ yield None
+ return
+ for relative, payload in zip(allowed, blobs, strict=True):
+ committed = _run_env_bytes(
+ ["git", "show", f"{head_sha}:{relative}"],
+ config.project_root,
+ runner,
+ os.environ,
+ )
+ if committed.returncode != 0 or not isinstance(committed.stdout, bytes) or committed.stdout != payload:
+ yield None
+ return
+ for relative in remove_paths:
+ committed = _run_env_bytes(
+ ["git", "show", f"{head_sha}:{relative}"],
+ config.project_root,
+ runner,
+ os.environ,
+ )
+ if committed.returncode == 0:
+ yield None
+ return
+ preliminary_remote = _run(
+ ["git", "rev-parse", f"refs/remotes/{config.git_remote}/{config.git_branch}"],
+ config.project_root,
+ runner,
+ )
+ preliminary_remote_sha = preliminary_remote.stdout.strip() if isinstance(preliminary_remote.stdout, str) else ""
+ if preliminary_remote.returncode != 0 or preliminary_remote_sha != head_sha:
+ raise PipelineError("E_GIT_PUSH", "exact artifacts are not bound to the published branch")
+ with _held_exact_git_artifacts(config, allowed, blobs), _held_git_ref_locks(config, runner):
+ rebound_head = _run(["git", "rev-parse", "HEAD"], config.project_root, runner)
+ remote = _run(
+ ["git", "rev-parse", f"refs/remotes/{config.git_remote}/{config.git_branch}"],
+ config.project_root,
+ runner,
+ )
+ rebound_sha = rebound_head.stdout.strip() if isinstance(rebound_head.stdout, str) else ""
+ remote_sha = remote.stdout.strip() if isinstance(remote.stdout, str) else ""
+ if rebound_head.returncode != 0 or rebound_sha != head_sha or remote.returncode != 0 or remote_sha != head_sha:
+ raise PipelineError("E_GIT_PUSH", "exact artifacts are not bound to the stable published branch")
+ for relative, payload in zip(allowed, blobs, strict=True):
+ committed = _run_env_bytes(
+ ["git", "show", f"{head_sha}:{relative}"],
+ config.project_root,
+ runner,
+ os.environ,
+ )
+ if committed.returncode != 0 or not isinstance(committed.stdout, bytes) or committed.stdout != payload:
+ raise PipelineError("E_GIT_SCOPE", "published artifact commit binding drifted")
+ for relative in remove_paths:
+ committed = _run_env_bytes(
+ ["git", "show", f"{head_sha}:{relative}"],
+ config.project_root,
+ runner,
+ os.environ,
+ )
+ target = Path(os.path.abspath(config.project_root / relative))
+ if committed.returncode == 0 or target.exists() or target.is_symlink():
+ raise PipelineError("E_GIT_SCOPE", "published relocation removal binding drifted")
+ yield head_sha
+
+
+def _read_git_guard(config: Config, path: Path) -> dict[str, Any]:
+ value, _ = _strict_json(path, "Git shared-index guard")
+ guard = _exact(value, {
+ "schema_version", "task_id", "creator_uid", "batch_id", "outbox_ids", "baseline_head",
+ "index_bytes", "index_sha256", "index_identity", "staged_paths",
+ }, "Git shared-index guard")
+ if (
+ type(guard["schema_version"]) is not int or guard["schema_version"] != SCHEMA
+ or guard["task_id"] != TASK_ID or guard["creator_uid"] != config.creator_uid
+ or not isinstance(guard["batch_id"], str) or not SHA256.fullmatch(guard["batch_id"])
+ or not isinstance(guard["outbox_ids"], list) or not guard["outbox_ids"]
+ or any(not isinstance(value, str) or not SHA256.fullmatch(value.upper()) for value in guard["outbox_ids"])
+ or not isinstance(guard["baseline_head"], str) or not re.fullmatch(r"[0-9a-f]{40,64}", guard["baseline_head"])
+ or type(guard["index_bytes"]) is not int or guard["index_bytes"] < 0
+ or not isinstance(guard["index_sha256"], str) or not SHA256.fullmatch(guard["index_sha256"])
+ or not isinstance(guard["index_identity"], list) or not guard["index_identity"]
+ or any(type(value) is not int for value in guard["index_identity"])
+ or not isinstance(guard["staged_paths"], list)
+ or any(not isinstance(value, str) for value in guard["staged_paths"])
+ ):
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git shared-index guard identity differs")
+ return dict(guard)
+
+
+def _ensure_git_index_guard(
+ config: Config,
+ rows: Sequence[Mapping[str, Any]],
+ item: Mapping[str, Any],
+ runner: Callable[..., subprocess.CompletedProcess[Any]],
+) -> dict[str, Any]:
+ batch_id, outbox_ids = _git_batch(rows, item)
+ guard_path = config.git_index_guard_path(batch_id)
+ if guard_path.exists() or guard_path.is_symlink():
+ _strict_chain(config.state_dir, guard_path, final_file=True)
+ guard = _read_git_guard(config, guard_path)
+ else:
+ batch_events = [
+ row for row in rows
+ if row.get("outbox_id") in outbox_ids and row.get("event") in {"GIT_COMMIT_INTENT", "COMMIT_CREATED", "COMPLETE"}
+ ]
+ if batch_events:
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git baseline guard is absent after batch progress")
+ head = _run(["git", "rev-parse", "HEAD"], config.project_root, runner)
+ baseline_head = head.stdout.strip() if isinstance(head.stdout, str) else ""
+ if head.returncode != 0 or not re.fullmatch(r"[0-9a-f]{40,64}", baseline_head):
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git baseline HEAD is unavailable")
+ snapshot = _shared_index_snapshot(config, baseline_head, runner)
+ guard = _git_guard_value(config, batch_id, outbox_ids, baseline_head, snapshot)
+ _create_new(guard_path, _canonical(guard))
+ _strict_chain(config.state_dir, guard_path, final_file=True)
+ guard = _read_git_guard(config, guard_path)
+ expected = _git_guard_value(
+ config,
+ batch_id,
+ outbox_ids,
+ guard["baseline_head"],
+ _shared_index_snapshot(config, guard["baseline_head"], runner),
+ )
+ if guard != expected:
+ raise PipelineError("E_GIT_INDEX_DIRTY", "shared Git index changed after the batch baseline")
+ return guard
+
+
+def recover_git_index_guard(
+ config: Config,
+ outbox_id: str,
+ baseline_head: str,
+ expected_bytes: int,
+ expected_sha256: str,
+ *,
+ runner: Callable[..., subprocess.CompletedProcess[Any]] = subprocess.run,
+) -> dict[str, Any]:
+ with _lock(config):
+ rows = _outbox_rows(config)
+ item = next((row for row in rows if row.get("event") == "CREATED" and row.get("outbox_id") == outbox_id), None)
+ if item is None or item.get("kind") != "GIT_DELIVERY_READY":
+ raise PipelineError("E_GIT_SCOPE", "Git recovery outbox differs")
+ batch_id, outbox_ids = _git_batch(rows, item)
+ guard_path = config.git_index_guard_path(batch_id)
+ if (
+ not isinstance(baseline_head, str) or not re.fullmatch(r"[0-9a-f]{40,64}", baseline_head)
+ or type(expected_bytes) is not int or expected_bytes < 0
+ or not isinstance(expected_sha256, str) or not SHA256.fullmatch(expected_sha256.upper())
+ ):
+ raise PipelineError("E_GIT_INDEX_DIRTY", "expected shared-index identity differs")
+ expected_sha256 = expected_sha256.upper()
+ if guard_path.exists() or guard_path.is_symlink():
+ try:
+ _strict_chain(config.state_dir, guard_path, final_file=True)
+ guard = _read_git_guard(config, guard_path)
+ if (
+ guard["batch_id"] != batch_id
+ or guard["outbox_ids"] != outbox_ids
+ or guard["baseline_head"] != baseline_head
+ or guard["index_bytes"] != expected_bytes
+ or guard["index_sha256"] != expected_sha256
+ ):
+ raise PipelineError("E_GIT_INDEX_DIRTY", "durable Git guard recovery binding differs")
+ fresh = _git_guard_value(
+ config,
+ batch_id,
+ outbox_ids,
+ guard["baseline_head"],
+ _shared_index_snapshot(config, guard["baseline_head"], runner),
+ )
+ if guard != fresh:
+ raise PipelineError("E_GIT_INDEX_DIRTY", "shared Git index changed after the durable guard")
+ except PipelineError as exc:
+ if exc.code == "E_GIT_INDEX_DIRTY":
+ raise
+ raise PipelineError("E_GIT_INDEX_DIRTY", "durable Git guard cannot be revalidated") from exc
+ return {"status": "GIT_INDEX_GUARD_ALREADY_DURABLE", "batch_id": batch_id}
+ batch_rows = [row for row in rows if row.get("outbox_id") in outbox_ids]
+ intents = [row for row in batch_rows if row.get("event") == "GIT_COMMIT_INTENT"]
+ commits = [row for row in batch_rows if row.get("event") == "COMMIT_CREATED"]
+ if not intents or len(intents) != len(commits) or intents[0].get("parent_sha") != baseline_head:
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git recovery history does not bind the baseline")
+ expected_parent = baseline_head
+ for intent, commit in zip(intents, commits, strict=True):
+ if intent.get("parent_sha") != expected_parent or commit.get("parent_sha") != expected_parent:
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git recovery commit chain differs")
+ expected_parent = commit.get("commit_sha")
+ head = _run(["git", "rev-parse", "HEAD"], config.project_root, runner)
+ if head.returncode != 0 or head.stdout.strip() != expected_parent:
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git recovery HEAD differs")
+ snapshot = _shared_index_snapshot(config, baseline_head, runner)
+ if snapshot["index_bytes"] != expected_bytes or snapshot["index_sha256"] != expected_sha256:
+ raise PipelineError("E_GIT_INDEX_DIRTY", "shared Git index differs from its frozen first-run bytes")
+ guard = _git_guard_value(config, batch_id, outbox_ids, baseline_head, snapshot)
+ _create_new(guard_path, _canonical(guard))
+ if _read_git_guard(config, guard_path) != guard:
+ raise PipelineError("E_DURABILITY", "Git shared-index guard readback differs")
+ return {"status": "GIT_INDEX_GUARD_RECOVERED", "batch_id": batch_id, "outbox_count": len(outbox_ids)}
+
+
+def git_preflight(
+ config: Config,
+ *,
+ runner: Callable[..., subprocess.CompletedProcess[Any]] = subprocess.run,
+) -> dict[str, Any]:
+ """Read-only shared-index diagnostics; never repairs or rewrites the index."""
+ index_path = Path(os.path.abspath(config.project_root / ".git" / "index"))
+ before_chain = _strict_chain(config.project_root, index_path, final_file=True)
+ with _open_no_write_handle(index_path, "E_GIT_INDEX_DIRTY") as index_stream:
+ before = _read_exact_held_file(
+ index_stream,
+ index_path,
+ config.project_root,
+ before_chain,
+ "E_GIT_INDEX_DIRTY",
+ )
+ index_view = index_path
+ if os.name != "nt":
+ proc_view = Path(f"/proc/self/fd/{index_stream.fileno()}")
+ if proc_view.exists():
+ index_view = proc_view
+ git_env = {
+ **os.environ,
+ "GIT_OPTIONAL_LOCKS": "0",
+ "GIT_INDEX_FILE": str(index_view),
+ }
+ try:
+ head_result = _run_env(["git", "rev-parse", "HEAD"], config.project_root, runner, git_env)
+ head = head_result.stdout.strip() if isinstance(head_result.stdout, str) else ""
+ if head_result.returncode != 0 or not re.fullmatch(r"[0-9a-f]{40,64}", head):
+ raise PipelineError("E_GIT_INDEX_DIRTY", "Git preflight HEAD is unavailable")
+ staged = _run_env_bytes(
+ ["git", "diff", "--cached", "--name-only", "-z", head, "--"],
+ config.project_root,
+ runner,
+ git_env,
+ )
+ staged_paths = _decode_git_paths(staged, "E_GIT_INDEX_DIRTY")
+ _read_exact_held_file(index_stream, index_path, config.project_root, before_chain, "E_GIT_INDEX_DIRTY")
+ archive_relative = config.archive_root.relative_to(config.project_root).as_posix()
+ deleted = _run_env_bytes(
+ ["git", "diff", "--cached", "--diff-filter=D", "--name-only", "-z", head, "--", archive_relative],
+ config.project_root,
+ runner,
+ git_env,
+ )
+ deleted_paths = _decode_git_paths(deleted, "E_GIT_INDEX_DIRTY")
+ _read_exact_held_file(index_stream, index_path, config.project_root, before_chain, "E_GIT_INDEX_DIRTY")
+ present_deletions: list[str] = []
+ for relative in deleted_paths:
+ target = Path(os.path.abspath(config.project_root / relative))
+ if target.exists() or target.is_symlink():
+ _strict_chain(config.archive_root, target, final_file=True)
+ present_deletions.append(relative)
+ lock_path = Path(os.path.abspath(config.project_root / ".git" / "index.lock"))
+ stale_lock: dict[str, Any] | None = None
+ if lock_path.exists() or lock_path.is_symlink():
+ lock_payload = _stable_artifact_bytes(lock_path, config.project_root)
+ lock_info = os.lstat(lock_path)
+ stale_lock = {
+ "path": ".git/index.lock",
+ "bytes": len(lock_payload),
+ "sha256": hashlib.sha256(lock_payload).hexdigest().upper(),
+ "mtime_ns": lock_info.st_mtime_ns,
+ }
+ return {
+ "status": "GIT_PREFLIGHT",
+ "head": head,
+ "index_bytes": len(before),
+ "index_sha256": hashlib.sha256(before).hexdigest().upper(),
+ "index_identity": list(before_chain[-1][1]),
+ "index_matches_head": not staged_paths,
+ "staged_paths": staged_paths,
+ "archive_staged_delete_present": present_deletions,
+ "stale_index_lock": stale_lock,
+ }
+ finally:
+ after = _read_exact_held_file(
+ index_stream,
+ index_path,
+ config.project_root,
+ before_chain,
+ "E_GIT_INDEX_MUTATION",
+ )
+ if before != after:
+ raise PipelineError("E_GIT_INDEX_MUTATION", "Git preflight changed the shared index")
+
+
+def _migration_file_identity(config: Config, path: Path, kind: str, root: Path) -> dict[str, Any]:
+ payload = _stable_artifact_bytes(path, root)
+ return {
+ "kind": kind,
+ "bytes": len(payload),
+ "sha256": hashlib.sha256(payload).hexdigest().upper(),
+ }
+
+
+def _video_metadata(config: Config) -> dict[str, dict[str, Any]]:
+ rows, _ = _read_lines(config.formal_manifest, "formal manifest", reject_secrets=False)
+ selected: dict[str, dict[str, Any]] = {}
+ for row in rows:
+ stable_id = row.get("stable_id")
+ row_creator_uid = _creator_uid(row)
+ creator_matches = row_creator_uid == config.creator_uid or (
+ row_creator_uid is None and row.get("creator") == config.creator_name
+ )
+ if (
+ row.get("item_type") != "video"
+ or not creator_matches
+ or not isinstance(stable_id, str)
+ or not BVID.fullmatch(stable_id)
+ ):
+ continue
+ title = row.get("title")
+ published_at = row.get("published_at")
+ if not isinstance(title, str) or not isinstance(published_at, str):
+ continue
+ _canonical_video_base(stable_id, title, published_at)
+ candidate = {
+ "stable_id": stable_id,
+ "title": title,
+ "published_at": published_at,
+ "complete": row.get("status") == VIDEO_COMPLETE,
+ }
+ existing = selected.get(stable_id)
+ if existing is not None and existing["complete"] and candidate["complete"] and (
+ existing["title"] != title or existing["published_at"] != published_at
+ ):
+ raise PipelineError("E_RELOCATION", "completed video metadata is ambiguous")
+ if candidate["complete"] or existing is None:
+ selected[stable_id] = candidate
+ return selected
+
+
+def _video_artifact_ids(config: Config) -> list[str]:
+ values: set[str] = set()
+ for suffix in (".transcript", ".minutes"):
+ for path in config.archive_root.glob(f"*{suffix}"):
+ name = path.name.removesuffix(suffix)
+ stable_id = name if BVID.fullmatch(name) else name.rsplit("_", 1)[-1]
+ if path.is_dir() and BVID.fullmatch(stable_id):
+ _strict_chain(config.archive_root, path, final_file=False)
+ values.add(stable_id)
+ return sorted(values)
+
+
+def _git_tracked(config: Config, relative: str, runner: Callable[..., subprocess.CompletedProcess[Any]]) -> bool:
+ result = _run(["git", "ls-files", "--error-unmatch", "--", relative], config.project_root, runner)
+ return result.returncode == 0
+
+
+def _git_head(config: Config, runner: Callable[..., subprocess.CompletedProcess[Any]]) -> str:
+ result = _run(["git", "rev-parse", "HEAD"], config.project_root, runner)
+ head = result.stdout.strip() if isinstance(result.stdout, str) else ""
+ if result.returncode != 0 or not re.fullmatch(r"[0-9a-f]{40,64}", head):
+ raise PipelineError("E_RELOCATION", "migration Git baseline is unavailable")
+ exists = _run(["git", "cat-file", "-e", f"{head}^{{commit}}"], config.project_root, runner)
+ if exists.returncode != 0:
+ raise PipelineError("E_RELOCATION", "migration Git baseline differs")
+ return head
+
+
+def _git_blob_at(
+ config: Config,
+ baseline_head: str,
+ relative: str,
+ runner: Callable[..., subprocess.CompletedProcess[Any]],
+) -> bytes | None:
+ result = _run_env_bytes(
+ ["git", "show", f"{baseline_head}:{relative}"],
+ config.project_root,
+ runner,
+ {**os.environ, "GIT_OPTIONAL_LOCKS": "0"},
+ )
+ if result.returncode != 0:
+ return None
+ if not isinstance(result.stdout, bytes):
+ raise PipelineError("E_RELOCATION", "migration Git blob encoding differs")
+ return result.stdout
+
+
+def _migration_current_identity(
+ config: Config,
+ old_path: Path,
+ new_path: Path,
+ kind: str,
+ old_root: Path,
+ new_root: Path,
+ *,
+ allow_recovery_pair: bool,
+) -> dict[str, Any]:
+ old_exists = old_path.exists() or old_path.is_symlink()
+ new_exists = new_path.exists() or new_path.is_symlink()
+ if not old_exists and not new_exists:
+ raise PipelineError("E_RELOCATION", "migration artifact is absent")
+ if old_exists and new_exists and not allow_recovery_pair:
+ raise PipelineError("E_RELOCATION", "legacy and canonical artifact both exist")
+ identities: list[dict[str, Any]] = []
+ if old_exists:
+ identities.append(_migration_file_identity(config, old_path, kind, old_root))
+ if new_exists:
+ identities.append(_migration_file_identity(config, new_path, kind, new_root))
+ if len(identities) == 2 and identities[0] != identities[1]:
+ raise PipelineError("E_RELOCATION", "migration recovery pair differs")
+ return identities[0]
+
+
+def _build_video_artifact_migration_report(
+ config: Config,
+ created_at: str,
+ baseline_head: str,
+ stable_ids: Sequence[str],
+ runner: Callable[..., subprocess.CompletedProcess[Any]],
+) -> dict[str, Any]:
+ _published_at(created_at, "created_at")
+ if (
+ not isinstance(baseline_head, str)
+ or not re.fullmatch(r"[0-9a-f]{40,64}", baseline_head)
+ or list(stable_ids) != sorted(set(stable_ids))
+ or not stable_ids
+ ):
+ raise PipelineError("E_RELOCATION", "migration plan identity differs")
+ exists = _run(["git", "cat-file", "-e", f"{baseline_head}^{{commit}}"], config.project_root, runner)
+ if exists.returncode != 0:
+ raise PipelineError("E_RELOCATION", "migration baseline commit is absent")
+ metadata = _video_metadata(config)
+ items: list[dict[str, Any]] = []
+ remove_paths: list[str] = []
+ kind_order = tuple(PUBLIC_VIDEO_KINDS)
+ for stable_id in stable_ids:
+ if not isinstance(stable_id, str) or not BVID.fullmatch(stable_id):
+ raise PipelineError("E_RELOCATION", "migration stable identity differs")
+ source = metadata.get(stable_id)
+ if source is None or not source["complete"]:
+ raise PipelineError("E_RELOCATION", "legacy video lacks a completed formal identity")
+ canonical_base = _canonical_video_base(stable_id, source["title"], source["published_at"])
+ aliases: list[dict[str, Any]] = []
+ intermediates: list[dict[str, Any]] = []
+ locations = {
+ "transcript_txt": (
+ config.archive_root / f"{stable_id}.transcript" / f"{stable_id}.txt",
+ config.archive_root / f"{canonical_base}.transcript" / f"{canonical_base}.txt",
+ ),
+ "transcript_srt": (
+ config.archive_root / f"{stable_id}.transcript" / f"{stable_id}.srt",
+ config.archive_root / f"{canonical_base}.transcript" / f"{canonical_base}.srt",
+ ),
+ "transcript_json": (
+ config.archive_root / f"{stable_id}.transcript" / f"{stable_id}.json",
+ config.archive_root / f"{canonical_base}.transcript" / f"{canonical_base}.json",
+ ),
+ "minutes_md": (
+ config.archive_root / f"{stable_id}.minutes" / f"{stable_id}.md",
+ config.archive_root / f"{canonical_base}.minutes" / f"{canonical_base}.md",
+ ),
+ "minutes_pdf": (
+ config.archive_root / f"{stable_id}.minutes" / f"{stable_id}.pdf",
+ config.archive_root / f"{canonical_base}.minutes" / f"{canonical_base}.pdf",
+ ),
+ }
+ for kind in kind_order:
+ old_path, new_path = locations[kind]
+ old_exists = old_path.exists() or old_path.is_symlink()
+ new_exists = new_path.exists() or new_path.is_symlink()
+ if not old_exists and not new_exists:
+ continue
+ identity = _migration_current_identity(
+ config,
+ old_path,
+ new_path,
+ kind,
+ config.archive_root,
+ config.archive_root,
+ allow_recovery_pair=True,
+ )
+ old_relative = _project_relative(config, old_path)
+ new_relative = _project_relative(config, new_path)
+ aliases.append({
+ "kind": kind,
+ "old_path": old_relative,
+ "new_path": new_relative,
+ "bytes": identity["bytes"],
+ "sha256": identity["sha256"],
+ })
+ committed = _git_blob_at(config, baseline_head, old_relative, runner)
+ if committed is not None:
+ remove_paths.append(old_relative)
+ transcript_kinds = {alias["kind"] for alias in aliases if alias["kind"].startswith("transcript_")}
+ minutes_kinds = {alias["kind"] for alias in aliases if alias["kind"].startswith("minutes_")}
+ if transcript_kinds != {"transcript_txt", "transcript_srt", "transcript_json"}:
+ raise PipelineError("E_RELOCATION", "legacy video transcript set is incomplete")
+ if minutes_kinds not in (set(), {"minutes_md", "minutes_pdf"}):
+ raise PipelineError("E_RELOCATION", "legacy video minutes set is incomplete")
+ old_flac = config.archive_root / f"{stable_id}.transcript" / f"{stable_id}.audio.flac"
+ new_flac = config.video_root / "intermediate" / "transcription" / f"{stable_id}.audio.flac"
+ if old_flac.exists() or old_flac.is_symlink() or new_flac.exists() or new_flac.is_symlink():
+ identity = _migration_current_identity(
+ config,
+ old_flac,
+ new_flac,
+ "audio_flac",
+ config.archive_root,
+ config.video_root,
+ allow_recovery_pair=True,
+ )
+ intermediates.append({
+ "kind": "audio_flac",
+ "old_path": _project_relative(config, old_flac),
+ "new_path": str(Path(os.path.abspath(new_flac))),
+ "bytes": identity["bytes"],
+ "sha256": identity["sha256"],
+ })
+ items.append({
+ "stable_id": stable_id,
+ "title": source["title"],
+ "published_at": source["published_at"],
+ "canonical_base": canonical_base,
+ "aliases": aliases,
+ "intermediates": intermediates,
+ })
+ docs: list[dict[str, Any]] = []
+ for relative in sorted(config.git_doc_paths):
+ path = Path(os.path.abspath(config.project_root / relative))
+ identity = _migration_file_identity(config, path, "documentation", config.project_root)
+ docs.append({"path": relative, **identity})
+ report: dict[str, Any] = {
+ "schema_version": SCHEMA,
+ "type": RELOCATION_REPORT_TYPE,
+ "task_id": TASK_ID,
+ "creator_uid": config.creator_uid,
+ "batch_id": "",
+ "created_at": created_at,
+ "baseline_head": baseline_head,
+ "items": items,
+ "docs": docs,
+ "remove_paths": sorted(set(remove_paths)),
+ }
+ report["batch_id"] = _relocation_batch_id(report)
+ return report
+
+
+def _relocation_outbox_exists(config: Config, report_path: Path, report: Mapping[str, Any]) -> bool:
+ if not config.outbox_path.exists():
+ return False
+ payload = _stable_artifact_bytes(report_path, config.relocation_root)
+ source = {
+ "journal": "relocation_report",
+ "path": _project_relative(config, report_path),
+ "bytes": len(payload),
+ "sha256": hashlib.sha256(payload).hexdigest().upper(),
+ }
+ expected_payload = _project_relocation_payload(config, source)
+ expected_id = _outbox_id("GIT_DELIVERY_READY", source, expected_payload)
+ rows = _outbox_rows(config)
+ created = [
+ row for row in rows
+ if row.get("event") == "CREATED" and row.get("outbox_id") == expected_id
+ ]
+ if len(created) > 1:
+ raise PipelineError("E_RELOCATION", "relocation outbox is duplicated")
+ return len(created) == 1
+
+
+def plan_video_artifact_migration(
+ config: Config,
+ now: datetime,
+ *,
+ runner: Callable[..., subprocess.CompletedProcess[Any]] = subprocess.run,
+) -> dict[str, Any]:
+ reports = _relocation_reports(config)
+ stable_ids = _video_artifact_ids(config)
+ if reports:
+ report_path, existing = reports[-1]
+ report_ids = [item["stable_id"] for item in existing["items"]]
+ outbox_exists = _relocation_outbox_exists(config, report_path, existing)
+ if not outbox_exists and not set(stable_ids).issubset(set(report_ids)):
+ raise PipelineError("E_RELOCATION", "durable relocation report omitted a legacy identity")
+ rebound = _build_video_artifact_migration_report(
+ config,
+ existing["created_at"],
+ existing["baseline_head"],
+ report_ids,
+ runner,
+ )
+ if rebound != existing:
+ raise PipelineError("E_RELOCATION", "durable relocation report differs from the external plan")
+ if not outbox_exists:
+ return existing
+ new_ids = sorted(set(stable_ids) - set(report_ids))
+ if not new_ids:
+ return existing
+ historical_ids = {
+ item["stable_id"]
+ for _, report in reports
+ for item in report["items"]
+ }
+ if any(stable_id in historical_ids for stable_id in new_ids):
+ raise PipelineError("E_RELOCATION", "legacy video identity was already migrated")
+ stable_ids = new_ids
+ if not stable_ids:
+ raise PipelineError("E_RELOCATION", "no legacy video artifacts were found")
+ return _build_video_artifact_migration_report(
+ config,
+ now.isoformat(),
+ _git_head(config, runner),
+ stable_ids,
+ runner,
+ )
+
+
+def _ensure_directory_chain(root: Path, target: Path) -> None:
+ if not _within(target, root):
+ raise PipelineError("E_RELOCATION", "migration directory escaped its root")
+ missing: list[Path] = []
+ current = target
+ while not current.exists() and current != root:
+ missing.append(current)
+ current = current.parent
+ _strict_chain(root, current, final_file=False)
+ for path in reversed(missing):
+ path.mkdir()
+ _strict_chain(root, path, final_file=False)
+
+
+def _move_relocation_file(
+ source_root: Path,
+ target_root: Path,
+ source: Path,
+ target: Path,
+ size: int,
+ digest: str,
+) -> None:
+ source_exists = source.exists() or source.is_symlink()
+ target_exists = target.exists() or target.is_symlink()
+ if not source_exists:
+ if not target_exists:
+ raise PipelineError("E_RELOCATION", "migration source and target are absent")
+ payload = _stable_artifact_bytes(target, target_root)
+ if len(payload) != size or hashlib.sha256(payload).hexdigest().upper() != digest:
+ raise PipelineError("E_RELOCATION", "migration target identity differs")
+ return
+ with _held_relocation_source(source, source_root, size, digest) as (stream, payload, source_chain):
+ if target_exists:
+ rebound = _stable_artifact_bytes(target, target_root)
+ if rebound != payload:
+ raise PipelineError("E_RELOCATION", "migration recovery target differs")
+ else:
+ _ensure_directory_chain(target_root, target.parent)
+ _create_new(target, payload)
+ rebound = _stable_artifact_bytes(target, target_root)
+ if rebound != payload:
+ raise PipelineError("E_RELOCATION", "migration target readback differs")
+ _delete_held_relocation_source(stream, source, source_root, payload, source_chain)
+ if source.exists() or source.is_symlink():
+ raise PipelineError("E_RELOCATION", "migration source remained after handle-bound delete")
+
+
+def migrate_video_artifacts(
+ config: Config,
+ now: datetime,
+ *,
+ runner: Callable[..., subprocess.CompletedProcess[Any]] = subprocess.run,
+) -> dict[str, Any]:
+ with _lock(config):
+ state = _load_state(config)
+ if state["active_run"] is not None:
+ raise PipelineError("E_RUN_ACTIVE", "artifact migration requires no active half-hour run")
+ report = plan_video_artifact_migration(config, now, runner=runner)
+ report_path = config.relocation_root / f"{report['batch_id']}.json"
+ if not report_path.exists() and not report_path.is_symlink():
+ _ensure_directory_chain(config.archive_root, config.relocation_root)
+ _create_new(report_path, _canonical(report))
+ rebound = _read_relocation_report(config, report_path)
+ if rebound != report:
+ raise PipelineError("E_RELOCATION", "durable relocation report differs")
+ for item in report["items"]:
+ for alias in item["aliases"]:
+ source = Path(os.path.abspath(config.project_root / alias["old_path"]))
+ target = Path(os.path.abspath(config.project_root / alias["new_path"]))
+ _move_relocation_file(
+ config.archive_root,
+ config.archive_root,
+ source,
+ target,
+ alias["bytes"],
+ alias["sha256"],
+ )
+ for intermediate in item["intermediates"]:
+ source = Path(os.path.abspath(config.project_root / intermediate["old_path"]))
+ target = Path(os.path.abspath(intermediate["new_path"]))
+ _move_relocation_file(
+ config.archive_root,
+ config.video_root,
+ source,
+ target,
+ intermediate["bytes"],
+ intermediate["sha256"],
+ )
+ for suffix in (".transcript", ".minutes"):
+ legacy = config.archive_root / f"{item['stable_id']}{suffix}"
+ if legacy.exists():
+ _strict_chain(config.archive_root, legacy, final_file=False)
+ try:
+ legacy.rmdir()
+ except OSError as exc:
+ raise PipelineError("E_RELOCATION", "legacy artifact directory is not empty") from exc
+ completed = _build_video_artifact_migration_report(
+ config,
+ report["created_at"],
+ report["baseline_head"],
+ [item["stable_id"] for item in report["items"]],
+ runner,
+ )
+ if completed != report:
+ raise PipelineError("E_RELOCATION", "completed migration differs from its durable external plan")
+ report_payload = _stable_artifact_bytes(report_path, config.relocation_root)
+ source = {
+ "journal": "relocation_report",
+ "path": _project_relative(config, report_path),
+ "bytes": len(report_payload),
+ "sha256": hashlib.sha256(report_payload).hexdigest().upper(),
+ }
+ payload = _project_relocation_payload(config, source)
+ outbox_id = _append_outbox(config, "GIT_DELIVERY_READY", source, payload, report["created_at"])
+ return {
+ "status": "VIDEO_ARTIFACTS_MIGRATED",
+ "batch_id": report["batch_id"],
+ "report": source,
+ "item_count": len(report["items"]),
+ "public_file_count": sum(len(item["aliases"]) for item in report["items"]),
+ "intermediate_count": sum(len(item["intermediates"]) for item in report["items"]),
+ "git_outbox_id": outbox_id,
+ }
+
+
+def git_deliver(config: Config, outbox_id: str, now: datetime, *, runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run) -> dict[str, Any]:
+ with _lock(config):
+ rows = _outbox_rows(config)
+ item = next((row for row in rows if row.get("event") == "CREATED" and row.get("outbox_id") == outbox_id), None)
+ if item is None or item.get("kind") != "GIT_DELIVERY_READY":
+ raise PipelineError("E_GIT_SCOPE", "Git outbox differs")
+ if any(row.get("event") == "COMPLETE" and row.get("outbox_id") == outbox_id for row in rows):
+ return {"status": "GIT_ALREADY_COMPLETE", "outbox_id": outbox_id}
+ _ensure_git_index_guard(config, rows, item, runner)
+ payload = item.get("payload", {})
+ files = payload.get("files")
+ allowed, _, blobs = _git_artifacts(config, files)
+ remove_paths = _git_remove_paths(config, payload.get("remove_paths"))
+ expected_paths = _git_expected_paths(payload)
+ if expected_paths != (allowed if not remove_paths else sorted([*allowed, *remove_paths])):
+ raise PipelineError("E_GIT_SCOPE", "Git staged scope differs from its payload")
+ with _published_head_with_exact_artifacts(config, allowed, blobs, remove_paths, runner) as published_head:
+ if published_head is not None:
+ _append(config.outbox_path, {
+ "schema_version": SCHEMA, "event": "COMPLETE", "outbox_id": outbox_id,
+ "result": "NO_CHANGES", "completed_at": now.isoformat(),
+ })
+ return {
+ "status": "GIT_NO_CHANGES", "outbox_id": outbox_id,
+ "commit_sha": published_head, "files": allowed,
+ }
+ git_intents = [row for row in rows if row.get("event") == "GIT_COMMIT_INTENT" and row.get("outbox_id") == outbox_id]
+ commit_events = [row for row in rows if row.get("event") == "COMMIT_CREATED" and row.get("outbox_id") == outbox_id]
+ if len(git_intents) > 1 or len(commit_events) > 1:
+ raise PipelineError("E_GIT_COMMIT", "Git commit journal is ambiguous")
+ if commit_events:
+ event = commit_events[0]
+ commit_sha = event.get("commit_sha")
+ parent_sha = event.get("parent_sha")
+ if (
+ not isinstance(commit_sha, str) or not re.fullmatch(r"[0-9a-f]{40,64}", commit_sha)
+ or not isinstance(parent_sha, str) or not re.fullmatch(r"[0-9a-f]{40,64}", parent_sha)
+ or event.get("files") != expected_paths
+ ):
+ raise PipelineError("E_GIT_COMMIT", "Git commit journal identity differs")
+ exists = _run(["git", "cat-file", "-e", f"{commit_sha}^{{commit}}"], config.project_root, runner)
+ head = _run(["git", "rev-parse", "HEAD"], config.project_root, runner)
+ current_head = head.stdout.strip()
+ if exists.returncode != 0 or head.returncode != 0 or current_head not in {parent_sha, commit_sha}:
+ raise PipelineError("E_GIT_COMMIT", "Git recovery identity differs")
+ if current_head == parent_sha:
+ _git_artifacts(config, files)
+ _git_remove_paths(config, payload.get("remove_paths"))
+ updated = _run(["git", "update-ref", "HEAD", commit_sha, parent_sha], config.project_root, runner)
+ if updated.returncode != 0:
+ raise PipelineError("E_GIT_COMMIT", "Git recovery ref update failed")
+ pushed = _run(["git", "push", config.git_remote, f"{commit_sha}:refs/heads/{config.git_branch}"], config.project_root, runner)
+ if pushed.returncode != 0:
+ raise PipelineError("E_GIT_PUSH", "non-force push failed")
+ _append(config.outbox_path, {
+ "schema_version": SCHEMA, "event": "COMPLETE", "outbox_id": outbox_id,
+ "result": "PUSHED", "commit_sha": commit_sha, "completed_at": now.isoformat(),
+ })
+ return {"status": "GIT_PUSHED", "outbox_id": outbox_id, "commit_sha": commit_sha, "files": allowed}
+ task_index = config.state_dir / f"git-index-{outbox_id}"
+ task_lock = Path(str(task_index) + ".lock")
+ task_env = dict(os.environ)
+ task_env["GIT_INDEX_FILE"] = str(task_index)
+ try:
+ git_intent = git_intents[0] if git_intents else None
+ if git_intent is None:
+ _remove_task_index(task_index, config.state_dir)
+ _remove_task_index(task_lock, config.state_dir)
+ read_tree = _run_env(["git", "read-tree", "HEAD"], config.project_root, runner, task_env)
+ added_ok = True
+ for relative, artifact_payload in zip(allowed, blobs, strict=True):
+ blob = _run_env_input(["git", "hash-object", "-w", "--stdin"], config.project_root, runner, task_env, artifact_payload)
+ blob_sha = blob.stdout.decode("ascii").strip() if isinstance(blob.stdout, bytes) else blob.stdout.strip()
+ if blob.returncode != 0 or not re.fullmatch(r"[0-9a-f]{40,64}", blob_sha):
+ added_ok = False
+ break
+ indexed = _run_env(["git", "update-index", "--add", "--cacheinfo", f"100644,{blob_sha},{relative}"], config.project_root, runner, task_env)
+ if indexed.returncode != 0:
+ added_ok = False
+ break
+ for relative in remove_paths:
+ removed = _run_env(
+ ["git", "update-index", "--force-remove", "--", relative],
+ config.project_root,
+ runner,
+ task_env,
+ )
+ if removed.returncode != 0:
+ added_ok = False
+ break
+ # `-z` returns repository path bytes without C quoting or console
+ # code-page conversion. Decode Git's UTF-8 path contract directly
+ # before enforcing the exact task-index allowlist.
+ staged_command = ["git", "diff", "--cached", "--name-only", "-z"]
+ if remove_paths:
+ staged_command.append("--no-renames")
+ staged_command.append("--")
+ staged = _run_env_bytes(
+ staged_command,
+ config.project_root,
+ runner,
+ task_env,
+ )
+ try:
+ staged_paths = [part.decode("utf-8").replace("\\", "/") for part in staged.stdout.split(b"\0") if part]
+ except (AttributeError, UnicodeDecodeError) as exc:
+ raise PipelineError("E_GIT_SCOPE", "task index path encoding differs") from exc
+ if read_tree.returncode != 0 or not added_ok:
+ raise PipelineError("E_GIT_ADD", "task-index preparation failed")
+ if staged.returncode != 0 or set(staged_paths) != set(expected_paths):
+ raise PipelineError(
+ "E_GIT_SCOPE",
+ f"task index escaped the exact allowlist ({len(staged_paths)}/{len(expected_paths)})",
+ )
+ if not staged_paths:
+ _append(config.outbox_path, {"schema_version": SCHEMA, "event": "COMPLETE", "outbox_id": outbox_id, "result": "NO_CHANGES", "completed_at": now.isoformat()})
+ return {"status": "GIT_NO_CHANGES", "outbox_id": outbox_id}
+ parent = _run(["git", "rev-parse", "HEAD"], config.project_root, runner)
+ tree = _run_env(["git", "write-tree"], config.project_root, runner, task_env)
+ parent_sha = parent.stdout.strip()
+ tree_sha = tree.stdout.strip()
+ if parent.returncode != 0 or not re.fullmatch(r"[0-9a-f]{40,64}", parent_sha) or tree.returncode != 0 or not re.fullmatch(r"[0-9a-f]{40,64}", tree_sha):
+ raise PipelineError("E_GIT_COMMIT", "Git parent or tree identity is unavailable")
+ git_intent = {
+ "schema_version": SCHEMA, "event": "GIT_COMMIT_INTENT", "outbox_id": outbox_id,
+ "parent_sha": parent_sha, "tree_sha": tree_sha, "files": expected_paths,
+ "message": (
+ f"chore(project-info): migrate Bilibili artifacts {outbox_id[:12]}"
+ if remove_paths else f"chore(project-info): archive Bilibili dynamic {outbox_id[:12]}"
+ ),
+ "author_name": "MB-X Bilibili Pipeline", "author_email": "mbx-bili-pipeline@localhost",
+ "authored_at": now.isoformat(), "created_at": now.isoformat(),
+ }
+ _append(config.outbox_path, git_intent)
+ parent_sha = git_intent["parent_sha"]
+ tree_sha = git_intent["tree_sha"]
+ head = _run(["git", "rev-parse", "HEAD"], config.project_root, runner)
+ tree_exists = _run(["git", "cat-file", "-e", f"{tree_sha}^{{tree}}"], config.project_root, runner)
+ if head.returncode != 0 or head.stdout.strip() != parent_sha or tree_exists.returncode != 0 or git_intent["files"] != expected_paths:
+ raise PipelineError("E_GIT_COMMIT", "Git commit intent recovery identity differs")
+ _git_artifacts(config, files)
+ _git_remove_paths(config, payload.get("remove_paths"))
+ commit_env = dict(os.environ)
+ commit_env.update({
+ "GIT_AUTHOR_NAME": git_intent["author_name"], "GIT_COMMITTER_NAME": git_intent["author_name"],
+ "GIT_AUTHOR_EMAIL": git_intent["author_email"], "GIT_COMMITTER_EMAIL": git_intent["author_email"],
+ "GIT_AUTHOR_DATE": git_intent["authored_at"], "GIT_COMMITTER_DATE": git_intent["authored_at"],
+ })
+ committed = _run_env(["git", "commit-tree", tree_sha, "-p", parent_sha, "-m", git_intent["message"]], config.project_root, runner, commit_env)
+ finally:
+ _remove_task_index(task_index, config.state_dir)
+ _remove_task_index(task_lock, config.state_dir)
+ commit_sha = committed.stdout.strip()
+ if committed.returncode != 0 or not re.fullmatch(r"[0-9a-f]{40,64}", commit_sha):
+ raise PipelineError("E_GIT_COMMIT", "commit identity is unavailable")
+ _append(config.outbox_path, {
+ "schema_version": SCHEMA, "event": "COMMIT_CREATED", "outbox_id": outbox_id,
+ "parent_sha": parent_sha, "tree_sha": tree_sha, "commit_sha": commit_sha,
+ "intent_sha256": hashlib.sha256(_canonical(git_intent)).hexdigest().upper(),
+ "files": expected_paths, "created_at": now.isoformat(),
+ })
+ _git_artifacts(config, files)
+ _git_remove_paths(config, payload.get("remove_paths"))
+ updated = _run(["git", "update-ref", "HEAD", commit_sha, parent_sha], config.project_root, runner)
+ if updated.returncode != 0:
+ raise PipelineError("E_GIT_COMMIT", "task-scoped ref update failed")
+ pushed = _run(["git", "push", config.git_remote, f"{commit_sha}:refs/heads/{config.git_branch}"], config.project_root, runner)
+ if pushed.returncode != 0:
+ raise PipelineError("E_GIT_PUSH", "non-force push failed")
+ event = {"schema_version": SCHEMA, "event": "COMPLETE", "outbox_id": outbox_id, "result": "PUSHED", "commit_sha": commit_sha, "completed_at": now.isoformat()}
+ _append(config.outbox_path, event)
+ return {"status": "GIT_PUSHED", "outbox_id": outbox_id, "commit_sha": commit_sha, "files": allowed}
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Durable Bilibili half-hour pipeline coordinator")
+ parser.add_argument("--config", type=Path, required=True)
+ sub = parser.add_subparsers(dest="command", required=True)
+ for name in ("init", "begin", "reconcile", "pending", "git-preflight", "plan-video-artifact-migration", "migrate-video-artifacts"):
+ child = sub.add_parser(name)
+ if name not in {"pending", "git-preflight"}:
+ child.add_argument("--now")
+ dispatch = sub.add_parser("dispatch-intent")
+ dispatch.add_argument("--outbox-id", required=True)
+ dispatch.add_argument("--now")
+ observed = sub.add_parser("observe-dispatch")
+ observed.add_argument("--outbox-id", required=True)
+ observed.add_argument("--delivery-id", required=True)
+ observed.add_argument("--now")
+ receipt = sub.add_parser("ingest-receipt")
+ receipt.add_argument("--receipt", type=Path, required=True)
+ receipt.add_argument("--now")
+ finish_parser = sub.add_parser("finish")
+ finish_parser.add_argument("--status", choices=["COMPLETE", "FAILED"], required=True)
+ finish_parser.add_argument("--now")
+ git_parser = sub.add_parser("git-deliver")
+ git_parser.add_argument("--outbox-id", required=True)
+ git_parser.add_argument("--now")
+ guard_parser = sub.add_parser("recover-git-index-guard")
+ guard_parser.add_argument("--outbox-id", required=True)
+ guard_parser.add_argument("--baseline-head", required=True)
+ guard_parser.add_argument("--expected-bytes", required=True, type=int)
+ guard_parser.add_argument("--expected-sha256", required=True)
+ return parser
+
+
+def run(argv: Sequence[str] | None = None) -> tuple[int, dict[str, Any]]:
+ args = build_parser().parse_args(argv)
+ try:
+ config = load_config(Path(os.path.abspath(args.config)))
+ if args.command == "init":
+ result = initialize(config, _now(args.now))
+ elif args.command == "begin":
+ result = begin(config, _now(args.now))
+ elif args.command == "reconcile":
+ result = reconcile(config, _now(args.now))
+ elif args.command == "pending":
+ result = pending(config)
+ elif args.command == "git-preflight":
+ result = git_preflight(config)
+ elif args.command == "plan-video-artifact-migration":
+ result = plan_video_artifact_migration(config, _now(args.now))
+ elif args.command == "migrate-video-artifacts":
+ result = migrate_video_artifacts(config, _now(args.now))
+ elif args.command == "dispatch-intent":
+ result = dispatch_intent(config, args.outbox_id, _now(args.now))
+ elif args.command == "observe-dispatch":
+ result = observe_dispatch(config, args.outbox_id, args.delivery_id, _now(args.now))
+ elif args.command == "ingest-receipt":
+ result = ingest_receipt(config, Path(os.path.abspath(args.receipt)), _now(args.now))
+ elif args.command == "finish":
+ result = finish(config, args.status, _now(args.now))
+ elif args.command == "recover-git-index-guard":
+ result = recover_git_index_guard(
+ config,
+ args.outbox_id,
+ args.baseline_head,
+ args.expected_bytes,
+ args.expected_sha256,
+ )
+ else:
+ result = git_deliver(config, args.outbox_id, _now(args.now))
+ return 0, result
+ except PipelineError as exc:
+ return 2, {"status": "FAILED", "error_code": exc.code}
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ code, result = run(argv)
+ sys.stdout.write(json.dumps(result, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n")
+ return code
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/dev/project-dev/bili_half_hour_pipeline_source_manifest.json b/dev/project-dev/bili_half_hour_pipeline_source_manifest.json
new file mode 100644
index 0000000..5976d5d
--- /dev/null
+++ b/dev/project-dev/bili_half_hour_pipeline_source_manifest.json
@@ -0,0 +1,50 @@
+{
+ "schema": 1,
+ "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "scope": "bilibili-job-owned-close-and-half-hour-pipeline",
+ "root": "dev/project-dev",
+ "tree_hash_algorithm": "sha256(path-utf8,nul,decimal-bytes-ascii,nul,payload-sha256-lower-hex-ascii,lf)-upper-hex-v1",
+ "tree_sha256": "8E3246BC37F0BE8C10BC0DF1ED5F5138F2E11E95986F858B82C2FCD7BC215005",
+ "files": [
+ {
+ "path": "bili_half_hour_pipeline.py",
+ "bytes": 150022,
+ "sha256": "498F36937E0AB9B87FF7C7E92D899F1B8E2F7F3752E52A1EC5629D36071973CE"
+ },
+ {
+ "path": "bili_half_hour_pipeline.config.json",
+ "bytes": 1398,
+ "sha256": "679241B516A183971DBE59AD159D19AC10C93C932244962971CCB7E9DF328A39"
+ },
+ {
+ "path": "test/test_bili_half_hour_pipeline.py",
+ "bytes": 93686,
+ "sha256": "09C9F3BC165BA0B9F3B414C73C03AE5729F07E84668E354CCC95DE5E936F4B60"
+ },
+ {
+ "path": "bili_dynamic_refresh.py",
+ "bytes": 126349,
+ "sha256": "52A83FF8064C1A7DEBE15F3B52363416E9FD308D0A0D7C54BBEAC041C9A0BFD8"
+ },
+ {
+ "path": "test/test_bili_dynamic_refresh.py",
+ "bytes": 79064,
+ "sha256": "AC2B44830D3301FD244CA5DA58CE732DD2A6CE5ABF4715CDA71291B07B98A32B"
+ },
+ {
+ "path": "bili_article_image_collector.py",
+ "bytes": 53497,
+ "sha256": "C8B9BB3EAA8BE580DF11A20DCA94B507DCE1414D62D65F42D09BDAAA5348BB93"
+ },
+ {
+ "path": "bili_article_image_source_manifest.json",
+ "bytes": 5407,
+ "sha256": "1E0671EB8796DFE6F94D68A473EF4F960506496B7E01053924C62B72326D0C68"
+ },
+ {
+ "path": "bili_authenticated_extension/source-artifact-manifest.json",
+ "bytes": 3770,
+ "sha256": "132B637634550A43B0EDD5E8E74C439205DE4BFECCE0A9276D8EBEF78445ECFC"
+ }
+ ]
+}
diff --git a/dev/project-dev/test/bili_authenticated_extension/test_successor_trust_gate.py b/dev/project-dev/test/bili_authenticated_extension/test_successor_trust_gate.py
new file mode 100644
index 0000000..3378e91
--- /dev/null
+++ b/dev/project-dev/test/bili_authenticated_extension/test_successor_trust_gate.py
@@ -0,0 +1,915 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import pathlib
+import tempfile
+import unittest
+from unittest import mock
+
+PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[4]
+PROJECT_DEV = PROJECT_ROOT / "dev" / "project-dev"
+if str(PROJECT_DEV) not in os.sys.path:
+ os.sys.path.insert(0, str(PROJECT_DEV))
+
+from bili_authenticated_extension.constants import ( # noqa: E402
+ EXTENSION_BUILD,
+ HOST_BUILD,
+ MANIFEST_PUBLIC_KEY,
+ RELOAD_GENERATION,
+ stable_job_id,
+)
+from bili_authenticated_extension.protocol import ProtocolError # noqa: E402
+from bili_authenticated_extension import queue_producer as producer # noqa: E402
+from bili_authenticated_extension.queue_producer import ( # noqa: E402
+ _load_release_approval,
+ _projection_tree,
+ _successor_scope_sha256,
+)
+from bili_authenticated_extension.queue_state import QueueStore # noqa: E402
+
+
+CREATOR = "246813579"
+BVID = "BV1AbCd2EfGh"
+AUTH_MESSAGE = "msg_20260817123456789_deadbeef"
+AUTH_HANDOFF = "HANDOFF-INFOADMIN-INFODEV2-SYNTHETIC-SUCCESSOR-20260817-001"
+SOURCE_FILES = (
+ "__init__.py", "background.js", "build_host.ps1", "config.example.json",
+ "constants.py", "dependencies/dependency-artifact-manifest.json",
+ "dependencies/yt_dlp-2026.7.4-py3-none-any.whl", "formal_legacy_identity_manifest.py",
+ "install_native_host.ps1",
+ "job.py", "manifest.json", "native-host-manifest.template.json", "native_host.py",
+ "protocol.py", "queue-producer.example.json", "queue_producer.py", "queue_state.py",
+ "sidepanel.css", "sidepanel.html", "sidepanel.js", "worker.py",
+)
+
+
+def encode(value: object) -> bytes:
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
+
+
+def write_json(path: pathlib.Path, value: object) -> bytes:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ payload = encode(value)
+ path.write_bytes(payload)
+ return payload
+
+
+def file_spec(root: pathlib.Path, path: pathlib.Path) -> dict[str, object]:
+ payload = path.read_bytes()
+ return {
+ "relative_path": path.relative_to(root).as_posix(),
+ "bytes": len(payload),
+ "sha256": hashlib.sha256(payload).hexdigest().upper(),
+ }
+
+
+def build_environment(
+ root: pathlib.Path, *, fast_path: bool = False,
+ fast_path_mode: str = "PRODUCER_ONLY_DELTA",
+) -> tuple[dict[str, object], pathlib.Path, pathlib.Path, pathlib.Path]:
+ source_root = root / "dev/project-dev/bili_authenticated_extension"
+ projection_root = root / "dev/project-dev/bili_authenticated_extension_unpacked"
+ runtime = root / "runtime"
+ runtime.mkdir(parents=True)
+ queue = runtime / "queue.jsonl"
+ state = runtime / "state.jsonl"
+ lock = runtime / "queue.lock"
+ reload = runtime / "reload.jsonl"
+ lock.write_bytes(b"\0")
+
+ manifest = {
+ "manifest_version": 3,
+ "name": "synthetic generic",
+ "version": "1.2.25",
+ "version_name": "1.2.25+20260829.generic.v027",
+ "key": MANIFEST_PUBLIC_KEY,
+ }
+ payloads: dict[str, bytes] = {
+ "background.js": f'const EXTENSION_BUILD = "{EXTENSION_BUILD}";\n'.encode("ascii"),
+ "manifest.json": encode(manifest),
+ "sidepanel.css": b"body{}\n",
+ "sidepanel.html": b"<!doctype html><title>synthetic</title>\n",
+ "sidepanel.js": b"export {};\n",
+ }
+ for name in SOURCE_FILES:
+ payloads.setdefault(name, f"synthetic:{name}\n".encode("ascii"))
+ dependency_payload = b'{"schema":1,"synthetic":true}\n'
+ payloads["dependencies/dependency-artifact-manifest.json"] = dependency_payload
+ entries = []
+ for name in ("background.js", "manifest.json", "sidepanel.css", "sidepanel.html", "sidepanel.js"):
+ payload = payloads[name]
+ for parent in (source_root, projection_root):
+ (parent / name).parent.mkdir(parents=True, exist_ok=True)
+ (parent / name).write_bytes(payload)
+ entries.append({
+ "path": name, "bytes": len(payload),
+ "sha256": hashlib.sha256(payload).hexdigest().upper(),
+ })
+ source_entries = []
+ for name in SOURCE_FILES:
+ payload = payloads[name]
+ candidate = source_root / name
+ candidate.parent.mkdir(parents=True, exist_ok=True)
+ candidate.write_bytes(payload)
+ source_entries.append({
+ "path": name, "bytes": len(payload),
+ "sha256": hashlib.sha256(payload).hexdigest().upper(),
+ })
+ tree_hash = _projection_tree(entries)
+ contract_payload = write_json(root / "dev/project-dev/bili_authenticated_extension_unpacked_contract.json", {
+ "schema": 1,
+ "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "project_id": "project-info",
+ "source_root": "dev/project-dev/bili_authenticated_extension",
+ "projection_root": "dev/project-dev/bili_authenticated_extension_unpacked",
+ "expected_extension_id": "oidmclckpdmpabbfedplkbdplmfcenbb",
+ "public_key_der_sha256": "E83C2B2AF3CF011543FBA13FBC524D1122EEA68548F9F27B9F7A82B5D594666C",
+ "tree_hash_algorithm": "path-nul-bytes-nul-sha256-upper-lf-v1",
+ "tree_sha256": tree_hash,
+ "files": sorted(entries, key=lambda item: item["path"]),
+ })
+ source_manifest_value = {
+ "schema": 1, "scope": "generic-bilibili-queue",
+ "extension_id": "oidmclckpdmpabbfedplkbdplmfcenbb",
+ "extension_build": EXTENSION_BUILD, "host_build": HOST_BUILD,
+ "archive_metadata_contract": {
+ "schema": 1, "root": "yt_dlp-2026.7.4.dist-info",
+ "relative_files": ["INSTALLER", "METADATA", "RECORD", "REQUESTED", "WHEEL", "entry_points.txt", "licenses/LICENSE"],
+ "distribution_name": "yt-dlp", "distribution_version": "2026.7.4",
+ "allowed_type_codes": ["b", "x"], "source_date_epoch": 1786207924,
+ "tree_hash_algorithm": "sha256(path-utf8,nul,decimal-bytes-ascii,nul,payload-sha256-lower-hex-ascii,lf)-upper-hex-v1",
+ "canonical_tree_sha256": "32F1DC23F6704966AE4511CB173318CD11FCBA031323028D268D9F2488589E70",
+ },
+ "dependency_artifact_manifest_bytes": len(dependency_payload),
+ "dependency_artifact_manifest_sha256": hashlib.sha256(dependency_payload).hexdigest().upper(),
+ "files": source_entries,
+ }
+ source_manifest_payload = write_json(
+ source_root / "source-artifact-manifest.json", source_manifest_value,
+ )
+ source_hash = hashlib.sha256(source_manifest_payload).hexdigest().upper()
+ host_build_source_manifest_payload = source_manifest_payload
+ host_build_source_manifest_hash = source_hash
+ host_build_source_manifest_path = root / "ai-infoadmin/worklog/synthetic-fast-path-host-build-source-manifest.json"
+ if fast_path:
+ host_source = json.loads(json.dumps(source_manifest_value))
+ if fast_path_mode == "PRODUCER_ONLY_DELTA":
+ for item in host_source["files"]:
+ if item["path"] == "queue_producer.py":
+ previous = b"synthetic:queue_producer.py:previous\n"
+ item["bytes"] = len(previous)
+ item["sha256"] = hashlib.sha256(previous).hexdigest().upper()
+ break
+ host_build_source_manifest_payload = write_json(host_build_source_manifest_path, host_source)
+ host_build_source_manifest_hash = hashlib.sha256(host_build_source_manifest_payload).hexdigest().upper()
+
+ build_root = root / "dev/tmp/synthetic-build"
+ exe = build_root / "project-info-bili-auth-native-host.exe"
+ exe.parent.mkdir(parents=True)
+ exe.write_bytes(b"MZsynthetic-v002")
+ exe_hash = hashlib.sha256(exe.read_bytes()).hexdigest().upper()
+ build_receipt_payload = write_json(build_root / "build-artifact-manifest.json", {
+ "schema": 2, "scope": "generic-bilibili-queue",
+ "extension_id": "oidmclckpdmpabbfedplkbdplmfcenbb",
+ "extension_build": EXTENSION_BUILD, "host_build": HOST_BUILD,
+ "packaging": "pyinstaller-onefile", "pyinstaller_version": "6.15.0",
+ "yt_dlp_version": "2026.7.4", "builder_python_sha256": "A" * 64,
+ "pyinstaller_executable_bytes": 1, "pyinstaller_executable_sha256": "B" * 64,
+ "builder_provision_receipt_bytes": 1, "builder_provision_receipt_sha256": "C" * 64,
+ "build_script_sha256": "D" * 64,
+ "source_artifact_manifest_bytes": len(host_build_source_manifest_payload),
+ "source_artifact_manifest_sha256": host_build_source_manifest_hash,
+ "dependency_artifact_manifest_bytes": len(dependency_payload),
+ "dependency_artifact_manifest_sha256": hashlib.sha256(dependency_payload).hexdigest().upper(),
+ "yt_dlp_wheel_sha256": "E" * 64,
+ "archive_verification": {
+ "status": "PASS", "method": "synthetic static archive",
+ "required_modules": ["bili_authenticated_extension.worker", "yt_dlp", "yt_dlp.downloader", "yt_dlp.globals", "yt_dlp.plugins", "yt_dlp.version"],
+ "metadata_entry": "yt_dlp-2026.7.4.dist-info/METADATA", "metadata_files": 7,
+ "metadata_type_codes": ["b"],
+ "metadata_tree_sha256": "32F1DC23F6704966AE4511CB173318CD11FCBA031323028D268D9F2488589E70",
+ },
+ "files": [{"path": exe.name, "bytes": exe.stat().st_size, "sha256": exe_hash}],
+ })
+ source_receipt_path = root / (
+ "ai-infoadmin/worklog/synthetic-fast-path-source-receipt.json"
+ if fast_path else "ai-inforev/worklog/synthetic-source-approval.json"
+ )
+ build_approval_payload = write_json(source_receipt_path, {
+ "schema": 1, "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "approval_scope": "controlled-build-source-manifest",
+ "approved_by_role": "project.admin" if fast_path else "dev.reviewer.project",
+ "status": "APPROVED",
+ "source_artifact_manifest_bytes": len(source_manifest_payload),
+ "source_artifact_manifest_sha256": source_hash,
+ })
+ build_validation_receipt_path = root / (
+ "ai-infoadmin/worklog/synthetic-fast-path-build-receipt.json"
+ if fast_path else "ai-inforev/worklog/synthetic-install-approval.json"
+ )
+ install_approval_payload = write_json(build_validation_receipt_path, {
+ "schema": 1, "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "approval_scope": "install-exact-build",
+ "approved_by_role": "project.admin" if fast_path else "dev.reviewer.project",
+ "status": "APPROVED",
+ "source_artifact_manifest_sha256": host_build_source_manifest_hash,
+ "build_artifact_manifest_bytes": len(build_receipt_payload),
+ "build_artifact_manifest_sha256": hashlib.sha256(build_receipt_payload).hexdigest().upper(),
+ "host_executable_bytes": len(exe.read_bytes()),
+ "host_executable_sha256": exe_hash,
+ })
+
+ installed_root = root / "installed/generic-v002"
+ installed_root.mkdir(parents=True)
+ installed_exe = installed_root / exe.name
+ installed_exe.write_bytes(exe.read_bytes())
+ host_config = installed_root / "config.json"
+ host_config_payload = write_json(host_config, {
+ "schema": 2, "creator_allowlist": [CREATOR],
+ "queue_path": str(queue), "queue_state_path": str(state), "queue_lock_path": str(lock),
+ "reload_state_path": str(reload), "reload_generation": RELOAD_GENERATION,
+ "required_extension_build": EXTENSION_BUILD,
+ "ffmpeg": str(root / "tools/ffmpeg.exe"), "ffmpeg_sha256": "1" * 64,
+ "ffprobe": str(root / "tools/ffprobe.exe"), "ffprobe_sha256": "2" * 64,
+ "bridge_python": str(root / "tools/python.exe"), "bridge_python_sha256": "3" * 64,
+ "bridge_script": str(root / "tools/bridge.py"), "bridge_script_sha256": "4" * 64,
+ "yt_dlp_executable": str(root / "tools/yt-dlp.exe"), "yt_dlp_executable_sha256": "5" * 64,
+ "destination": str(root / "destination"),
+ "creator_name": "Synthetic Creator",
+ "formal_manifest_path": str(root / "formal/manifest.jsonl"),
+ "processing_handoff_path": str(root / "formal/video-processing-handoffs.jsonl"),
+ })
+ native_manifest = installed_root / "native-host-manifest.json"
+ native_manifest_payload = write_json(native_manifest, {
+ "name": "com.project_info.bili_auth_ingress", "description": "synthetic",
+ "path": str(installed_exe), "type": "stdio",
+ "allowed_origins": ["chrome-extension://oidmclckpdmpabbfedplkbdplmfcenbb/"],
+ })
+ installed_files = []
+ for candidate in (installed_exe, host_config, native_manifest):
+ payload = candidate.read_bytes()
+ installed_files.append({
+ "path": candidate.name, "bytes": len(payload),
+ "sha256": hashlib.sha256(payload).hexdigest().upper(),
+ })
+ install_receipt_path = root / (
+ "ai-infoadmin/worklog/synthetic-fast-path-install-readiness-receipt.json"
+ if fast_path else "ai-infoadmin/worklog/synthetic-install-receipt.json"
+ )
+ if not fast_path:
+ install_receipt_payload = write_json(install_receipt_path, {
+ "schema": 1,
+ "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "host_build": HOST_BUILD, "required_extension_build": EXTENSION_BUILD,
+ "extension_id": "oidmclckpdmpabbfedplkbdplmfcenbb",
+ "host_name": "com.project_info.bili_auth_ingress",
+ "installed_root": str(installed_root), "installed_files": installed_files,
+ })
+ write_json(reload, {
+ "schema": 1, "generation": RELOAD_GENERATION, "event": "APPLIED",
+ "token": "a" * 32, "from_build": EXTENSION_BUILD, "to_build": EXTENSION_BUILD,
+ "at_unix_ms": 2_000_000_000_000,
+ })
+
+ ingress = {
+ "schema": 1, "bvid": BVID, "creator_uid": CREATOR,
+ "expected_duration_ms": 180_000, "discovered_at_unix_ms": 1_900_000_000_000,
+ "published_at": "2026-08-17T12:00:00+08:00", "title": "synthetic title",
+ }
+ queue.write_bytes(encode(ingress))
+ store = QueueStore(queue, state, lock, frozenset({CREATOR}))
+ claimed = store.claim_next(2_000_000_000_100)
+ assert claimed is not None
+ store.reject_claim(claimed[0], claimed[1], 2_000_000_000_101, "E_PAGE_PROOF", {
+ "attempts": 1, "elapsed_ms": 1, "state": "PAGE_REJECTED",
+ "reason": "PAGE_IDENTITY_REJECTED",
+ })
+
+ repair_audit_id = "DEV-AUDIT-PROJECT-INFO-SYNTHETIC-PAGE-PROOF-20260817-001"
+ implementation_audit_id = "DEV-AUDIT-PROJECT-INFO-SYNTHETIC-LINEAGE-IMPLEMENTATION-20260817-001"
+ repair_prefix = f"# audit\n\n## {repair_audit_id}\nPASS/0\n".encode("utf-8")
+ implementation_prefix = repair_prefix + f"\n## {implementation_audit_id}\nPASS/0\n".encode("utf-8")
+ audit_path = root / "dev-doc/开发审计报告.md"
+ audit_path.parent.mkdir(parents=True, exist_ok=True)
+ audit_path.write_bytes(implementation_prefix)
+ repair = {
+ "review_result_message_id": "msg_20260817111111111_a1b2c3d4",
+ "audit_id": repair_audit_id,
+ "audit_bytes": len(repair_prefix),
+ "audit_sha256": hashlib.sha256(repair_prefix).hexdigest().upper(),
+ "verdict": "PASS/0", "blocking_findings": 0,
+ }
+ authorization_path = root / "ai-infoadmin/worklog/synthetic-authorization.json"
+ authorization_value = {
+ "schema": 1, "scope": "bili-auth-successor-lineage-v1",
+ "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "authorized_by_role": "project.admin", "authorization_message_id": AUTH_MESSAGE,
+ "authorization_handoff_id": AUTH_HANDOFF, "repair": repair,
+ "successors": [{
+ "creator_uid": CREATOR, "bvid": BVID,
+ "predecessor_job_id": stable_job_id(CREATOR, BVID),
+ "retry_generation": 1, "terminal_error_code": "E_PAGE_PROOF",
+ }],
+ }
+ authorization_payload = write_json(authorization_path, authorization_value)
+ review = {
+ "result_message_id": "msg_20260817122222222_11223344",
+ "audit_id": implementation_audit_id,
+ "audit_bytes": len(implementation_prefix),
+ "audit_sha256": hashlib.sha256(implementation_prefix).hexdigest().upper(),
+ "verdict": "PASS/0", "blocking_findings": 0,
+ }
+ formal = root / "formal/manifest.jsonl"
+ formal.parent.mkdir(parents=True, exist_ok=True)
+ formal.write_bytes(b'{"schema":1,"synthetic":true}\n')
+ if fast_path:
+ def frozen_fields(path: pathlib.Path, prefix: str, *, exact: bool) -> dict[str, object]:
+ payload = path.read_bytes()
+ size_name = f"{prefix}_bytes" if exact else f"{prefix}_prefix_bytes"
+ lines_name = f"{prefix}_lines" if exact else f"{prefix}_prefix_lines"
+ sha_name = f"{prefix}_sha256" if exact else f"{prefix}_prefix_sha256"
+ return {
+ f"{prefix}_path": str(path), size_name: len(payload),
+ lines_name: payload.count(b"\n"),
+ sha_name: hashlib.sha256(payload).hexdigest().upper(),
+ }
+
+ install_receipt = {
+ "schema": 2,
+ "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "validation_scope": "continuous-fast-path-installed-readiness-v1",
+ "validated_by_role": "project.admin", "status": "VALIDATED",
+ "continuous_authorization_handoff_id": producer._CONTINUOUS_FAST_PATH_HANDOFF_ID,
+ "owner_ai_id": "infodev-2",
+ "owner_thread_id": "019fbcbb-bed7-7c90-83ab-f50610f80d3a",
+ "owner_role_instance_id": "dev.developer.project.secondary",
+ "authorization_file_relative_path": authorization_path.relative_to(root).as_posix(),
+ "authorization_file_bytes": len(authorization_payload),
+ "authorization_file_sha256": hashlib.sha256(authorization_payload).hexdigest().upper(),
+ "successor_scope_sha256": _successor_scope_sha256(authorization_value["successors"]),
+ "implementation_review_audit_id": review["audit_id"],
+ "implementation_review_audit_bytes": review["audit_bytes"],
+ "implementation_review_audit_sha256": review["audit_sha256"],
+ "source_artifact_manifest_bytes": len(source_manifest_payload),
+ "source_artifact_manifest_sha256": source_hash,
+ "source_receipt_bytes": len(build_approval_payload),
+ "source_receipt_sha256": hashlib.sha256(build_approval_payload).hexdigest().upper(),
+ "build_artifact_manifest_bytes": len(build_receipt_payload),
+ "build_artifact_manifest_sha256": hashlib.sha256(build_receipt_payload).hexdigest().upper(),
+ "build_receipt_bytes": len(install_approval_payload),
+ "build_receipt_sha256": hashlib.sha256(install_approval_payload).hexdigest().upper(),
+ "host_executable_bytes": len(exe.read_bytes()), "host_executable_sha256": exe_hash,
+ "host_build": HOST_BUILD, "required_extension_build": EXTENSION_BUILD,
+ "extension_id": "oidmclckpdmpabbfedplkbdplmfcenbb",
+ "host_name": "com.project_info.bili_auth_ingress",
+ "installed_root": str(installed_root), "installed_files": installed_files,
+ "native_messaging_host_manifest": str(native_manifest),
+ "host_build_source_manifest": file_spec(root, host_build_source_manifest_path),
+ "host_source_binding_mode": fast_path_mode,
+ "producer_only_changed_files": ["queue_producer.py"] if fast_path_mode == "PRODUCER_ONLY_DELTA" else [],
+ "host_archive_excluded_modules": ["bili_authenticated_extension.queue_producer"],
+ "projection_contract_sha256": hashlib.sha256(contract_payload).hexdigest().upper(),
+ "projection_tree_sha256": tree_hash, "secret_field_count": 0,
+ }
+ install_receipt.update(frozen_fields(reload, "reload_state", exact=True))
+ install_receipt.update(frozen_fields(queue, "queue", exact=False))
+ install_receipt.update(frozen_fields(state, "queue_state", exact=False))
+ install_receipt.update(frozen_fields(formal, "formal_manifest", exact=False))
+ install_receipt_payload = write_json(install_receipt_path, install_receipt)
+ deployment = {
+ "extension_build": EXTENSION_BUILD, "host_build": HOST_BUILD,
+ "reload_generation": RELOAD_GENERATION, "source_manifest_sha256": source_hash,
+ "build_approval": file_spec(root, source_receipt_path),
+ "build_receipt": file_spec(root, build_root / "build-artifact-manifest.json"),
+ "exe": file_spec(root, exe),
+ "install_approval": file_spec(root, build_validation_receipt_path),
+ "install_receipt": file_spec(root, install_receipt_path),
+ "installed_config_sha256": hashlib.sha256(host_config_payload).hexdigest().upper(),
+ "installed_manifest_sha256": hashlib.sha256(native_manifest_payload).hexdigest().upper(),
+ "installed_exe_sha256": exe_hash,
+ "projection_contract_sha256": hashlib.sha256(contract_payload).hexdigest().upper(),
+ "projection_tree_sha256": tree_hash,
+ "extension_id": "oidmclckpdmpabbfedplkbdplmfcenbb",
+ }
+ write_json(root / f"ai-infoadmin/worklog/bili-auth-successor-release-approval-{AUTH_MESSAGE}.json", {
+ "schema": 1, "scope": "bili-auth-successor-exact-release-v1",
+ "project_id": "project-info",
+ "task_id": "DEV-PROJECT-INFO-BILI-AUTHENTICATED-SESSION-DOWNLOAD-20260805-001",
+ "approved_by_role": "project.admin", "authorization_message_id": AUTH_MESSAGE,
+ "authorization_handoff_id": AUTH_HANDOFF,
+ "authorization_file": file_spec(root, authorization_path),
+ "implementation_review": review, "deployment": deployment,
+ })
+ config = {
+ "schema": 2, "project_root": root, "host_config_path": host_config,
+ "creator_uid": CREATOR,
+ "successor_authorization_message_id": AUTH_MESSAGE,
+ "queue_paths": {
+ "queue_path": queue, "queue_state_path": state, "queue_lock_path": lock,
+ "reload_state_path": reload,
+ },
+ "_host_config_sha256": hashlib.sha256(host_config_payload).hexdigest().upper(),
+ }
+ return config, queue, state, native_manifest
+
+
+def refresh_installed_identity(root: pathlib.Path, name: str) -> None:
+ installed_root = root / "installed/generic-v002"
+ receipt_path = root / "ai-infoadmin/worklog/synthetic-install-receipt.json"
+ receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
+ candidate = installed_root / name
+ payload = candidate.read_bytes()
+ for item in receipt["installed_files"]:
+ if item["path"] == name:
+ item["bytes"] = len(payload)
+ item["sha256"] = hashlib.sha256(payload).hexdigest().upper()
+ break
+ write_json(receipt_path, receipt)
+ approval_path = root / f"ai-infoadmin/worklog/bili-auth-successor-release-approval-{AUTH_MESSAGE}.json"
+ approval = json.loads(approval_path.read_text(encoding="utf-8"))
+ approval["deployment"]["install_receipt"] = file_spec(root, receipt_path)
+ field = {
+ "config.json": "installed_config_sha256",
+ "project-info-bili-auth-native-host.exe": "installed_exe_sha256",
+ "native-host-manifest.json": "installed_manifest_sha256",
+ }[name]
+ approval["deployment"][field] = hashlib.sha256(payload).hexdigest().upper()
+ write_json(approval_path, approval)
+
+
+def rebind_deployment_file(root: pathlib.Path, field: str, path: pathlib.Path) -> None:
+ approval_path = root / f"ai-infoadmin/worklog/bili-auth-successor-release-approval-{AUTH_MESSAGE}.json"
+ approval = json.loads(approval_path.read_text(encoding="utf-8"))
+ approval["deployment"][field] = file_spec(root, path)
+ write_json(approval_path, approval)
+
+
+def rewrite_fast_receipt(root: pathlib.Path, value: dict[str, object]) -> pathlib.Path:
+ path = root / "ai-infoadmin/worklog/synthetic-fast-path-install-readiness-receipt.json"
+ write_json(path, value)
+ rebind_deployment_file(root, "install_receipt", path)
+ return path
+
+
+def assert_deployment_rejected(
+ testcase: unittest.TestCase,
+ root: pathlib.Path,
+ config: dict[str, object],
+ queue: pathlib.Path,
+ state: pathlib.Path,
+ native_manifest: pathlib.Path,
+) -> None:
+ frozen = (queue.read_bytes(), state.read_bytes())
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=str(native_manifest)),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_exact", return_value=str(native_manifest)),
+ mock.patch("bili_authenticated_extension.queue_producer._fast_path_canonical_paths", return_value={
+ "queue": queue,
+ "queue_state": state,
+ "reload_state": config["queue_paths"]["reload_state_path"],
+ "formal_manifest": root / "formal/manifest.jsonl",
+ }),
+ testcase.assertRaisesRegex(ProtocolError, "E_DEPLOYMENT_NOT_READY"),
+ ):
+ store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ testcase.assertEqual(frozen, (queue.read_bytes(), state.read_bytes()))
+
+
+class SuccessorTrustGateTests(unittest.TestCase):
+ def test_exact_release_and_v002_deployment_pass_before_append(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root)
+ before_queue, before_state = queue.read_bytes(), state.read_bytes()
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=str(native_manifest)),
+ ):
+ result = store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ replay = store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ self.assertEqual({"appended": 1, "unchanged": 0, "recovered": 0}, result)
+ self.assertEqual({"appended": 0, "unchanged": 1, "recovered": 0}, replay)
+ self.assertTrue(queue.read_bytes().startswith(before_queue))
+ self.assertEqual(before_state, state.read_bytes())
+
+ def test_exact_project_admin_continuous_fast_path_passes_and_replays(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root, fast_path=True)
+ before_queue, before_state = queue.read_bytes(), state.read_bytes()
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=str(native_manifest)),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_exact", return_value=str(native_manifest)),
+ mock.patch("bili_authenticated_extension.queue_producer._fast_path_canonical_paths", return_value={
+ "queue": queue,
+ "queue_state": state,
+ "reload_state": config["queue_paths"]["reload_state_path"],
+ "formal_manifest": root / "formal/manifest.jsonl",
+ }),
+ ):
+ result = store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ replay = store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ self.assertEqual({"appended": 1, "unchanged": 0, "recovered": 0}, result)
+ self.assertEqual({"appended": 0, "unchanged": 1, "recovered": 0}, replay)
+ self.assertTrue(queue.read_bytes().startswith(before_queue))
+ self.assertEqual(before_state, state.read_bytes())
+
+ def test_project_admin_same_source_host_build_passes_and_replays(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(
+ root, fast_path=True, fast_path_mode="SAME_SOURCE_HOST_BUILD",
+ )
+ before_queue, before_state = queue.read_bytes(), state.read_bytes()
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=str(native_manifest)),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_exact", return_value=str(native_manifest)),
+ mock.patch("bili_authenticated_extension.queue_producer._fast_path_canonical_paths", return_value={
+ "queue": queue,
+ "queue_state": state,
+ "reload_state": config["queue_paths"]["reload_state_path"],
+ "formal_manifest": root / "formal/manifest.jsonl",
+ }),
+ ):
+ result = store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ replay = store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ self.assertEqual({"appended": 1, "unchanged": 0, "recovered": 0}, result)
+ self.assertEqual({"appended": 0, "unchanged": 1, "recovered": 0}, replay)
+ self.assertTrue(queue.read_bytes().startswith(before_queue))
+ self.assertEqual(before_state, state.read_bytes())
+
+ def test_project_admin_fast_path_spoof_drift_and_cross_branch_matrix_is_mutation_zero(self) -> None:
+ mutations = (
+ "source-path-shadow", "cross-branch-mix", "wrong-source-role", "wrong-source-task",
+ "wrong-continuous-authorization", "wrong-scope", "wrong-status", "wrong-schema",
+ "wrong-owner", "wrong-successor-scope", "changed-authorization-generation",
+ "extra-receipt-field",
+ "drift-source-binding", "drift-build-binding", "drift-installed-binding",
+ "drift-readiness", "drift-queue-prefix", "drift-state-prefix",
+ "drift-formal-prefix", "shadow-queue-path", "nonzero-secret-field-count",
+ "host-source-nonproducer-delta", "wrong-host-source-mode",
+ "same-source-declared-for-delta",
+ )
+ for mutation in mutations:
+ with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root, fast_path=True)
+ receipt_path = root / "ai-infoadmin/worklog/synthetic-fast-path-install-readiness-receipt.json"
+ receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
+ if mutation == "source-path-shadow":
+ source = root / "ai-infoadmin/worklog/synthetic-fast-path-source-receipt.json"
+ shadow = root / "ai-infoadmin/worklog/shadow/equal-fast-path-source-receipt.json"
+ shadow.parent.mkdir(parents=True)
+ shadow.write_bytes(source.read_bytes())
+ rebind_deployment_file(root, "build_approval", shadow)
+ elif mutation == "cross-branch-mix":
+ source = root / "ai-infoadmin/worklog/synthetic-fast-path-build-receipt.json"
+ mixed = root / "ai-inforev/worklog/synthetic-install-approval.json"
+ mixed.parent.mkdir(parents=True, exist_ok=True)
+ mixed.write_bytes(source.read_bytes())
+ rebind_deployment_file(root, "install_approval", mixed)
+ elif mutation in {"wrong-source-role", "wrong-source-task"}:
+ source = root / "ai-infoadmin/worklog/synthetic-fast-path-source-receipt.json"
+ source_value = json.loads(source.read_text(encoding="utf-8"))
+ if mutation == "wrong-source-role":
+ source_value["approved_by_role"] = "dev.reviewer.project"
+ else:
+ source_value["task_id"] = "DEV-ANOTHER-TASK"
+ source_payload = write_json(source, source_value)
+ receipt["source_receipt_bytes"] = len(source_payload)
+ receipt["source_receipt_sha256"] = hashlib.sha256(source_payload).hexdigest().upper()
+ rewrite_fast_receipt(root, receipt)
+ rebind_deployment_file(root, "build_approval", source)
+ elif mutation == "changed-authorization-generation":
+ authorization_path = root / "ai-infoadmin/worklog/synthetic-authorization.json"
+ authorization = json.loads(authorization_path.read_text(encoding="utf-8"))
+ authorization["successors"][0]["retry_generation"] = 2
+ authorization_payload = write_json(authorization_path, authorization)
+ release_path = root / f"ai-infoadmin/worklog/bili-auth-successor-release-approval-{AUTH_MESSAGE}.json"
+ release = json.loads(release_path.read_text(encoding="utf-8"))
+ release["authorization_file"] = file_spec(root, authorization_path)
+ receipt["authorization_file_bytes"] = len(authorization_payload)
+ receipt["authorization_file_sha256"] = hashlib.sha256(authorization_payload).hexdigest().upper()
+ write_json(release_path, release)
+ rewrite_fast_receipt(root, receipt)
+ elif mutation == "host-source-nonproducer-delta":
+ host_source_path = root / "ai-infoadmin/worklog/synthetic-fast-path-host-build-source-manifest.json"
+ host_source = json.loads(host_source_path.read_text(encoding="utf-8"))
+ for item in host_source["files"]:
+ if item["path"] == "worker.py":
+ item["sha256"] = "2" * 64
+ break
+ host_source_payload = write_json(host_source_path, host_source)
+ host_source_hash = hashlib.sha256(host_source_payload).hexdigest().upper()
+ artifact_path = root / "dev/tmp/synthetic-build/build-artifact-manifest.json"
+ artifact = json.loads(artifact_path.read_text(encoding="utf-8"))
+ artifact["source_artifact_manifest_bytes"] = len(host_source_payload)
+ artifact["source_artifact_manifest_sha256"] = host_source_hash
+ artifact_payload = write_json(artifact_path, artifact)
+ build_path = root / "ai-infoadmin/worklog/synthetic-fast-path-build-receipt.json"
+ build_value = json.loads(build_path.read_text(encoding="utf-8"))
+ build_value["source_artifact_manifest_sha256"] = host_source_hash
+ build_value["build_artifact_manifest_bytes"] = len(artifact_payload)
+ build_value["build_artifact_manifest_sha256"] = hashlib.sha256(artifact_payload).hexdigest().upper()
+ build_payload = write_json(build_path, build_value)
+ receipt["host_build_source_manifest"] = file_spec(root, host_source_path)
+ receipt["build_artifact_manifest_bytes"] = len(artifact_payload)
+ receipt["build_artifact_manifest_sha256"] = hashlib.sha256(artifact_payload).hexdigest().upper()
+ receipt["build_receipt_bytes"] = len(build_payload)
+ receipt["build_receipt_sha256"] = hashlib.sha256(build_payload).hexdigest().upper()
+ rewrite_fast_receipt(root, receipt)
+ rebind_deployment_file(root, "build_receipt", artifact_path)
+ rebind_deployment_file(root, "install_approval", build_path)
+ else:
+ if mutation == "wrong-continuous-authorization":
+ receipt["continuous_authorization_handoff_id"] = "HANDOFF-OTHER-CONTINUOUS-AUTH"
+ elif mutation == "wrong-scope":
+ receipt["validation_scope"] = "other-scope"
+ elif mutation == "wrong-status":
+ receipt["status"] = "PENDING"
+ elif mutation == "wrong-schema":
+ receipt["schema"] = 1
+ elif mutation == "wrong-owner":
+ receipt["owner_thread_id"] = "01900000-0000-0000-0000-000000000000"
+ elif mutation == "wrong-successor-scope":
+ receipt["successor_scope_sha256"] = "A" * 64
+ elif mutation == "extra-receipt-field":
+ receipt["unexpected"] = "forbidden"
+ elif mutation == "drift-source-binding":
+ receipt["source_artifact_manifest_sha256"] = "B" * 64
+ elif mutation == "drift-build-binding":
+ receipt["build_artifact_manifest_sha256"] = "C" * 64
+ elif mutation == "drift-installed-binding":
+ receipt["installed_files"][0]["sha256"] = "D" * 64
+ elif mutation == "drift-readiness":
+ receipt["reload_state_sha256"] = "E" * 64
+ elif mutation == "drift-queue-prefix":
+ receipt["queue_prefix_sha256"] = "F" * 64
+ elif mutation == "drift-state-prefix":
+ receipt["queue_state_prefix_sha256"] = "0" * 64
+ elif mutation == "drift-formal-prefix":
+ receipt["formal_manifest_prefix_sha256"] = "1" * 64
+ elif mutation == "shadow-queue-path":
+ shadow = root / "shadow/queue.jsonl"
+ shadow.parent.mkdir(parents=True)
+ shadow.write_bytes(queue.read_bytes())
+ receipt["queue_path"] = str(shadow)
+ elif mutation == "nonzero-secret-field-count":
+ receipt["secret_field_count"] = 1
+ elif mutation == "wrong-host-source-mode":
+ receipt["host_source_binding_mode"] = "UNRESTRICTED"
+ elif mutation == "same-source-declared-for-delta":
+ receipt["host_source_binding_mode"] = "SAME_SOURCE_HOST_BUILD"
+ receipt["producer_only_changed_files"] = []
+ rewrite_fast_receipt(root, receipt)
+ assert_deployment_rejected(self, root, config, queue, state, native_manifest)
+
+ def test_producer_only_delta_declared_for_same_source_is_mutation_zero(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(
+ root, fast_path=True, fast_path_mode="SAME_SOURCE_HOST_BUILD",
+ )
+ receipt_path = root / "ai-infoadmin/worklog/synthetic-fast-path-install-readiness-receipt.json"
+ receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
+ receipt["host_source_binding_mode"] = "PRODUCER_ONLY_DELTA"
+ receipt["producer_only_changed_files"] = ["queue_producer.py"]
+ rewrite_fast_receipt(root, receipt)
+ assert_deployment_rejected(self, root, config, queue, state, native_manifest)
+
+ def test_project_admin_fast_path_exact_registry_gate_rejects_extra_state_before_append(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root, fast_path=True)
+ frozen = (queue.read_bytes(), state.read_bytes())
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=str(native_manifest)),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_exact", side_effect=ProtocolError("E_DEPLOYMENT_NOT_READY")),
+ mock.patch("bili_authenticated_extension.queue_producer._fast_path_canonical_paths", return_value={
+ "queue": queue, "queue_state": state,
+ "reload_state": config["queue_paths"]["reload_state_path"],
+ "formal_manifest": root / "formal/manifest.jsonl",
+ }),
+ self.assertRaisesRegex(ProtocolError, "E_DEPLOYMENT_NOT_READY"),
+ ):
+ store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ self.assertEqual(frozen, (queue.read_bytes(), state.read_bytes()))
+
+ def test_project_admin_fast_path_reparse_receipt_is_rejected_before_append(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root, fast_path=True)
+ source_receipt = root / "ai-infoadmin/worklog/synthetic-fast-path-source-receipt.json"
+ real_lstat = os.lstat
+
+ class ReparseStat:
+ def __init__(self, value: os.stat_result) -> None:
+ self._value = value
+ self.st_file_attributes = 0x400
+
+ def __getattr__(self, name: str) -> object:
+ return getattr(self._value, name)
+
+ def marked_lstat(path: os.PathLike[str] | str) -> os.stat_result | ReparseStat:
+ value = real_lstat(path)
+ if pathlib.Path(path) == source_receipt:
+ return ReparseStat(value)
+ return value
+
+ frozen = (queue.read_bytes(), state.read_bytes())
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer.os.lstat", side_effect=marked_lstat),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=str(native_manifest)),
+ self.assertRaisesRegex(ProtocolError, "E_DEPLOYMENT_NOT_READY"),
+ ):
+ store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ self.assertEqual(frozen, (queue.read_bytes(), state.read_bytes()))
+
+ def test_missing_fixed_approval_and_caller_override_are_not_trust_seams(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root)
+ approval_path = root / f"ai-infoadmin/worklog/bili-auth-successor-release-approval-{AUTH_MESSAGE}.json"
+ approval_path.rename(approval_path.with_suffix(".absent"))
+ frozen = (queue.read_bytes(), state.read_bytes())
+ config["approval_path"] = str(approval_path.with_suffix(".absent"))
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=str(native_manifest)),
+ self.assertRaisesRegex(ProtocolError, "E_AUTH_TRUST"),
+ ):
+ store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ self.assertEqual(frozen, (queue.read_bytes(), state.read_bytes()))
+ source = (PROJECT_DEV / "bili_authenticated_extension" / "queue_producer.py").read_text(encoding="utf-8")
+ self.assertNotIn("--approval-path", source)
+ self.assertNotIn("--approval-hash", source)
+ self.assertNotIn("--trust-root", source)
+
+ def test_mixed_versions_reload_pending_registry_and_projection_drift_fail_before_begin(self) -> None:
+ mutations = ("host-build", "reload-begin", "registry", "projection")
+ for mutation in mutations:
+ with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root)
+ if mutation == "host-build":
+ receipt = root / "ai-infoadmin/worklog/synthetic-install-receipt.json"
+ raw = json.loads(receipt.read_text(encoding="utf-8"))
+ raw["host_build"] = "project-info-bili-auth-native-host/1.1.0+20260816.generic.v001"
+ write_json(receipt, raw)
+ elif mutation == "reload-begin":
+ reload = config["queue_paths"]["reload_state_path"]
+ write_json(reload, {
+ "schema": 1, "generation": RELOAD_GENERATION, "event": "BEGIN",
+ "token": "a" * 32, "from_build": EXTENSION_BUILD,
+ "to_build": EXTENSION_BUILD, "at_unix_ms": 2_000_000_000_000,
+ })
+ elif mutation == "projection":
+ (root / "dev/project-dev/bili_authenticated_extension_unpacked/background.js").write_text("drift\n", encoding="utf-8")
+ registry = str(native_manifest) if mutation != "registry" else str(root / "wrong.json")
+ frozen = (queue.read_bytes(), state.read_bytes())
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=registry),
+ self.assertRaisesRegex(ProtocolError, "E_DEPLOYMENT_NOT_READY"),
+ ):
+ store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ self.assertEqual(frozen, (queue.read_bytes(), state.read_bytes()))
+
+ def test_projection_manifest_semantic_version_must_match_v024_before_begin(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, _native_manifest = build_environment(root)
+ source_manifest_path = root / "dev/project-dev/bili_authenticated_extension/manifest.json"
+ projection_manifest_path = root / "dev/project-dev/bili_authenticated_extension_unpacked/manifest.json"
+ manifest = json.loads(source_manifest_path.read_text(encoding="utf-8"))
+ manifest["version"] = "1.2.15"
+ manifest_payload = encode(manifest)
+ source_manifest_path.write_bytes(manifest_payload)
+ projection_manifest_path.write_bytes(manifest_payload)
+
+ contract_path = root / "dev/project-dev/bili_authenticated_extension_unpacked_contract.json"
+ contract = json.loads(contract_path.read_text(encoding="utf-8"))
+ for item in contract["files"]:
+ if item["path"] == "manifest.json":
+ item["bytes"] = len(manifest_payload)
+ item["sha256"] = hashlib.sha256(manifest_payload).hexdigest().upper()
+ contract["tree_sha256"] = _projection_tree(contract["files"])
+ contract_payload = write_json(contract_path, contract)
+ deployment = {
+ "projection_contract_sha256": hashlib.sha256(contract_payload).hexdigest().upper(),
+ "projection_tree_sha256": contract["tree_sha256"],
+ }
+ frozen = (queue.read_bytes(), state.read_bytes())
+ with self.assertRaisesRegex(ProtocolError, "E_DEPLOYMENT_NOT_READY"):
+ producer._validate_projection(root, deployment)
+ self.assertEqual(frozen, (queue.read_bytes(), state.read_bytes()))
+
+ def test_admin_role_authorization_hash_and_audit_prefix_tamper_fail_trust(self) -> None:
+ for mutation in ("role", "authorization", "audit"):
+ with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root)
+ approval_path = root / f"ai-infoadmin/worklog/bili-auth-successor-release-approval-{AUTH_MESSAGE}.json"
+ if mutation == "role":
+ value = json.loads(approval_path.read_text(encoding="utf-8"))
+ value["approved_by_role"] = "dev.developer.project.secondary"
+ write_json(approval_path, value)
+ elif mutation == "authorization":
+ authorization_path = root / "ai-infoadmin/worklog/synthetic-authorization.json"
+ authorization_path.write_bytes(authorization_path.read_bytes() + b" ")
+ else:
+ audit_path = root / "dev-doc/开发审计报告.md"
+ payload = bytearray(audit_path.read_bytes())
+ payload[0] ^= 1
+ audit_path.write_bytes(payload)
+ ingress = json.loads(queue.read_bytes().splitlines()[0])
+ store = QueueStore(queue, state, config["queue_paths"]["queue_lock_path"], frozenset({CREATOR}))
+ frozen = (queue.read_bytes(), state.read_bytes())
+ with (
+ mock.patch("bili_authenticated_extension.queue_producer._project_root_from_source", return_value=root),
+ mock.patch("bili_authenticated_extension.queue_producer._read_native_host_registry_default", return_value=str(native_manifest)),
+ self.assertRaisesRegex(ProtocolError, "E_AUTH_TRUST"),
+ ):
+ store.append_authorized_successors(lambda: _load_release_approval(config), [ingress])
+ self.assertEqual(frozen, (queue.read_bytes(), state.read_bytes()))
+
+ def test_actual_source_and_projection_exact_sets_reject_unmanifested_files_before_begin(self) -> None:
+ mutations = (
+ "dev/project-dev/bili_authenticated_extension/unreviewed.py",
+ "dev/project-dev/bili_authenticated_extension_unpacked/unreviewed.js",
+ )
+ for relative in mutations:
+ with self.subTest(relative=relative), tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root)
+ candidate = root / relative
+ candidate.parent.mkdir(parents=True, exist_ok=True)
+ candidate.write_bytes(b"unreviewed\n")
+ assert_deployment_rejected(self, root, config, queue, state, native_manifest)
+
+ def test_different_installed_exe_is_rejected_even_when_receipt_and_release_self_agree(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root)
+ installed_exe = root / "installed/generic-v002/project-info-bili-auth-native-host.exe"
+ installed_exe.write_bytes(b"MZdifferent-installed-executable")
+ refresh_installed_identity(root, installed_exe.name)
+ assert_deployment_rejected(self, root, config, queue, state, native_manifest)
+
+ def test_host_config_a_b_swap_after_initial_snapshot_is_rejected_before_begin(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root)
+ host_config = pathlib.Path(config["host_config_path"])
+ replacement = json.loads(host_config.read_text(encoding="utf-8"))
+ replacement["destination"] = str(root / "other-destination")
+ write_json(host_config, replacement)
+ refresh_installed_identity(root, host_config.name)
+ assert_deployment_rejected(self, root, config, queue, state, native_manifest)
+
+ def test_build_receipt_schema_drift_is_rejected_even_when_outer_hashes_are_rebound(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = pathlib.Path(temporary)
+ config, queue, state, native_manifest = build_environment(root)
+ build_receipt_path = root / "dev/tmp/synthetic-build/build-artifact-manifest.json"
+ build_receipt = json.loads(build_receipt_path.read_text(encoding="utf-8"))
+ build_receipt["unexpected"] = "schema-drift"
+ build_payload = write_json(build_receipt_path, build_receipt)
+ install_approval_path = root / "ai-inforev/worklog/synthetic-install-approval.json"
+ install_approval = json.loads(install_approval_path.read_text(encoding="utf-8"))
+ install_approval["build_artifact_manifest_bytes"] = len(build_payload)
+ install_approval["build_artifact_manifest_sha256"] = hashlib.sha256(build_payload).hexdigest().upper()
+ write_json(install_approval_path, install_approval)
+ release_path = root / f"ai-infoadmin/worklog/bili-auth-successor-release-approval-{AUTH_MESSAGE}.json"
+ release = json.loads(release_path.read_text(encoding="utf-8"))
+ release["deployment"]["build_receipt"] = file_spec(root, build_receipt_path)
+ release["deployment"]["install_approval"] = file_spec(root, install_approval_path)
+ write_json(release_path, release)
+ assert_deployment_rejected(self, root, config, queue, state, native_manifest)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/dev/project-dev/test/bili_authenticated_extension/test_unpacked_projection.py b/dev/project-dev/test/bili_authenticated_extension/test_unpacked_projection.py
new file mode 100644
index 0000000..019b8a2
--- /dev/null
+++ b/dev/project-dev/test/bili_authenticated_extension/test_unpacked_projection.py
@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+from dataclasses import replace
+from contextlib import redirect_stdout
+import hashlib
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+
+PROJECT_ROOT = Path(__file__).parents[4]
+VALIDATOR_PATH = PROJECT_ROOT / "dev" / "project-dev" / "bili_authenticated_extension_unpacked_validator.py"
+SPEC = importlib.util.spec_from_file_location("bili_authenticated_extension_unpacked_validator", VALIDATOR_PATH)
+assert SPEC is not None and SPEC.loader is not None
+validator = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = validator
+SPEC.loader.exec_module(validator)
+
+
+def _remove_reparse(path: Path) -> None:
+ try:
+ if path.is_symlink():
+ path.unlink()
+ elif path.exists():
+ os.rmdir(path)
+ except FileNotFoundError:
+ pass
+
+
+class UnpackedProjectionTests(unittest.TestCase):
+ maxDiff = None
+
+ def setUp(self) -> None:
+ temp_parent = PROJECT_ROOT / "dev" / "tmp"
+ temp_parent.mkdir(parents=True, exist_ok=True)
+ self.temp = tempfile.TemporaryDirectory(dir=temp_parent)
+ self.addCleanup(self.temp.cleanup)
+ self.temp_root = Path(self.temp.name)
+
+ def _project(self, parent: str = "case") -> Path:
+ root = self.temp_root / parent / validator.PROJECT_ID
+ source = root.joinpath(*validator.SOURCE_ROOT_REL.parts)
+ projection = root.joinpath(*validator.PROJECTION_ROOT_REL.parts)
+ source.mkdir(parents=True)
+ projection.mkdir(parents=True)
+ (root / "mbx.project.yaml").write_text("schema_version: '1.0'\nproject:\n id: project-info\n", encoding="utf-8")
+ authoritative = PROJECT_ROOT / "dev" / "project-dev" / "bili_authenticated_extension"
+ for name in validator.EXPECTED_FILES:
+ shutil.copyfile(authoritative / name, source / name)
+ shutil.copyfile(authoritative / name, projection / name)
+ shutil.copyfile(
+ authoritative / "source-artifact-manifest.json",
+ root.joinpath(*validator.SOURCE_ARTIFACT_MANIFEST_REL.parts),
+ )
+ shutil.copyfile(
+ PROJECT_ROOT / "dev" / "project-dev" / "bili_authenticated_extension_unpacked_contract.json",
+ root.joinpath(*validator.CONTRACT_REL.parts),
+ )
+ return root
+
+ def _assert_error(self, root: Path, expected: str, **kwargs: object) -> None:
+ with self.assertRaises(validator.ValidationError) as caught:
+ validator.validate_project(str(root), **kwargs)
+ self.assertEqual(caught.exception.code, expected)
+
+ def _assert_cli_safety_stop(self, root: Path, expected: str) -> None:
+ output = io.StringIO()
+ with redirect_stdout(output):
+ code = validator.main(["--project-root", str(root)])
+ body = output.getvalue()
+ self.assertEqual(code, 3)
+ self.assertNotIn("VALIDATION_PASS_ONLY", body)
+ self.assertEqual(json.loads(body), {"schema": 1, "status": "SAFETY_STOP", "error_code": expected})
+
+ def _case_only_rename(self, path: Path, new_name: str) -> Path:
+ temporary = path.with_name(f"case-rename-{path.name}.tmp")
+ renamed = path.with_name(new_name)
+ os.replace(path, temporary)
+ os.replace(temporary, renamed)
+ self.assertFalse(path.exists() and path.name == new_name)
+ actual_names = tuple(entry.name for entry in os.scandir(renamed.parent))
+ self.assertIn(new_name, actual_names)
+ return renamed
+
+ def _junction(self, link: Path, target: Path) -> None:
+ result = subprocess.run(
+ ["cmd.exe", "/d", "/c", "mklink", "/J", str(link), str(target)],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ if result.returncode != 0:
+ self.fail(f"real junction creation failed: {result.returncode}")
+ self.addCleanup(_remove_reparse, link)
+ attributes = os.lstat(link).st_file_attributes
+ self.assertTrue(attributes & validator.FILE_ATTRIBUTE_REPARSE_POINT)
+
+ def _leaf_reparse(self, link: Path, target: Path) -> None:
+ result = subprocess.run(
+ ["cmd.exe", "/d", "/c", "mklink", str(link), str(target)],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ if result.returncode != 0:
+ junction_target = target.with_name(f"{target.name}-junction-target")
+ junction_target.mkdir()
+ self._junction(link, junction_target)
+ else:
+ self.addCleanup(_remove_reparse, link)
+ attributes = os.lstat(link).st_file_attributes
+ self.assertTrue(attributes & validator.FILE_ATTRIBUTE_REPARSE_POINT)
+
+ def test_01_real_projection_passes_exact_set_bytes_tree_and_id(self) -> None:
+ result = validator.validate_project(str(PROJECT_ROOT))
+ self.assertEqual(result["status"], "VALIDATION_PASS_ONLY")
+ self.assertEqual(result["extension_id"], "oidmclckpdmpabbfedplkbdplmfcenbb")
+ self.assertEqual(result["file_count"], 5)
+ self.assertEqual(
+ result["tree_sha256"],
+ "0F5095CFC0CD62E0165E0A990104D9E46DA5F04B42C89B16D3F7A3420ADDB968",
+ )
+ projection = PROJECT_ROOT / "dev" / "project-dev" / "bili_authenticated_extension_unpacked"
+ self.assertEqual(tuple(sorted(path.name for path in projection.iterdir())), validator.EXPECTED_FILES)
+ self.assertFalse(any(part.startswith("_") for path in projection.iterdir() for part in path.parts[-1:]))
+ self.assertEqual(
+ result["source_artifact_manifest"],
+ {
+ "bytes": 3770,
+ "sha256": "132B637634550A43B0EDD5E8E74C439205DE4BFECCE0A9276D8EBEF78445ECFC",
+ },
+ )
+
+ def test_02_reserved_extra_missing_and_byte_drift_fail_closed(self) -> None:
+ root = self._project("reserved")
+ projection = root.joinpath(*validator.PROJECTION_ROOT_REL.parts)
+ (projection / "__reserved").write_bytes(b"x")
+ self._assert_error(root, "E_RESERVED_NAME")
+
+ root = self._project("missing")
+ root.joinpath(*validator.PROJECTION_ROOT_REL.parts, "sidepanel.css").unlink()
+ self._assert_error(root, "E_PATH_MISSING")
+
+ root = self._project("drift")
+ with root.joinpath(*validator.PROJECTION_ROOT_REL.parts, "background.js").open("ab") as handle:
+ handle.write(b"\n")
+ self._assert_error(root, "E_BYTE_DRIFT")
+
+ def test_03_project_root_reparse_fails_before_content_acceptance(self) -> None:
+ target = self._project("target")
+ alias_parent = self.temp_root / "alias"
+ alias_parent.mkdir()
+ alias = alias_parent / validator.PROJECT_ID
+ self._junction(alias, target)
+ self._assert_error(alias, "E_REPARSE")
+
+ def test_04_source_root_reparse_fails_before_source_read(self) -> None:
+ root = self._project("source-root")
+ source = root.joinpath(*validator.SOURCE_ROOT_REL.parts)
+ external = self.temp_root / "source-external"
+ shutil.copytree(source, external)
+ shutil.rmtree(source)
+ self._junction(source, external)
+ with mock.patch.object(Path, "read_bytes", side_effect=AssertionError("content read before gate")):
+ self._assert_error(root, "E_REPARSE")
+
+ def test_05_projection_root_reparse_fails_before_handoff(self) -> None:
+ root = self._project("projection-root")
+ projection = root.joinpath(*validator.PROJECTION_ROOT_REL.parts)
+ external = self.temp_root / "projection-external"
+ shutil.copytree(projection, external)
+ shutil.rmtree(projection)
+ self._junction(projection, external)
+ self._assert_error(root, "E_REPARSE")
+
+ def test_06_intermediate_parent_junction_fails(self) -> None:
+ root = self._project("parent")
+ dev = root / "dev"
+ external = self.temp_root / "dev-external"
+ shutil.move(str(dev), str(external))
+ self._junction(dev, external)
+ self._assert_error(root, "E_REPARSE")
+
+ def test_07_source_and_projection_leaf_real_reparse_points_fail(self) -> None:
+ for parent_rel, case_name in (
+ (validator.SOURCE_ROOT_REL, "source-leaf"),
+ (validator.PROJECTION_ROOT_REL, "projection-leaf"),
+ ):
+ with self.subTest(parent=parent_rel.as_posix()):
+ root = self._project(case_name)
+ leaf = root.joinpath(*parent_rel.parts, "manifest.json")
+ external = self.temp_root / f"{case_name}-manifest.json"
+ shutil.copyfile(leaf, external)
+ leaf.unlink()
+ self._leaf_reparse(leaf, external)
+ self._assert_error(root, "E_REPARSE")
+
+ def test_08_unknown_reparse_tag_adapter_fails(self) -> None:
+ root = self._project("unknown-tag")
+ target = root.joinpath(*validator.PROJECTION_ROOT_REL.parts, "background.js")
+ target_inode = os.lstat(target).st_ino
+ original = validator._reparse_tag
+
+ def probe(st: os.stat_result) -> int:
+ if st.st_ino == target_inode:
+ return 0xDEADBEEF
+ return original(st)
+
+ with mock.patch.object(validator, "_reparse_tag", side_effect=probe):
+ self._assert_error(root, "E_REPARSE")
+
+ def test_09_physical_escape_adapter_fails_component_boundary(self) -> None:
+ root = self._project("physical-escape")
+ original = validator._open_identity
+
+ def open_identity(*args: object, **kwargs: object):
+ identity = original(*args, **kwargs)
+ if identity.relative_path == validator.PROJECTION_ROOT_REL.as_posix():
+ return replace(identity, final_path=str(root.parent / "project-info-escape"))
+ return identity
+
+ with mock.patch.object(validator, "_open_identity", side_effect=open_identity):
+ self._assert_error(root, "E_PATH_ESCAPE")
+
+ def test_10_same_byte_leaf_replacement_between_passes_fails(self) -> None:
+ root = self._project("identity-race")
+ leaf = root.joinpath(*validator.PROJECTION_ROOT_REL.parts, "sidepanel.js")
+
+ def replace_same_bytes() -> None:
+ replacement = leaf.with_suffix(".replacement")
+ replacement.write_bytes(leaf.read_bytes())
+ os.replace(replacement, leaf)
+
+ self._assert_error(root, "E_PATH_IDENTITY_DRIFT", between_passes=replace_same_bytes)
+
+ def test_11_manifest_key_drift_cannot_derive_accepted_identity(self) -> None:
+ manifest_path = PROJECT_ROOT / "dev" / "project-dev" / "bili_authenticated_extension_unpacked" / "manifest.json"
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+ manifest["key"] = manifest["key"][:-1] + ("A" if manifest["key"][-1] != "A" else "B")
+ with self.assertRaises(validator.ValidationError) as caught:
+ validator._validate_manifest(json.dumps(manifest).encode("utf-8"))
+ self.assertEqual(caught.exception.code, "E_MANIFEST")
+
+ def test_12_source_parent_root_and_all_five_leaf_case_drift_fail_closed(self) -> None:
+ cases = (
+ (Path("dev"), "Dev"),
+ (Path(*validator.SOURCE_ROOT_REL.parts), "Bili_authenticated_extension"),
+ *(
+ (Path(*validator.SOURCE_ROOT_REL.parts, name), name[0].upper() + name[1:])
+ for name in validator.EXPECTED_FILES
+ ),
+ )
+ for index, (relative, new_name) in enumerate(cases):
+ with self.subTest(relative=relative.as_posix()):
+ root = self._project(f"source-case-{index}")
+ target = root / relative
+ self._case_only_rename(target, new_name)
+ self._assert_cli_safety_stop(root, "E_PATH_CASE_DRIFT")
+
+ def test_13_projection_case_only_rename_fails_closed(self) -> None:
+ root = self._project("projection-case")
+ target = root.joinpath(*validator.PROJECTION_ROOT_REL.parts, "manifest.json")
+ self._case_only_rename(target, "Manifest.json")
+ self._assert_cli_safety_stop(root, "E_PATH_CASE_DRIFT")
+
+ def test_14_source_artifact_manifest_exactly_binds_five_payload_entries(self) -> None:
+ path = PROJECT_ROOT / "dev" / "project-dev" / "bili_authenticated_extension" / "source-artifact-manifest.json"
+ data = path.read_bytes()
+ self.assertEqual(len(data), validator.SOURCE_ARTIFACT_MANIFEST_BYTES)
+ self.assertEqual(hashlib.sha256(data).hexdigest().upper(), validator.SOURCE_ARTIFACT_MANIFEST_SHA256)
+ source_manifest = json.loads(data)
+ contract = json.loads(
+ (PROJECT_ROOT / "dev" / "project-dev" / "bili_authenticated_extension_unpacked_contract.json").read_bytes()
+ )
+ expected = {entry["path"]: entry for entry in contract["files"]}
+ observed = {entry["path"]: entry for entry in source_manifest["files"] if entry["path"] in expected}
+ self.assertEqual(tuple(sorted(observed)), validator.EXPECTED_FILES)
+ self.assertEqual(observed, expected)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/codex_stdio_race_repro.mjs b/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/codex_stdio_race_repro.mjs
new file mode 100644
index 0000000..e7644e3
--- /dev/null
+++ b/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/codex_stdio_race_repro.mjs
@@ -0,0 +1,53 @@
+import assert from "node:assert/strict";
+import {EventEmitter} from "node:events";
+
+class PendingWriteStream extends EventEmitter {
+ constructor() {
+ super();
+ this.destroyed = false;
+ this.writable = true;
+ this.writableEnded = false;
+ this.writableFinished = false;
+ }
+
+ write(_message, callback = null) {
+ setImmediate(() => {
+ const error = Object.assign(new Error("write EOF"), {code: "EOF"});
+ if (typeof callback === "function") callback(error);
+ else this.emit("error", error);
+ });
+ return true;
+ }
+}
+
+async function extractedOrdering() {
+ const stream = new PendingWriteStream();
+ const onError = () => undefined;
+ stream.on("error", onError);
+ stream.write("frame");
+ // Models finishProcessExitIfReady -> cleanupProcessListeners before the
+ // pending Windows WriteWrap completion is delivered.
+ stream.removeListener("error", onError);
+ return await new Promise((resolve) => {
+ process.once("uncaughtException", (error) => resolve(error.message === "write EOF"));
+ });
+}
+
+async function boundedOrdering() {
+ const stream = new PendingWriteStream();
+ let fixedCode = null;
+ const onError = () => { fixedCode = "E_NATIVE_PEER_CLOSED"; };
+ stream.on("error", onError);
+ await new Promise((resolve) => {
+ stream.write("frame", (error) => {
+ if (error?.message === "write EOF") fixedCode = "E_NATIVE_PEER_CLOSED";
+ resolve();
+ });
+ });
+ stream.removeListener("error", onError);
+ return fixedCode;
+}
+
+assert.equal(true, await extractedOrdering());
+assert.equal("E_NATIVE_PEER_CLOSED", await boundedOrdering());
+process.stdout.write(JSON.stringify({old_ordering_uncaught: true, bounded_error_code: "E_NATIVE_PEER_CLOSED"}) + "\n");
diff --git a/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/js_contract.mjs b/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/js_contract.mjs
new file mode 100644
index 0000000..5a10023
--- /dev/null
+++ b/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/js_contract.mjs
@@ -0,0 +1,57 @@
+import assert from "node:assert/strict";
+import cryptoModule from "node:crypto";
+import {pathToFileURL} from "node:url";
+
+const protocol = await import(pathToFileURL(process.argv[2]));
+const extractor = await import(pathToFileURL(process.argv[3]));
+
+function stable(value) {
+ if (Array.isArray(value)) return value.map(stable);
+ if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])]));
+ return value;
+}
+function canonical(value) { return JSON.stringify(stable(value)); }
+function signed(secret, value) {
+ return {...value, hmac: cryptoModule.createHmac("sha256", Buffer.from(secret, "hex")).update(canonical(value)).digest("hex")};
+}
+
+const calls = {prepare: 0, dispatch: 0, observe: 0};
+const api = {
+ async prepare(action) { calls.prepare += 1; return {...action, tab_id: 7}; },
+ async dispatch(prepared) { calls.dispatch += 1; return {action_id: prepared.action_id, tab_id: 7, url: prepared.url}; },
+ async observe(result) { calls.observe += 1; return {schema_version: 1, final_url: result.url}; }
+};
+const hello = {extension_id: "gllihoanalkiollgpeggamfhajmmnmml", version: "1.0.0", manifest_name: "project-info Bilibili dynamic refresh trusted adapter"};
+const session = new protocol.TrustedRuntimeSession(api, hello);
+assert.equal(session.helloFrame().type, "EXTENSION_HELLO");
+const secret = "11".repeat(32);
+const challenge = signed(secret, {schema_version: 1, type: "HOST_CHALLENGE", run_id: "run", request_id: "request", sequence: 2, challenge_id: "challenge", secret});
+const accepted = await session.accept(challenge);
+assert.equal(accepted.type, "EXTENSION_CHALLENGE_ACCEPTED");
+const prepare = signed(secret, {schema_version: 1, type: "HOST_ACTION_PREPARE", run_id: "run", request_id: "request", sequence: 4, action: {action_id: "action", kind: "reload", url: "https://space.bilibili.com/1420210197/dynamic"}});
+const ready = await session.accept(prepare);
+assert.equal(ready.type, "EXTENSION_READY_TO_DISPATCH");
+assert.deepEqual(calls, {prepare: 1, dispatch: 0, observe: 0});
+const permit = signed(secret, {schema_version: 1, type: "HOST_DISPATCH_PERMIT", run_id: "run", request_id: "request", sequence: 6, permit: {permit_id: "permit", action_id: "action", deadline_at: "2999-01-01T00:00:00Z"}});
+const action = await session.accept(permit);
+assert.equal(action.type, "EXTENSION_ACTION_RESULT");
+assert.deepEqual(calls, {prepare: 1, dispatch: 1, observe: 0});
+await assert.rejects(session.accept(permit), /E_SEQUENCE|E_PERMIT|E_REPLAY/u);
+assert.deepEqual(calls, {prepare: 1, dispatch: 1, observe: 0});
+const request = signed(secret, {schema_version: 1, type: "HOST_OBSERVATION_REQUEST", run_id: "run", request_id: "request", sequence: 8, action_id: "action"});
+const observation = await session.accept(request);
+assert.equal(observation.type, "EXTENSION_OBSERVATION");
+assert.deepEqual(calls, {prepare: 1, dispatch: 1, observe: 1});
+
+const target = "https://space.bilibili.com/1420210197/dynamic";
+const normalized = extractor.canonicalizeSnapshot({
+ final_url: "https://space.bilibili.com/1420210197/dynamic#x",
+ page_title: " title ", ready_state: "complete", visibility_state: "visible",
+ creator: {uid: "1420210197", name: "青枫浦上Q", profile_url: "https://space.bilibili.com/1420210197"},
+ cards: [{dynamic_id: "123", published_at: "2026-08-14T00:00:00+08:00", text: " hello world ", url: "https://space.bilibili.com/1420210197/dynamic/123"}],
+ unparsed_nodes: 0, terminal_marker_text: "已经到底了"
+}, target);
+assert.equal(normalized.cards[0].text, "hello world");
+assert.equal(normalized.coverage_complete, true);
+await assert.rejects(async () => extractor.canonicalizeSnapshot({...normalized, cards: [normalized.cards[0], normalized.cards[0]]}, target), /E_CARD_DUPLICATE/u);
+console.log(JSON.stringify({ok: true, calls, normalized_id: normalized.cards[0].dynamic_id}));
diff --git a/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/runtime_lifecycle.mjs b/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/runtime_lifecycle.mjs
new file mode 100644
index 0000000..8818fb9
--- /dev/null
+++ b/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/runtime_lifecycle.mjs
@@ -0,0 +1,439 @@
+import assert from "node:assert/strict";
+import {pathToFileURL} from "node:url";
+
+const runtime = await import(pathToFileURL(process.argv[2]));
+const TARGET = "https://space.bilibili.com/1420210197/dynamic";
+
+function listenerSet() {
+ const listeners = new Set();
+ return {
+ addListener(listener) { listeners.add(listener); },
+ emit(value) { for (const listener of [...listeners]) listener(value); },
+ get size() { return listeners.size; }
+ };
+}
+
+function fakeChrome({createStatus = "complete", driftOnGet = false, markerMode = "normal"} = {}) {
+ const tabs = new Map([[1, {id: 1, windowId: 1, status: "complete", url: "https://example.com/user", marker: null}]]);
+ const sessionStorage = new Map();
+ const localStorage = new Map();
+ const stats = {created: 0, removed: 0, updated: 0, closeAttempts: 0};
+ let nextTab = 10;
+ let failRemove = false;
+ let driftPending = driftOnGet;
+ function storageArea(storage) {
+ return {
+ async get(key) { return storage.has(key) ? {[key]: storage.get(key)} : {}; },
+ async set(value) { for (const [key, item] of Object.entries(value)) storage.set(key, structuredClone(item)); },
+ async remove(key) { storage.delete(key); }
+ };
+ }
+ return {
+ tabs: {
+ async create({url, active}) {
+ assert.equal(false, active);
+ const tab = {id: nextTab++, windowId: 2, status: createStatus, url, marker: null};
+ tabs.set(tab.id, tab);
+ stats.created += 1;
+ return {...tab};
+ },
+ async get(id) {
+ if (!tabs.has(id)) throw new Error("missing");
+ if (driftPending && id !== 1) {
+ driftPending = false;
+ tabs.get(id).url = "https://example.com/drifted";
+ }
+ return {...tabs.get(id)};
+ },
+ async remove(id) {
+ stats.closeAttempts += 1;
+ if (failRemove) throw new Error("remove failed");
+ if (!tabs.delete(id)) throw new Error("missing");
+ stats.removed += 1;
+ },
+ async update() { stats.updated += 1; throw new Error("forbidden"); }
+ },
+ scripting: {
+ async executeScript({target, func, args = []}) {
+ const tab = tabs.get(target.tabId);
+ if (!tab) throw new Error("missing");
+ if (func.name === "setOwnershipMarker") {
+ if (markerMode === "throw") throw new Error("injected marker failure");
+ tab.marker = args[0];
+ return [{result: markerMode === "mismatch" ? "mismatch" : tab.marker}];
+ }
+ if (func.name === "readOwnershipMarker") return [{result: tab.marker}];
+ return [{result: {schema_version: 1, final_url: tab.url}}];
+ }
+ },
+ storage: {session: storageArea(sessionStorage), local: storageArea(localStorage)},
+ _tabs: tabs,
+ _storage: sessionStorage,
+ _local: localStorage,
+ _stats: stats,
+ _setFailRemove(value) { failRemove = value; }
+ };
+}
+
+function fakePort(mode = "open") {
+ const onMessage = listenerSet();
+ const onDisconnect = listenerSet();
+ const stats = {writes: 0, disconnects: 0};
+ let disconnected = mode === "before";
+ return {
+ onMessage,
+ onDisconnect,
+ postMessage() {
+ if (disconnected) throw new Error("peer closed");
+ stats.writes += 1;
+ if (mode === "during") {
+ disconnected = true;
+ onDisconnect.emit();
+ }
+ },
+ disconnect() {
+ if (disconnected) return;
+ disconnected = true;
+ stats.disconnects += 1;
+ onDisconnect.emit();
+ },
+ closeFromPeer() {
+ if (disconnected) return;
+ disconnected = true;
+ onDisconnect.emit();
+ },
+ stats
+ };
+}
+
+const chrome = fakeChrome();
+const lifecycle = new runtime.OwnedTabLifecycle(chrome, {
+ randomHex: () => "ab".repeat(16),
+ delay: async () => undefined,
+ maxPolls: 1
+});
+
+// One hundred independent scheduled slots leave no tab, storage, or global
+// process-like handle behind and never touch the user's pre-existing tab.
+for (let index = 0; index < 100; index += 1) {
+ const slot = index.toString(16).padStart(64, "0");
+ const created = await lifecycle.create(slot, TARGET);
+ assert.equal(runtime.FIXED_LIFECYCLE.CLOSED, await lifecycle.cleanup(created.record));
+ assert.deepEqual([...chrome._tabs.keys()], [1]);
+ assert.equal(0, chrome._storage.size);
+}
+assert.deepEqual(chrome._stats, {created: 100, removed: 100, updated: 0, closeAttempts: 100});
+
+// Every post-create failure is closed inside create's unescaped capability.
+// The ordinary recovery path below remains marker-bound and fail closed.
+for (const [options, pattern] of [
+ [{createStatus: "loading"}, /E_TAB_TIMEOUT/u],
+ [{driftOnGet: true}, /E_TAB_IDENTITY/u],
+ [{markerMode: "throw"}, /injected marker failure/u],
+ [{markerMode: "mismatch"}, /E_TAB_MARKER/u]
+]) {
+ const failedChrome = fakeChrome(options);
+ const failedLifecycle = new runtime.OwnedTabLifecycle(failedChrome, {
+ randomHex: () => "ba".repeat(16), delay: async () => undefined, maxPolls: 1
+ });
+ await assert.rejects(failedLifecycle.create("a".repeat(64), TARGET), pattern);
+ assert.deepEqual([...failedChrome._tabs.keys()], [1]);
+ assert.equal(0, failedChrome._storage.size);
+ assert.deepEqual(failedChrome._stats, {created: 1, removed: 1, updated: 0, closeAttempts: 1});
+}
+
+// A removed tab is idempotent success.
+const missing = await lifecycle.create("f".repeat(64), TARGET);
+chrome._tabs.delete(missing.record.tab_id);
+assert.equal(runtime.FIXED_LIFECYCLE.ALREADY_CLOSED, await lifecycle.cleanup(missing.record));
+
+// A same-id, same-URL user replacement cannot satisfy the unforgeable marker.
+const drift = await lifecycle.create("e".repeat(64), TARGET);
+chrome._tabs.set(drift.record.tab_id, {
+ id: drift.record.tab_id, windowId: drift.record.window_id,
+ status: "complete", url: TARGET, marker: "user-replacement"
+});
+const removesBeforeDrift = chrome._stats.closeAttempts;
+assert.equal(runtime.FIXED_LIFECYCLE.IDENTITY_DRIFT, await lifecycle.cleanup(drift.record));
+assert.equal(removesBeforeDrift, chrome._stats.closeAttempts);
+assert.equal(0, chrome._stats.updated);
+
+// A remove failure retires ownership and performs no retry or about:blank fallback.
+const failed = await lifecycle.create("d".repeat(64), TARGET);
+chrome._setFailRemove(true);
+assert.equal(runtime.FIXED_LIFECYCLE.CLOSE_FAILED, await lifecycle.cleanup(failed.record));
+assert.equal(0, chrome._storage.size);
+const attemptsAfterFailure = chrome._stats.closeAttempts;
+assert.equal(runtime.FIXED_LIFECYCLE.ALREADY_CLOSED, await lifecycle.cleanup());
+assert.equal(attemptsAfterFailure, chrome._stats.closeAttempts);
+assert.equal(0, chrome._stats.updated);
+chrome._setFailRemove(false);
+
+// Service-worker restart recovery uses the durable marker-bound record once.
+const recoverable = await lifecycle.create("c".repeat(64), TARGET);
+const restarted = new runtime.OwnedTabLifecycle(chrome, {
+ randomHex: () => "cd".repeat(16), delay: async () => undefined, maxPolls: 1
+});
+assert.equal(runtime.FIXED_LIFECYCLE.CLOSED, await restarted.cleanup());
+assert.equal(false, chrome._tabs.has(recoverable.record.tab_id));
+
+// Native-port handlers exist before the first write, writes serialize, and
+// peer exits before/during/after a write become one fixed sanitized error.
+for (const mode of ["before", "during"]) {
+ const port = fakePort(mode);
+ const transport = new runtime.NativePortTransport(port);
+ assert.equal(1, port.onMessage.size);
+ assert.equal(1, port.onDisconnect.size);
+ await assert.rejects(transport.send({schema_version: 1}), /E_NATIVE_PEER_CLOSED/u);
+}
+const afterPort = fakePort();
+const after = new runtime.NativePortTransport(afterPort);
+await after.send({sequence: 1});
+afterPort.closeFromPeer();
+await assert.rejects(after.send({sequence: 2}), /E_NATIVE_PEER_CLOSED/u);
+assert.equal(1, afterPort.stats.writes);
+
+const serializedPort = fakePort();
+const serialized = new runtime.NativePortTransport(serializedPort);
+await Promise.all([serialized.send({sequence: 1}), serialized.send({sequence: 2})]);
+assert.equal(2, serializedPort.stats.writes);
+serialized.disconnect();
+serialized.disconnect();
+assert.equal(1, serializedPort.stats.disconnects);
+
+function hashSlot(material) {
+ const match = /^dynamic-slot:([0-9]+)$/u.exec(material);
+ assert.ok(match);
+ return Promise.resolve(BigInt(match[1]).toString(16).padStart(64, "0"));
+}
+
+function successfulCoordinator(testChrome, {connectStats = {count: 0}, control = null} = {}) {
+ const slotStore = new runtime.SlotStateStore(testChrome, {randomHex: () => "ef".repeat(16)});
+ const slotLifecycle = new runtime.OwnedTabLifecycle(testChrome, {
+ randomHex: () => "de".repeat(16), delay: async () => undefined, maxPolls: 1
+ });
+ return {
+ slotStore,
+ coordinator: new runtime.SlotCoordinator({
+ lifecycle: slotLifecycle,
+ slotStore,
+ hashSlot,
+ collectPage: () => ({schema_version: 1}),
+ connect() {
+ connectStats.count += 1;
+ const port = fakePort();
+ const original = port.postMessage.bind(port);
+ port.postMessage = (message) => {
+ original(message);
+ if (message?.type === "CLIENT_HELLO") {
+ if (control?.onHello) control.onHello();
+ queueMicrotask(() => port.onMessage.emit({
+ type: "HOST_ACTION", action_id: "action-1", kind: "goto", url: TARGET
+ }));
+ } else if (message?.type === "CLIENT_ACTION_RESULT") {
+ if (control?.onOutcome) control.onOutcome();
+ const release = control?.commitRelease || Promise.resolve();
+ void Promise.resolve(release).then(() => {
+ queueMicrotask(() => port.onMessage.emit({type: "HOST_COMMIT_RESULT", result: {accepted: true}}));
+ });
+ }
+ };
+ return new runtime.NativePortTransport(port, {timeoutMilliseconds: 1000});
+ },
+ createSession(api) {
+ return {
+ helloFrame() { return {type: "CLIENT_HELLO"}; },
+ async accept(message) {
+ if (message?.type === "HOST_ACTION") {
+ const prepared = await api.prepare(message);
+ const dispatched = await api.dispatch(prepared);
+ await api.observe(dispatched);
+ return {type: "CLIENT_ACTION_RESULT"};
+ }
+ return null;
+ }
+ };
+ }
+ })
+ };
+}
+
+// A durable terminal suppresses serial replay, a fresh service-worker instance,
+// and duplicate-alarm delivery before any Native connection or browser action.
+const slotChrome = fakeChrome();
+const connectStats = {count: 0};
+const firstWorker = successfulCoordinator(slotChrome, {connectStats});
+assert.equal("COMPLETE", (await firstWorker.coordinator.run("dynamic-slot:100")).status);
+const afterFirst = {...slotChrome._stats, connects: connectStats.count};
+assert.equal("SKIPPED_TERMINAL", (await firstWorker.coordinator.run("dynamic-slot:100")).status);
+const secondWorker = successfulCoordinator(slotChrome, {connectStats});
+assert.equal("SKIPPED_TERMINAL", (await secondWorker.coordinator.run("dynamic-slot:100")).status);
+assert.equal("SKIPPED_TERMINAL", (await secondWorker.coordinator.run("dynamic-slot:100")).status);
+assert.deepEqual({...slotChrome._stats, connects: connectStats.count}, afterFirst);
+
+// Restart before terminal resumes only the same lease for cleanup and records a
+// fixed interrupted terminal; it never creates another tab or Native session.
+const slot101Id = await hashSlot("dynamic-slot:101");
+const interruptedClaim = await secondWorker.slotStore.claim("dynamic-slot:101", slot101Id);
+assert.equal("CLAIMED", interruptedClaim.disposition);
+const interruptedLifecycle = secondWorker.coordinator.lifecycle;
+await interruptedLifecycle.create(slot101Id, TARGET, interruptedClaim.entry.lease_id);
+const beforeResume = {...slotChrome._stats, connects: connectStats.count};
+const thirdWorker = successfulCoordinator(slotChrome, {connectStats});
+assert.deepEqual(await thirdWorker.coordinator.run("dynamic-slot:101"), {
+ status: "FAILED", error_code: "E_SLOT_INTERRUPTED"
+});
+assert.equal(beforeResume.created, slotChrome._stats.created);
+assert.equal(beforeResume.connects, connectStats.count);
+assert.equal(beforeResume.removed + 1, slotChrome._stats.removed);
+assert.equal("SKIPPED_TERMINAL", (await thirdWorker.coordinator.run("dynamic-slot:101")).status);
+
+// The production startup path terminalizes an old STARTED lease before a
+// different, normally scheduled next slot is claimed. Cover no-tab, exact-tab,
+// and one-shot cleanup-failure recovery without any Native action for the old
+// lease or a permanent BUSY state.
+const startupChrome = fakeChrome();
+const startupConnects = {count: 0};
+let startupWorker = successfulCoordinator(startupChrome, {connectStats: startupConnects});
+const old200Id = await hashSlot("dynamic-slot:200");
+assert.equal("CLAIMED", (await startupWorker.slotStore.claim("dynamic-slot:200", old200Id)).disposition);
+assert.equal("E_SLOT_INTERRUPTED", (await startupWorker.coordinator.recover()).result_code);
+assert.equal(0, startupConnects.count);
+assert.equal("COMPLETE", (await startupWorker.coordinator.run("dynamic-slot:201")).status);
+
+const old202Id = await hashSlot("dynamic-slot:202");
+const old202 = await startupWorker.slotStore.claim("dynamic-slot:202", old202Id);
+await startupWorker.coordinator.lifecycle.create(old202Id, TARGET, old202.entry.lease_id);
+const removesBeforeStartup = startupChrome._stats.removed;
+startupWorker = successfulCoordinator(startupChrome, {connectStats: startupConnects});
+assert.equal("E_SLOT_INTERRUPTED", (await startupWorker.coordinator.recover()).result_code);
+assert.equal(removesBeforeStartup + 1, startupChrome._stats.removed);
+assert.equal("COMPLETE", (await startupWorker.coordinator.run("dynamic-slot:203")).status);
+
+const old204Id = await hashSlot("dynamic-slot:204");
+const old204 = await startupWorker.slotStore.claim("dynamic-slot:204", old204Id);
+await startupWorker.coordinator.lifecycle.create(old204Id, TARGET, old204.entry.lease_id);
+startupChrome._setFailRemove(true);
+const failedCloseRecovery = await successfulCoordinator(startupChrome, {connectStats: startupConnects}).coordinator.recover();
+assert.equal(runtime.FIXED_LIFECYCLE.CLOSE_FAILED, failedCloseRecovery.lifecycle_code);
+startupChrome._setFailRemove(false);
+assert.equal("COMPLETE", (await successfulCoordinator(startupChrome, {connectStats: startupConnects}).coordinator.run("dynamic-slot:205")).status);
+
+// Terminal state/result pairs are semantic, not merely syntactic. Every cross
+// pair, null, unknown, and wrong-type value fails strict read with mutation0.
+for (const [state, result_code] of [
+ ["COMPLETE", "E_SLOT_FAILED"],
+ ["COMPLETE", null],
+ ["COMPLETE", 7],
+ ["FAILED", "SLOT_COMPLETE"],
+ ["FAILED", null],
+ ["FAILED", "UNKNOWN"]
+]) {
+ const pairChrome = fakeChrome();
+ const corrupt = {
+ schema: 1,
+ entries: [{
+ schema: 1,
+ slot_material: "dynamic-slot:300",
+ slot_number: 300,
+ slot_id: await hashSlot("dynamic-slot:300"),
+ lease_id: "aa".repeat(16),
+ state,
+ result_code
+ }]
+ };
+ pairChrome._local.set(runtime.SLOT_STATE_STORAGE_KEY, corrupt);
+ const before = JSON.stringify(corrupt);
+ await assert.rejects(new runtime.SlotStateStore(pairChrome).read(), /E_SLOT_STATE/u);
+ assert.equal(before, JSON.stringify(pairChrome._local.get(runtime.SLOT_STATE_STORAGE_KEY)));
+}
+
+// Malformed durable state is rejected without repair, browser/native action, or
+// storage mutation.
+const driftChrome = fakeChrome();
+const driftStats = {count: 0};
+driftChrome._local.set(runtime.SLOT_STATE_STORAGE_KEY, {schema: 1, entries: [], extra: true});
+const driftBytes = JSON.stringify(driftChrome._local.get(runtime.SLOT_STATE_STORAGE_KEY));
+const driftWorker = successfulCoordinator(driftChrome, {connectStats: driftStats});
+assert.deepEqual(await driftWorker.coordinator.run("dynamic-slot:102"), {
+ status: "FAILED", error_code: "E_SLOT_STATE"
+});
+assert.equal(driftBytes, JSON.stringify(driftChrome._local.get(runtime.SLOT_STATE_STORAGE_KEY)));
+assert.deepEqual(driftChrome._stats, {created: 0, removed: 0, updated: 0, closeAttempts: 0});
+assert.equal(0, driftStats.count);
+
+// The coordinator-wide gate includes the whole active run. A startup recovery
+// arriving after CLIENT_HELLO waits until the active lease has appended and
+// rebound its terminal; it cannot relabel that lease as interrupted.
+const helloRaceChrome = fakeChrome();
+const helloRaceStats = {count: 0};
+let signalHello;
+let releaseCommit;
+const helloSeen = new Promise((resolve) => { signalHello = resolve; });
+const commitRelease = new Promise((resolve) => { releaseCommit = resolve; });
+const helloRaceWorker = successfulCoordinator(helloRaceChrome, {
+ connectStats: helloRaceStats,
+ control: {onHello: signalHello, commitRelease}
+});
+const helloRun = helloRaceWorker.coordinator.run("dynamic-slot:400");
+await helloSeen;
+let helloRecoverySettled = false;
+const helloRecovery = helloRaceWorker.coordinator.recover().then((value) => {
+ helloRecoverySettled = true;
+ return value;
+});
+await new Promise((resolve) => setImmediate(resolve));
+assert.equal(false, helloRecoverySettled);
+releaseCommit();
+assert.equal("COMPLETE", (await helloRun).status);
+assert.equal(null, await helloRecovery);
+const helloRoot = await helloRaceWorker.slotStore.read();
+assert.deepEqual(helloRoot.entries.filter((entry) => entry.slot_material === "dynamic-slot:400").map((entry) => [entry.state, entry.result_code]), [["COMPLETE", "SLOT_COMPLETE"]]);
+
+// The gate also spans outcome-to-terminal append/readback. A startup event
+// queues behind that append, while a duplicate alarm is skipped without action.
+const finishRaceChrome = fakeChrome();
+const finishRaceStats = {count: 0};
+const finishRaceWorker = successfulCoordinator(finishRaceChrome, {connectStats: finishRaceStats});
+const originalFinish = finishRaceWorker.slotStore.finish.bind(finishRaceWorker.slotStore);
+let signalFinish;
+let releaseFinish;
+const finishSeen = new Promise((resolve) => { signalFinish = resolve; });
+const finishRelease = new Promise((resolve) => { releaseFinish = resolve; });
+finishRaceWorker.slotStore.finish = async (...args) => {
+ if (args[1] === "COMPLETE") {
+ signalFinish();
+ await finishRelease;
+ }
+ return await originalFinish(...args);
+};
+const finishRun = finishRaceWorker.coordinator.run("dynamic-slot:401");
+await finishSeen;
+let finishRecoverySettled = false;
+const finishRecovery = finishRaceWorker.coordinator.recover().then((value) => {
+ finishRecoverySettled = true;
+ return value;
+});
+assert.deepEqual(await finishRaceWorker.coordinator.run("dynamic-slot:402"), {status: "SKIPPED_OVERLAP"});
+await new Promise((resolve) => setImmediate(resolve));
+assert.equal(false, finishRecoverySettled);
+releaseFinish();
+assert.equal("COMPLETE", (await finishRun).status);
+assert.equal(null, await finishRecovery);
+assert.equal("COMPLETE", (await finishRaceWorker.coordinator.run("dynamic-slot:402")).status);
+const finishRoot = await finishRaceWorker.slotStore.read();
+assert.deepEqual(finishRoot.entries.filter((entry) => entry.slot_material === "dynamic-slot:401").map((entry) => [entry.state, entry.result_code]), [["COMPLETE", "SLOT_COMPLETE"]]);
+
+assert.equal(0, chrome._stats.updated);
+assert.deepEqual(chrome._tabs.get(1), {id: 1, windowId: 1, status: "complete", url: "https://example.com/user", marker: null});
+process.stdout.write(JSON.stringify({
+ ok: true,
+ slots: 100,
+ create_failure_cleanup: 4,
+ durable_slot_replay: true,
+ startup_next_slot_recovery: 3,
+ terminal_cross_pair_rejections: 6,
+ coordinator_wide_races: 2,
+ tabs_update: 0,
+ user_tab_preserved: true
+}) + "\n");
diff --git a/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/test_v009_contract.py b/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/test_v009_contract.py
new file mode 100644
index 0000000..70bbfe7
--- /dev/null
+++ b/dev/project-dev/test/bili_dynamic_refresh_trusted_adapter/test_v009_contract.py
@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import errno
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from datetime import datetime, timezone
+from pathlib import Path
+
+PROJECT_DEV = Path(__file__).resolve().parents[2]
+ROOT = PROJECT_DEV.parents[1]
+sys.path.insert(0, str(PROJECT_DEV))
+
+import bili_dynamic_collector as collector
+import bili_dynamic_refresh_controller as public_controller
+from bili_dynamic_refresh_native_host.constants import EXTENSION_ID, EXTENSION_ORIGIN, EXTENSION_NAME, HOST_NAME
+from bili_dynamic_refresh_native_host.durable import PendingStore
+from bili_dynamic_refresh_native_host.identity import IdentityError, verify_local_identity, verify_source_tree
+from bili_dynamic_refresh_native_host.native_host import NativeFrameWriter
+from bili_dynamic_refresh_native_host.protocol import HostSession, ProtocolError, sign_extension_frame
+
+EXTENSION_ROOT = PROJECT_DEV / "bili_dynamic_refresh_extension"
+HOST_ROOT = PROJECT_DEV / "bili_dynamic_refresh_native_host"
+
+
+class V009ContractTests(unittest.TestCase):
+ def copy_extension(self, root: Path) -> Path:
+ target = root / "extension"
+ shutil.copytree(EXTENSION_ROOT, target)
+ return target
+
+ def facts(self, source_root: Path) -> dict[str, object]:
+ source = verify_source_tree(source_root)
+ manifest_payload = (source_root / "manifest.json").read_bytes()
+ receipt_sha = "2" * 64
+ host_manifest_sha = "3" * 64
+ chrome_sha = "4" * 64
+ started = "2026-08-14T16:00:00+08:00"
+ source_approval = {
+ "schema": 1, "task_id": "DEV-PROJECT-INFO-BILI-DYNAMIC-REFRESH-COLLECTOR-20260813-001",
+ "scope": "controlled-local-unpacked-source-manifest",
+ "source_manifest_path": str(source_root / "source-artifact-manifest.json"),
+ "source_manifest_bytes": source["manifest_bytes"], "source_manifest_sha256": source["manifest_sha256"],
+ "payload_tree_sha256": source["payload_tree_sha256"], "manifest_bytes": len(manifest_payload),
+ "manifest_sha256": hashlib.sha256(manifest_payload).hexdigest(), "extension_id": EXTENSION_ID,
+ "approved_by_role": "dev.reviewer.project", "review_audit_id": "AUDIT", "created_at": started,
+ }
+ load = {
+ "schema": 1, "task_id": source_approval["task_id"], "scope": "local-unpacked-load",
+ "account_holder_confirmed": True, "observed_extension_id": EXTENSION_ID,
+ "observed_name": EXTENSION_NAME, "observed_version": "1.0.0", "observed_enabled": True,
+ "observed_error_count": 0, "source_root_absolute": str(source_root),
+ "source_manifest_bytes": source["manifest_bytes"], "source_manifest_sha256": source["manifest_sha256"],
+ "payload_tree_sha256": source["payload_tree_sha256"], "host_install_receipt_bytes": 100,
+ "host_install_receipt_sha256": receipt_sha, "host_manifest_sha256": host_manifest_sha,
+ "chrome_parent_pid": 1234, "chrome_parent_path_sha256": chrome_sha,
+ "chrome_parent_signature_verified": True, "chrome_parent_started_at": started,
+ "observed_at": started, "approved_by_role": "project.admin", "approval_reason": "synthetic-test-only",
+ }
+ host = {
+ "host_name": HOST_NAME, "allowed_origins": [EXTENSION_ORIGIN], "install_receipt_bytes": 100,
+ "install_receipt_sha256": receipt_sha, "host_manifest_sha256": host_manifest_sha,
+ "chrome_parent_pid": 1234, "chrome_parent_path_sha256": chrome_sha,
+ "chrome_parent_signature_verified": True, "chrome_parent_started_at": started,
+ }
+ return {"source_approval": source_approval, "load_approval": load, "host": host}
+
+ def test_manifest_identity_exact_set_and_forbidden_surfaces(self) -> None:
+ manifest = json.loads((EXTENSION_ROOT / "manifest.json").read_text(encoding="utf-8"))
+ self.assertEqual((3, "1.0.0", "120"), (manifest["manifest_version"], manifest["version"], manifest["minimum_chrome_version"]))
+ self.assertEqual(["alarms", "nativeMessaging", "scripting", "storage", "tabs"], manifest["permissions"])
+ self.assertEqual(["https://space.bilibili.com/*/dynamic*"], manifest["host_permissions"])
+ self.assertEqual({"service_worker": "service_worker.js", "type": "module"}, manifest["background"])
+ self.assertEqual(EXTENSION_ID, verify_source_tree(EXTENSION_ROOT) and EXTENSION_ID)
+ verify_source_tree(HOST_ROOT)
+ product = "\n".join(path.read_text(encoding="utf-8") for root in (EXTENSION_ROOT, HOST_ROOT) for path in root.rglob("*") if path.is_file())
+ for forbidden in ("chrome.cookies", "localStorage", "remote-debugging", "ExtensionInstallForcelist", "ExtensionSettings", "clients2.google.com"):
+ self.assertNotIn(forbidden, product)
+ self.assertEqual([EXTENSION_ORIGIN], json.loads((HOST_ROOT / "native-host-manifest.template.json").read_text(encoding="utf-8"))["allowed_origins"])
+ native_main = (HOST_ROOT / "native_host.py").read_text(encoding="utf-8")
+ for forbidden in ("argparse", "os.environ", "--test-peer", "--fixture", "--transcript"):
+ self.assertNotIn(forbidden, native_main)
+
+ runtime_source = (EXTENSION_ROOT / "runtime.js").read_text(encoding="utf-8")
+ worker_source = (EXTENSION_ROOT / "service_worker.js").read_text(encoding="utf-8")
+ self.assertNotIn("about:blank", runtime_source + worker_source)
+ self.assertNotIn("tabs.update", runtime_source + worker_source)
+ self.assertNotIn("node_repl", runtime_source + worker_source)
+ self.assertIn("periodInMinutes: PERIOD_MINUTES", worker_source)
+
+ def test_external_source_and_visible_load_identity_fail_closed(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ source_root = self.copy_extension(Path(raw))
+ facts = self.facts(source_root)
+ verified = verify_local_identity(source_root, facts)
+ self.assertEqual((EXTENSION_ID, 1234), (verified.extension_id, verified.chrome_parent_pid))
+ mutations = [
+ ("source_approval", "approved_by_role", "dev.developer.project.secondary"),
+ ("source_approval", "scope", "caller-self-report"),
+ ("load_approval", "observed_extension_id", "a" * 32),
+ ("load_approval", "observed_enabled", False),
+ ("load_approval", "observed_error_count", 1),
+ ("load_approval", "approved_by_role", "extension"),
+ ("host", "allowed_origins", ["chrome-extension://" + "a" * 32 + "/"]),
+ ("host", "chrome_parent_pid", 5678),
+ ]
+ for section, key, value in mutations:
+ with self.subTest(section=section, key=key):
+ changed = json.loads(json.dumps(facts))
+ changed[section][key] = value
+ with self.assertRaises(IdentityError):
+ verify_local_identity(source_root, changed)
+ (source_root / "extra.js").write_text("caller-authored", encoding="utf-8")
+ with self.assertRaises(IdentityError):
+ verify_local_identity(source_root, facts)
+
+ def test_actual_host_protocol_durable_permit_and_single_commit(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ temp = Path(raw)
+ source_root = self.copy_extension(temp)
+ identity = verify_local_identity(source_root, self.facts(source_root))
+ committed: list[dict[str, object]] = []
+ session = HostSession(identity, PendingStore(temp / "pending.json"), lambda evidence: committed.append(dict(evidence)) or {"status": "REFRESH_CONFIRMED_NO_NEW"}, "https://space.bilibili.com/1420210197/dynamic", now=lambda: datetime(2026, 8, 14, 8, tzinfo=timezone.utc))
+ hello = {"schema_version": 1, "type": "EXTENSION_HELLO", "sequence": 1, "extension_id": EXTENSION_ID, "version": "1.0.0", "manifest_name": EXTENSION_NAME}
+ challenge = session.start(hello, run_id="run", request_id="request", deadline_at="2026-08-14T08:02:00Z")
+ secret = challenge["secret"]
+ accepted = sign_extension_frame(secret, {"schema_version": 1, "type": "EXTENSION_CHALLENGE_ACCEPTED", "run_id": "run", "request_id": "request", "sequence": 3, "challenge_id": challenge["challenge_id"]})
+ prepare = session.accept(accepted)
+ ready = sign_extension_frame(secret, {"schema_version": 1, "type": "EXTENSION_READY_TO_DISPATCH", "run_id": "run", "request_id": "request", "sequence": 5, "action_id": prepare["action"]["action_id"], "prepared": {"action_id": prepare["action"]["action_id"]}})
+ permit = session.accept(ready)
+ pending = PendingStore(temp / "pending.json").load()
+ self.assertEqual((True, 1, 0), (pending["action_budget_consumed"], pending["refresh_count"], pending["retry_count"]))
+ with self.assertRaises(Exception):
+ session.accept(ready)
+ result = sign_extension_frame(secret, {"schema_version": 1, "type": "EXTENSION_ACTION_RESULT", "run_id": "run", "request_id": "request", "sequence": 7, "permit_id": permit["permit"]["permit_id"], "result": {"tab_id": 7}})
+ request = session.accept(result)
+ observation = sign_extension_frame(secret, {"schema_version": 1, "type": "EXTENSION_OBSERVATION", "run_id": "run", "request_id": "request", "sequence": 9, "observation": {"cards": [], "coverage_complete": True}})
+ final = session.accept(observation)
+ self.assertEqual("HOST_COMMIT_RESULT", final["type"])
+ self.assertEqual(1, len(committed))
+
+ def test_expired_original_deadline_stops_before_dispatch_permit(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ temp = Path(raw)
+ source_root = self.copy_extension(temp)
+ identity = verify_local_identity(source_root, self.facts(source_root))
+ current = [datetime(2026, 8, 14, 8, tzinfo=timezone.utc)]
+ store = PendingStore(temp / "pending.json")
+ session = HostSession(identity, store, lambda evidence: {}, "https://space.bilibili.com/1420210197/dynamic", now=lambda: current[0])
+ hello = {"schema_version": 1, "type": "EXTENSION_HELLO", "sequence": 1, "extension_id": EXTENSION_ID, "version": "1.0.0", "manifest_name": EXTENSION_NAME}
+ challenge = session.start(hello, run_id="run", request_id="request", deadline_at="2026-08-14T08:02:00Z")
+ secret = challenge["secret"]
+ accepted = sign_extension_frame(secret, {"schema_version": 1, "type": "EXTENSION_CHALLENGE_ACCEPTED", "run_id": "run", "request_id": "request", "sequence": 3, "challenge_id": challenge["challenge_id"]})
+ prepare = session.accept(accepted)
+ current[0] = datetime(2026, 8, 14, 8, 2, 0, 1, tzinfo=timezone.utc)
+ ready = sign_extension_frame(secret, {"schema_version": 1, "type": "EXTENSION_READY_TO_DISPATCH", "run_id": "run", "request_id": "request", "sequence": 5, "action_id": prepare["action"]["action_id"], "prepared": {}})
+ with self.assertRaises(ProtocolError) as failure:
+ session.accept(ready)
+ self.assertEqual("E_OVERALL_DEADLINE", failure.exception.code)
+ pending = store.load()
+ self.assertEqual((False, 0, 0), (pending["action_budget_consumed"], pending["refresh_count"], pending["retry_count"]))
+
+ def test_real_subprocess_crash_recovery_is_bounded(self) -> None:
+ child = Path(__file__).with_name("crash_child.py")
+ cases = {
+ "after_open": (0, False),
+ "after_file_fsync": (0, False),
+ "after_replace": (1, True),
+ "after_directory_fsync": (1, True),
+ }
+ for phase, expected in cases.items():
+ with self.subTest(phase=phase), tempfile.TemporaryDirectory() as raw:
+ path = Path(raw) / "pending.json"
+ completed = subprocess.run([sys.executable, "-B", str(child), str(path), phase], timeout=20)
+ self.assertEqual(79, completed.returncode)
+ projection = PendingStore(path).recovery_projection()
+ self.assertEqual(expected, (projection["refresh_count"], projection["may_have_dispatched"]))
+ self.assertEqual(0, projection["retry_count"])
+
+ def test_corrupt_durable_state_projects_conservative_single_action(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ path = Path(raw) / "pending.json"
+ path.write_bytes(b'{"schema":1,"broken":true}\n')
+ projection = PendingStore(path).recovery_projection()
+ self.assertEqual(("E_DISPATCH_DURABILITY_AMBIGUOUS", 1, 0, True), (projection["error_code"], projection["refresh_count"], projection["retry_count"], projection["may_have_dispatched"]))
+
+ def test_real_service_worker_reducer_and_extractor(self) -> None:
+ completed = subprocess.run(
+ ["node", str(Path(__file__).with_name("js_contract.mjs")), str(EXTENSION_ROOT / "protocol.js"), str(EXTENSION_ROOT / "page_extract.js")],
+ text=True, capture_output=True, timeout=20, check=True,
+ )
+ result = json.loads(completed.stdout)
+ self.assertEqual({"prepare": 1, "dispatch": 1, "observe": 1}, result["calls"])
+
+ lifecycle = subprocess.run(
+ ["node", str(Path(__file__).with_name("runtime_lifecycle.mjs")), str(EXTENSION_ROOT / "runtime.js")],
+ text=True, capture_output=True, timeout=30, check=True,
+ )
+ lifecycle_result = json.loads(lifecycle.stdout)
+ self.assertEqual((100, 0, True), (
+ lifecycle_result["slots"], lifecycle_result["tabs_update"], lifecycle_result["user_tab_preserved"]
+ ))
+
+ reproduction = subprocess.run(
+ ["node", str(Path(__file__).with_name("codex_stdio_race_repro.mjs"))],
+ text=True, capture_output=True, timeout=20, check=True,
+ )
+ reproduction_result = json.loads(reproduction.stdout)
+ self.assertEqual((True, "E_NATIVE_PEER_CLOSED"), (
+ reproduction_result["old_ordering_uncaught"], reproduction_result["bounded_error_code"]
+ ))
+
+ def test_native_frame_writer_sanitizes_only_peer_close(self) -> None:
+ class Stream:
+ def __init__(self, *, write_error: BaseException | None = None, flush_error: BaseException | None = None) -> None:
+ self.write_error = write_error
+ self.flush_error = flush_error
+ self.payloads: list[bytes] = []
+ self.flushes = 0
+
+ def write(self, payload: bytes) -> None:
+ if self.write_error is not None:
+ raise self.write_error
+ self.payloads.append(payload)
+
+ def flush(self) -> None:
+ self.flushes += 1
+ if self.flush_error is not None:
+ raise self.flush_error
+
+ for error in (BrokenPipeError(), EOFError(), OSError(errno.EPIPE, "pipe closed")):
+ with self.subTest(error=type(error).__name__):
+ stream = Stream(write_error=error)
+ writer = NativeFrameWriter(stream) # type: ignore[arg-type]
+ first = writer.write_frame({"schema_version": 1})
+ second = writer.write_frame({"schema_version": 1})
+ self.assertEqual((False, "E_NATIVE_PEER_CLOSED"), (first.written, first.error_code))
+ self.assertEqual(first, second)
+ self.assertEqual([], stream.payloads)
+
+ flush_stream = Stream(flush_error=BrokenPipeError())
+ flushed = NativeFrameWriter(flush_stream).write_frame({"schema_version": 1}) # type: ignore[arg-type]
+ self.assertEqual((False, "E_NATIVE_PEER_CLOSED", 1, 1), (
+ flushed.written, flushed.error_code, len(flush_stream.payloads), flush_stream.flushes
+ ))
+
+ with self.assertRaises(PermissionError):
+ NativeFrameWriter(Stream(write_error=PermissionError("denied"))).write_frame({"schema_version": 1}) # type: ignore[arg-type]
+
+ def test_public_controller_does_not_read_caller_transcript(self) -> None:
+ stream = io.StringIO('{"schema_version":4,"authoritative":true,"saved":true,"no_new":true}\n')
+ with self.assertRaises(collector.CollectorError) as failure:
+ public_controller.run_product(None, Path("unused"), datetime.now(timezone.utc), input_stream=stream, output_stream=io.StringIO()) # type: ignore[arg-type]
+ self.assertEqual("E_TRUSTED_ADAPTER_REQUIRED", failure.exception.code)
+ self.assertEqual({"authoritative": False, "saved": False, "no_new": False}, failure.exception.details)
+ self.assertEqual(0, stream.tell())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/dev/project-dev/test/test_bili_article_image_capture.mjs b/dev/project-dev/test/test_bili_article_image_capture.mjs
new file mode 100644
index 0000000..a8c7586
--- /dev/null
+++ b/dev/project-dev/test/test_bili_article_image_capture.mjs
@@ -0,0 +1,102 @@
+import assert from "node:assert/strict";
+import cryptoModule from "node:crypto";
+import {createRequire} from "node:module";
+
+globalThis.crypto ??= cryptoModule.webcrypto;
+globalThis.CSS ??= {escape: (value) => String(value)};
+const require = createRequire(import.meta.url);
+const capture = require(process.argv[2]);
+
+function config(uid, name) {
+ return {
+ creator_uid: uid,
+ creator_name: name,
+ dynamic_url: `https://space.bilibili.com/${uid}/dynamic`,
+ profile_url: `https://space.bilibili.com/${uid}`,
+ include_types: ["article", "text", "image"],
+ deadline_ms: 30000,
+ observation_interval_ms: 100,
+ stable_observations: 3,
+ };
+}
+
+function documentFixture(uid, body, imageUrl) {
+ const nodes = {
+ ".opus-module-content": {innerText: body},
+ "h1": {innerText: "Full article"},
+ "time[datetime], [data-published-at]": {getAttribute: (name) => name === "datetime" ? "2026-08-12T10:00:00+08:00" : null},
+ ".article-content, .opus-module-title": {},
+ };
+ return {
+ body: {innerText: "ordinary visible article"},
+ querySelector(selector) {
+ if (selector.includes(`data-mid=\"${uid}\"`)) return {};
+ return nodes[selector] ?? null;
+ },
+ querySelectorAll(selector) {
+ if (selector.includes("img[src]")) return [{naturalWidth: 100, naturalHeight: 80, currentSrc: imageUrl, src: imageUrl}];
+ return [];
+ },
+ };
+}
+
+let stableItem;
+for (const [uid, name] of [["10001", "Creator A"], ["20002", "创作者乙"]]) {
+ const body = `完整正文-${name}\n`.repeat(100);
+ const result = capture.currentOpusSnapshot(
+ documentFixture(uid, body, "https://i1.hdslb.com/bfs/new_dyn/sample.png"),
+ {href: "https://www.bilibili.com/opus/sample_1"},
+ config(uid, name),
+ );
+ assert.equal(result.state, "READY");
+ assert.equal(result.item.body_text, body.trimEnd());
+ assert.deepEqual(result.item.original_image_candidates, ["https://i1.hdslb.com/bfs/new_dyn/sample.png"]);
+ if (uid === "10001") stableItem = result.item;
+}
+
+assert.ok(stableItem);
+const sharedVectorDigest = await capture.acceptedSnapshotFingerprint([{
+ stable_id: "snap_1",
+ item_type: "article",
+ title: "Shared",
+ source_url: "https://www.bilibili.com/opus/snap_1",
+ published_at: "2026-08-12T02:00:00.000Z",
+ body_text: "正文\nline",
+ body_complete: true,
+ original_image_candidates: ["https://i1.hdslb.com/bfs/new_dyn/shared.png"],
+}]);
+assert.equal(sharedVectorDigest, "4F0616496B08F8537A1C9F17B48D53BC2F93A26EB15AB5977C6E891ABA411454");
+const stableDigest = await capture.acceptedSnapshotFingerprint([stableItem]);
+assert.match(stableDigest, /^[0-9A-F]{64}$/u);
+assert.notEqual(
+ stableDigest,
+ await capture.acceptedSnapshotFingerprint([{...stableItem, body_text: `${stableItem.body_text}-changed`}]),
+);
+assert.notEqual(
+ stableDigest,
+ await capture.acceptedSnapshotFingerprint([stableItem, {...stableItem, stable_id: "sample_2", source_url: "https://www.bilibili.com/opus/sample_2"}]),
+);
+
+const samples = [
+ {state: "METADATA_NOT_READY", reason: "METADATA_NOT_READY", item: null},
+ {state: "READY", reason: "READY", item: stableItem},
+ {state: "READY", reason: "READY", item: stableItem},
+ {state: "DIMENSIONS_PENDING", reason: "DIMENSIONS_PENDING", item: null},
+ {state: "READY", reason: "READY", item: stableItem},
+ {state: "READY", reason: "READY", item: stableItem},
+ {state: "READY", reason: "READY", item: stableItem},
+];
+const stable = await capture.observeUntilStable(config("30003", "Stable"), async () => samples.shift(), async () => {});
+assert.equal(stable.observations.length, 7);
+
+await assert.rejects(
+ capture.observeUntilStable(config("30003", "Stable"), async () => ({state: "ACCESS_BLOCKED", reason: "ACCESS_BLOCKED", item: null}), async () => {}),
+ /E_ACCESS_CONTROL/u,
+);
+assert.throws(
+ () => capture.currentOpusSnapshot(documentFixture("10001", "body", "https://i1.hdslb.com/bfs/new_dyn/sample.png?token=synthetic"), {href: "https://www.bilibili.com/opus/sample_1"}, config("10001", "Creator A")),
+ /E_IMAGE_IDENTITY/u,
+);
+assert.throws(() => capture.validatePublicConfig({...config("10001", "Creator A"), dynamic_url: "https://space.bilibili.com/20002/dynamic"}), /E_PAGE_IDENTITY/u);
+
+console.log(JSON.stringify({status: "PASS", creators: 2, readiness_observations: stable.observations.length, network_requests: 0}));
diff --git a/dev/project-dev/test/test_bili_article_image_collector.py b/dev/project-dev/test/test_bili_article_image_collector.py
new file mode 100644
index 0000000..ef41426
--- /dev/null
+++ b/dev/project-dev/test/test_bili_article_image_collector.py
@@ -0,0 +1,407 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from datetime import datetime, timezone
+from pathlib import Path
+from unittest import mock
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[3]
+PROJECT_DEV = Path(__file__).resolve().parents[1]
+TMP_ROOT = PROJECT_ROOT / "dev" / "tmp"
+MODULE_PATH = PROJECT_DEV / "bili_article_image_collector.py"
+SPEC = importlib.util.spec_from_file_location("bili_article_image_collector", MODULE_PATH)
+assert SPEC and SPEC.loader
+collector = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = collector
+SPEC.loader.exec_module(collector)
+
+HOST_PATH = PROJECT_DEV / "bili_article_image_native_host.py"
+HOST_SPEC = importlib.util.spec_from_file_location("bili_article_image_native_host", HOST_PATH)
+assert HOST_SPEC and HOST_SPEC.loader
+native_host = importlib.util.module_from_spec(HOST_SPEC)
+sys.modules[HOST_SPEC.name] = native_host
+HOST_SPEC.loader.exec_module(native_host)
+
+VALIDATOR_PATH = PROJECT_DEV / "bili_article_image_source_validator.py"
+VALIDATOR_SPEC = importlib.util.spec_from_file_location("bili_article_image_source_validator", VALIDATOR_PATH)
+assert VALIDATOR_SPEC and VALIDATOR_SPEC.loader
+source_validator = importlib.util.module_from_spec(VALIDATOR_SPEC)
+sys.modules[VALIDATOR_SPEC.name] = source_validator
+VALIDATOR_SPEC.loader.exec_module(source_validator)
+
+
+def canonical_json(value: object) -> bytes:
+ return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
+
+
+def accepted_snapshot_sha256(items: list[dict[str, object]]) -> str:
+ canonical_items = []
+ for item in items:
+ published = datetime.fromisoformat(str(item["published_at"]))
+ canonical_items.append({
+ "body_complete": True,
+ "body_text": str(item["body_text"]).replace("\r\n", "\n").replace("\r", "\n").rstrip("\n"),
+ "image_count": len(item["images"]),
+ "item_type": item["item_type"],
+ "published_at_epoch_ms": int(published.timestamp() * 1000),
+ "source_url": item["source_url"],
+ "stable_id": item["stable_id"],
+ "title": str(item["title"]).strip(),
+ })
+ canonical_items.sort(key=lambda item: str(item["stable_id"]))
+ payload = json.dumps({"items": canonical_items, "schema_version": 1}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ return hashlib.sha256(payload).hexdigest().upper()
+
+
+class GenericArticleImageCollectorTests(unittest.TestCase):
+ def setUp(self) -> None:
+ TMP_ROOT.mkdir(parents=True, exist_ok=True)
+ self.temp = tempfile.TemporaryDirectory(dir=TMP_ROOT)
+ self.root = Path(self.temp.name)
+ self.intake = self.root / "intake"
+ self.intake.mkdir()
+
+ def tearDown(self) -> None:
+ self.temp.cleanup()
+
+ def write_json(self, path: Path, value: object) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(canonical_json(value))
+
+ def config(self, uid: str, name: str, *, policy: str = "verify_or_append", output: Path | None = None) -> tuple[Path, Path]:
+ output_root = output or (self.root / f"out-{uid}")
+ path = self.root / f"config-{uid}.json"
+ self.write_json(
+ path,
+ {
+ "schema_version": 1,
+ "creator": {"uid": uid, "name": name},
+ "page": {
+ "dynamic_url": f"https://space.bilibili.com/{uid}/dynamic",
+ "profile_url": f"https://space.bilibili.com/{uid}",
+ },
+ "output": {"root": str(output_root), "intake_root": str(self.intake), "manifest_name": "manifest.jsonl"},
+ "selection": {
+ "date_start": "2026-08-01T00:00:00+08:00",
+ "date_end": "2026-08-31T23:59:59+08:00",
+ "window_days": None,
+ "timezone": "Asia/Shanghai",
+ "include_types": ["article", "text", "image"],
+ },
+ "readiness": {"deadline_seconds": 30, "observation_interval_ms": 100, "stable_observations": 3},
+ "rerun": {"policy": policy},
+ "verification": {"summary_path": None},
+ "limits": {"max_items": 20, "max_body_bytes": 1048576, "max_images_per_item": 8, "max_image_bytes": 1048576},
+ },
+ )
+ return path, output_root
+
+ def image(self, name: str, extension: str) -> dict[str, object]:
+ if extension == ".png":
+ payload = b"\x89PNG\r\n\x1a\n" + b"p" * 24
+ elif extension == ".webp":
+ payload = b"RIFF" + b"\x10\x00\x00\x00" + b"WEBP" + b"w" * 20
+ else:
+ payload = b"\xff\xd8\xff" + b"j" * 29
+ path = self.intake / f"{name}{extension}"
+ path.write_bytes(payload)
+ return {"path": path.name, "bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest().upper(), "extension": extension}
+
+ @staticmethod
+ def observations(items: list[dict[str, object]], *states: str, ready_sha: str | None = None) -> list[dict[str, object]]:
+ result = []
+ fingerprint = ready_sha or accepted_snapshot_sha256(items)
+ for index, state in enumerate(states):
+ result.append(
+ {
+ "elapsed_ms": index * 100 + 1,
+ "state": state,
+ "reason": state,
+ "snapshot_sha256": fingerprint if state == "READY" else None,
+ }
+ )
+ return result
+
+ def capture(
+ self,
+ uid: str,
+ name: str,
+ items: list[dict[str, object]],
+ states: tuple[str, ...] = ("METADATA_NOT_READY", "READY", "READY", "READY"),
+ *,
+ ready_sha: str | None = None,
+ ) -> Path:
+ path = self.root / f"capture-{uid}-{len(list(self.root.glob('capture-*.json')))}.json"
+ self.write_json(
+ path,
+ {
+ "schema_version": 1,
+ "creator_uid": uid,
+ "creator_name": name,
+ "dynamic_url": f"https://space.bilibili.com/{uid}/dynamic",
+ "profile_url": f"https://space.bilibili.com/{uid}",
+ "observations": self.observations(items, *states, ready_sha=ready_sha),
+ "items": items,
+ },
+ )
+ return path
+
+ @staticmethod
+ def item(stable_id: str, item_type: str, title: str, body: str, images: list[dict[str, object]] | None = None) -> dict[str, object]:
+ return {
+ "stable_id": stable_id,
+ "item_type": item_type,
+ "title": title,
+ "source_url": f"https://www.bilibili.com/opus/{stable_id}",
+ "published_at": "2026-08-12T10:00:00+08:00",
+ "body_text": body,
+ "body_complete": True,
+ "images": images or [],
+ }
+
+ def test_two_configured_creators_full_body_images_and_safe_rerun(self) -> None:
+ config_a, output_a = self.config("10001", "创作者甲")
+ long_body = "正文段落\n" * 300
+ capture_a = self.capture(
+ "10001",
+ "创作者甲",
+ [
+ self.item("opusA1", "article", "完整文章", long_body, [self.image("a1", ".png")]),
+ self.item("opusA2", "text", "文字动态", "完整文字动态"),
+ self.item("opusA3", "image", "图片动态", "图片说明", [self.image("a3", ".jpg")]),
+ ],
+ )
+ code, result = collector.run(["--config", str(config_a), "collect", "--capture", str(capture_a)])
+ self.assertEqual(0, code, result)
+ self.assertEqual((3, 5), (result["new_items"], result["artifact_count"]))
+ self.assertIn(long_body.encode("utf-8"), next(output_a.glob("*opusA1.txt")).read_bytes())
+ code, verified = collector.run(["--config", str(config_a), "verify"])
+ self.assertEqual(0, code, verified)
+ self.assertEqual((3, 1, 2, 2), (verified["item_count"], verified["article_count"], verified["text_image_dynamic_count"], verified["original_image_count"]))
+ code, rerun = collector.run(["--config", str(config_a), "collect", "--capture", str(capture_a)])
+ self.assertEqual(0, code, rerun)
+ self.assertEqual(("NO_NEW_ITEMS", 0), (rerun["status"], rerun["mutation_count"]))
+
+ config_b, output_b = self.config("20002", "Creator-B")
+ capture_b = self.capture("20002", "Creator-B", [self.item("opusB1", "text", "Second creator", "Independent corpus")])
+ code, result_b = collector.run(["--config", str(config_b), "collect", "--capture", str(capture_b)])
+ self.assertEqual(0, code, result_b)
+ self.assertEqual(1, len(list(output_b.glob("*.txt"))))
+ self.assertNotEqual(output_a, output_b)
+
+ def test_pending_intermission_requires_three_consecutive_ready_samples(self) -> None:
+ config_path, _ = self.config("30003", "稳定性样例")
+ item = self.item("stable1", "text", "稳定", "正文")
+ valid = self.capture("30003", "稳定性样例", [item], ("OWNER_PENDING", "READY", "READY", "DIMENSIONS_PENDING", "READY", "READY", "READY"))
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(valid)])
+ self.assertEqual(0, code, result)
+ self.assertEqual(7, result["readiness_attempts"])
+ invalid = self.capture("30003", "稳定性样例", [item], ("READY", "READY", "METADATA_NOT_READY", "READY", "READY"))
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(invalid)])
+ self.assertEqual((3, "E_READINESS_TIMEOUT", 0), (code, result["error_code"], result["mutation_count"]))
+
+ def test_ready_tail_binds_exact_canonical_items_and_rejects_trailing_evidence(self) -> None:
+ config_path, output_root = self.config("31003", "快照绑定样例")
+ shared_vector = {
+ "stable_id": "snap_1",
+ "item_type": "article",
+ "title": "Shared",
+ "source_url": "https://www.bilibili.com/opus/snap_1",
+ "published_at": datetime.fromisoformat("2026-08-12T10:00:00+08:00"),
+ "body": "正文\nline\n".encode("utf-8"),
+ "images": [{}],
+ }
+ self.assertEqual("4F0616496B08F8537A1C9F17B48D53BC2F93A26EB15AB5977C6E891ABA411454", collector._accepted_snapshot_sha256([shared_vector]))
+ first = self.item("snapshot1", "article", "快照一", "BODY-A")
+ second = self.item("snapshot2", "text", "快照二", "BODY-B")
+ unrelated = hashlib.sha256(b"unrelated").hexdigest().upper()
+ digest_mismatch = self.capture("31003", "快照绑定样例", [first], ready_sha=unrelated)
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(digest_mismatch)])
+ self.assertEqual((3, "E_READINESS_DIGEST", 0), (code, result["error_code"], result["mutation_count"]))
+
+ original_digest = accepted_snapshot_sha256([first])
+ changed_body = dict(first)
+ changed_body["body_text"] = "BODY-B-NOT-OBSERVED"
+ body_drift = self.capture("31003", "快照绑定样例", [changed_body], ready_sha=original_digest)
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(body_drift)])
+ self.assertEqual((3, "E_READINESS_DIGEST", 0), (code, result["error_code"], result["mutation_count"]))
+
+ multi_not_shared = self.capture("31003", "快照绑定样例", [first, second], ready_sha=original_digest)
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(multi_not_shared)])
+ self.assertEqual((3, "E_READINESS_DIGEST", 0), (code, result["error_code"], result["mutation_count"]))
+
+ trailing_access = self.capture("31003", "快照绑定样例", [first], ("READY", "READY", "READY", "ACCESS_BLOCKED"))
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(trailing_access)])
+ self.assertEqual((3, "E_ACCESS_CONTROL", 0), (code, result["error_code"], result["mutation_count"]))
+
+ trailing_pending = self.capture("31003", "快照绑定样例", [first], ("READY", "READY", "READY", "METADATA_NOT_READY"))
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(trailing_pending)])
+ self.assertEqual((3, "E_READINESS_TIMEOUT", 0), (code, result["error_code"], result["mutation_count"]))
+ self.assertFalse(output_root.exists())
+
+ def test_post_validation_image_drift_fails_before_pending_or_formal_mutation(self) -> None:
+ config_path, output_root = self.config("32003", "图片冻结样例")
+ image = self.image("drift", ".png")
+ source_path = self.intake / str(image["path"])
+ capture_path = self.capture("32003", "图片冻结样例", [self.item("drift1", "image", "图片漂移", "完整正文", [image])])
+ original_validate = collector.validate_capture
+
+ def validate_then_drift(config: collector.CollectorConfig, path: Path) -> dict[str, object]:
+ value = original_validate(config, path)
+ source_path.write_bytes(b"\x89PNG\r\n\x1a\n" + b"q" * 24)
+ return value
+
+ with mock.patch.object(collector, "validate_capture", side_effect=validate_then_drift):
+ code, result = collector.run(["--config", str(config_path), "collect", "--capture", str(capture_path)])
+ self.assertEqual((3, "E_ARTIFACT_DRIFT", 0), (code, result["error_code"], result["mutation_count"]))
+ self.assertFalse(output_root.exists())
+ self.assertFalse((output_root / "manifest.jsonl").exists())
+ self.assertEqual([], list(output_root.glob(".bili-article-image.pending.*.json")))
+ self.assertEqual([], list(output_root.glob("*.txt")))
+ self.assertEqual([], list(output_root.glob("*.png")))
+
+ config_frozen, output_frozen = self.config("32004", "冻结载荷样例")
+ frozen_image = self.image("frozen", ".png")
+ frozen_source = self.intake / str(frozen_image["path"])
+ expected_payload = frozen_source.read_bytes()
+ frozen_capture = self.capture("32004", "冻结载荷样例", [self.item("frozen1", "image", "冻结载荷", "完整正文", [frozen_image])])
+ original_create_new = collector._create_new
+ mutated_after_freeze = False
+
+ def create_then_mutate_intake(path: Path, payload: bytes) -> None:
+ nonlocal mutated_after_freeze
+ original_create_new(path, payload)
+ if collector.OWNED_PENDING.fullmatch(path.name) and not mutated_after_freeze:
+ frozen_source.write_bytes(b"\x89PNG\r\n\x1a\n" + b"r" * 24)
+ mutated_after_freeze = True
+
+ with mock.patch.object(collector, "_create_new", side_effect=create_then_mutate_intake):
+ code, result = collector.run(["--config", str(config_frozen), "collect", "--capture", str(frozen_capture)])
+ self.assertEqual((0, "CONTENT_SAVED", True), (code, result["status"], mutated_after_freeze))
+ published = next(output_frozen.glob("*.png"))
+ self.assertEqual(expected_payload, published.read_bytes())
+ manifest_row = json.loads((output_frozen / "manifest.jsonl").read_text(encoding="utf-8"))
+ self.assertEqual(
+ (len(expected_payload), hashlib.sha256(expected_payload).hexdigest().upper()),
+ (manifest_row["images"][0]["bytes"], manifest_row["images"][0]["sha256"]),
+ )
+
+ def test_identity_access_secret_partial_and_path_escape_fail_closed(self) -> None:
+ config_path, output_root = self.config("40004", "安全样例")
+ wrong = self.capture("40005", "安全样例", [self.item("bad1", "text", "bad", "body")])
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(wrong)])
+ self.assertEqual((3, "E_CREATOR_IDENTITY"), (code, result["error_code"]))
+ blocked = self.capture("40004", "安全样例", [self.item("bad2", "text", "bad", "body")], ("ACCESS_BLOCKED",))
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(blocked)])
+ self.assertEqual((3, "E_ACCESS_CONTROL"), (code, result["error_code"]))
+ partial = self.capture("40004", "安全样例", [self.item("bad3", "text", "bad", " ")])
+ code, result = collector.run(["--config", str(config_path), "validate-capture", "--capture", str(partial)])
+ self.assertEqual((3, "E_CONTENT_INCOMPLETE"), (code, result["error_code"]))
+ config_value = json.loads(config_path.read_text(encoding="utf-8"))
+ config_value["cookie"] = "synthetic-marker"
+ self.write_json(config_path, config_value)
+ code, result = collector.run(["--config", str(config_path), "verify"])
+ self.assertEqual((3, "E_SECRET_FIELD"), (code, result["error_code"]))
+ self.assertFalse(output_root.exists())
+ self.config("40004", "安全样例", output=PROJECT_ROOT.parent / "escape")
+ code, result = collector.run(["--config", str(self.root / "config-40004.json"), "verify"])
+ self.assertEqual((3, "E_PATH_ESCAPE"), (code, result["error_code"]))
+
+ config_path, _ = self.config("40004", "安全样例")
+ outside_terminal = PROJECT_ROOT.parent / f"bili-article-image-outside-terminal-{os.getpid()}.json"
+ self.assertFalse(outside_terminal.exists())
+ code, result = collector.run(["--config", str(config_path), "verify", "--terminal", str(outside_terminal)])
+ self.assertEqual((3, "E_PATH_ESCAPE", 0), (code, result["error_code"], result["mutation_count"]))
+ self.assertFalse(outside_terminal.exists())
+
+ def test_collision_precommit_recovery_and_reparse_are_mutation_zero(self) -> None:
+ config_path, output_root = self.config("50005", "恢复样例")
+ capture_path = self.capture("50005", "恢复样例", [self.item("recover1", "article", "恢复", "完整正文")])
+ config = collector.load_config(config_path)
+ empty = hashlib.sha256(b"").hexdigest().upper()
+ with mock.patch.object(collector, "_manifest_snapshot", side_effect=[(0, empty), (1, "A" * 64), (0, empty)]):
+ with self.assertRaises(collector.CollectorError) as raised:
+ collector.collect(config, capture_path, None)
+ self.assertEqual("E_PRECOMMIT_DRIFT", raised.exception.code)
+ self.assertEqual([], list(output_root.glob("*.txt")))
+ self.assertFalse((output_root / "manifest.jsonl").exists())
+ self.assertEqual([], list(output_root.glob(".bili-article-image.pending.*.json")))
+
+ capture = collector.validate_capture(config, capture_path)
+ item = capture["items"][0]
+ local = item["published_at"].astimezone(config.tz)
+ stem = f"{local:%Y%m%d-%H%M%S}_{item['item_type']}_{collector._safe_component(item['title'], max_length=48)}_{item['stable_id']}"
+ output_root.mkdir(parents=True, exist_ok=True)
+ (output_root / f"{stem}.txt").write_bytes(b"collision")
+ code, result = collector.run(["--config", str(config_path), "collect", "--capture", str(capture_path)])
+ self.assertEqual((3, "E_TARGET_EXISTS", 0), (code, result["error_code"], result["mutation_count"]))
+
+ probe = output_root / "reparse-probe"
+ probe.mkdir()
+ with mock.patch.object(collector, "_is_reparse", side_effect=lambda path: path == probe):
+ with self.assertRaises(collector.CollectorError) as reparse:
+ collector._safe_existing_chain(probe / "child")
+ self.assertEqual("E_PATH_REPARSE", reparse.exception.code)
+
+ pending = output_root / f".bili-article-image.pending.{'a' * 32}.json"
+ pending.write_bytes(canonical_json({"schema_version": 1, "status": "PUBLISH_PENDING"}))
+ before = pending.read_bytes()
+ code, result = collector.run(["--config", str(config_path), "verify"])
+ self.assertEqual((3, "E_RECOVERY_REQUIRED", 0), (code, result["error_code"], result["mutation_count"]))
+ self.assertEqual(before, pending.read_bytes())
+
+ def test_native_host_strict_boundary_and_real_corpus_readback(self) -> None:
+ real_config = PROJECT_ROOT / "dev" / "tmp" / "bili-article-image-generic-real-validation-config-20260825.json"
+ code, result = native_host.run_request(
+ {"schema_version": 1, "action": "verify", "config_path": str(real_config), "capture_path": None, "terminal_path": None}
+ )
+ self.assertEqual(0, code, result)
+ self.assertEqual((85, 22, 63, 16), (result["item_count"], result["article_count"], result["text_image_dynamic_count"], result["original_image_count"]))
+ self.assertEqual("4CF0BB9936431C24278A1209C0E4CB1BD3645EF8E14BBC4B32F9B22629211EE3", result["manifest_sha256"])
+ code, blocked = native_host.run_request(
+ {"schema_version": 1, "action": "verify", "config_path": str(real_config), "capture_path": None, "terminal_path": None, "session_token": "synthetic"}
+ )
+ self.assertEqual((3, "E_HOST_SCHEMA", 0), (code, blocked["error_code"], blocked["mutation_count"]))
+
+ def test_runtime_sources_have_no_current_creator_constant_or_secret_api(self) -> None:
+ runtime_paths = [
+ MODULE_PATH,
+ PROJECT_DEV / "bili_article_image_capture.js",
+ HOST_PATH,
+ PROJECT_DEV / "bili_dynamic_collector.py",
+ PROJECT_DEV / "bili_dynamic_refresh_extension" / "service_worker.js",
+ PROJECT_DEV / "bili_dynamic_refresh_extension" / "page_extract.js",
+ PROJECT_DEV / "bili_dynamic_refresh_native_host" / "constants.py",
+ PROJECT_DEV / "bili_dynamic_refresh_native_host" / "protocol.py",
+ PROJECT_DEV / "bili_article_image_collector.example.json",
+ ]
+ text = "\n".join(path.read_text(encoding="utf-8") for path in runtime_paths)
+ for forbidden in ["1420210197", "青枫浦上Q", "document.cookie", "localStorage", "Profile", "--cookies", "Cookie:"]:
+ self.assertNotIn(forbidden, text)
+
+ def test_browser_capture_two_creator_and_readiness_contract(self) -> None:
+ completed = subprocess.run(
+ ["node", str(Path(__file__).with_name("test_bili_article_image_capture.mjs")), str(PROJECT_DEV / "bili_article_image_capture.js")],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=20,
+ )
+ result = json.loads(completed.stdout)
+ self.assertEqual({"status": "PASS", "creators": 2, "readiness_observations": 7, "network_requests": 0}, result)
+
+ def test_exact_source_manifest(self) -> None:
+ result = source_validator.validate()
+ self.assertEqual(("SOURCE_VALID", 28, 0), (result["status"], result["file_count"], result["mutation_count"]))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/dev/project-dev/test/test_bili_dynamic_refresh.py b/dev/project-dev/test/test_bili_dynamic_refresh.py
index 9d05344..669e44a 100644
--- a/dev/project-dev/test/test_bili_dynamic_refresh.py
+++ b/dev/project-dev/test/test_bili_dynamic_refresh.py
@@ -1,11 +1,14 @@
from __future__ import annotations
import hashlib
+import hmac
import importlib.util
+import io
import json
import gc
import os
import shutil
+import secrets
import subprocess
import sys
import tempfile
@@ -42,6 +45,7 @@
for path in (self.archive, self.intake, self.downloads, self.videos):
path.mkdir()
self.config_path = self.root / "config.json"
+ self.controller_keys: dict[str, bytes] = {}
self.write_json(
self.config_path,
{
@@ -93,20 +97,7 @@
def write_json(self, path: Path, value: object) -> None:
if isinstance(value, dict) and value.get("schema_version") == 3 and isinstance(value.get("controller_attestation"), dict):
- config = collector.load_config(self.config_path)
- pending = refresh._load_pending(config)
- if pending is not None:
- attestation = value["controller_attestation"]
- refresh._attest_controller_evidence(
- pending, value,
- action_dispatched=bool(attestation["action_dispatched"]),
- monotonic_run_started_ms=int(attestation["monotonic_run_started_ms"]),
- monotonic_action_started_ms=attestation["monotonic_action_started_ms"],
- monotonic_action_finished_ms=attestation["monotonic_action_finished_ms"],
- monotonic_observation_started_ms=attestation["monotonic_observation_started_ms"],
- monotonic_observation_finished_ms=attestation["monotonic_observation_finished_ms"],
- monotonic_evidence_write_started_ms=int(attestation["monotonic_evidence_write_started_ms"]),
- )
+ self.sign_evidence(value)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
@@ -116,12 +107,30 @@
if path.exists():
shutil.rmtree(path)
path.mkdir()
+ self.controller_keys.clear()
def begin(self) -> dict[str, object]:
- code, result = collector.run(["--config", str(self.config_path), "refresh-begin", "--now", NOW])
- self.assertEqual(0, code)
+ key = secrets.token_bytes(32)
+ config = collector.load_config(self.config_path)
+ result = refresh.refresh_begin(
+ config,
+ self.config_path,
+ collector.parse_datetime(NOW, "test now"),
+ _controller_key_commitment=hashlib.sha256(key).hexdigest(),
+ )
self.assertEqual("BROWSER_REFRESH_REQUIRED", result["status"])
+ self.controller_keys[str(result["run_id"])] = key
return result
+
+ def sign_evidence(self, evidence: dict[str, object]) -> None:
+ key = self.controller_keys.get(str(evidence.get("run_id")))
+ attestation = evidence.get("controller_attestation")
+ if key is None or not isinstance(attestation, dict):
+ return
+ attestation["binding_sha256"] = None
+ attestation["binding_sha256"] = hmac.new(
+ key, refresh._controller_attestation_payload(evidence), hashlib.sha256
+ ).hexdigest()
def formal_pending(self, config: collector.CollectorConfig) -> dict[str, object]:
return {
@@ -244,9 +253,6 @@
"observation_count": observation_count,
},
}
- config = collector.load_config(self.config_path)
- pending = refresh._load_pending(config)
- assert pending is not None
if action_outcome == "PRE_DISPATCH_ERROR":
action_started = action_finished = observation_started = observation_finished = None
else:
@@ -262,17 +268,22 @@
action_finished or 0,
observation_finished or 0,
)
- refresh._attest_controller_evidence(
- pending,
- evidence,
- action_dispatched=action_outcome != "PRE_DISPATCH_ERROR",
- monotonic_run_started_ms=0,
- monotonic_action_started_ms=action_started,
- monotonic_action_finished_ms=action_finished,
- monotonic_observation_started_ms=observation_started,
- monotonic_observation_finished_ms=observation_finished,
- monotonic_evidence_write_started_ms=write_started,
- )
+ runtime = collector.load_config(self.config_path).refresh
+ assert runtime is not None
+ evidence["controller_attestation"] = {
+ "controller_id": "bili-supported-chrome-controller-v1",
+ "controller_sha256": hashlib.sha256(refresh.CONTROLLER_SOURCE.read_bytes()).hexdigest(),
+ "binding_algorithm": "hmac-sha256-controller-envelope-v1",
+ "action_dispatched": action_outcome != "PRE_DISPATCH_ERROR",
+ "monotonic_run_started_ms": 0,
+ "monotonic_action_started_ms": action_started,
+ "monotonic_action_finished_ms": action_finished,
+ "monotonic_observation_started_ms": observation_started,
+ "monotonic_observation_finished_ms": observation_finished,
+ "monotonic_evidence_write_started_ms": write_started,
+ "binding_sha256": None,
+ }
+ self.sign_evidence(evidence)
return evidence
def set_runtime_diagnostic(self, evidence: dict[str, object], code: str) -> None:
@@ -297,33 +308,36 @@
}
def commit(self, begin: dict[str, object], evidence: dict[str, object]) -> tuple[int, dict[str, object]]:
- if "controller_attestation" in evidence:
- config = collector.load_config(self.config_path)
- pending = refresh._load_pending(config)
- assert pending is not None
- attestation = evidence["controller_attestation"]
- assert isinstance(attestation, dict)
- refresh._attest_controller_evidence(
- pending,
- evidence,
- action_dispatched=bool(attestation["action_dispatched"]),
- monotonic_run_started_ms=int(attestation["monotonic_run_started_ms"]),
- monotonic_action_started_ms=attestation["monotonic_action_started_ms"],
- monotonic_action_finished_ms=attestation["monotonic_action_finished_ms"],
- monotonic_observation_started_ms=attestation["monotonic_observation_started_ms"],
- monotonic_observation_finished_ms=attestation["monotonic_observation_finished_ms"],
- monotonic_evidence_write_started_ms=int(attestation["monotonic_evidence_write_started_ms"]),
- )
+ path = self.bind_for_test(begin, evidence)
+ return collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
+
+ def bind_for_test(self, begin: dict[str, object], evidence: dict[str, object]) -> Path:
+ self.sign_evidence(evidence)
path = Path(str(begin["evidence_path"]))
self.write_json(path, evidence)
config = collector.load_config(self.config_path)
pending = refresh._load_pending(config)
assert pending is not None
- refresh.bind_controller_evidence(
- config, pending, path,
- transitioned_at=collector.parse_datetime("2026-08-13T10:00:24+08:00", "test transitioned_at"),
+ key = self.controller_keys[str(begin["run_id"])]
+ value, payload, _, _ = refresh._validate_evidence(config, pending, path, _controller_key=key)
+ pending["evidence_identity"] = {
+ "bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest()
+ }
+ pending["controller_binding_sha256"] = value["controller_attestation"]["binding_sha256"]
+ pending["phase"] = "EVIDENCE_BOUND"
+ pending["last_transition_at"] = "2026-08-13T02:00:24Z"
+ refresh._write_pending(config, pending)
+ return path
+
+ def validate_for_test(self, begin: dict[str, object], evidence: dict[str, object]) -> None:
+ path = Path(str(begin["evidence_path"]))
+ self.write_json(path, evidence)
+ config = collector.load_config(self.config_path)
+ pending = refresh._load_pending(config)
+ assert pending is not None
+ refresh._validate_evidence(
+ config, pending, path, _controller_key=self.controller_keys[str(begin["run_id"])]
)
- return collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
@staticmethod
def bound_card_item(identifier: str = "456") -> tuple[dict[str, object], dict[str, object]]:
@@ -447,10 +461,10 @@
}
node = {"position": 0, "node_fingerprint_sha256": "b" * 64, "reason_code": "PARSER_REJECTED"}
observation = self.observation([card], [node])
- path = Path(str(begin["evidence_path"]))
- self.write_json(path, self.evidence(begin, outcome="READABLE", observations=[observation], marker=None))
with self.assertRaisesRegex(collector.CollectorError, "positions overlap"):
- collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
+ self.validate_for_test(
+ begin, self.evidence(begin, outcome="READABLE", observations=[observation], marker=None)
+ )
self.assertFalse((self.archive / "manifest.jsonl").exists())
def test_complete_observed_card_with_missing_item_cannot_claim_no_new(self) -> None:
@@ -460,10 +474,8 @@
begin, outcome="READABLE", observations=[self.observation([card])],
marker=self.end_marker(), items=[],
)
- path = Path(str(begin["evidence_path"]))
- self.write_json(path, evidence)
with self.assertRaises(collector.CollectorError) as failure:
- collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
+ self.validate_for_test(begin, evidence)
self.assertEqual("E_EVIDENCE_ITEM_BINDING", failure.exception.code)
self.assertFalse((self.archive / "manifest.jsonl").exists())
@@ -474,10 +486,8 @@
begin, outcome="READABLE", observations=[self.observation([])],
marker=self.end_marker(), items=[item],
)
- path = Path(str(begin["evidence_path"]))
- self.write_json(path, evidence)
with self.assertRaises(collector.CollectorError) as failure:
- collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
+ self.validate_for_test(begin, evidence)
self.assertEqual("E_EVIDENCE_ITEM_BINDING", failure.exception.code)
self.assertFalse((self.archive / "manifest.jsonl").exists())
@@ -489,10 +499,8 @@
begin, outcome="READABLE", observations=[self.observation([card])],
marker=self.end_marker(), items=[item],
)
- path = Path(str(begin["evidence_path"]))
- self.write_json(path, evidence)
with self.assertRaises(collector.CollectorError) as failure:
- collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
+ self.validate_for_test(begin, evidence)
self.assertEqual("E_EVIDENCE_ITEM_BINDING", failure.exception.code)
def test_observation_overlapping_component_identity_fails_closed(self) -> None:
@@ -510,10 +518,8 @@
begin, outcome="READABLE", observations=[self.observation([card, other_card])],
marker=self.end_marker(), items=[item, other_item],
)
- path = Path(str(begin["evidence_path"]))
- self.write_json(path, evidence)
with self.assertRaises(collector.CollectorError) as failure:
- collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
+ self.validate_for_test(begin, evidence)
self.assertEqual("E_EVIDENCE_ITEM_BINDING", failure.exception.code)
def test_new_text_commits_both_manifests_and_artifact(self) -> None:
@@ -565,24 +571,23 @@
self.assertEqual(4, code)
self.assertEqual("E_EVIDENCE_MISSING_AFTER_DEADLINE", terminal["error_code"])
with self.assertRaises(collector.CollectorError) as occupied:
- collector.run(["--config", str(self.config_path), "refresh-begin", "--now", "2026-08-13T10:02:10+08:00"])
+ refresh.refresh_begin(
+ collector.load_config(self.config_path), self.config_path,
+ collector.parse_now("2026-08-13T10:02:10+08:00"),
+ _controller_key_commitment=hashlib.sha256(b"next-controller").hexdigest(),
+ )
self.assertEqual("E_RUN_HOUR_OCCUPIED", occupied.exception.code)
def test_latest_commit_failure_retains_pending_and_reopen_rebuilds_index(self) -> None:
begin = self.begin()
+ config = collector.load_config(self.config_path)
observation = self.observation([])
evidence = self.evidence(
begin, outcome="READABLE", observations=[observation], marker=self.end_marker(), items=[]
)
path = Path(str(begin["evidence_path"]))
self.write_json(path, evidence)
- config = collector.load_config(self.config_path)
- pending = refresh._load_pending(config)
- assert pending is not None
- refresh.bind_controller_evidence(
- config, pending, path,
- transitioned_at=collector.parse_datetime("2026-08-13T10:00:24+08:00", "test transitioned_at"),
- )
+ self.bind_for_test(begin, evidence)
original = refresh._write_readback
injected = {"done": False}
@@ -628,7 +633,7 @@
self.assertTrue(config.lock_path.is_file())
self.assertFalse((config.state_dir / ".collector.lock.owner.json").exists())
- def test_frozen_formal_catalog_shape_is_48_22_16_5_1_read_only(self) -> None:
+ def test_current_formal_catalog_shape_is_140_111_103_8_0_read_only(self) -> None:
formal = Path(__file__).resolve().parents[3] / "ana-data" / "news-青枫浦上Q" / "manifest.jsonl"
if not formal.exists():
self.skipTest("formal read-only acceptance snapshot is not present")
@@ -648,7 +653,7 @@
),
)
_, _, counts = refresh.load_formal_catalog(rebound)
- self.assertEqual({"events": 48, "components": 22, "saved": 16, "video": 5, "retryable": 1}, counts)
+ self.assertEqual({"events": 140, "components": 111, "saved": 103, "video": 8, "retryable": 0}, counts)
self.assertEqual(before, (formal.stat().st_size, hashlib.sha256(formal.read_bytes()).hexdigest()))
def test_mixed_history_union_saved_precedence_and_schema2_identity(self) -> None:
@@ -723,7 +728,10 @@
refresh._ensure_started_slot = lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("after pending"))
try:
with self.assertRaisesRegex(OSError, "after pending"):
- refresh.refresh_begin(config, self.config_path, collector.parse_now(NOW))
+ refresh.refresh_begin(
+ config, self.config_path, collector.parse_now(NOW),
+ _controller_key_commitment=hashlib.sha256(b"test-controller").hexdigest(),
+ )
finally:
refresh._ensure_started_slot = original
pending = refresh._load_pending(config)
@@ -1174,10 +1182,8 @@
self.set_runtime_diagnostic(evidence, "ACTION_PRE_DISPATCH")
evidence["runtime_observation"]["observation_outcome"] = observation_outcome
evidence["runtime_observation"]["observation_count"] = 1
- path = Path(str(begin["evidence_path"]))
- self.write_json(path, evidence)
with self.assertRaises(collector.CollectorError) as invalid:
- collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
+ self.validate_for_test(begin, evidence)
self.assertEqual("E_EVIDENCE_SCHEMA", invalid.exception.code)
refresh._pending_path(collector.load_config(self.config_path)).unlink()
Path(str(begin["run_evidence_path"])).unlink()
@@ -1261,127 +1267,109 @@
self.assertFalse(result["coverage_complete"])
def test_reviewer_trusted_controller_calls_once_and_caller_forgery_fails(self) -> None:
- begin = self.begin()
- calls = {"tabs": 0, "reload": 0, "goto": 0, "evaluate": 0}
-
- class Fake:
- def open_tabs(inner) -> list[dict[str, object]]:
- calls["tabs"] += 1
- return [{"url": "https://space.bilibili.com/1420210197/dynamic"}]
-
- def reload(inner, tab: dict[str, object], timeout_seconds: int) -> None:
- calls["reload"] += 1
-
- def goto(inner, url: str, timeout_seconds: int) -> dict[str, object]:
- calls["goto"] += 1
- return {"url": url}
-
- def evaluate(inner, tab: dict[str, object], source: str, timeout_seconds: int) -> dict[str, object]:
- calls["evaluate"] += 1
- return {
- "schema_version": 1,
- "ready_state": "complete", "visibility_state": "visible",
- "final_url": "https://space.bilibili.com/1420210197/dynamic",
- "page_title": "青枫浦上Q个人动态-青枫浦上Q动态记录-哔哩哔哩视频",
- "creator": {"uid": "1420210197", "name": "青枫浦上Q", "profile_url": "https://space.bilibili.com/1420210197"},
- "cards": [], "unparsed_nodes": [], "terminal_marker_text": "已经到底了", "limit_hit": "NONE",
- }
-
- ticks = iter([0.0, 0.001, 0.011, 0.012, 0.022, 0.023, 0.024])
- path = controller.run_once(
- collector.load_config(self.config_path), begin, Fake(), monotonic=lambda: next(ticks),
- wall_now=lambda: collector.parse_datetime("2026-08-13T10:00:24+08:00", "wall"),
- )
- self.assertEqual({"tabs": 1, "reload": 1, "goto": 0, "evaluate": 1}, calls)
- code, result = collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
- self.assertEqual((0, "REFRESH_CONFIRMED_NO_NEW"), (code, result["status"]))
-
- Path(str(result["run_evidence_path"])).unlink()
- (self.state / "refresh" / "runs" / "latest.json").unlink()
- begin = self.begin()
- forged = self.evidence(begin, outcome="READABLE", observations=[self.observation([])], marker=self.end_marker())
- forged["controller_attestation"]["binding_sha256"] = "0" * 64
- path = Path(str(begin["evidence_path"]))
- path.parent.mkdir(parents=True, exist_ok=True)
- path.write_text(json.dumps(forged), encoding="utf-8")
+ input_stream = io.StringIO('{"page_authoritative":true,"no_new":true}\n')
+ output_stream = io.StringIO()
with self.assertRaises(collector.CollectorError) as failure:
- collector.run(["--config", str(self.config_path), "refresh-commit", "--input", str(path), "--now", "2026-08-13T10:00:25+08:00"])
- self.assertIn(failure.exception.code, {"E_CONTROLLER_ATTESTATION", "E_CONTROLLER_REQUIRED"})
+ controller.run_product(
+ collector.load_config(self.config_path), self.config_path,
+ collector.parse_datetime(NOW, "now"), input_stream=input_stream,
+ output_stream=output_stream,
+ )
+ self.assertEqual("E_TRUSTED_ADAPTER_REQUIRED", failure.exception.code)
+ self.assertEqual({"authoritative": False, "saved": False, "no_new": False}, failure.exception.details)
+ self.assertEqual(0, input_stream.tell())
+ self.assertEqual("", output_stream.getvalue())
+ self.assertFalse((self.state / "refresh" / "pending.json").exists())
def test_reviewer_real_js_fixture_adapts_to_accepted_schema(self) -> None:
runner = PROJECT_DEV / "test" / "fixtures" / "bili_dynamic_collector" / "page_extract_fixture_runner.js"
- expected = {
- "new": (0, "NEW_ITEMS_SAVED", None),
- "empty": (0, "REFRESH_CONFIRMED_NO_NEW", None),
- "unparsed": (4, "PARTIAL_DISCOVERY_UNCONFIRMED", "E_COVERAGE_INCOMPLETE"),
- "identity": (3, "REFRESH_BLOCKED_AUTH_OR_ACCESS", "E_CREATOR_MISMATCH"),
- }
- for fixture_case, wanted in expected.items():
+ for fixture_case in ("new", "empty", "unparsed", "identity", "access"):
with self.subTest(fixture_case=fixture_case):
self.reset_runtime_fixture()
+ raw = None
+ if fixture_case != "access":
+ completed = subprocess.run(
+ ["node", str(runner), str(refresh.EXTRACTOR_SOURCE), fixture_case],
+ capture_output=True, check=True, timeout=10,
+ )
+ raw = json.loads(completed.stdout.decode("utf-8"))
+
+ def response(request_id: int, result: object, *, error: str | None = None) -> str:
+ return json.dumps({
+ "schema_version": 1,
+ "type": "supported_chrome_response",
+ "request_id": request_id,
+ "ok": error is None,
+ "result": result,
+ "error_code": error,
+ }, ensure_ascii=False)
+
+ protocol_input = "\n".join([
+ response(1, [{"tab_id": "fixture-tab", "url": "https://space.bilibili.com/1420210197/dynamic"}]),
+ response(2, None),
+ response(3, raw, error="ACCESS_BLOCKED" if fixture_case == "access" else None),
+ ]) + "\n"
completed = subprocess.run(
- ["node", str(runner), str(refresh.EXTRACTOR_SOURCE), fixture_case],
- capture_output=True, check=True, timeout=10,
+ [
+ sys.executable, "-B", str(COLLECTOR_PATH), "--config", str(self.config_path),
+ "refresh-run", "--now", NOW,
+ ],
+ input=protocol_input, text=True, capture_output=True, timeout=20,
)
- raw = json.loads(completed.stdout.decode("utf-8"))
- begin = self.begin()
- calls = {"reload": 0, "evaluate": 0}
+ lines = [json.loads(line) for line in completed.stdout.splitlines()]
+ requests = [line for line in lines if line.get("type") == "supported_chrome_request"]
+ terminal = lines[-1]
+ self.assertEqual([], requests)
+ self.assertEqual((3, "SAFETY_STOP", "E_TRUSTED_ADAPTER_REQUIRED"), (completed.returncode, terminal["status"], terminal["error_code"]))
+ self.assertEqual({"authoritative": False, "saved": False, "no_new": False}, terminal["details"])
+ self.assertFalse((self.state / "refresh" / "pending.json").exists())
- class Fake:
- def open_tabs(inner) -> list[dict[str, object]]:
- return [{"url": "https://space.bilibili.com/1420210197/dynamic"}]
+ def test_reviewer_public_cli_cannot_create_or_bind_caller_schema3(self) -> None:
+ self.assertFalse(hasattr(refresh, "_attest_controller_evidence"))
+ self.assertFalse(hasattr(refresh, "bind_controller_evidence"))
+ self.assertFalse(hasattr(controller, "_attest_controller_evidence"))
+ self.assertFalse(hasattr(controller, "bind_controller_evidence"))
+ with self.assertRaises(collector.CollectorError) as blocked:
+ collector.run(["--config", str(self.config_path), "refresh-begin", "--now", NOW])
+ self.assertEqual("E_CONTROLLER_ENTRY_REQUIRED", blocked.exception.code)
+ config = collector.load_config(self.config_path)
+ self.assertFalse(refresh._pending_path(config).exists())
- def reload(inner, tab: dict[str, object], timeout_seconds: int) -> None:
- calls["reload"] += 1
-
- def goto(inner, url: str, timeout_seconds: int) -> dict[str, object]:
- raise AssertionError("exact fixture tab must use reload")
-
- def evaluate(inner, tab: dict[str, object], source: str, timeout_seconds: int) -> dict[str, object]:
- calls["evaluate"] += 1
- self.assertIn("projectInfoCollectVisibleDynamicNodes({page_internal_settle_timeout_ms:15000})", source)
- return raw
-
- ticks = iter([0.0, 0.001, 0.011, 0.012, 0.022, 0.023, 0.024])
- path = controller.run_once(
- collector.load_config(self.config_path), begin, Fake(),
- monotonic=lambda: next(ticks),
- wall_now=lambda: collector.parse_datetime("2026-08-13T10:00:24+08:00", "wall"),
- )
- code, result = collector.run([
- "--config", str(self.config_path), "refresh-commit", "--input", str(path),
- "--now", "2026-08-13T10:00:25+08:00",
- ])
- self.assertEqual(wanted, (code, result["status"], result["error_code"]))
- self.assertEqual({"reload": 1, "evaluate": 1}, calls)
-
- self.reset_runtime_fixture()
begin = self.begin()
+ pending = refresh._load_pending(config)
+ self.assertNotIn("controller_capability", pending)
+ self.assertRegex(str(pending["controller_key_commitment"]), r"^[0-9a-f]{64}$")
- class AccessBlocked:
- def open_tabs(inner) -> list[dict[str, object]]:
- return [{"url": "https://space.bilibili.com/1420210197/dynamic"}]
-
- def reload(inner, tab: dict[str, object], timeout_seconds: int) -> None:
- return None
-
- def goto(inner, url: str, timeout_seconds: int) -> dict[str, object]:
- raise AssertionError("exact fixture tab must use reload")
-
- def evaluate(inner, tab: dict[str, object], source: str, timeout_seconds: int) -> dict[str, object]:
- raise PermissionError("synthetic access interstitial")
-
- ticks = iter([0.0, 0.001, 0.011, 0.012, 0.022, 0.023, 0.024])
- path = controller.run_once(
- collector.load_config(self.config_path), begin, AccessBlocked(),
- monotonic=lambda: next(ticks),
- wall_now=lambda: collector.parse_datetime("2026-08-13T10:00:24+08:00", "wall"),
+ forged = self.evidence(
+ begin, outcome="READABLE", observations=[self.observation([])], marker=self.end_marker()
)
- code, result = collector.run([
- "--config", str(self.config_path), "refresh-commit", "--input", str(path),
- "--now", "2026-08-13T10:00:25+08:00",
- ])
- self.assertEqual((3, "REFRESH_BLOCKED_AUTH_OR_ACCESS", "E_ACCESS_BLOCKED"), (code, result["status"], result["error_code"]))
+ path = Path(str(begin["evidence_path"]))
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(forged, ensure_ascii=False), encoding="utf-8")
+ with self.assertRaises(collector.CollectorError) as rejected:
+ collector.run([
+ "--config", str(self.config_path), "refresh-commit", "--input", str(path),
+ "--now", "2026-08-13T10:00:25+08:00",
+ ])
+ self.assertEqual("E_CONTROLLER_REQUIRED", rejected.exception.code)
+ self.assertFalse((self.archive / "manifest.jsonl").exists())
+
+ def test_reviewer_expired_pending_is_zero_adapter_calls_and_zero_evidence(self) -> None:
+ begin = self.begin()
+ protocol_input = io.StringIO('{"page_authoritative":true}\n')
+ protocol_output = io.StringIO()
+ with self.assertRaises(collector.CollectorError) as expired:
+ controller.run_product(
+ collector.load_config(self.config_path), self.config_path,
+ collector.parse_datetime("2026-08-13T10:02:00.001+08:00", "wall"),
+ input_stream=protocol_input, output_stream=protocol_output,
+ )
+ self.assertEqual("E_TRUSTED_ADAPTER_REQUIRED", expired.exception.code)
+ self.assertEqual(0, protocol_input.tell())
+ self.assertEqual("", protocol_output.getvalue())
+ self.assertFalse(Path(str(begin["evidence_path"])).exists())
+ pending = refresh._load_pending(collector.load_config(self.config_path))
+ self.assertEqual("AWAITING_EVIDENCE", pending["phase"])
def test_reviewer_deadline_plus_epsilon_never_binds_or_saves(self) -> None:
begin = self.begin()
@@ -1395,34 +1383,12 @@
def test_reviewer_controller_commit_crossing_deadline_removes_unbound_evidence(self) -> None:
begin = self.begin()
-
- class Fake:
- def open_tabs(inner) -> list[dict[str, object]]:
- return [{"url": "https://space.bilibili.com/1420210197/dynamic"}]
-
- def reload(inner, tab: dict[str, object], timeout_seconds: int) -> None:
- return None
-
- def goto(inner, url: str, timeout_seconds: int) -> dict[str, object]:
- raise AssertionError("exact fixture tab must use reload")
-
- def evaluate(inner, tab: dict[str, object], source: str, timeout_seconds: int) -> dict[str, object]:
- return {
- "schema_version": 1, "ready_state": "complete", "visibility_state": "visible",
- "final_url": "https://space.bilibili.com/1420210197/dynamic",
- "page_title": "青枫浦上Q个人动态-青枫浦上Q动态记录-哔哩哔哩视频",
- "creator": {"uid": "1420210197", "name": "青枫浦上Q", "profile_url": "https://space.bilibili.com/1420210197"},
- "cards": [], "unparsed_nodes": [], "terminal_marker_text": "已经到底了", "limit_hit": "NONE",
- }
-
- ticks = iter([0.0, 0.001, 0.011, 0.012, 0.022, 119.999, 120.001])
with self.assertRaises(collector.CollectorError) as failure:
- controller.run_once(
- collector.load_config(self.config_path), begin, Fake(),
- monotonic=lambda: next(ticks),
- wall_now=lambda: collector.parse_datetime("2026-08-13T10:00:24+08:00", "wall"),
+ controller.run_product(
+ collector.load_config(self.config_path), self.config_path,
+ collector.parse_datetime("2026-08-13T10:00:24+08:00", "wall"),
)
- self.assertEqual("E_OVERALL_DEADLINE", failure.exception.code)
+ self.assertEqual("E_TRUSTED_ADAPTER_REQUIRED", failure.exception.code)
self.assertFalse(Path(str(begin["evidence_path"])).exists())
pending = refresh._load_pending(collector.load_config(self.config_path))
self.assertIsNotNone(pending)
@@ -1441,19 +1407,16 @@
assert pending is not None
pending["schema_version"] = refresh.LEGACY_PENDING_SCHEMA
pending.pop("runtime_contract")
- pending.pop("controller_capability")
+ pending.pop("controller_key_commitment")
+ pending.pop("controller_binding_sha256")
refresh._write_pending(config, pending)
if not slot_present:
Path(str(begin["run_evidence_path"])).unlink()
now_text = "2026-08-13T10:02:01+08:00" if after_deadline else "2026-08-13T10:00:30+08:00"
if entrypoint == "controller":
- class NeverCalled:
- def open_tabs(inner) -> list[dict[str, object]]:
- raise AssertionError("legacy pending must not touch the browser adapter")
-
with self.assertRaises(collector.CollectorError) as failure:
- controller.run_once(config, begin, NeverCalled())
- self.assertEqual("E_CONTROLLER_PENDING", failure.exception.code)
+ controller.run_product(config, self.config_path, collector.parse_datetime(now_text, "now"))
+ self.assertEqual("E_TRUSTED_ADAPTER_REQUIRED", failure.exception.code)
continue
if entrypoint == "begin":
invoke = lambda: refresh.refresh_begin(config, self.config_path, collector.parse_datetime(now_text, "now"))
diff --git a/dev/project-dev/test/test_bili_half_hour_pipeline.py b/dev/project-dev/test/test_bili_half_hour_pipeline.py
new file mode 100644
index 0000000..9df7ff7
--- /dev/null
+++ b/dev/project-dev/test/test_bili_half_hour_pipeline.py
@@ -0,0 +1,1555 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+PROJECT_DEV = Path(__file__).resolve().parents[1]
+PROJECT_ROOT = PROJECT_DEV.parents[1]
+sys.path.insert(0, str(PROJECT_DEV))
+
+import bili_half_hour_pipeline as product
+
+
+NOW = datetime(2026, 8, 29, 8, 0, tzinfo=timezone.utc)
+UID = "1420210197"
+BVID = "BV1Q541167Qg"
+
+
+def canonical(value: object) -> bytes:
+ return (json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii")
+
+
+class PipelineFixture:
+ def __init__(self, root: Path) -> None:
+ self.root = root
+ self.archive = root / "ana-data" / "news-fixture"
+ self.state = root / "dev" / "tmp" / "pipeline"
+ self.archive.mkdir(parents=True)
+ self.formal = self.archive / "manifest.jsonl"
+ self.handoffs = self.archive / "video-processing-handoffs.jsonl"
+ self.formal.write_bytes(canonical({
+ "schema_version": 1, "stable_id": "OLD", "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": "old.txt", "bytes": 3, "sha256": hashlib.sha256(b"old").hexdigest().upper()
+ }))
+ (self.archive / "old.txt").write_bytes(b"old")
+ self.handoffs.write_bytes(canonical({
+ "schema_version": 1, "type": "media-processing-handoff", "status": "READY",
+ "handoff_id": "HANDOFF-OLD", "queue_job_id": "0" * 64, "creator_uid": UID,
+ "bvid": "BV1Q541167Qf", "source_url": "https://www.bilibili.com/video/BV1Q541167Qf",
+ "media_path": "F:/video/old.mkv", "mapping_path": "F:/video/old.download.json",
+ "bytes": 1, "sha256": "A" * 64, "duration_seconds": 1.0,
+ "video_codec": "av1", "audio_codec": "aac", "created_at": NOW.isoformat()
+ }))
+ self.config_path = root / "pipeline.json"
+ self.config_path.write_bytes(canonical({
+ "schema_version": 1,
+ "task_id": product.TASK_ID,
+ "interval_minutes": 30,
+ "creator": {"uid": UID, "name": "fixture", "dynamic_url": f"https://space.bilibili.com/{UID}/dynamic"},
+ "paths": {
+ "project_root": str(root), "archive_root": str(self.archive),
+ "formal_manifest": str(self.formal), "processing_handoffs": str(self.handoffs),
+ "state_dir": str(self.state), "video_root": str(root / "external-video")
+ },
+ "downstream": {
+ "video_downloader_thread_id": "019fcc5d-798f-7ea1-8325-3a4d1f2dc5a5",
+ "media_processor_thread_id": "019fb7a4-bdfd-79f2-bd6b-e67e2b7d8efd",
+ "minutes_thread_id": "019fae88-ef98-7f83-b964-dcd4c5f842ef",
+ "reply_thread_id": "019fbcbb-bed7-7c90-83ab-f50610f80d3a"
+ },
+ "git": {
+ "remote": "origin",
+ "branch": "main",
+ "allowed_extensions": [".txt", ".md", ".json", ".jsonl", ".srt", ".pdf", ".png", ".jpg"],
+ "allowed_docs": ["ana-data/news-fixture/目录导读.md"],
+ }
+ }))
+ (self.archive / "目录导读.md").write_bytes(b"# fixture\n")
+ self.config = product.load_config(self.config_path)
+
+ def append_formal(self, value: dict[str, object]) -> None:
+ with self.formal.open("ab") as stream:
+ stream.write(canonical(value))
+
+ def append_handoff(self, value: dict[str, object]) -> None:
+ with self.handoffs.open("ab") as stream:
+ stream.write(canonical(value))
+
+
+def prepare_migration_fixture(root: Path) -> tuple[PipelineFixture, dict[str, Path], dict[str, bytes], Path]:
+ fixture = PipelineFixture(root)
+ (fixture.root / "external-video").mkdir()
+ fixture.append_formal({
+ "schema_version": 1,
+ "stable_id": BVID,
+ "item_type": "video",
+ "status": product.VIDEO_COMPLETE,
+ "creator": "fixture",
+ "title": "A / canonical: title?",
+ "published_at": "2026-08-30T12:34:56+08:00",
+ })
+ transcript_dir = fixture.archive / f"{BVID}.transcript"
+ minutes_dir = fixture.archive / f"{BVID}.minutes"
+ transcript_dir.mkdir()
+ minutes_dir.mkdir()
+ payloads = {
+ "transcript_txt": b"transcript\n",
+ "transcript_srt": b"1\n00:00:00,000 --> 00:00:01,000\ntext\n",
+ "transcript_json": b'{"segments":[]}\n',
+ "minutes_md": b"# minutes\n",
+ "minutes_pdf": b"%PDF-fixture\n",
+ }
+ legacy_paths = {
+ "transcript_txt": transcript_dir / f"{BVID}.txt",
+ "transcript_srt": transcript_dir / f"{BVID}.srt",
+ "transcript_json": transcript_dir / f"{BVID}.json",
+ "minutes_md": minutes_dir / f"{BVID}.md",
+ "minutes_pdf": minutes_dir / f"{BVID}.pdf",
+ }
+ for kind, path in legacy_paths.items():
+ path.write_bytes(payloads[kind])
+ flac = transcript_dir / f"{BVID}.audio.flac"
+ flac.write_bytes(b"fLaC-fixture")
+ subprocess.run(["git", "init", "-b", "main"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "config", "user.name", "Fixture"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "add", "."], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "commit", "-m", "legacy baseline"], cwd=fixture.root, check=True, capture_output=True)
+ payloads["minutes_pdf"] = b"%PDF-modified-worktree-fixture\n"
+ legacy_paths["minutes_pdf"].write_bytes(payloads["minutes_pdf"])
+ (fixture.archive / "目录导读.md").write_bytes(b"# canonical naming guide\n")
+ product.initialize(fixture.config, NOW)
+ return fixture, legacy_paths, payloads, flac
+
+
+class HalfHourPipelineTests(unittest.TestCase):
+ @staticmethod
+ def _new_handoff(fixture: PipelineFixture, *, source_url: str | None = None) -> None:
+ fixture.append_handoff({
+ "schema_version": 1, "type": "media-processing-handoff", "status": "READY",
+ "handoff_id": "HANDOFF-NEW", "queue_job_id": "1" * 64, "creator_uid": UID, "bvid": BVID,
+ "source_url": source_url or f"https://www.bilibili.com/video/{BVID}", "media_path": "F:/video/new.mkv",
+ "mapping_path": "F:/video/new.download.json", "bytes": 100, "sha256": "B" * 64,
+ "duration_seconds": 60.0, "video_codec": "av1", "audio_codec": "aac", "created_at": NOW.isoformat(),
+ })
+
+ def test_external_schema_integers_are_strict_and_fail_before_mutation(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ original = json.loads(fixture.config_path.read_text(encoding="ascii"))
+ for field, bad_values in {
+ "schema_version": [True, 1.0, "1", None],
+ "interval_minutes": [True, 30.0, "30", None],
+ }.items():
+ for bad in bad_values:
+ candidate = dict(original)
+ candidate[field] = bad
+ fixture.config_path.write_bytes(canonical(candidate))
+ with self.assertRaises(product.PipelineError) as rejected:
+ product.load_config(fixture.config_path)
+ self.assertEqual("E_CONFIG", rejected.exception.code)
+ self.assertFalse(fixture.state.exists())
+ fixture.config_path.write_bytes(canonical(original))
+ config = product.load_config(fixture.config_path)
+ product.initialize(config, NOW)
+ state_before = config.state_path.read_bytes()
+ state = json.loads(state_before)
+ state["schema_version"] = True
+ config.state_path.write_bytes(canonical(state))
+ with self.assertRaises(product.PipelineError) as rejected:
+ product.begin(config, NOW)
+ self.assertEqual("E_STATE", rejected.exception.code)
+ self.assertFalse(config.runs_path.exists())
+
+ def test_processing_handoff_accepts_exact_schema_aliases_and_rejects_schema_drift(self) -> None:
+ def handoff(schema_key: str | None, schema_value: object = 1) -> dict[str, object]:
+ value: dict[str, object] = {
+ "type": "media-processing-handoff", "status": "READY",
+ "handoff_id": "HANDOFF-SCHEMA", "queue_job_id": "1" * 64,
+ "creator_uid": UID, "bvid": BVID, "source_url": f"https://www.bilibili.com/video/{BVID}",
+ "media_path": "F:/video/schema.mkv", "mapping_path": "F:/video/schema.download.json",
+ "bytes": 100, "sha256": "B" * 64, "duration_seconds": 60.0,
+ "video_codec": "av1", "audio_codec": "aac", "created_at": NOW.isoformat(),
+ }
+ if schema_key is not None:
+ value[schema_key] = schema_value
+ return value
+
+ positive_sha = (
+ ("schema", "b" * 64),
+ ("schema_version", "B" * 64),
+ ("schema_version", "Aa" * 32),
+ )
+ for schema_key, sha256 in positive_sha:
+ with self.subTest(positive=(schema_key, sha256[:2])), tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ row = handoff(schema_key)
+ row["sha256"] = sha256
+ fixture.append_handoff(row)
+ created = product.reconcile(fixture.config, NOW)["created_outbox_ids"]
+ self.assertEqual(1, len(created))
+ item = product.pending(fixture.config)["items"][0]
+ self.assertEqual(("VIDEO_TRANSCRIPTION_READY", BVID), (item["kind"], item["payload"]["bvid"]))
+ self.assertEqual(sha256.upper(), item["payload"]["sha256"])
+
+ invalid: list[tuple[str, dict[str, object]]] = [
+ ("bool", handoff("schema", True)),
+ ("float", handoff("schema", 1.0)),
+ ("string", handoff("schema", "1")),
+ ("null", handoff("schema", None)),
+ ("wrong_integer", handoff("schema", 2)),
+ ("missing", handoff(None)),
+ ]
+ invalid_sha = handoff("schema")
+ invalid_sha["sha256"] = "G" * 64
+ invalid.append(("invalid_sha", invalid_sha))
+ unicode_expansion_sha = handoff("schema")
+ unicode_expansion_sha["sha256"] = "0" * 62 + "\ufb00"
+ invalid.append(("unicode_expansion_sha", unicode_expansion_sha))
+ dual = handoff("schema")
+ dual["schema_version"] = 1
+ invalid.append(("dual", dual))
+ extra = handoff("schema")
+ extra["schema_extra"] = 1
+ invalid.append(("extra", extra))
+ for label, row in invalid:
+ with self.subTest(negative=label), tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ fixture.append_handoff(row)
+ state_before = fixture.config.state_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as rejected:
+ product.reconcile(fixture.config, NOW)
+ self.assertEqual("E_SOURCE_BINDING", rejected.exception.code)
+ self.assertEqual(state_before, fixture.config.state_path.read_bytes())
+ self.assertFalse(fixture.config.outbox_path.exists())
+
+ def test_outbox_history_schema_tamper_fails_without_append(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ fixture.append_handoff({
+ "schema_version": 1, "type": "media-processing-handoff", "status": "READY",
+ "handoff_id": "HANDOFF-NEW", "queue_job_id": "1" * 64, "creator_uid": UID, "bvid": BVID,
+ "source_url": f"https://www.bilibili.com/video/{BVID}", "media_path": "F:/video/new.mkv",
+ "mapping_path": "F:/video/new.download.json", "bytes": 100, "sha256": "B" * 64,
+ "duration_seconds": 60.0, "video_codec": "av1", "audio_codec": "aac", "created_at": NOW.isoformat(),
+ })
+ product.reconcile(fixture.config, NOW)
+ rows = [json.loads(line) for line in fixture.config.outbox_path.read_text(encoding="ascii").splitlines()]
+ rows[0]["schema_version"] = True
+ fixture.config.outbox_path.write_bytes(b"".join(canonical(row) for row in rows))
+ before = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as rejected:
+ product.pending(fixture.config)
+ self.assertEqual("E_OUTBOX", rejected.exception.code)
+ self.assertEqual(before, fixture.config.outbox_path.read_bytes())
+
+ def test_baseline_nonoverlap_reconcile_and_exact_once_outbox(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ initialized = product.initialize(fixture.config, NOW)
+ self.assertEqual(("INITIALIZED", 1, 1), (
+ initialized["status"], initialized["baseline"]["formal"]["lines"], initialized["baseline"]["handoff"]["lines"]
+ ))
+ self.assertEqual("RUN_STARTED", product.begin(fixture.config, NOW)["status"])
+ self.assertEqual("RUN_RESUMED", product.begin(fixture.config, NOW)["status"])
+ with self.assertRaises(product.PipelineError) as active:
+ product.begin(fixture.config, datetime(2026, 8, 29, 8, 30, tzinfo=timezone.utc))
+ self.assertEqual("E_RUN_ACTIVE", active.exception.code)
+
+ body = b"new text\n"
+ image = b"\x89PNG\r\nfixture"
+ (fixture.archive / "new.txt").write_bytes(body)
+ (fixture.archive / "new.png").write_bytes(image)
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "123456", "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": "new.txt", "bytes": len(body), "sha256": hashlib.sha256(body).hexdigest().upper(),
+ "image_path": "new.png", "image_bytes": len(image), "image_sha256": hashlib.sha256(image).hexdigest().upper()
+ })
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": BVID, "item_type": "video", "status": "VIDEO_DOWNLOAD_PENDING_EXTENSION",
+ "creator_uid": UID, "source_url": f"https://www.bilibili.com/video/{BVID}", "title": "fixture video",
+ "published_at": NOW.isoformat(), "expected_duration_seconds": 60
+ })
+ fixture.append_handoff({
+ "schema_version": 1, "type": "media-processing-handoff", "status": "READY",
+ "handoff_id": "HANDOFF-NEW", "queue_job_id": "1" * 64, "creator_uid": UID, "bvid": BVID,
+ "source_url": f"https://www.bilibili.com/video/{BVID}", "media_path": "F:/video/new.mkv",
+ "mapping_path": "F:/video/new.download.json", "bytes": 100, "sha256": "B" * 64,
+ "duration_seconds": 60.0, "video_codec": "av1", "audio_codec": "aac", "created_at": NOW.isoformat()
+ })
+ result = product.reconcile(fixture.config, NOW)
+ self.assertEqual(3, len(result["created_outbox_ids"]))
+ self.assertEqual(0, len(product.reconcile(fixture.config, NOW)["created_outbox_ids"]))
+ pending = product.pending(fixture.config)
+ self.assertEqual(3, pending["count"])
+ self.assertEqual(
+ {"GIT_DELIVERY_READY", "VIDEO_DOWNLOAD_READY", "VIDEO_TRANSCRIPTION_READY"},
+ {item["kind"] for item in pending["items"]},
+ )
+ video_item = next(item for item in pending["items"] if item["kind"] == "VIDEO_DOWNLOAD_READY")
+ self.assertEqual(
+ fixture.config.video_downloader_thread_id,
+ product.dispatch_intent(fixture.config, video_item["outbox_id"], NOW)["target_thread_id"],
+ )
+ serialized = fixture.config.outbox_path.read_text(encoding="ascii").lower()
+ for secret in ("cookie", "sessdata", "localstorage", "profile"):
+ self.assertNotIn(secret, serialized)
+
+ def test_dispatch_intent_is_restart_safe_and_observed_exact_once(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ fixture.append_handoff({
+ "schema_version": 1, "type": "media-processing-handoff", "status": "READY",
+ "handoff_id": "HANDOFF-NEW", "queue_job_id": "1" * 64, "creator_uid": UID, "bvid": BVID,
+ "source_url": f"https://www.bilibili.com/video/{BVID}", "media_path": "F:/video/new.mkv",
+ "mapping_path": "F:/video/new.download.json", "bytes": 100, "sha256": "B" * 64,
+ "duration_seconds": 60.0, "video_codec": "av1", "audio_codec": "aac", "created_at": NOW.isoformat()
+ })
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ intent = product.dispatch_intent(fixture.config, outbox_id, NOW)
+ self.assertEqual(("DISPATCH_INTENT_DURABLE", fixture.config.media_thread_id), (intent["status"], intent["target_thread_id"]))
+ self.assertEqual(outbox_id, intent["envelope"]["outbox_id"])
+ self.assertEqual("DISPATCH_INTENT", product.pending(fixture.config)["items"][0]["delivery_state"])
+ resumed = product.dispatch_intent(fixture.config, outbox_id, NOW)
+ self.assertEqual(("DISPATCH_INTENT_RESUMED", intent["envelope"]), (resumed["status"], resumed["envelope"]))
+ observed = product.observe_dispatch(fixture.config, outbox_id, "DELIVERY-1", NOW)
+ self.assertEqual("DISPATCH_OBSERVED", observed["status"])
+ self.assertEqual("DISPATCH_ALREADY_OBSERVED", product.observe_dispatch(fixture.config, outbox_id, "DELIVERY-1", NOW)["status"])
+ self.assertEqual(0, product.pending(fixture.config)["count"])
+ with self.assertRaises(product.PipelineError) as conflict:
+ product.observe_dispatch(fixture.config, outbox_id, "DELIVERY-2", NOW)
+ self.assertEqual("E_DISPATCH_RECEIPT", conflict.exception.code)
+
+ def test_transcript_and_minutes_receipts_create_restart_safe_downstream(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ fixture.append_handoff({
+ "schema_version": 1, "type": "media-processing-handoff", "status": "READY",
+ "handoff_id": "HANDOFF-NEW", "queue_job_id": "1" * 64, "creator_uid": UID, "bvid": BVID,
+ "source_url": f"https://www.bilibili.com/video/{BVID}", "media_path": "F:/video/new.mkv",
+ "mapping_path": "F:/video/new.download.json", "bytes": 100, "sha256": "B" * 64,
+ "duration_seconds": 60.0, "video_codec": "av1", "audio_codec": "aac", "created_at": NOW.isoformat()
+ })
+ source_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ product.dispatch_intent(fixture.config, source_id, NOW)
+ product.observe_dispatch(fixture.config, source_id, "TRANSCRIPT-DISPATCH-1", NOW)
+ transcript = fixture.archive / "new.transcript.txt"
+ transcript_srt = fixture.archive / "new.transcript.srt"
+ transcript_json = fixture.archive / "new.transcript.json"
+ transcript.write_bytes(b"transcript\n")
+ transcript_srt.write_bytes(b"1\n00:00:00,000 --> 00:00:01,000\ntranscript\n")
+ transcript_json.write_bytes(b'{"segments":[]}\n')
+ receipt = fixture.root / "transcript-receipt.json"
+ receipt.write_bytes(canonical({
+ "schema_version": 1, "type": "TRANSCRIPTION_COMPLETE", "outbox_id": source_id,
+ "stable_id": BVID, "terminal_id": "TRANSCRIPT-1", "status": "COMPLETE",
+ "files": [
+ {"path": str(transcript), "bytes": transcript.stat().st_size,
+ "sha256": hashlib.sha256(transcript.read_bytes()).hexdigest().upper(), "kind": "transcript_txt"},
+ {"path": str(transcript_srt), "bytes": transcript_srt.stat().st_size,
+ "sha256": hashlib.sha256(transcript_srt.read_bytes()).hexdigest().upper(), "kind": "transcript_srt"},
+ {"path": str(transcript_json), "bytes": transcript_json.stat().st_size,
+ "sha256": hashlib.sha256(transcript_json.read_bytes()).hexdigest().upper(), "kind": "transcript_json"},
+ ],
+ "created_at": NOW.isoformat()
+ }))
+ committed = product.ingest_receipt(fixture.config, receipt, NOW)
+ self.assertEqual(("RECEIPT_COMMITTED", 2), (committed["status"], len(committed["created_outbox_ids"])))
+ replay = product.ingest_receipt(fixture.config, receipt, NOW)
+ self.assertEqual("RECEIPT_ALREADY_COMMITTED", replay["status"])
+ terminal_before = fixture.config.terminals_path.read_bytes()
+ outbox_before = fixture.config.outbox_path.read_bytes()
+ conflicting = json.loads(receipt.read_text(encoding="ascii"))
+ conflicting["terminal_id"] = "TRANSCRIPT-CONFLICT"
+ receipt.write_bytes(canonical(conflicting))
+ with self.assertRaises(product.PipelineError) as conflict:
+ product.ingest_receipt(fixture.config, receipt, NOW)
+ self.assertEqual("E_RECEIPT_CONFLICT", conflict.exception.code)
+ self.assertEqual(terminal_before, fixture.config.terminals_path.read_bytes())
+ self.assertEqual(outbox_before, fixture.config.outbox_path.read_bytes())
+ items = product.pending(fixture.config)["items"]
+ minutes_item = next(item for item in items if item["kind"] == "MINUTES_READY")
+ product.dispatch_intent(fixture.config, minutes_item["outbox_id"], NOW)
+ product.observe_dispatch(fixture.config, minutes_item["outbox_id"], "MINUTES-DISPATCH-1", NOW)
+ minutes = fixture.archive / "new.minutes.md"
+ minutes_pdf = fixture.archive / "new.minutes.pdf"
+ minutes.write_bytes(b"# minutes\n")
+ minutes_pdf.write_bytes(b"%PDF-fixture\n")
+ minutes_receipt = fixture.root / "minutes-receipt.json"
+ minutes_receipt.write_bytes(canonical({
+ "schema_version": 1, "type": "MINUTES_COMPLETE", "outbox_id": minutes_item["outbox_id"],
+ "stable_id": BVID, "terminal_id": "MINUTES-1", "status": "COMPLETE",
+ "files": [
+ {"path": str(minutes), "bytes": minutes.stat().st_size,
+ "sha256": hashlib.sha256(minutes.read_bytes()).hexdigest().upper(), "kind": "minutes_md"},
+ {"path": str(minutes_pdf), "bytes": minutes_pdf.stat().st_size,
+ "sha256": hashlib.sha256(minutes_pdf.read_bytes()).hexdigest().upper(), "kind": "minutes_pdf"},
+ ],
+ "created_at": NOW.isoformat()
+ }))
+ result = product.ingest_receipt(fixture.config, minutes_receipt, NOW)
+ self.assertEqual(1, len(result["created_outbox_ids"]))
+ git_items = [
+ row for row in product._outbox_rows(fixture.config)
+ if row.get("event") == "CREATED" and row.get("kind") == "GIT_DELIVERY_READY"
+ ]
+ self.assertEqual(2, len(git_items))
+ self.assertEqual([3, 2], [len(product._git_artifacts(fixture.config, row["payload"]["files"])[0]) for row in git_items])
+ mismatched = dict(git_items[0]["payload"]["files"][0])
+ mismatched["kind"] = "minutes_pdf"
+ with self.assertRaises(product.PipelineError) as kind_drift:
+ product._git_artifacts(fixture.config, [mismatched])
+ self.assertEqual("E_GIT_SCOPE", kind_drift.exception.code)
+
+ def test_git_delivery_uses_exact_paths_and_nonforce_push(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ (fixture.root / ".git").mkdir()
+ (fixture.root / ".git" / "index").write_bytes(b"USER-INDEX")
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ artifact = fixture.archive / "new.txt"
+ artifact.write_bytes(b"content\n")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "123", "item_type": "text", "status": "SAVED", "creator_uid": UID,
+ "path": "new.txt", "bytes": artifact.stat().st_size, "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper()
+ })
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ commands: list[list[str]] = []
+ rel = artifact.relative_to(fixture.root).as_posix()
+
+ def runner(command: list[str], **_: object) -> subprocess.CompletedProcess[str]:
+ commands.append(command)
+ if command[:5] == ["git", "diff", "--cached", "--name-only", "-z"] and command[5] != "--":
+ return subprocess.CompletedProcess(command, 0, b"", b"")
+ if command[:6] == ["git", "diff", "--cached", "--name-only", "-z", "--"]:
+ return subprocess.CompletedProcess(command, 0, rel.encode("utf-8") + b"\0", b"")
+ if command[:4] == ["git", "diff", "--cached", "--name-only"]:
+ return subprocess.CompletedProcess(command, 0, "", "")
+ if command[:3] == ["git", "rev-parse", "HEAD"]:
+ return subprocess.CompletedProcess(command, 0, "a" * 40 + "\n", "")
+ if command[:2] == ["git", "hash-object"]:
+ return subprocess.CompletedProcess(command, 0, "d" * 40 + "\n", "")
+ if command[:2] == ["git", "write-tree"]:
+ return subprocess.CompletedProcess(command, 0, "b" * 40 + "\n", "")
+ if command[:2] == ["git", "commit-tree"]:
+ return subprocess.CompletedProcess(command, 0, "c" * 40 + "\n", "")
+ return subprocess.CompletedProcess(command, 0, "", "")
+
+ result = product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)
+ self.assertEqual("GIT_PUSHED", result["status"])
+ self.assertIn(["git", "update-index", "--add", "--cacheinfo", f"100644,{'d' * 40},{rel}"], commands)
+ self.assertIn(["git", "push", "origin", "c" * 40 + ":refs/heads/main"], commands)
+ flattened = "\n".join(" ".join(command) for command in commands)
+ self.assertNotIn("git add .", flattened)
+ self.assertNotIn("--force", flattened)
+
+ def test_git_delivery_accepts_exact_published_head_without_git_writes(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ for command in (
+ ["git", "init", "-b", "main"],
+ ["git", "config", "user.name", "Fixture"],
+ ["git", "config", "user.email", "fixture@example.invalid"],
+ ["git", "add", "."],
+ ["git", "commit", "-m", "fixture baseline"],
+ ):
+ subprocess.run(command, cwd=fixture.root, check=True, capture_output=True)
+ remote = fixture.root / "remote.git"
+ subprocess.run(["git", "init", "--bare", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "remote", "add", "origin", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "push", "-u", "origin", "main"], cwd=fixture.root, check=True, capture_output=True)
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ artifact = fixture.archive / "已交付.txt"
+ artifact.write_bytes(b"already published\n")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "published", "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": artifact.name, "bytes": artifact.stat().st_size,
+ "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper(),
+ })
+ subprocess.run(
+ ["git", "add", "--", artifact.relative_to(fixture.root).as_posix(), fixture.formal.relative_to(fixture.root).as_posix()],
+ cwd=fixture.root, check=True, capture_output=True,
+ )
+ subprocess.run(["git", "commit", "-m", "accepted data delivery"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "push", "origin", "main"], cwd=fixture.root, check=True, capture_output=True)
+ head_before = subprocess.run(
+ ["git", "rev-parse", "HEAD"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.strip()
+ shared_index = fixture.root / ".git" / "index"
+ index_before = shared_index.read_bytes()
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ commands: list[list[str]] = []
+
+ def runner(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[Any]:
+ commands.append(command)
+ return subprocess.run(command, **kwargs)
+
+ result = product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)
+ self.assertEqual(("GIT_NO_CHANGES", head_before), (result["status"], result["commit_sha"]))
+ self.assertEqual(index_before, shared_index.read_bytes())
+ self.assertEqual(head_before, subprocess.run(
+ ["git", "rev-parse", "HEAD"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.strip())
+ self.assertEqual(head_before, subprocess.run(
+ ["git", "rev-parse", "refs/remotes/origin/main"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.strip())
+ forbidden = {"hash-object", "update-index", "commit-tree", "update-ref", "push"}
+ self.assertFalse(any(len(command) > 1 and command[1] in forbidden for command in commands))
+ self.assertEqual(0, len(list(fixture.config.state_dir.glob(f"git-index-{outbox_id}*"))))
+ events = [json.loads(line) for line in fixture.config.outbox_path.read_text(encoding="ascii").splitlines()]
+ complete = [row for row in events if row.get("outbox_id") == outbox_id and row.get("event") == "COMPLETE"]
+ self.assertEqual(["NO_CHANGES"], [row["result"] for row in complete])
+
+ local_only = fixture.archive / "local-only.txt"
+ local_only.write_bytes(b"not published\n")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "local-only", "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": local_only.name, "bytes": local_only.stat().st_size,
+ "sha256": hashlib.sha256(local_only.read_bytes()).hexdigest().upper(),
+ })
+ subprocess.run(
+ ["git", "add", "--", local_only.relative_to(fixture.root).as_posix(), fixture.formal.relative_to(fixture.root).as_posix()],
+ cwd=fixture.root, check=True, capture_output=True,
+ )
+ subprocess.run(["git", "commit", "-m", "local only"], cwd=fixture.root, check=True, capture_output=True)
+ later = datetime(2026, 8, 29, 8, 30, tzinfo=timezone.utc)
+ local_outbox = product.reconcile(fixture.config, later)["created_outbox_ids"][0]
+ before = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as unpublished:
+ product.git_deliver(fixture.config, local_outbox, later)
+ self.assertEqual("E_GIT_PUSH", unpublished.exception.code)
+ self.assertEqual(before, fixture.config.outbox_path.read_bytes())
+
+ def test_git_no_changes_rejects_terminal_boundary_artifact_and_ref_drift(self) -> None:
+ for mode, expected_code in (
+ ("artifact_after_read", "E_GIT_SCOPE"),
+ ("head_after_capture", "E_GIT_PUSH"),
+ ("remote_after_read", "E_GIT_PUSH"),
+ ):
+ with self.subTest(mode=mode), tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ for command in (
+ ["git", "init", "-b", "main"],
+ ["git", "config", "user.name", "Fixture"],
+ ["git", "config", "user.email", "fixture@example.invalid"],
+ ["git", "add", "."],
+ ["git", "commit", "-m", "fixture baseline"],
+ ):
+ subprocess.run(command, cwd=fixture.root, check=True, capture_output=True)
+ remote = fixture.root / "remote.git"
+ subprocess.run(["git", "init", "--bare", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "remote", "add", "origin", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "push", "-u", "origin", "main"], cwd=fixture.root, check=True, capture_output=True)
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ artifact = fixture.archive / "终态竞态.txt"
+ artifact.write_bytes(b"published artifact\n")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": f"race-{mode}", "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": artifact.name, "bytes": artifact.stat().st_size,
+ "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper(),
+ })
+ subprocess.run(
+ ["git", "add", "--", artifact.relative_to(fixture.root).as_posix(), fixture.formal.relative_to(fixture.root).as_posix()],
+ cwd=fixture.root, check=True, capture_output=True,
+ )
+ subprocess.run(["git", "commit", "-m", "accepted published artifact"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "push", "origin", "main"], cwd=fixture.root, check=True, capture_output=True)
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ outbox_before = fixture.config.outbox_path.read_bytes()
+ injected = False
+
+ def runner(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[Any]:
+ nonlocal injected
+ result = subprocess.run(command, **kwargs)
+ if not injected and mode in {"artifact_after_read", "head_after_capture"} and command[:2] == ["git", "show"]:
+ injected = True
+ if mode == "artifact_after_read":
+ identity = artifact.stat()
+ payload = artifact.read_bytes()
+ artifact.write_bytes(b"X" + payload[1:])
+ os.utime(artifact, ns=(identity.st_atime_ns, identity.st_mtime_ns))
+ else:
+ old_head = subprocess.run(
+ ["git", "rev-parse", "HEAD"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.strip()
+ tree = subprocess.run(
+ ["git", "rev-parse", "HEAD^{tree}"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.strip()
+ env = dict(os.environ)
+ env.update({
+ "GIT_AUTHOR_NAME": "Race", "GIT_COMMITTER_NAME": "Race",
+ "GIT_AUTHOR_EMAIL": "race@example.invalid", "GIT_COMMITTER_EMAIL": "race@example.invalid",
+ })
+ new_head = subprocess.run(
+ ["git", "commit-tree", tree, "-p", old_head, "-m", "same tree local race"],
+ cwd=fixture.root, check=True, text=True, capture_output=True, env=env,
+ ).stdout.strip()
+ subprocess.run(["git", "update-ref", "HEAD", new_head, old_head], cwd=fixture.root, check=True, capture_output=True)
+ elif (
+ not injected
+ and mode == "remote_after_read"
+ and command[:3] == ["git", "rev-parse", "refs/remotes/origin/main"]
+ ):
+ injected = True
+ old_remote = result.stdout.strip()
+ parent = subprocess.run(
+ ["git", "rev-parse", "HEAD^"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.strip()
+ subprocess.run(
+ ["git", "update-ref", "refs/remotes/origin/main", parent, old_remote],
+ cwd=fixture.root, check=True, capture_output=True,
+ )
+ return result
+
+ with self.assertRaises(product.PipelineError) as drift:
+ product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)
+ self.assertTrue(injected)
+ self.assertEqual(expected_code, drift.exception.code)
+ self.assertEqual(outbox_before, fixture.config.outbox_path.read_bytes())
+ self.assertEqual(0, len(list(fixture.config.state_dir.glob(f"git-index-{outbox_id}*"))))
+ self.assertEqual(0, len(list((fixture.root / ".git").rglob("*.lock"))))
+
+ def test_append_only_rewrite_and_unsafe_artifact_fail_closed(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ fixture.formal.write_bytes(b"")
+ with self.assertRaises(product.PipelineError) as rewritten:
+ product.reconcile(fixture.config, NOW)
+ self.assertEqual("E_HISTORY_REWRITE", rewritten.exception.code)
+
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ original = fixture.formal.read_bytes()
+ replacement = original.replace(b'"stable_id":"OLD"', b'"stable_id":"NEW"')
+ self.assertEqual(len(original), len(replacement))
+ fixture.formal.write_bytes(replacement)
+ with self.assertRaises(product.PipelineError) as prefix:
+ product.reconcile(fixture.config, NOW)
+ self.assertEqual("E_HISTORY_REWRITE", prefix.exception.code)
+
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ outside = fixture.root / "outside.txt"
+ outside.write_bytes(b"outside")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "123", "item_type": "text", "status": "SAVED", "creator_uid": UID,
+ "path": str(outside), "bytes": outside.stat().st_size, "sha256": hashlib.sha256(outside.read_bytes()).hexdigest().upper()
+ })
+ with self.assertRaises(product.PipelineError) as escaped:
+ product.reconcile(fixture.config, NOW)
+ self.assertEqual("E_ARTIFACT", escaped.exception.code)
+
+ def test_git_commit_journal_recovers_push_without_second_commit(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ (fixture.root / ".git").mkdir()
+ (fixture.root / ".git" / "index").write_bytes(b"USER-INDEX")
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ artifact = fixture.archive / "recover.txt"
+ artifact.write_bytes(b"recover\n")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "recover", "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": "recover.txt", "bytes": artifact.stat().st_size,
+ "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper(),
+ })
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ rel = artifact.relative_to(fixture.root).as_posix()
+ commands: list[list[str]] = []
+ push_count = 0
+
+ def runner(command: list[str], **_: object) -> subprocess.CompletedProcess[str]:
+ nonlocal push_count
+ commands.append(command)
+ if command[:5] == ["git", "diff", "--cached", "--name-only", "-z"] and command[5] != "--":
+ return subprocess.CompletedProcess(command, 0, b"", b"")
+ if command[:6] == ["git", "diff", "--cached", "--name-only", "-z", "--"]:
+ return subprocess.CompletedProcess(command, 0, rel.encode("utf-8") + b"\0", b"")
+ if command[:4] == ["git", "diff", "--cached", "--name-only"]:
+ return subprocess.CompletedProcess(command, 0, "", "")
+ if command[:3] == ["git", "rev-parse", "HEAD"]:
+ return subprocess.CompletedProcess(command, 0, ("a" if push_count == 0 else "c") * 40 + "\n", "")
+ if command[:2] == ["git", "hash-object"]:
+ return subprocess.CompletedProcess(command, 0, "d" * 40 + "\n", "")
+ if command[:2] == ["git", "write-tree"]:
+ return subprocess.CompletedProcess(command, 0, "b" * 40 + "\n", "")
+ if command[:2] == ["git", "commit-tree"]:
+ return subprocess.CompletedProcess(command, 0, "c" * 40 + "\n", "")
+ if command[:2] == ["git", "push"]:
+ push_count += 1
+ return subprocess.CompletedProcess(command, 1 if push_count == 1 else 0, "", "")
+ return subprocess.CompletedProcess(command, 0, "", "")
+
+ with self.assertRaises(product.PipelineError) as first:
+ product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)
+ self.assertEqual("E_GIT_PUSH", first.exception.code)
+ self.assertEqual("GIT_PUSHED", product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)["status"])
+ self.assertEqual(1, sum(1 for command in commands if command[:2] == ["git", "commit-tree"]))
+ self.assertEqual(1, sum(1 for command in commands if command[:2] == ["git", "update-index"]))
+
+ def test_payload_source_rebinding_and_signed_url_reject_before_dispatch(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": BVID, "item_type": "video", "status": "VIDEO_DOWNLOAD_PENDING_EXTENSION",
+ "creator_uid": UID, "source_url": f"https://www.bilibili.com/video/{BVID}?token=SYNTHETIC_SECRET",
+ "title": "fixture", "published_at": NOW.isoformat(), "expected_duration_seconds": 60,
+ })
+ with self.assertRaises(product.PipelineError) as secret:
+ product.reconcile(fixture.config, NOW)
+ self.assertEqual("E_SECRET_FIELD", secret.exception.code)
+ self.assertFalse(fixture.config.outbox_path.exists())
+
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ self._new_handoff(fixture)
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ rows = [json.loads(line) for line in fixture.config.outbox_path.read_text(encoding="ascii").splitlines()]
+ rows[0]["payload"]["bvid"] = "BV1Q541167Qf"
+ fixture.config.outbox_path.write_bytes(b"".join(canonical(row) for row in rows))
+ before = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as tampered:
+ product.dispatch_intent(fixture.config, outbox_id, NOW)
+ self.assertEqual("E_OUTBOX", tampered.exception.code)
+ self.assertEqual(before, fixture.config.outbox_path.read_bytes())
+
+ def test_legacy_audit_handoff_identity_key_is_narrowly_projected(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": BVID, "item_type": "video",
+ "status": "VIDEO_DOWNLOAD_PENDING_EXTENSION", "creator_uid": UID,
+ "source_url": f"https://www.bilibili.com/video/{BVID}",
+ "title": "fixture", "published_at": NOW.isoformat(),
+ "expected_duration_seconds": 60,
+ "runtime_authorization_handoff_id": "HANDOFF-LEGACY-AUDIT-IDENTITY",
+ })
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ pending = product.pending(fixture.config)
+ self.assertEqual((1, outbox_id), (pending["count"], pending["items"][0]["outbox_id"]))
+ serialized = fixture.config.outbox_path.read_text(encoding="ascii").lower()
+ self.assertNotIn("authorization", serialized)
+ self.assertEqual(
+ "DISPATCH_INTENT_DURABLE",
+ product.dispatch_intent(fixture.config, outbox_id, NOW)["status"],
+ )
+
+ def test_receipt_requires_observed_and_late_observed_is_mutation_zero(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ self._new_handoff(fixture)
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ product.dispatch_intent(fixture.config, outbox_id, NOW)
+ transcript = fixture.archive / "ordered.transcript.txt"
+ transcript.write_bytes(b"ordered\n")
+ receipt = fixture.root / "receipt.json"
+ receipt.write_bytes(canonical({
+ "schema_version": 1, "type": "TRANSCRIPTION_COMPLETE", "outbox_id": outbox_id,
+ "stable_id": BVID, "terminal_id": "TRANSCRIPT-ORDER", "status": "COMPLETE",
+ "files": [{"path": str(transcript), "bytes": transcript.stat().st_size,
+ "sha256": hashlib.sha256(transcript.read_bytes()).hexdigest().upper(), "kind": "transcript"}],
+ "created_at": NOW.isoformat(),
+ }))
+ outbox_before = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as early:
+ product.ingest_receipt(fixture.config, receipt, NOW)
+ self.assertEqual("E_RECEIPT", early.exception.code)
+ self.assertEqual(outbox_before, fixture.config.outbox_path.read_bytes())
+ self.assertFalse(fixture.config.terminals_path.exists())
+ product._append(fixture.config.outbox_path, {
+ "schema_version": 1, "event": "COMPLETE", "outbox_id": outbox_id,
+ "result": "TRANSCRIPTION_COMPLETE", "terminal_id": "TRANSCRIPT-ORDER",
+ "receipt_sha256": hashlib.sha256(receipt.read_bytes()).hexdigest().upper(), "completed_at": NOW.isoformat(),
+ })
+ malformed = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as late:
+ product.observe_dispatch(fixture.config, outbox_id, "DELIVERY-LATE", NOW)
+ self.assertEqual("E_OUTBOX", late.exception.code)
+ self.assertEqual(malformed, fixture.config.outbox_path.read_bytes())
+
+ def test_git_commit_failure_uses_only_task_index_and_retry_is_deterministic(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ (fixture.root / ".git").mkdir()
+ shared_index = fixture.root / ".git" / "index"
+ shared_index.write_bytes(b"USER-INDEX")
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ artifact = fixture.archive / "failure.txt"
+ artifact.write_bytes(b"failure\n")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "failure", "item_type": "text", "status": "SAVED", "creator_uid": UID,
+ "path": "failure.txt", "bytes": artifact.stat().st_size, "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper(),
+ })
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ rel = artifact.relative_to(fixture.root).as_posix()
+ calls: list[tuple[list[str], dict[str, object]]] = []
+ commit_attempt = 0
+
+ def runner(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
+ nonlocal commit_attempt
+ calls.append((command, kwargs))
+ if command[:5] == ["git", "diff", "--cached", "--name-only", "-z"] and command[5] != "--":
+ return subprocess.CompletedProcess(command, 0, b"", b"")
+ if command[:6] == ["git", "diff", "--cached", "--name-only", "-z", "--"]:
+ return subprocess.CompletedProcess(command, 0, rel.encode("utf-8") + b"\0", b"")
+ if command[:4] == ["git", "diff", "--cached", "--name-only"]:
+ return subprocess.CompletedProcess(command, 0, "", "")
+ if command[:3] == ["git", "rev-parse", "HEAD"]:
+ return subprocess.CompletedProcess(command, 0, "a" * 40 + "\n", "")
+ if command[:2] == ["git", "hash-object"]:
+ return subprocess.CompletedProcess(command, 0, "d" * 40 + "\n", "")
+ if command[:2] == ["git", "write-tree"]:
+ return subprocess.CompletedProcess(command, 0, "b" * 40 + "\n", "")
+ if command[:2] == ["git", "commit-tree"]:
+ commit_attempt += 1
+ return subprocess.CompletedProcess(command, 1 if commit_attempt == 1 else 0, "" if commit_attempt == 1 else "c" * 40 + "\n", "")
+ return subprocess.CompletedProcess(command, 0, "", "")
+
+ with self.assertRaises(product.PipelineError) as failed:
+ product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)
+ self.assertEqual("E_GIT_COMMIT", failed.exception.code)
+ self.assertEqual(b"USER-INDEX", shared_index.read_bytes())
+ self.assertEqual(1, fixture.config.outbox_path.read_text(encoding="ascii").count('"event":"GIT_COMMIT_INTENT"'))
+ self.assertEqual("GIT_PUSHED", product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)["status"])
+ self.assertEqual(b"USER-INDEX", shared_index.read_bytes())
+ self.assertEqual(1, sum(command[:2] == ["git", "update-index"] for command, _ in calls))
+ task_envs = [kwargs["env"] for command, kwargs in calls if command[:2] == ["git", "update-index"]]
+ self.assertTrue(all(str(env["GIT_INDEX_FILE"]).startswith(str(fixture.config.state_dir)) for env in task_envs))
+
+ def test_real_git_commit_failure_preserves_shared_index_and_exact_retry_pushes(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ for command in (
+ ["git", "init", "-b", "main"],
+ ["git", "config", "user.name", "Fixture"],
+ ["git", "config", "user.email", "fixture@example.invalid"],
+ ["git", "add", "."],
+ ["git", "commit", "-m", "fixture baseline"],
+ ):
+ subprocess.run(command, cwd=fixture.root, check=True, capture_output=True)
+ remote = fixture.root / "remote.git"
+ subprocess.run(["git", "init", "--bare", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "remote", "add", "origin", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ artifact = fixture.archive / "真实-交付.txt"
+ artifact.write_bytes(b"real failure\n")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "real-failure", "item_type": "text", "status": "SAVED", "creator_uid": UID,
+ "path": artifact.name, "bytes": artifact.stat().st_size,
+ "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper(),
+ })
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ shared_index = fixture.root / ".git" / "index"
+ index_before = shared_index.read_bytes()
+ failed_once = False
+
+ def runner(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
+ nonlocal failed_once
+ if command[:2] == ["git", "commit-tree"] and not failed_once:
+ failed_once = True
+ return subprocess.CompletedProcess(command, 1, "", "injected")
+ return subprocess.run(command, **kwargs)
+
+ with self.assertRaises(product.PipelineError) as failure:
+ product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)
+ self.assertEqual("E_GIT_COMMIT", failure.exception.code)
+ self.assertEqual(index_before, shared_index.read_bytes())
+ staged = subprocess.run(["git", "diff", "--cached", "--name-only"], cwd=fixture.root, check=True, text=True, capture_output=True)
+ self.assertEqual("", staged.stdout)
+ result = product.git_deliver(fixture.config, outbox_id, NOW, runner=runner)
+ self.assertEqual("GIT_PUSHED", result["status"])
+ self.assertEqual(index_before, shared_index.read_bytes())
+ remote_head = subprocess.run(
+ ["git", "--git-dir", str(remote), "rev-parse", "refs/heads/main"], check=True, text=True, capture_output=True,
+ ).stdout.strip()
+ self.assertEqual(result["commit_sha"], remote_head)
+
+ def test_real_git_batch_freezes_shared_index_across_moving_head_and_rejects_drift(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ for command in (
+ ["git", "init", "-b", "main"],
+ ["git", "config", "user.name", "Fixture"],
+ ["git", "config", "user.email", "fixture@example.invalid"],
+ ["git", "add", "."],
+ ["git", "commit", "-m", "fixture baseline"],
+ ):
+ subprocess.run(command, cwd=fixture.root, check=True, capture_output=True)
+ remote = fixture.root / "remote.git"
+ subprocess.run(["git", "init", "--bare", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "remote", "add", "origin", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ user_file = fixture.root / "user-staged.txt"
+ user_file.write_bytes(b"user staged preimage\n")
+ subprocess.run(["git", "add", user_file.name], cwd=fixture.root, check=True, capture_output=True)
+ shared_index = fixture.root / ".git" / "index"
+ index_before = shared_index.read_bytes()
+ baseline_head = subprocess.run(
+ ["git", "rev-parse", "HEAD"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.strip()
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ for stable_id in ("batch-one", "batch-two"):
+ artifact = fixture.archive / f"{stable_id}.txt"
+ artifact.write_bytes((stable_id + "\n").encode("ascii"))
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": stable_id, "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": artifact.name, "bytes": artifact.stat().st_size,
+ "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper(),
+ })
+ outbox_ids = product.reconcile(fixture.config, NOW)["created_outbox_ids"]
+ self.assertEqual(2, len(outbox_ids))
+ first = product.git_deliver(fixture.config, outbox_ids[0], NOW)
+ self.assertEqual(index_before, shared_index.read_bytes())
+ interpreted_after_head_move = subprocess.run(
+ ["git", "diff", "--cached", "--name-only"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.splitlines()
+ self.assertIn("user-staged.txt", interpreted_after_head_move)
+ guards = list(fixture.config.state_dir.glob("git-shared-index-guard-*.json"))
+ self.assertEqual(1, len(guards))
+ guards[0].unlink()
+ outbox_before_recovery = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as wrong_frozen_bytes:
+ product.recover_git_index_guard(
+ fixture.config,
+ outbox_ids[1],
+ baseline_head,
+ len(index_before),
+ "0" * 64,
+ )
+ self.assertEqual("E_GIT_INDEX_DIRTY", wrong_frozen_bytes.exception.code)
+ self.assertEqual(outbox_before_recovery, fixture.config.outbox_path.read_bytes())
+ self.assertEqual([], list(fixture.config.state_dir.glob("git-shared-index-guard-*.json")))
+ recovered = product.recover_git_index_guard(
+ fixture.config,
+ outbox_ids[1],
+ baseline_head,
+ len(index_before),
+ hashlib.sha256(index_before).hexdigest().upper(),
+ )
+ self.assertEqual("GIT_INDEX_GUARD_RECOVERED", recovered["status"])
+ second = product.git_deliver(fixture.config, outbox_ids[1], NOW)
+ self.assertEqual(index_before, shared_index.read_bytes())
+ self.assertNotEqual(first["commit_sha"], second["commit_sha"])
+ self.assertEqual([], list(fixture.config.state_dir.glob("git-index-*")))
+ remote_head = subprocess.run(
+ ["git", "--git-dir", str(remote), "rev-parse", "refs/heads/main"], check=True, text=True, capture_output=True,
+ ).stdout.strip()
+ self.assertEqual(second["commit_sha"], remote_head)
+
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ for command in (
+ ["git", "init", "-b", "main"],
+ ["git", "config", "user.name", "Fixture"],
+ ["git", "config", "user.email", "fixture@example.invalid"],
+ ["git", "add", "."],
+ ["git", "commit", "-m", "fixture baseline"],
+ ):
+ subprocess.run(command, cwd=fixture.root, check=True, capture_output=True)
+ remote = fixture.root / "remote.git"
+ subprocess.run(["git", "init", "--bare", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "remote", "add", "origin", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ for stable_id in ("drift-one", "drift-two"):
+ artifact = fixture.archive / f"{stable_id}.txt"
+ artifact.write_bytes((stable_id + "\n").encode("ascii"))
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": stable_id, "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": artifact.name, "bytes": artifact.stat().st_size,
+ "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper(),
+ })
+ outbox_ids = product.reconcile(fixture.config, NOW)["created_outbox_ids"]
+ first = product.git_deliver(fixture.config, outbox_ids[0], NOW)
+ user_file = fixture.root / "late-user-stage.txt"
+ user_file.write_bytes(b"late user stage\n")
+ subprocess.run(["git", "add", user_file.name], cwd=fixture.root, check=True, capture_output=True)
+ outbox_before = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as drift:
+ product.git_deliver(fixture.config, outbox_ids[1], NOW)
+ self.assertEqual("E_GIT_INDEX_DIRTY", drift.exception.code)
+ self.assertEqual(outbox_before, fixture.config.outbox_path.read_bytes())
+ self.assertEqual(first["commit_sha"], subprocess.run(
+ ["git", "rev-parse", "HEAD"], cwd=fixture.root, check=True, text=True, capture_output=True,
+ ).stdout.strip())
+ self.assertEqual([], list(fixture.config.state_dir.glob("git-index-*")))
+
+ def test_existing_git_guard_recovery_revalidates_guard_and_shared_index(self) -> None:
+ baseline_head = "a" * 40
+
+ def prepare(raw: str) -> tuple[PipelineFixture, Path, Path, str, bytes, dict[str, bytes]]:
+ fixture = PipelineFixture(Path(raw))
+ (fixture.root / ".git").mkdir()
+ shared_index = fixture.root / ".git" / "index"
+ index_bytes = b"INDEX-A"
+ shared_index.write_bytes(index_bytes)
+ product.initialize(fixture.config, NOW)
+ product.begin(fixture.config, NOW)
+ artifact = fixture.archive / "guarded.txt"
+ artifact.write_bytes(b"guarded\n")
+ fixture.append_formal({
+ "schema_version": 1, "stable_id": "guarded", "item_type": "text", "status": "SAVED",
+ "creator_uid": UID, "path": artifact.name, "bytes": artifact.stat().st_size,
+ "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest().upper(),
+ })
+ outbox_id = product.reconcile(fixture.config, NOW)["created_outbox_ids"][0]
+ staged = {"value": b"user-staged.txt\0"}
+
+ def runner(command: list[str], **_: object) -> subprocess.CompletedProcess[object]:
+ if command[:3] == ["git", "rev-parse", "HEAD"]:
+ return subprocess.CompletedProcess(command, 0, baseline_head + "\n", "")
+ if command[:5] == ["git", "diff", "--cached", "--name-only", "-z"]:
+ return subprocess.CompletedProcess(command, 0, staged["value"], b"")
+ return subprocess.CompletedProcess(command, 0, "", "")
+
+ rows = product._outbox_rows(fixture.config)
+ item = next(row for row in rows if row.get("event") == "CREATED" and row.get("outbox_id") == outbox_id)
+ guard = product._ensure_git_index_guard(fixture.config, rows, item, runner)
+ guard_path = fixture.config.git_index_guard_path(guard["batch_id"])
+ fixture.guard_runner = runner # type: ignore[attr-defined]
+ return fixture, shared_index, guard_path, outbox_id, index_bytes, staged
+
+ with tempfile.TemporaryDirectory() as raw:
+ fixture, _, _, outbox_id, index_bytes, _ = prepare(raw)
+ result = product.recover_git_index_guard(
+ fixture.config, outbox_id, baseline_head, len(index_bytes), hashlib.sha256(index_bytes).hexdigest().upper(),
+ runner=fixture.guard_runner, # type: ignore[attr-defined]
+ )
+ self.assertEqual("GIT_INDEX_GUARD_ALREADY_DURABLE", result["status"])
+
+ for case in ("bytes", "staged_paths", "file_identity", "guard_field"):
+ with self.subTest(case=case), tempfile.TemporaryDirectory() as raw:
+ fixture, shared_index, guard_path, outbox_id, index_bytes, staged = prepare(raw)
+ if case == "bytes":
+ shared_index.write_bytes(b"INDEX-B")
+ elif case == "staged_paths":
+ staged["value"] = b"other-user-staged.txt\0"
+ elif case == "file_identity":
+ replacement = shared_index.with_name("index-replacement")
+ replacement.write_bytes(index_bytes)
+ os.replace(replacement, shared_index)
+ else:
+ guard = json.loads(guard_path.read_text(encoding="ascii"))
+ guard["outbox_ids"] = ["F" * 64]
+ guard_path.write_bytes(canonical(guard))
+ outbox_before = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as rejected:
+ product.recover_git_index_guard(
+ fixture.config, outbox_id, baseline_head, len(index_bytes),
+ hashlib.sha256(index_bytes).hexdigest().upper(),
+ runner=fixture.guard_runner, # type: ignore[attr-defined]
+ )
+ self.assertEqual("E_GIT_INDEX_DIRTY", rejected.exception.code)
+ self.assertEqual(outbox_before, fixture.config.outbox_path.read_bytes())
+
+ for broken in (False, True):
+ with self.subTest(symlink_broken=broken), tempfile.TemporaryDirectory() as raw:
+ fixture, _, guard_path, outbox_id, index_bytes, _ = prepare(raw)
+ target = guard_path.with_name("guard-link-target.json")
+ if not broken:
+ target.write_bytes(guard_path.read_bytes())
+ guard_path.unlink()
+ try:
+ os.symlink(target, guard_path)
+ except OSError:
+ continue
+ outbox_before = fixture.config.outbox_path.read_bytes()
+ with self.assertRaises(product.PipelineError) as rejected:
+ product.recover_git_index_guard(
+ fixture.config, outbox_id, baseline_head, len(index_bytes),
+ hashlib.sha256(index_bytes).hexdigest().upper(),
+ runner=fixture.guard_runner, # type: ignore[attr-defined]
+ )
+ self.assertEqual("E_GIT_INDEX_DIRTY", rejected.exception.code)
+ self.assertEqual(outbox_before, fixture.config.outbox_path.read_bytes())
+
+ def test_full_chain_reparse_and_identity_drift_fail_closed(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ artifact = fixture.archive / "stable.txt"
+ artifact.write_bytes(b"stable\n")
+ digest = hashlib.sha256(artifact.read_bytes()).hexdigest().upper()
+ original = product._strict_chain
+ calls = 0
+
+ def drifting(root: Path, target: Path, *, final_file: bool) -> tuple[tuple[str, tuple[int, ...]], ...]:
+ nonlocal calls
+ calls += 1
+ value = original(root, target, final_file=final_file)
+ if calls == 2:
+ mutated = list(value)
+ name, identity = mutated[-2]
+ mutated[-2] = (name, identity[:-1] + (identity[-1] + 1,))
+ return tuple(mutated)
+ return value
+
+ with mock.patch.object(product, "_strict_chain", side_effect=drifting):
+ with self.assertRaises(product.PipelineError) as drift:
+ product._artifact("stable.txt", artifact.stat().st_size, digest, fixture.config)
+ self.assertEqual("E_ARTIFACT", drift.exception.code)
+ outside = fixture.root / "outside.txt"
+ outside.write_bytes(b"outside")
+ link = fixture.archive / "linked.txt"
+ try:
+ os.symlink(outside, link)
+ except OSError:
+ return
+ with self.assertRaises(product.PipelineError) as reparse:
+ product._artifact("linked.txt", outside.stat().st_size, hashlib.sha256(outside.read_bytes()).hexdigest().upper(), fixture.config)
+ self.assertEqual("E_ARTIFACT", reparse.exception.code)
+
+ def test_finish_allows_next_slot_and_cli_is_ascii_only(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ product.initialize(fixture.config, NOW)
+ first = product.begin(fixture.config, NOW)
+ finished = product.finish(fixture.config, "COMPLETE", NOW)
+ self.assertEqual(first["run"]["run_id"], finished["run_id"])
+ second = product.begin(fixture.config, datetime(2026, 8, 29, 8, 30, tzinfo=timezone.utc))
+ self.assertNotEqual(first["run"]["run_id"], second["run"]["run_id"])
+ code, result = product.run(["--config", str(fixture.config_path), "pending"])
+ self.assertEqual((0, "PENDING"), (code, result["status"]))
+ product._canonical(result).decode("ascii")
+
+ def test_git_preflight_detects_index_regression_without_mutating_shared_index(self) -> None:
+ class Interrupted(BaseException):
+ pass
+
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ subprocess.run(["git", "init", "-b", "main"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "config", "user.name", "Fixture"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "add", "."], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "commit", "-m", "baseline"], cwd=fixture.root, check=True, capture_output=True)
+ relative = (fixture.archive / "old.txt").relative_to(fixture.root).as_posix()
+ subprocess.run(["git", "rm", "--cached", "--", relative], cwd=fixture.root, check=True, capture_output=True)
+ index_path = fixture.root / ".git" / "index"
+ lock_path = fixture.root / ".git" / "index.lock"
+ lock_path.write_bytes(b"")
+ before = index_path.read_bytes()
+ before_stat = os.stat(index_path)
+ result = product.git_preflight(fixture.config)
+ self.assertFalse(result["index_matches_head"])
+ self.assertEqual([relative], result["archive_staged_delete_present"])
+ self.assertEqual((0, hashlib.sha256(b"").hexdigest().upper()), (
+ result["stale_index_lock"]["bytes"], result["stale_index_lock"]["sha256"]
+ ))
+ self.assertEqual(before, index_path.read_bytes())
+ self.assertEqual((before_stat.st_dev, before_stat.st_ino), (os.stat(index_path).st_dev, os.stat(index_path).st_ino))
+
+ def fail_runner(*_args: object, **_kwargs: object) -> object:
+ raise RuntimeError("synthetic read failure")
+
+ with self.assertRaises(RuntimeError):
+ product.git_preflight(fixture.config, runner=fail_runner) # type: ignore[arg-type]
+ self.assertEqual(before, index_path.read_bytes())
+
+ def interrupt_runner(*_args: object, **_kwargs: object) -> object:
+ raise Interrupted()
+
+ with self.assertRaises(Interrupted):
+ product.git_preflight(fixture.config, runner=interrupt_runner) # type: ignore[arg-type]
+ self.assertEqual(before, index_path.read_bytes())
+
+ def test_git_preflight_uses_one_held_index_during_transient_path_swap(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ subprocess.run(["git", "init", "-b", "main"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "config", "user.name", "Fixture"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "add", "."], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "commit", "-m", "baseline"], cwd=fixture.root, check=True, capture_output=True)
+ index_path = fixture.root / ".git" / "index"
+ alternate = fixture.root / ".git" / "alternate-index"
+ parked = fixture.root / ".git" / "original-index-held-test"
+ shutil.copyfile(index_path, alternate)
+ relative = (fixture.archive / "old.txt").relative_to(fixture.root).as_posix()
+ alternate_env = {**os.environ, "GIT_INDEX_FILE": str(alternate)}
+ subprocess.run(
+ ["git", "rm", "--cached", "--", relative],
+ cwd=fixture.root,
+ env=alternate_env,
+ check=True,
+ capture_output=True,
+ )
+ alternate_staged = subprocess.run(
+ ["git", "diff", "--cached", "--name-only", "HEAD", "--"],
+ cwd=fixture.root,
+ env=alternate_env,
+ check=True,
+ text=True,
+ capture_output=True,
+ ).stdout.splitlines()
+ self.assertEqual([relative], alternate_staged)
+ before = index_path.read_bytes()
+ before_stat = os.stat(index_path)
+ race = {"attempted": False, "performed": False, "denied": False}
+
+ def swap_runner(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[object]:
+ if not race["attempted"] and command[1:2] == ["diff"]:
+ race["attempted"] = True
+ try:
+ os.replace(index_path, parked)
+ os.replace(alternate, index_path)
+ race["performed"] = True
+ except OSError:
+ race["denied"] = True
+ try:
+ return subprocess.run(command, **kwargs) # type: ignore[arg-type]
+ finally:
+ if race["performed"]:
+ os.replace(index_path, alternate)
+ os.replace(parked, index_path)
+ return subprocess.run(command, **kwargs) # type: ignore[arg-type]
+
+ result = product.git_preflight(fixture.config, runner=swap_runner) # type: ignore[arg-type]
+ self.assertTrue(race["attempted"])
+ self.assertTrue(race["performed"] or race["denied"])
+ self.assertTrue(result["index_matches_head"])
+ self.assertEqual([], result["staged_paths"])
+ self.assertEqual([], result["archive_staged_delete_present"])
+ self.assertEqual(before, index_path.read_bytes())
+ self.assertEqual(
+ (before_stat.st_dev, before_stat.st_ino),
+ (os.stat(index_path).st_dev, os.stat(index_path).st_ino),
+ )
+
+ def test_migration_restarts_from_one_durable_report_at_every_write_boundary(self) -> None:
+ class Interrupted(BaseException):
+ pass
+
+ boundaries = [
+ *(f"move-{index}" for index in range(0, 7)),
+ "rmdir-1",
+ "rmdir-2",
+ "outbox",
+ ]
+ for boundary in boundaries:
+ with self.subTest(boundary=boundary), tempfile.TemporaryDirectory() as raw:
+ fixture, legacy_paths, payloads, flac = prepare_migration_fixture(Path(raw))
+ initial_plan = product.plan_video_artifact_migration(fixture.config, NOW)
+ original_move = product._move_relocation_file
+ original_rmdir = Path.rmdir
+ original_append = product._append_outbox
+ move_calls = 0
+ rmdir_calls = 0
+
+ def interrupted_move(*args: object, **kwargs: object) -> None:
+ nonlocal move_calls
+ if boundary == "move-0" and move_calls == 0:
+ raise Interrupted()
+ original_move(*args, **kwargs)
+ move_calls += 1
+ if boundary == f"move-{move_calls}":
+ raise Interrupted()
+
+ def interrupted_rmdir(path: Path) -> None:
+ nonlocal rmdir_calls
+ original_rmdir(path)
+ if path.name in {f"{BVID}.transcript", f"{BVID}.minutes"}:
+ rmdir_calls += 1
+ if boundary == f"rmdir-{rmdir_calls}":
+ raise Interrupted()
+
+ def interrupted_append(*args: object, **kwargs: object) -> str:
+ value = original_append(*args, **kwargs)
+ if boundary == "outbox":
+ raise Interrupted()
+ return value
+
+ with (
+ mock.patch.object(product, "_move_relocation_file", side_effect=interrupted_move),
+ mock.patch.object(Path, "rmdir", new=interrupted_rmdir),
+ mock.patch.object(product, "_append_outbox", side_effect=interrupted_append),
+ ):
+ with self.assertRaises(Interrupted):
+ product.migrate_video_artifacts(fixture.config, NOW)
+ reports = list(fixture.config.relocation_root.glob("*.json"))
+ self.assertEqual(1, len(reports))
+ self.assertEqual(initial_plan, product._read_relocation_report(fixture.config, reports[0]))
+
+ completed = product.migrate_video_artifacts(
+ fixture.config,
+ datetime(2026, 8, 29, 8, 1, tzinfo=timezone.utc),
+ )
+ self.assertEqual(initial_plan["batch_id"], completed["batch_id"])
+ self.assertEqual(initial_plan["created_at"], product._read_relocation_report(fixture.config, reports[0])["created_at"])
+ rows = product._outbox_rows(fixture.config)
+ created = [
+ row for row in rows
+ if row.get("event") == "CREATED" and row.get("outbox_id") == completed["git_outbox_id"]
+ ]
+ self.assertEqual(1, len(created))
+ for alias in initial_plan["items"][0]["aliases"]:
+ self.assertFalse((fixture.root / alias["old_path"]).exists())
+ self.assertEqual(payloads[alias["kind"]], (fixture.root / alias["new_path"]).read_bytes())
+ target_flac = fixture.root / "external-video" / "intermediate" / "transcription" / f"{BVID}.audio.flac"
+ self.assertFalse(flac.exists())
+ self.assertEqual(b"fLaC-fixture", target_flac.read_bytes())
+ self.assertFalse((fixture.archive / f"{BVID}.transcript").exists())
+ self.assertFalse((fixture.archive / f"{BVID}.minutes").exists())
+ self.assertTrue(all(not path.exists() for path in legacy_paths.values()))
+
+ def test_relocation_source_replacement_cannot_delete_replacement_bytes(self) -> None:
+ source_tmp_root = PROJECT_ROOT / "dev" / "tmp"
+ source_tmp_root.mkdir(parents=True, exist_ok=True)
+ target_parents = [source_tmp_root]
+ if Path("F:/").is_dir():
+ target_parents.append(Path("F:/"))
+ for target_parent in target_parents:
+ with (
+ self.subTest(target_parent=str(target_parent)),
+ tempfile.TemporaryDirectory(dir=source_tmp_root) as source_raw,
+ tempfile.TemporaryDirectory(dir=target_parent) as target_raw,
+ ):
+ source_root = Path(source_raw)
+ target_root = Path(target_raw)
+ source = source_root / "source.bin"
+ target = target_root / "target.bin"
+ parked = source_root / "parked-original.bin"
+ source.write_bytes(b"AAAA")
+ original_delete = product._delete_held_relocation_source
+ race = {"swapped": False, "denied": False}
+
+ def replace_before_delete(*args: object, **kwargs: object) -> None:
+ try:
+ os.replace(source, parked)
+ source.write_bytes(b"BBBB")
+ race["swapped"] = True
+ except OSError:
+ race["denied"] = True
+ original_delete(*args, **kwargs)
+
+ if os.name == "nt":
+ with mock.patch.object(product, "_delete_held_relocation_source", side_effect=replace_before_delete):
+ product._move_relocation_file(
+ source_root,
+ target_root,
+ source,
+ target,
+ 4,
+ hashlib.sha256(b"AAAA").hexdigest().upper(),
+ )
+ self.assertTrue(race["denied"])
+ self.assertFalse(race["swapped"])
+ self.assertFalse(source.exists())
+ else:
+ with (
+ mock.patch.object(product, "_delete_held_relocation_source", side_effect=replace_before_delete),
+ self.assertRaises(product.PipelineError) as rejected,
+ ):
+ product._move_relocation_file(
+ source_root,
+ target_root,
+ source,
+ target,
+ 4,
+ hashlib.sha256(b"AAAA").hexdigest().upper(),
+ )
+ self.assertEqual("E_RELOCATION", rejected.exception.code)
+ self.assertTrue(race["swapped"])
+ self.assertEqual(b"BBBB", source.read_bytes())
+ self.assertEqual(b"AAAA", parked.read_bytes())
+ self.assertEqual(b"AAAA", target.read_bytes())
+
+ def test_relocation_report_is_rebound_to_the_exact_external_plan(self) -> None:
+ mutations = (
+ "missing_intermediate", "missing_remove", "cross_path", "extra_alias",
+ "omitted_legacy_id", "omitted_canonical_id",
+ )
+ for mutation in mutations:
+ with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as raw:
+ fixture, legacy_paths, payloads, flac = prepare_migration_fixture(Path(raw))
+ report = json.loads(json.dumps(product.plan_video_artifact_migration(fixture.config, NOW)))
+ if mutation == "missing_intermediate":
+ report["items"][0]["intermediates"] = []
+ elif mutation == "missing_remove":
+ report["remove_paths"] = report["remove_paths"][1:]
+ elif mutation == "cross_path":
+ report["items"][0]["aliases"][0]["old_path"] = fixture.formal.relative_to(fixture.root).as_posix()
+ elif mutation == "extra_alias":
+ report["items"][0]["aliases"].append(dict(report["items"][0]["aliases"][0]))
+ else:
+ omitted = "BV1Q541167Qf"
+ omitted_title = "Omitted legacy item"
+ omitted_published = "2026-08-31T01:02:03+08:00"
+ fixture.append_formal({
+ "schema_version": 1,
+ "stable_id": omitted,
+ "item_type": "video",
+ "status": product.VIDEO_COMPLETE,
+ "creator": "fixture",
+ "title": omitted_title,
+ "published_at": omitted_published,
+ })
+ omitted_base = omitted if mutation == "omitted_legacy_id" else product._canonical_video_base(
+ omitted,
+ omitted_title,
+ omitted_published,
+ )
+ omitted_dir = fixture.archive / f"{omitted_base}.transcript"
+ omitted_dir.mkdir()
+ (omitted_dir / f"{omitted_base}.txt").write_bytes(b"omitted\n")
+ (omitted_dir / f"{omitted_base}.srt").write_bytes(b"1\n00:00:00,000 --> 00:00:01,000\nomitted\n")
+ (omitted_dir / f"{omitted_base}.json").write_bytes(b'{"segments":[]}\n')
+ report["batch_id"] = product._relocation_batch_id(report)
+ fixture.config.relocation_root.mkdir(parents=True)
+ forged_path = fixture.config.relocation_root / f"{report['batch_id']}.json"
+ forged_path.write_bytes(canonical(report))
+ state_before = fixture.config.state_path.read_bytes()
+ formal_before = fixture.formal.read_bytes()
+ with self.assertRaises(product.PipelineError) as rejected:
+ product.migrate_video_artifacts(fixture.config, NOW)
+ self.assertEqual("E_RELOCATION", rejected.exception.code)
+ self.assertEqual(state_before, fixture.config.state_path.read_bytes())
+ self.assertEqual(formal_before, fixture.formal.read_bytes())
+ self.assertFalse(fixture.config.outbox_path.exists())
+ self.assertEqual(b"fLaC-fixture", flac.read_bytes())
+ for kind, path in legacy_paths.items():
+ self.assertEqual(payloads[kind], path.read_bytes())
+ canonical_base = product._canonical_video_base(BVID, "A / canonical: title?", "2026-08-30T12:34:56+08:00")
+ self.assertFalse((fixture.archive / f"{canonical_base}.transcript").exists())
+ self.assertFalse((fixture.archive / f"{canonical_base}.minutes").exists())
+
+ def test_video_artifact_migration_is_byte_exact_alias_resolvable_and_git_scoped(self) -> None:
+ with tempfile.TemporaryDirectory() as raw:
+ fixture = PipelineFixture(Path(raw))
+ (fixture.root / "external-video").mkdir()
+ stable_id = BVID
+ title = "A / canonical: title?"
+ published_at = "2026-08-30T12:34:56+08:00"
+ fixture.append_formal({
+ "schema_version": 1,
+ "stable_id": stable_id,
+ "item_type": "video",
+ "status": product.VIDEO_COMPLETE,
+ "creator_uid": UID,
+ "title": title,
+ "published_at": published_at,
+ })
+ transcript_dir = fixture.archive / f"{stable_id}.transcript"
+ minutes_dir = fixture.archive / f"{stable_id}.minutes"
+ transcript_dir.mkdir()
+ minutes_dir.mkdir()
+ payloads = {
+ "transcript_txt": b"transcript\n",
+ "transcript_srt": b"1\n00:00:00,000 --> 00:00:01,000\ntext\n",
+ "transcript_json": b'{"segments":[]}\n',
+ "minutes_md": b"# minutes\n",
+ "minutes_pdf": b"%PDF-fixture\n",
+ }
+ legacy_paths = {
+ "transcript_txt": transcript_dir / f"{stable_id}.txt",
+ "transcript_srt": transcript_dir / f"{stable_id}.srt",
+ "transcript_json": transcript_dir / f"{stable_id}.json",
+ "minutes_md": minutes_dir / f"{stable_id}.md",
+ "minutes_pdf": minutes_dir / f"{stable_id}.pdf",
+ }
+ for kind, path in legacy_paths.items():
+ path.write_bytes(payloads[kind])
+ flac = transcript_dir / f"{stable_id}.audio.flac"
+ flac.write_bytes(b"fLaC-fixture")
+
+ subprocess.run(["git", "init", "-b", "main"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "config", "user.name", "Fixture"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "add", "."], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "commit", "-m", "legacy baseline"], cwd=fixture.root, check=True, capture_output=True)
+ remote = fixture.root / "remote.git"
+ subprocess.run(["git", "init", "--bare", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "remote", "add", "origin", str(remote)], cwd=fixture.root, check=True, capture_output=True)
+ subprocess.run(["git", "push", "-u", "origin", "main"], cwd=fixture.root, check=True, capture_output=True)
+ (fixture.archive / "目录导读.md").write_bytes(b"# canonical naming guide\n")
+ product.initialize(fixture.config, NOW)
+ index_path = fixture.root / ".git" / "index"
+ index_before = index_path.read_bytes()
+
+ plan = product.plan_video_artifact_migration(fixture.config, NOW)
+ expected_base = "20260830-123456_video_A _ canonical_ title__BV1Q541167Qg"
+ self.assertEqual(expected_base, plan["items"][0]["canonical_base"])
+ result = product.migrate_video_artifacts(fixture.config, NOW)
+ self.assertEqual(("VIDEO_ARTIFACTS_MIGRATED", 1, 5, 1), (
+ result["status"], result["item_count"], result["public_file_count"], result["intermediate_count"]
+ ))
+ self.assertEqual(index_before, index_path.read_bytes())
+ for alias in plan["items"][0]["aliases"]:
+ old_path = fixture.root / alias["old_path"]
+ new_path = fixture.root / alias["new_path"]
+ self.assertFalse(old_path.exists())
+ self.assertEqual(payloads[alias["kind"]], new_path.read_bytes())
+ rebound = product._artifact(alias["old_path"], alias["bytes"], alias["sha256"], fixture.config)
+ self.assertEqual(alias["old_path"], rebound["path"])
+ external_flac = fixture.root / "external-video" / "intermediate" / "transcription" / f"{stable_id}.audio.flac"
+ self.assertEqual(b"fLaC-fixture", external_flac.read_bytes())
+ self.assertFalse(flac.exists())
+ report_path = fixture.root / result["report"]["path"]
+ self.assertTrue(report_path.is_file())
+ rerun = product.migrate_video_artifacts(fixture.config, NOW)
+ self.assertEqual(result["git_outbox_id"], rerun["git_outbox_id"])
+
+ delivered = product.git_deliver(fixture.config, result["git_outbox_id"], NOW)
+ self.assertEqual("GIT_PUSHED", delivered["status"])
+ self.assertEqual(index_before, index_path.read_bytes())
+ changed = subprocess.run(
+ ["git", "diff-tree", "--no-commit-id", "--name-status", "-r", "HEAD"],
+ cwd=fixture.root,
+ check=True,
+ text=True,
+ capture_output=True,
+ ).stdout
+ for alias in plan["items"][0]["aliases"]:
+ self.assertIn(alias["old_path"], changed)
+ self.assertIn(alias["new_path"], changed)
+ self.assertNotIn(".flac", changed.lower())
+ local_head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=fixture.root, check=True, text=True, capture_output=True).stdout.strip()
+ remote_head = subprocess.run(
+ ["git", "--git-dir", str(remote), "rev-parse", "refs/heads/main"],
+ check=True,
+ text=True,
+ capture_output=True,
+ ).stdout.strip()
+ self.assertEqual(local_head, remote_head)
+
+
+if __name__ == "__main__":
+ unittest.main()
--
Gitblit v1.9.3