Added more folders and restructured classes inside them.
This commit is contained in:
1
src/ptlinux/game/__init__.py
Normal file
1
src/ptlinux/game/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Launcher-owned game data, synchronization, and compatibility services."""
|
||||
1
src/ptlinux/game/compatibility/__init__.py
Normal file
1
src/ptlinux/game/compatibility/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Verified compatibility transforms for publisher game data."""
|
||||
1081
src/ptlinux/game/compatibility/assbin.py
Normal file
1081
src/ptlinux/game/compatibility/assbin.py
Normal file
File diff suppressed because it is too large
Load Diff
87
src/ptlinux/game/depot/__init__.py
Normal file
87
src/ptlinux/game/depot/__init__.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""Public models for the verified game-depot feature."""
|
||||
|
||||
from .catalogs import launch_catalog_from_snapshot, select_depot_start_option
|
||||
from .filesystem import (
|
||||
apply_create_if_missing,
|
||||
apply_deletions,
|
||||
ensure_target_parent,
|
||||
quarantine_unmanaged_files,
|
||||
recover_crlf_manifest_file,
|
||||
safe_imported_user_file,
|
||||
write_json,
|
||||
)
|
||||
from .integrity import (
|
||||
build_depot_file_catalog,
|
||||
inspect_managed_game,
|
||||
plan_update,
|
||||
)
|
||||
from .manifest import parse_snapshot
|
||||
from .models import (
|
||||
DEFAULT_DEPOT_START_OPTION,
|
||||
CreateIfMissing,
|
||||
DepotDownloadMetrics,
|
||||
DepotError,
|
||||
DepotFile,
|
||||
DepotFileCatalog,
|
||||
DepotFileCatalogEntry,
|
||||
DepotIntegrityIssue,
|
||||
DepotLaunchCatalog,
|
||||
DepotPreflightResult,
|
||||
DepotSnapshot,
|
||||
DepotStartOption,
|
||||
DepotSyncResult,
|
||||
PlannedDownload,
|
||||
ProgressCallback,
|
||||
QuarantineResult,
|
||||
SyncProgress,
|
||||
UpdatePlan,
|
||||
)
|
||||
from .transfer import (
|
||||
AdaptiveConcurrencyController,
|
||||
BandwidthLimiter,
|
||||
DepotHttpClient,
|
||||
DownloadTelemetry,
|
||||
HttpOpener,
|
||||
HttpResponse,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_DEPOT_START_OPTION",
|
||||
"AdaptiveConcurrencyController",
|
||||
"BandwidthLimiter",
|
||||
"CreateIfMissing",
|
||||
"DepotDownloadMetrics",
|
||||
"DepotError",
|
||||
"DepotFile",
|
||||
"DepotFileCatalog",
|
||||
"DepotFileCatalogEntry",
|
||||
"DepotHttpClient",
|
||||
"DepotIntegrityIssue",
|
||||
"DepotLaunchCatalog",
|
||||
"DepotPreflightResult",
|
||||
"DepotSnapshot",
|
||||
"DepotStartOption",
|
||||
"DepotSyncResult",
|
||||
"DownloadTelemetry",
|
||||
"HttpOpener",
|
||||
"HttpResponse",
|
||||
"PlannedDownload",
|
||||
"ProgressCallback",
|
||||
"QuarantineResult",
|
||||
"SyncProgress",
|
||||
"UpdatePlan",
|
||||
"apply_create_if_missing",
|
||||
"apply_deletions",
|
||||
"build_depot_file_catalog",
|
||||
"ensure_target_parent",
|
||||
"inspect_managed_game",
|
||||
"launch_catalog_from_snapshot",
|
||||
"plan_update",
|
||||
"parse_snapshot",
|
||||
"quarantine_unmanaged_files",
|
||||
"recover_crlf_manifest_file",
|
||||
"safe_imported_user_file",
|
||||
"select_depot_start_option",
|
||||
"write_json",
|
||||
]
|
||||
208
src/ptlinux/game/depot/catalogs.py
Normal file
208
src/ptlinux/game/depot/catalogs.py
Normal file
@@ -0,0 +1,208 @@
|
||||
"""Pure serialization and strict parsing for cached depot catalogs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict
|
||||
|
||||
from .manifest import (
|
||||
SHA1_PATTERN,
|
||||
non_negative_int,
|
||||
normalize_relative_path,
|
||||
parse_start_options,
|
||||
positive_int,
|
||||
required_string,
|
||||
)
|
||||
from .models import (
|
||||
DEFAULT_DEPOT_START_OPTION,
|
||||
DepotError,
|
||||
DepotFileCatalog,
|
||||
DepotFileCatalogEntry,
|
||||
DepotLaunchCatalog,
|
||||
DepotSnapshot,
|
||||
DepotStartOption,
|
||||
)
|
||||
from .realms import launcher_launch_catalog, launcher_realm_name
|
||||
|
||||
|
||||
DEPOT_LAUNCH_CATALOG_SCHEMA = 1
|
||||
DEPOT_FILE_CATALOG_SCHEMA = 1
|
||||
|
||||
|
||||
def launch_catalog_from_snapshot(snapshot: DepotSnapshot) -> DepotLaunchCatalog:
|
||||
return launcher_launch_catalog(
|
||||
DepotLaunchCatalog(
|
||||
build=snapshot.build,
|
||||
start_options=snapshot.start_options,
|
||||
default_start_option=snapshot.default_start_option,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def select_depot_start_option(
|
||||
catalog: DepotLaunchCatalog | None,
|
||||
preferred_realm: str | None = None,
|
||||
) -> DepotStartOption:
|
||||
"""Choose a published realm, falling back to Babel when possible."""
|
||||
|
||||
current = launcher_launch_catalog(
|
||||
catalog
|
||||
or DepotLaunchCatalog(
|
||||
build=0,
|
||||
start_options=(DEFAULT_DEPOT_START_OPTION,),
|
||||
default_start_option=DEFAULT_DEPOT_START_OPTION.name,
|
||||
)
|
||||
)
|
||||
if preferred_realm:
|
||||
preferred_names = {
|
||||
preferred_realm.casefold(),
|
||||
launcher_realm_name(preferred_realm).casefold(),
|
||||
}
|
||||
for option in current.start_options:
|
||||
if option.name.casefold() in preferred_names:
|
||||
return option
|
||||
for option in current.start_options:
|
||||
if option.name.casefold() == DEFAULT_DEPOT_START_OPTION.name.casefold():
|
||||
return option
|
||||
for option in current.start_options:
|
||||
if option.name.casefold() == current.default_start_option.casefold():
|
||||
return option
|
||||
return current.start_options[0]
|
||||
|
||||
|
||||
def launch_catalog_payload(snapshot: DepotSnapshot) -> dict[str, object]:
|
||||
catalog = launch_catalog_from_snapshot(snapshot)
|
||||
return {
|
||||
"schema": DEPOT_LAUNCH_CATALOG_SCHEMA,
|
||||
"build": catalog.build,
|
||||
"default_start_option": catalog.default_start_option,
|
||||
"start_options": [asdict(option) for option in catalog.start_options],
|
||||
}
|
||||
|
||||
|
||||
def parse_launch_catalog(value: object) -> DepotLaunchCatalog | None:
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
if value.get("schema") != DEPOT_LAUNCH_CATALOG_SCHEMA:
|
||||
return None
|
||||
build = positive_int(value.get("build"), "cached depot build")
|
||||
raw_options = value.get("start_options")
|
||||
if not isinstance(raw_options, list):
|
||||
return None
|
||||
manifest_options = [
|
||||
{
|
||||
"Name": item.get("name"),
|
||||
"Path": item.get("path"),
|
||||
"Arguments": item.get("arguments"),
|
||||
}
|
||||
if isinstance(item, dict)
|
||||
else item
|
||||
for item in raw_options
|
||||
]
|
||||
options, default_name = parse_start_options(
|
||||
manifest_options,
|
||||
value.get("default_start_option"),
|
||||
)
|
||||
return launcher_launch_catalog(
|
||||
DepotLaunchCatalog(
|
||||
build=build,
|
||||
start_options=options,
|
||||
default_start_option=default_name,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def depot_file_catalog_payload(catalog: DepotFileCatalog) -> dict[str, object]:
|
||||
"""Use compact rows because this cache contains tens of thousands of files."""
|
||||
|
||||
return {
|
||||
"schema": DEPOT_FILE_CATALOG_SCHEMA,
|
||||
"build": catalog.build,
|
||||
"created_at": catalog.created_at,
|
||||
"files": [
|
||||
[
|
||||
entry.path,
|
||||
entry.size,
|
||||
entry.sha1,
|
||||
entry.device,
|
||||
entry.inode,
|
||||
entry.mtime_ns,
|
||||
entry.ctime_ns,
|
||||
entry.verified,
|
||||
]
|
||||
for entry in catalog.files
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def parse_depot_file_catalog_entry(value: object) -> DepotFileCatalogEntry:
|
||||
if not isinstance(value, list) or len(value) != 8:
|
||||
raise DepotError("cached depot file entry must be an eight-value row")
|
||||
path = normalize_relative_path(value[0], "cached depot file path")
|
||||
size = non_negative_int(value[1], f"cached depot file size for {path}")
|
||||
sha1 = required_string(value[2], f"cached depot file hash for {path}").casefold()
|
||||
if not SHA1_PATTERN.fullmatch(sha1):
|
||||
raise DepotError(f"invalid cached SHA-1 hash for {path}")
|
||||
identity: dict[str, int] = {}
|
||||
for field_name, field_value in zip(
|
||||
("device", "inode", "mtime_ns", "ctime_ns"),
|
||||
value[3:7],
|
||||
strict=True,
|
||||
):
|
||||
identity[field_name] = non_negative_int(
|
||||
field_value,
|
||||
f"cached depot file {field_name} for {path}",
|
||||
)
|
||||
verified = value[7]
|
||||
if not isinstance(verified, bool):
|
||||
raise DepotError(f"cached depot verification flag is invalid for {path}")
|
||||
return DepotFileCatalogEntry(
|
||||
path=path,
|
||||
size=size,
|
||||
sha1=sha1,
|
||||
device=identity["device"],
|
||||
inode=identity["inode"],
|
||||
mtime_ns=identity["mtime_ns"],
|
||||
ctime_ns=identity["ctime_ns"],
|
||||
verified=verified,
|
||||
)
|
||||
|
||||
|
||||
def parse_depot_file_catalog(value: object) -> DepotFileCatalog | None:
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
if value.get("schema") != DEPOT_FILE_CATALOG_SCHEMA:
|
||||
return None
|
||||
build = positive_int(value.get("build"), "cached depot file build")
|
||||
created_at = required_string(
|
||||
value.get("created_at"),
|
||||
"cached depot file timestamp",
|
||||
)
|
||||
raw_files = value.get("files")
|
||||
if not isinstance(raw_files, list):
|
||||
return None
|
||||
entries: list[DepotFileCatalogEntry] = []
|
||||
seen: set[str] = set()
|
||||
for raw in raw_files:
|
||||
entry = parse_depot_file_catalog_entry(raw)
|
||||
if entry.path.casefold() in seen:
|
||||
return None
|
||||
seen.add(entry.path.casefold())
|
||||
entries.append(entry)
|
||||
return DepotFileCatalog(
|
||||
build=build,
|
||||
files=tuple(entries),
|
||||
created_at=created_at,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEPOT_FILE_CATALOG_SCHEMA",
|
||||
"DEPOT_LAUNCH_CATALOG_SCHEMA",
|
||||
"depot_file_catalog_payload",
|
||||
"launch_catalog_from_snapshot",
|
||||
"launch_catalog_payload",
|
||||
"parse_depot_file_catalog",
|
||||
"parse_depot_file_catalog_entry",
|
||||
"parse_launch_catalog",
|
||||
"select_depot_start_option",
|
||||
]
|
||||
338
src/ptlinux/game/depot/filesystem.py
Normal file
338
src/ptlinux/game/depot/filesystem.py
Normal file
@@ -0,0 +1,338 @@
|
||||
"""Filesystem mutation and quarantine operations for the game depot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import stat
|
||||
from typing import Callable, Iterable
|
||||
from uuid import uuid4
|
||||
|
||||
from .integrity import prepare_game_root, safe_target
|
||||
from .manifest import decode_create_if_missing
|
||||
from .models import CreateIfMissing, DepotError, DepotFile, DepotSnapshot, QuarantineResult
|
||||
|
||||
|
||||
DEFAULT_MAX_LINE_ENDING_RECOVERY_BYTES = 1024 * 1024
|
||||
DEFAULT_UNMANAGED_EXECUTABLE_SUFFIXES = frozenset(
|
||||
{
|
||||
".asi",
|
||||
".bat",
|
||||
".cmd",
|
||||
".com",
|
||||
".cpl",
|
||||
".desktop",
|
||||
".dll",
|
||||
".drv",
|
||||
".exe",
|
||||
".jar",
|
||||
".js",
|
||||
".lnk",
|
||||
".lua",
|
||||
".msi",
|
||||
".ocx",
|
||||
".ps1",
|
||||
".py",
|
||||
".reg",
|
||||
".scr",
|
||||
".sh",
|
||||
".so",
|
||||
".url",
|
||||
".vbe",
|
||||
".vbs",
|
||||
".wsf",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def quarantine_unmanaged_files(
|
||||
source_game: Path,
|
||||
quarantine_base: Path,
|
||||
data_root: Path,
|
||||
snapshot: DepotSnapshot,
|
||||
*,
|
||||
unmanaged_executable_suffixes: frozenset[str] = (
|
||||
DEFAULT_UNMANAGED_EXECUTABLE_SUFFIXES
|
||||
),
|
||||
prepare_root: Callable[..., Path] = prepare_game_root,
|
||||
safe_user_file: Callable[[Path, str], bool] | None = None,
|
||||
write_manifest: Callable[[Path, object], None] | None = None,
|
||||
clock: Callable[[], datetime] | None = None,
|
||||
identifier: Callable[[], str] | None = None,
|
||||
) -> QuarantineResult:
|
||||
"""Move non-depot import contents aside without following links."""
|
||||
|
||||
root = prepare_root(source_game, create=False)
|
||||
if not root.exists():
|
||||
return QuarantineResult(0, None)
|
||||
allowed = {file.path for file in snapshot.files} | {
|
||||
entry.path for entry in snapshot.create_if_missing
|
||||
}
|
||||
candidates: list[tuple[Path, str]] = []
|
||||
user_file_is_safe = safe_user_file or (
|
||||
lambda candidate_path, candidate_relative: safe_imported_user_file(
|
||||
candidate_path,
|
||||
candidate_relative,
|
||||
executable_suffixes=unmanaged_executable_suffixes,
|
||||
)
|
||||
)
|
||||
|
||||
def walk_error(error: OSError) -> None:
|
||||
raise DepotError(f"cannot inspect imported game tree: {error}") from error
|
||||
|
||||
for directory, directory_names, file_names in os.walk(
|
||||
root,
|
||||
topdown=True,
|
||||
followlinks=False,
|
||||
onerror=walk_error,
|
||||
):
|
||||
parent = Path(directory)
|
||||
for name in tuple(directory_names):
|
||||
path = parent / name
|
||||
try:
|
||||
status = path.lstat()
|
||||
except OSError as exc:
|
||||
raise DepotError(
|
||||
f"cannot inspect imported entry {path}: {exc}"
|
||||
) from exc
|
||||
if stat.S_ISDIR(status.st_mode):
|
||||
continue
|
||||
directory_names.remove(name)
|
||||
candidates.append((path, path.relative_to(root).as_posix()))
|
||||
for name in file_names:
|
||||
path = parent / name
|
||||
relative = path.relative_to(root).as_posix()
|
||||
try:
|
||||
status = path.lstat()
|
||||
except OSError as exc:
|
||||
raise DepotError(
|
||||
f"cannot inspect imported entry {path}: {exc}"
|
||||
) from exc
|
||||
if not stat.S_ISREG(status.st_mode):
|
||||
candidates.append((path, relative))
|
||||
continue
|
||||
if relative in allowed:
|
||||
continue
|
||||
if user_file_is_safe(path, relative):
|
||||
continue
|
||||
candidates.append((path, relative))
|
||||
|
||||
if not candidates:
|
||||
return QuarantineResult(0, None)
|
||||
base = quarantine_base.resolve(strict=False)
|
||||
resolved_data_root = data_root.resolve(strict=False)
|
||||
try:
|
||||
base.relative_to(resolved_data_root)
|
||||
except ValueError as exc:
|
||||
raise DepotError(f"import quarantine is outside the data root: {base}") from exc
|
||||
now = clock or (lambda: datetime.now().astimezone())
|
||||
new_identifier = identifier or (lambda: uuid4().hex)
|
||||
stamp = now().strftime("%Y%m%d-%H%M%S")
|
||||
quarantine = base / f"{stamp}-{new_identifier()}"
|
||||
files_root = quarantine / "files"
|
||||
files_root.mkdir(parents=True, exist_ok=False)
|
||||
manifest = {
|
||||
"schema": 1,
|
||||
"complete": False,
|
||||
"created_at": now().isoformat(),
|
||||
"source": str(root),
|
||||
"files": [relative for _path, relative in candidates],
|
||||
}
|
||||
write = write_manifest or write_json
|
||||
write(quarantine / "quarantine.json", manifest)
|
||||
for source, relative in candidates:
|
||||
destination = files_root.joinpath(*PurePosixPath(relative).parts)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
source.replace(destination)
|
||||
except OSError as exc:
|
||||
raise DepotError(
|
||||
f"could not quarantine imported entry {relative}: {exc}"
|
||||
) from exc
|
||||
manifest["complete"] = True
|
||||
write(quarantine / "quarantine.json", manifest)
|
||||
return QuarantineResult(len(candidates), str(quarantine))
|
||||
|
||||
|
||||
def safe_imported_user_file(
|
||||
path: Path,
|
||||
relative: str,
|
||||
*,
|
||||
executable_suffixes: frozenset[str] = DEFAULT_UNMANAGED_EXECUTABLE_SUFFIXES,
|
||||
) -> bool:
|
||||
"""Return whether an unmanaged save entry is safe to leave in place."""
|
||||
|
||||
parts = PurePosixPath(relative).parts
|
||||
if not parts or parts[0].casefold() != "save":
|
||||
return False
|
||||
if path.suffix.casefold() in executable_suffixes:
|
||||
return False
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
prefix = handle.read(4)
|
||||
except OSError as exc:
|
||||
raise DepotError(f"cannot inspect imported user file {path}: {exc}") from exc
|
||||
return not (
|
||||
prefix.startswith(b"MZ")
|
||||
or prefix.startswith(b"\x7fELF")
|
||||
or prefix.startswith(b"#!")
|
||||
)
|
||||
|
||||
|
||||
def apply_deletions(
|
||||
root: Path,
|
||||
relative_paths: Iterable[str],
|
||||
*,
|
||||
target_for: Callable[[Path, str], Path] = safe_target,
|
||||
) -> int:
|
||||
"""Delete publisher-retired files and links within the managed root."""
|
||||
|
||||
deleted = 0
|
||||
for relative_path in relative_paths:
|
||||
target = target_for(root, relative_path)
|
||||
if target.is_symlink() or target.is_file():
|
||||
target.unlink()
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
|
||||
def apply_create_if_missing(
|
||||
root: Path,
|
||||
entries: Iterable[CreateIfMissing],
|
||||
*,
|
||||
target_for: Callable[[Path, str], Path] = safe_target,
|
||||
decode_entry: Callable[[CreateIfMissing], bytes] = decode_create_if_missing,
|
||||
ensure_parent: Callable[[Path, Path], None] | None = None,
|
||||
identifier: Callable[[], str] | None = None,
|
||||
) -> int:
|
||||
"""Atomically create publisher defaults while preserving existing files."""
|
||||
|
||||
created = 0
|
||||
prepare_parent = ensure_parent or ensure_target_parent
|
||||
new_identifier = identifier or (lambda: uuid4().hex)
|
||||
for entry in entries:
|
||||
target = target_for(root, entry.path)
|
||||
if target.is_symlink():
|
||||
raise DepotError(f"refusing to follow symbolic link for {entry.path}")
|
||||
if target.exists():
|
||||
continue
|
||||
content = decode_entry(entry)
|
||||
prepare_parent(root, target)
|
||||
temporary = target.with_name(f".{target.name}.{new_identifier()}.partial")
|
||||
try:
|
||||
with temporary.open("xb") as handle:
|
||||
handle.write(content)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
temporary.replace(target)
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
created += 1
|
||||
return created
|
||||
|
||||
|
||||
def recover_crlf_manifest_file(
|
||||
temporary: Path,
|
||||
destination: Path,
|
||||
file: DepotFile,
|
||||
*,
|
||||
maximum_bytes: int = DEFAULT_MAX_LINE_ENDING_RECOVERY_BYTES,
|
||||
process_id: Callable[[], int] = os.getpid,
|
||||
identifier: Callable[[], str] | None = None,
|
||||
) -> bool:
|
||||
"""Restore publisher-normalized legacy text only on an exact hash match."""
|
||||
|
||||
try:
|
||||
if temporary.stat().st_size > maximum_bytes:
|
||||
return False
|
||||
data = temporary.read_bytes()
|
||||
except OSError as exc:
|
||||
raise DepotError(f"could not inspect downloaded {file.path}: {exc}") from exc
|
||||
if b"\r" in data:
|
||||
return False
|
||||
recovered = data.replace(b"\n", b"\r\n")
|
||||
if len(recovered) != file.size:
|
||||
return False
|
||||
if (
|
||||
hashlib.sha1(recovered, usedforsecurity=False).hexdigest().casefold()
|
||||
!= file.sha1.casefold()
|
||||
):
|
||||
return False
|
||||
new_identifier = identifier or (lambda: uuid4().hex)
|
||||
recovered_temporary = destination.with_name(
|
||||
f".{destination.name}.{process_id()}.{new_identifier()}.recovered"
|
||||
)
|
||||
try:
|
||||
with recovered_temporary.open("xb") as handle:
|
||||
handle.write(recovered)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
recovered_temporary.replace(destination)
|
||||
except BaseException:
|
||||
recovered_temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
def ensure_target_parent(root: Path, target: Path) -> None:
|
||||
"""Create a target's parent directories without traversing symlinks."""
|
||||
|
||||
try:
|
||||
relative_parent = target.parent.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise DepotError(f"target escapes managed game directory: {target}") from exc
|
||||
cursor = root
|
||||
for part in relative_parent.parts:
|
||||
cursor = cursor / part
|
||||
if cursor.is_symlink():
|
||||
raise DepotError(f"refusing to traverse symbolic link in game tree: {cursor}")
|
||||
if cursor.exists():
|
||||
if not cursor.is_dir():
|
||||
raise DepotError(f"cannot create game directory over file: {cursor}")
|
||||
else:
|
||||
try:
|
||||
cursor.mkdir()
|
||||
except FileExistsError:
|
||||
if cursor.is_symlink():
|
||||
raise DepotError(
|
||||
f"refusing to traverse symbolic link in game tree: {cursor}"
|
||||
)
|
||||
if not cursor.is_dir():
|
||||
raise DepotError(f"cannot create game directory over file: {cursor}")
|
||||
except OSError as exc:
|
||||
raise DepotError(f"cannot create game directory {cursor}: {exc}") from exc
|
||||
|
||||
|
||||
def write_json(path: Path, value: object, *, compact: bool = False) -> None:
|
||||
"""Atomically serialize one JSON document beside its destination."""
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
try:
|
||||
payload = (
|
||||
json.dumps(value, separators=(",", ":"), sort_keys=True)
|
||||
if compact
|
||||
else json.dumps(value, indent=2, sort_keys=True)
|
||||
)
|
||||
temporary.write_text(payload + "\n", encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_MAX_LINE_ENDING_RECOVERY_BYTES",
|
||||
"DEFAULT_UNMANAGED_EXECUTABLE_SUFFIXES",
|
||||
"apply_create_if_missing",
|
||||
"apply_deletions",
|
||||
"ensure_target_parent",
|
||||
"quarantine_unmanaged_files",
|
||||
"recover_crlf_manifest_file",
|
||||
"safe_imported_user_file",
|
||||
"write_json",
|
||||
]
|
||||
413
src/ptlinux/game/depot/integrity.py
Normal file
413
src/ptlinux/game/depot/integrity.py
Normal file
@@ -0,0 +1,413 @@
|
||||
"""Read-only update planning and managed-game integrity inspection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import stat
|
||||
import time
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from .manifest import relative_parts
|
||||
from .models import (
|
||||
DepotError,
|
||||
DepotFile,
|
||||
DepotFileCatalog,
|
||||
DepotFileCatalogEntry,
|
||||
DepotIntegrityIssue,
|
||||
DepotPreflightResult,
|
||||
DepotSnapshot,
|
||||
PlannedDownload,
|
||||
UpdatePlan,
|
||||
)
|
||||
|
||||
|
||||
HashFile = Callable[[Path], str]
|
||||
Clock = Callable[[], float]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScannedGameEntry:
|
||||
path: Path
|
||||
state: os.stat_result
|
||||
kind: str
|
||||
|
||||
|
||||
def sha1_file(path: Path, *, read_size: int = 1024 * 1024) -> str:
|
||||
digest = hashlib.sha1(usedforsecurity=False)
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(read_size), b""):
|
||||
digest.update(chunk)
|
||||
except OSError as exc:
|
||||
raise DepotError(f"cannot hash {path}: {exc}") from exc
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def prepare_game_root(game_directory: Path, *, create: bool) -> Path:
|
||||
root = game_directory.expanduser()
|
||||
if root.is_symlink():
|
||||
raise DepotError(f"managed game directory must not be a symbolic link: {root}")
|
||||
if root.exists() and not root.is_dir():
|
||||
raise DepotError(f"managed game path is not a directory: {root}")
|
||||
if create:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root.resolve(strict=False)
|
||||
|
||||
|
||||
def safe_target(root: Path, relative_path: str) -> Path:
|
||||
parts = relative_parts(relative_path, "depot path")
|
||||
target = root.joinpath(*parts)
|
||||
try:
|
||||
target.resolve(strict=False).relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise DepotError(
|
||||
f"depot path escapes the managed game directory: {relative_path}"
|
||||
) from exc
|
||||
return target
|
||||
|
||||
|
||||
def scan_game_tree(root: Path) -> dict[str, ScannedGameEntry]:
|
||||
"""Inventory files without following symlinks or resolving every manifest path."""
|
||||
|
||||
entries: dict[str, ScannedGameEntry] = {}
|
||||
|
||||
def scan(directory: Path, relative_parent: PurePosixPath) -> None:
|
||||
children = scan_directory(directory)
|
||||
for child in children:
|
||||
relative = relative_parent / child.name
|
||||
key = relative.as_posix()
|
||||
try:
|
||||
state = child.stat(follow_symlinks=False)
|
||||
except OSError as exc:
|
||||
raise DepotError(
|
||||
f"cannot inspect managed game path {key}: {exc}"
|
||||
) from exc
|
||||
path = Path(child.path)
|
||||
if stat.S_ISLNK(state.st_mode):
|
||||
entries[key] = ScannedGameEntry(path, state, "symbolic link")
|
||||
elif stat.S_ISDIR(state.st_mode):
|
||||
scan(path, relative)
|
||||
elif stat.S_ISREG(state.st_mode):
|
||||
entries[key] = ScannedGameEntry(path, state, "file")
|
||||
else:
|
||||
entries[key] = ScannedGameEntry(path, state, "not a regular file")
|
||||
|
||||
scan(root, PurePosixPath())
|
||||
return entries
|
||||
|
||||
|
||||
def scan_directory(directory: Path) -> list[os.DirEntry[str]]:
|
||||
try:
|
||||
with os.scandir(directory) as iterator:
|
||||
return list(iterator)
|
||||
except OSError as exc:
|
||||
raise DepotError(f"cannot scan managed game directory {directory}: {exc}") from exc
|
||||
|
||||
|
||||
def catalog_entry(
|
||||
file: DepotFile,
|
||||
state: os.stat_result,
|
||||
*,
|
||||
verified: bool,
|
||||
) -> DepotFileCatalogEntry:
|
||||
return DepotFileCatalogEntry(
|
||||
path=file.path,
|
||||
size=file.size,
|
||||
sha1=file.sha1.casefold(),
|
||||
device=state.st_dev,
|
||||
inode=state.st_ino,
|
||||
mtime_ns=state.st_mtime_ns,
|
||||
ctime_ns=state.st_ctime_ns,
|
||||
verified=verified,
|
||||
)
|
||||
|
||||
|
||||
def catalog_entry_from_cached(
|
||||
entry: DepotFileCatalogEntry,
|
||||
state: os.stat_result,
|
||||
) -> DepotFileCatalogEntry:
|
||||
return DepotFileCatalogEntry(
|
||||
path=entry.path,
|
||||
size=entry.size,
|
||||
sha1=entry.sha1,
|
||||
device=state.st_dev,
|
||||
inode=state.st_ino,
|
||||
mtime_ns=state.st_mtime_ns,
|
||||
ctime_ns=state.st_ctime_ns,
|
||||
verified=True,
|
||||
)
|
||||
|
||||
|
||||
def catalog_identity_matches(
|
||||
entry: DepotFileCatalogEntry,
|
||||
state: os.stat_result,
|
||||
) -> bool:
|
||||
return (
|
||||
entry.device == state.st_dev
|
||||
and entry.inode == state.st_ino
|
||||
and entry.size == state.st_size
|
||||
and entry.mtime_ns == state.st_mtime_ns
|
||||
and entry.ctime_ns == state.st_ctime_ns
|
||||
)
|
||||
|
||||
|
||||
def build_depot_file_catalog(
|
||||
game_directory: Path,
|
||||
snapshot: DepotSnapshot,
|
||||
*,
|
||||
verified_paths: Iterable[str],
|
||||
previous: DepotFileCatalog | None = None,
|
||||
created_at: str | None = None,
|
||||
) -> DepotFileCatalog:
|
||||
"""Build an identity catalog without rehashing updater-verified files."""
|
||||
|
||||
root = prepare_game_root(game_directory, create=False)
|
||||
scanned_files = scan_game_tree(root)
|
||||
explicitly_verified = {
|
||||
verified_path.casefold() for verified_path in verified_paths
|
||||
}
|
||||
previous_files = (
|
||||
{entry.path.casefold(): entry for entry in previous.files}
|
||||
if previous is not None
|
||||
else {}
|
||||
)
|
||||
entries: list[DepotFileCatalogEntry] = []
|
||||
for file in snapshot.files:
|
||||
scanned = scanned_files.get(file.path)
|
||||
if scanned is None:
|
||||
raise DepotError(
|
||||
f"cannot cache missing depot file identity: {file.path}"
|
||||
)
|
||||
state = scanned.state
|
||||
if scanned.kind != "file" or state.st_size != file.size:
|
||||
raise DepotError(
|
||||
f"cannot cache invalid depot file identity: {file.path}"
|
||||
)
|
||||
prior = previous_files.get(file.path.casefold())
|
||||
verified = file.path.casefold() in explicitly_verified
|
||||
if (
|
||||
not verified
|
||||
and prior is not None
|
||||
and prior.verified
|
||||
and prior.size == file.size
|
||||
and prior.sha1 == file.sha1
|
||||
and catalog_identity_matches(prior, state)
|
||||
):
|
||||
verified = True
|
||||
entries.append(catalog_entry(file, state, verified=verified))
|
||||
|
||||
return DepotFileCatalog(
|
||||
build=snapshot.build,
|
||||
files=tuple(entries),
|
||||
created_at=(
|
||||
created_at
|
||||
if created_at is not None
|
||||
else datetime.now().astimezone().isoformat()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def plan_update(
|
||||
game_directory: Path,
|
||||
snapshot: DepotSnapshot,
|
||||
*,
|
||||
repair: bool = False,
|
||||
hash_file: HashFile = sha1_file,
|
||||
always_verify_paths: frozenset[str] = frozenset(
|
||||
{"game.exe", "game.dll", "gamese.exe", "gamese.dll"}
|
||||
),
|
||||
) -> UpdatePlan:
|
||||
"""Inspect a local game tree without writing to it."""
|
||||
|
||||
root = prepare_game_root(game_directory, create=False)
|
||||
downloads: list[PlannedDownload] = []
|
||||
hash_verified_paths: list[str] = []
|
||||
for file in snapshot.files:
|
||||
target = safe_target(root, file.path)
|
||||
if target.is_symlink():
|
||||
downloads.append(PlannedDownload(file, "symbolic link"))
|
||||
continue
|
||||
if not target.is_file():
|
||||
downloads.append(PlannedDownload(file, "missing"))
|
||||
continue
|
||||
try:
|
||||
if target.stat().st_size != file.size:
|
||||
downloads.append(PlannedDownload(file, "size mismatch"))
|
||||
continue
|
||||
except OSError as exc:
|
||||
raise DepotError(f"cannot inspect {file.path}: {exc}") from exc
|
||||
if (
|
||||
repair
|
||||
or file.build == snapshot.build
|
||||
or file.path.casefold() in always_verify_paths
|
||||
):
|
||||
if hash_file(target).casefold() != file.sha1.casefold():
|
||||
downloads.append(PlannedDownload(file, "hash mismatch"))
|
||||
else:
|
||||
hash_verified_paths.append(file.path)
|
||||
|
||||
deletes: list[str] = []
|
||||
for relative_path in snapshot.deleted_files:
|
||||
target = safe_target(root, relative_path)
|
||||
if target.is_symlink() or target.is_file():
|
||||
deletes.append(relative_path)
|
||||
elif target.exists():
|
||||
raise DepotError(
|
||||
f"refusing to delete non-file depot target: {relative_path}"
|
||||
)
|
||||
|
||||
return UpdatePlan(
|
||||
downloads=tuple(downloads),
|
||||
deletes=tuple(deletes),
|
||||
files_total=len(snapshot.files),
|
||||
bytes_total=sum(item.file.size for item in downloads),
|
||||
hash_verified_paths=tuple(hash_verified_paths),
|
||||
)
|
||||
|
||||
|
||||
def inspect_managed_game(
|
||||
game_directory: Path,
|
||||
catalog: DepotFileCatalog,
|
||||
*,
|
||||
hash_file: HashFile = sha1_file,
|
||||
max_issues: int = 32,
|
||||
clock: Clock = time.perf_counter,
|
||||
) -> tuple[DepotPreflightResult, DepotFileCatalog | None]:
|
||||
"""Inspect one catalog and return its report plus any refreshed identities."""
|
||||
|
||||
started = clock()
|
||||
root = prepare_game_root(game_directory, create=False)
|
||||
if not root.is_dir():
|
||||
return (
|
||||
DepotPreflightResult(
|
||||
build=catalog.build,
|
||||
files_total=len(catalog.files),
|
||||
verified_files=0,
|
||||
unverified_files=sum(
|
||||
not entry.verified for entry in catalog.files
|
||||
),
|
||||
changed_files_hashed=0,
|
||||
issue_count=1,
|
||||
issues=(
|
||||
DepotIntegrityIssue(
|
||||
".",
|
||||
"managed game directory is missing",
|
||||
),
|
||||
),
|
||||
duration_ms=elapsed_milliseconds(started, clock),
|
||||
),
|
||||
None,
|
||||
)
|
||||
try:
|
||||
scanned_files = scan_game_tree(root)
|
||||
except DepotError as exc:
|
||||
return (
|
||||
DepotPreflightResult(
|
||||
build=catalog.build,
|
||||
files_total=len(catalog.files),
|
||||
verified_files=0,
|
||||
unverified_files=sum(
|
||||
not entry.verified for entry in catalog.files
|
||||
),
|
||||
changed_files_hashed=0,
|
||||
issue_count=1,
|
||||
issues=(DepotIntegrityIssue(".", str(exc)),),
|
||||
duration_ms=elapsed_milliseconds(started, clock),
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
verified_files = 0
|
||||
unverified_files = 0
|
||||
changed_files_hashed = 0
|
||||
issue_count = 0
|
||||
issues: list[DepotIntegrityIssue] = []
|
||||
updated_entries: list[DepotFileCatalogEntry] = []
|
||||
cache_changed = False
|
||||
for entry in catalog.files:
|
||||
reason = None
|
||||
state: os.stat_result | None = None
|
||||
scanned = scanned_files.get(entry.path)
|
||||
if scanned is None:
|
||||
reason = "missing"
|
||||
elif scanned.kind != "file":
|
||||
reason = scanned.kind
|
||||
elif scanned.state.st_size != entry.size:
|
||||
reason = "size mismatch"
|
||||
else:
|
||||
state = scanned.state
|
||||
|
||||
updated_entry = entry
|
||||
if reason is None and state is not None:
|
||||
if not entry.verified:
|
||||
unverified_files += 1
|
||||
elif catalog_identity_matches(entry, state):
|
||||
verified_files += 1
|
||||
else:
|
||||
changed_files_hashed += 1
|
||||
try:
|
||||
assert scanned is not None
|
||||
matches = hash_file(scanned.path).casefold() == entry.sha1
|
||||
except DepotError as exc:
|
||||
reason = f"cannot verify changed file: {exc}"
|
||||
else:
|
||||
if matches:
|
||||
verified_files += 1
|
||||
updated_entry = catalog_entry_from_cached(entry, state)
|
||||
cache_changed = True
|
||||
else:
|
||||
reason = "hash mismatch"
|
||||
|
||||
if reason is not None:
|
||||
issue_count += 1
|
||||
if len(issues) < max_issues:
|
||||
issues.append(DepotIntegrityIssue(entry.path, reason))
|
||||
updated_entries.append(updated_entry)
|
||||
|
||||
updated_catalog = (
|
||||
DepotFileCatalog(
|
||||
build=catalog.build,
|
||||
files=tuple(updated_entries),
|
||||
created_at=datetime.now().astimezone().isoformat(),
|
||||
)
|
||||
if cache_changed
|
||||
else None
|
||||
)
|
||||
return (
|
||||
DepotPreflightResult(
|
||||
build=catalog.build,
|
||||
files_total=len(catalog.files),
|
||||
verified_files=verified_files,
|
||||
unverified_files=unverified_files,
|
||||
changed_files_hashed=changed_files_hashed,
|
||||
issue_count=issue_count,
|
||||
issues=tuple(issues),
|
||||
duration_ms=elapsed_milliseconds(started, clock),
|
||||
),
|
||||
updated_catalog,
|
||||
)
|
||||
|
||||
|
||||
def elapsed_milliseconds(started: float, clock: Clock = time.perf_counter) -> int:
|
||||
return max(0, round((clock() - started) * 1000))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"HashFile",
|
||||
"ScannedGameEntry",
|
||||
"build_depot_file_catalog",
|
||||
"catalog_entry",
|
||||
"catalog_entry_from_cached",
|
||||
"catalog_identity_matches",
|
||||
"inspect_managed_game",
|
||||
"plan_update",
|
||||
"prepare_game_root",
|
||||
"safe_target",
|
||||
"scan_directory",
|
||||
"scan_game_tree",
|
||||
"sha1_file",
|
||||
]
|
||||
279
src/ptlinux/game/depot/manifest.py
Normal file
279
src/ptlinux/game/depot/manifest.py
Normal file
@@ -0,0 +1,279 @@
|
||||
"""Strict parsing for publisher depot manifests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from base64 import b64decode
|
||||
from binascii import Error as BinAsciiError
|
||||
from pathlib import PurePosixPath
|
||||
import re
|
||||
from typing import Mapping
|
||||
|
||||
from .models import (
|
||||
DEFAULT_DEPOT_START_OPTION,
|
||||
CreateIfMissing,
|
||||
DepotError,
|
||||
DepotFile,
|
||||
DepotSnapshot,
|
||||
DepotStartOption,
|
||||
)
|
||||
|
||||
|
||||
MAX_CREATE_IF_MISSING_BYTES = 16 * 1024 * 1024
|
||||
SHA1_PATTERN = re.compile(r"^[0-9a-fA-F]{40}$")
|
||||
|
||||
|
||||
def parse_snapshot(
|
||||
region: str,
|
||||
server: str,
|
||||
depot_manifest: Mapping[str, object],
|
||||
build_manifest: Mapping[str, object],
|
||||
) -> DepotSnapshot:
|
||||
"""Validate publisher manifests and return their canonical snapshot."""
|
||||
|
||||
build = positive_int(depot_manifest.get("Build"), "depot Build")
|
||||
build_manifest_build = positive_int(build_manifest.get("b"), "build manifest b")
|
||||
if build_manifest_build != build:
|
||||
raise DepotError(
|
||||
f"depot build {build} does not match build manifest {build_manifest_build}"
|
||||
)
|
||||
name = required_string(depot_manifest.get("Name"), "depot Name")
|
||||
version = required_string(
|
||||
depot_manifest.get("BuildVersionText"),
|
||||
"depot BuildVersionText",
|
||||
)
|
||||
files_value = build_manifest.get("f")
|
||||
if not isinstance(files_value, list):
|
||||
raise DepotError("build manifest f must be an array")
|
||||
files: list[DepotFile] = []
|
||||
seen_paths: set[str] = set()
|
||||
for value in files_value:
|
||||
if not isinstance(value, dict):
|
||||
raise DepotError("build manifest contains a non-object file entry")
|
||||
path = normalize_relative_path(value.get("p"), "file path")
|
||||
if path.casefold() in seen_paths:
|
||||
raise DepotError(f"build manifest has duplicate file path: {path}")
|
||||
seen_paths.add(path.casefold())
|
||||
file_build = positive_int(value.get("b"), f"file build for {path}")
|
||||
size = non_negative_int(value.get("s"), f"file size for {path}")
|
||||
sha1 = required_string(value.get("h"), f"file hash for {path}")
|
||||
if not SHA1_PATTERN.fullmatch(sha1):
|
||||
raise DepotError(f"invalid SHA-1 hash for {path}")
|
||||
files.append(DepotFile(path, file_build, size, sha1.casefold()))
|
||||
|
||||
deleted = parse_deleted_paths(build_manifest.get("df"), seen_paths)
|
||||
create_if_missing = parse_create_if_missing(
|
||||
depot_manifest.get("CreateIfMissing")
|
||||
)
|
||||
start_options, default_start_option = parse_start_options(
|
||||
depot_manifest.get("StartOptions"),
|
||||
depot_manifest.get("DefaultStartOption"),
|
||||
)
|
||||
canonical_file_paths = {file.path.casefold(): file.path for file in files}
|
||||
start_options = tuple(
|
||||
DepotStartOption(
|
||||
name=option.name,
|
||||
path=canonical_file_paths.get(option.path.casefold(), option.path),
|
||||
arguments=option.arguments,
|
||||
)
|
||||
for option in start_options
|
||||
)
|
||||
return DepotSnapshot(
|
||||
region=region,
|
||||
server=server,
|
||||
name=name,
|
||||
build=build,
|
||||
version=version,
|
||||
files=tuple(files),
|
||||
deleted_files=deleted,
|
||||
create_if_missing=create_if_missing,
|
||||
start_options=start_options,
|
||||
default_start_option=default_start_option,
|
||||
)
|
||||
|
||||
|
||||
def parse_deleted_paths(value: object, known_paths: set[str]) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if not isinstance(value, list):
|
||||
raise DepotError("build manifest df must be an array")
|
||||
paths: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in value:
|
||||
path = normalize_relative_path(item, "deleted file path")
|
||||
folded = path.casefold()
|
||||
if folded in known_paths:
|
||||
raise DepotError(
|
||||
f"manifest lists a file as both present and deleted: {path}"
|
||||
)
|
||||
if folded in seen:
|
||||
continue
|
||||
seen.add(folded)
|
||||
paths.append(path)
|
||||
return tuple(paths)
|
||||
|
||||
|
||||
def parse_create_if_missing(value: object) -> tuple[CreateIfMissing, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if not isinstance(value, list):
|
||||
raise DepotError("CreateIfMissing must be an array")
|
||||
entries: list[CreateIfMissing] = []
|
||||
seen: set[str] = set()
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
raise DepotError("CreateIfMissing contains a non-object entry")
|
||||
path = normalize_relative_path(item.get("Path"), "CreateIfMissing path")
|
||||
folded = path.casefold()
|
||||
if folded in seen:
|
||||
raise DepotError(f"CreateIfMissing has duplicate path: {path}")
|
||||
seen.add(folded)
|
||||
content = required_string(
|
||||
item.get("Content"),
|
||||
f"CreateIfMissing content for {path}",
|
||||
)
|
||||
entry = CreateIfMissing(path, content)
|
||||
decode_create_if_missing(entry)
|
||||
entries.append(entry)
|
||||
return tuple(entries)
|
||||
|
||||
|
||||
def parse_start_options(
|
||||
value: object,
|
||||
default_value: object,
|
||||
) -> tuple[tuple[DepotStartOption, ...], str]:
|
||||
"""Validate the launch targets published by the official depot manifest."""
|
||||
|
||||
if value is None:
|
||||
return (DEFAULT_DEPOT_START_OPTION,), DEFAULT_DEPOT_START_OPTION.name
|
||||
if not isinstance(value, list) or not value:
|
||||
raise DepotError("StartOptions must be a non-empty array")
|
||||
options: list[DepotStartOption] = []
|
||||
seen_names: set[str] = set()
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
raise DepotError("StartOptions contains a non-object entry")
|
||||
name = start_option_text(item.get("Name"), "StartOptions Name", 64)
|
||||
folded_name = name.casefold()
|
||||
if folded_name in seen_names:
|
||||
raise DepotError(f"StartOptions has duplicate realm name: {name}")
|
||||
seen_names.add(folded_name)
|
||||
path = normalize_relative_path(
|
||||
item.get("Path"),
|
||||
f"StartOptions path for {name}",
|
||||
)
|
||||
if PurePosixPath(path).suffix.casefold() != ".exe":
|
||||
raise DepotError(
|
||||
f"StartOptions path is not an executable for {name}: {path}"
|
||||
)
|
||||
arguments = start_option_text(
|
||||
item.get("Arguments"),
|
||||
f"StartOptions arguments for {name}",
|
||||
2048,
|
||||
)
|
||||
options.append(DepotStartOption(name=name, path=path, arguments=arguments))
|
||||
|
||||
if default_value is None:
|
||||
default_name = next(
|
||||
(
|
||||
option.name
|
||||
for option in options
|
||||
if option.name.casefold()
|
||||
== DEFAULT_DEPOT_START_OPTION.name.casefold()
|
||||
),
|
||||
options[0].name,
|
||||
)
|
||||
else:
|
||||
default_name = start_option_text(default_value, "DefaultStartOption", 64)
|
||||
matching_default = next(
|
||||
(
|
||||
option.name
|
||||
for option in options
|
||||
if option.name.casefold() == default_name.casefold()
|
||||
),
|
||||
None,
|
||||
)
|
||||
if matching_default is None:
|
||||
raise DepotError(
|
||||
"DefaultStartOption does not match an advertised realm: "
|
||||
f"{default_name}"
|
||||
)
|
||||
default_name = matching_default
|
||||
return tuple(options), default_name
|
||||
|
||||
|
||||
def start_option_text(value: object, field: str, maximum_length: int) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise DepotError(f"{field} must be a string")
|
||||
text = value.strip()
|
||||
if (
|
||||
not text
|
||||
or len(text) > maximum_length
|
||||
or any(ord(character) < 32 for character in text)
|
||||
):
|
||||
raise DepotError(f"invalid {field}")
|
||||
return text
|
||||
|
||||
|
||||
def decode_create_if_missing(entry: CreateIfMissing) -> bytes:
|
||||
try:
|
||||
content = b64decode(entry.content_base64, validate=True)
|
||||
except (ValueError, BinAsciiError) as exc:
|
||||
raise DepotError(
|
||||
f"invalid base64 create-if-missing entry: {entry.path}"
|
||||
) from exc
|
||||
if len(content) > MAX_CREATE_IF_MISSING_BYTES:
|
||||
raise DepotError(f"create-if-missing entry is too large: {entry.path}")
|
||||
return content
|
||||
|
||||
|
||||
def normalize_relative_path(value: object, field: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise DepotError(f"{field} must be a string")
|
||||
return "/".join(relative_parts(value, field))
|
||||
|
||||
|
||||
def relative_parts(value: str, field: str) -> tuple[str, ...]:
|
||||
if not value or "\x00" in value:
|
||||
raise DepotError(f"invalid {field}")
|
||||
normalized = value.replace("\\", "/")
|
||||
if normalized.startswith("/") or normalized.startswith("//"):
|
||||
raise DepotError(f"{field} must be relative: {value!r}")
|
||||
parts = PurePosixPath(normalized).parts
|
||||
if not parts or any(part in ("", ".", "..") or ":" in part for part in parts):
|
||||
raise DepotError(f"invalid {field}: {value!r}")
|
||||
return tuple(parts)
|
||||
|
||||
|
||||
def positive_int(value: object, field: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
||||
raise DepotError(f"{field} must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def non_negative_int(value: object, field: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise DepotError(f"{field} must be a non-negative integer")
|
||||
return value
|
||||
|
||||
|
||||
def required_string(value: object, field: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise DepotError(f"{field} must be a non-empty string")
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_CREATE_IF_MISSING_BYTES",
|
||||
"SHA1_PATTERN",
|
||||
"decode_create_if_missing",
|
||||
"non_negative_int",
|
||||
"normalize_relative_path",
|
||||
"parse_create_if_missing",
|
||||
"parse_deleted_paths",
|
||||
"parse_snapshot",
|
||||
"parse_start_options",
|
||||
"positive_int",
|
||||
"relative_parts",
|
||||
"required_string",
|
||||
"start_option_text",
|
||||
]
|
||||
238
src/ptlinux/game/depot/models.py
Normal file
238
src/ptlinux/game/depot/models.py
Normal file
@@ -0,0 +1,238 @@
|
||||
"""Immutable models and public validation errors for depot synchronization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
|
||||
class DepotError(RuntimeError):
|
||||
"""Raised when a depot response or update operation cannot be trusted."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotFile:
|
||||
"""One exact file in the published depot build."""
|
||||
|
||||
path: str
|
||||
build: int
|
||||
size: int
|
||||
sha1: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CreateIfMissing:
|
||||
"""A small publisher-provided file created only when absent."""
|
||||
|
||||
path: str
|
||||
content_base64: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotStartOption:
|
||||
"""One publisher-advertised realm launch target."""
|
||||
|
||||
name: str
|
||||
path: str
|
||||
arguments: str
|
||||
|
||||
|
||||
DEFAULT_DEPOT_START_OPTION = DepotStartOption(
|
||||
name="Babel",
|
||||
path="Game.exe",
|
||||
arguments="PTEU_REBORN",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotSnapshot:
|
||||
"""The current, verified shape of the public game depot."""
|
||||
|
||||
region: str
|
||||
server: str
|
||||
name: str
|
||||
build: int
|
||||
version: str
|
||||
files: tuple[DepotFile, ...]
|
||||
deleted_files: tuple[str, ...]
|
||||
create_if_missing: tuple[CreateIfMissing, ...]
|
||||
start_options: tuple[DepotStartOption, ...] = (DEFAULT_DEPOT_START_OPTION,)
|
||||
default_start_option: str = DEFAULT_DEPOT_START_OPTION.name
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotLaunchCatalog:
|
||||
"""Small cached subset of a depot manifest needed for offline Play."""
|
||||
|
||||
build: int
|
||||
start_options: tuple[DepotStartOption, ...]
|
||||
default_start_option: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotFileCatalogEntry:
|
||||
"""One manifest file plus the identity recorded after verified installation."""
|
||||
|
||||
path: str
|
||||
size: int
|
||||
sha1: str
|
||||
device: int
|
||||
inode: int
|
||||
mtime_ns: int
|
||||
ctime_ns: int
|
||||
verified: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotFileCatalog:
|
||||
"""Cached manifest used for fast, network-free launch preflight checks."""
|
||||
|
||||
build: int
|
||||
files: tuple[DepotFileCatalogEntry, ...]
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotIntegrityIssue:
|
||||
path: str
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotPreflightResult:
|
||||
"""Outcome of validating cached file identities before starting the game."""
|
||||
|
||||
build: int
|
||||
files_total: int
|
||||
verified_files: int
|
||||
unverified_files: int
|
||||
changed_files_hashed: int
|
||||
issue_count: int
|
||||
issues: tuple[DepotIntegrityIssue, ...]
|
||||
duration_ms: int
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self.issue_count == 0
|
||||
|
||||
@property
|
||||
def fully_verified(self) -> bool:
|
||||
return self.ready and self.unverified_files == 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlannedDownload:
|
||||
file: DepotFile
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UpdatePlan:
|
||||
"""The safe changes needed to make a local tree match one depot snapshot."""
|
||||
|
||||
downloads: tuple[PlannedDownload, ...]
|
||||
deletes: tuple[str, ...]
|
||||
files_total: int
|
||||
bytes_total: int
|
||||
hash_verified_paths: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SyncProgress:
|
||||
"""A thread-safe progress snapshot emitted by depot synchronization."""
|
||||
|
||||
phase: str
|
||||
files_completed: int
|
||||
files_total: int
|
||||
bytes_completed: int
|
||||
bytes_total: int
|
||||
current_file: str | None
|
||||
bytes_transferred: int = 0
|
||||
bytes_remaining: int = 0
|
||||
transfer_rate_bytes_per_second: float | None = None
|
||||
eta_seconds: float | None = None
|
||||
current_server: str | None = None
|
||||
retry_count: int = 0
|
||||
retry_reason: str | None = None
|
||||
active_workers: int = 0
|
||||
target_workers: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotSyncResult:
|
||||
"""Persistent result of one native depot synchronization."""
|
||||
|
||||
region: str
|
||||
server: str
|
||||
name: str
|
||||
build: int
|
||||
version: str
|
||||
files_total: int
|
||||
files_downloaded: int
|
||||
bytes_downloaded: int
|
||||
files_deleted: int
|
||||
files_created: int
|
||||
files_quarantined: int
|
||||
quarantine_directory: str | None
|
||||
repair: bool
|
||||
dry_run: bool
|
||||
completed_at: str
|
||||
files_planned: int = 0
|
||||
bytes_planned: int = 0
|
||||
manifest_duration_ms: int = 0
|
||||
scan_duration_ms: int = 0
|
||||
download_duration_ms: int = 0
|
||||
apply_duration_ms: int = 0
|
||||
update_duration_ms: int = 0
|
||||
network_bytes_downloaded: int = 0
|
||||
download_retry_count: int = 0
|
||||
download_resume_used: bool = False
|
||||
download_average_mbps: float | None = None
|
||||
mirror_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotDownloadMetrics:
|
||||
"""Aggregate network measurements for one depot download phase."""
|
||||
|
||||
duration_ms: int = 0
|
||||
transferred_bytes: int = 0
|
||||
retry_count: int = 0
|
||||
resume_used: bool = False
|
||||
resumed_bytes: int = 0
|
||||
average_mbps: float | None = None
|
||||
mirror_id: str | None = None
|
||||
http_status: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuarantineResult:
|
||||
"""Recoverable unmanaged entries removed before authenticating an import."""
|
||||
|
||||
files: int
|
||||
directory: str | None
|
||||
|
||||
|
||||
ProgressCallback = Callable[[SyncProgress], None]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_DEPOT_START_OPTION",
|
||||
"CreateIfMissing",
|
||||
"DepotDownloadMetrics",
|
||||
"DepotError",
|
||||
"DepotFile",
|
||||
"DepotFileCatalog",
|
||||
"DepotFileCatalogEntry",
|
||||
"DepotIntegrityIssue",
|
||||
"DepotLaunchCatalog",
|
||||
"DepotPreflightResult",
|
||||
"DepotSnapshot",
|
||||
"DepotStartOption",
|
||||
"DepotSyncResult",
|
||||
"PlannedDownload",
|
||||
"ProgressCallback",
|
||||
"QuarantineResult",
|
||||
"SyncProgress",
|
||||
"UpdatePlan",
|
||||
]
|
||||
82
src/ptlinux/game/depot/realms.py
Normal file
82
src/ptlinux/game/depot/realms.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Launcher labels and runtime requirements for publisher realm options."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
from .models import DepotLaunchCatalog, DepotStartOption
|
||||
|
||||
|
||||
PTR_BABEL_REALM_NAME = "PTR Babel"
|
||||
PUBLISHER_TEST_BABEL_REALM_NAME = "Test Babel"
|
||||
PTR_BABEL_EXECUTABLE_PATH = PurePosixPath("Gamese.exe")
|
||||
PTR_BABEL_DLL_PATH = PurePosixPath("gamese.dll")
|
||||
|
||||
|
||||
def launcher_realm_name(name: str) -> str:
|
||||
"""Return the launcher-facing name for one publisher realm name."""
|
||||
|
||||
if name.casefold() == PUBLISHER_TEST_BABEL_REALM_NAME.casefold():
|
||||
return PTR_BABEL_REALM_NAME
|
||||
return name
|
||||
|
||||
|
||||
def launcher_launch_catalog(catalog: DepotLaunchCatalog) -> DepotLaunchCatalog:
|
||||
"""Apply launcher presentation names without changing launch metadata."""
|
||||
|
||||
options = tuple(_launcher_start_option(option) for option in catalog.start_options)
|
||||
default_name = catalog.default_start_option
|
||||
for publisher_option, launcher_option in zip(
|
||||
catalog.start_options,
|
||||
options,
|
||||
strict=True,
|
||||
):
|
||||
if publisher_option.name.casefold() == default_name.casefold():
|
||||
default_name = launcher_option.name
|
||||
break
|
||||
return DepotLaunchCatalog(
|
||||
build=catalog.build,
|
||||
start_options=options,
|
||||
default_start_option=default_name,
|
||||
)
|
||||
|
||||
|
||||
def required_start_option_files(
|
||||
option: DepotStartOption,
|
||||
) -> tuple[PurePosixPath, ...]:
|
||||
"""Return files required by launcher-defined realm behavior."""
|
||||
|
||||
executable = PurePosixPath(option.path)
|
||||
if (
|
||||
option.name.casefold() != PTR_BABEL_REALM_NAME.casefold()
|
||||
or executable.as_posix().casefold()
|
||||
!= PTR_BABEL_EXECUTABLE_PATH.as_posix().casefold()
|
||||
):
|
||||
return ()
|
||||
return executable, PTR_BABEL_DLL_PATH
|
||||
|
||||
|
||||
def _launcher_start_option(option: DepotStartOption) -> DepotStartOption:
|
||||
name = launcher_realm_name(option.name)
|
||||
if (
|
||||
name == option.name
|
||||
or PurePosixPath(option.path).as_posix().casefold()
|
||||
!= PTR_BABEL_EXECUTABLE_PATH.as_posix().casefold()
|
||||
):
|
||||
return option
|
||||
return DepotStartOption(
|
||||
name=name,
|
||||
path=option.path,
|
||||
arguments=option.arguments,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PTR_BABEL_DLL_PATH",
|
||||
"PTR_BABEL_EXECUTABLE_PATH",
|
||||
"PTR_BABEL_REALM_NAME",
|
||||
"PUBLISHER_TEST_BABEL_REALM_NAME",
|
||||
"launcher_launch_catalog",
|
||||
"launcher_realm_name",
|
||||
"required_start_option_files",
|
||||
]
|
||||
913
src/ptlinux/game/depot/transfer.py
Normal file
913
src/ptlinux/game/depot/transfer.py
Normal file
@@ -0,0 +1,913 @@
|
||||
"""Verified HTTP transfer and adaptive scheduling for the game depot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from http.client import HTTPException
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, Iterable, Mapping, Protocol
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import quote, urlparse
|
||||
from urllib.request import Request
|
||||
|
||||
from .models import DepotError, DepotFile, DepotSnapshot
|
||||
|
||||
|
||||
class DownloadLimits:
|
||||
"""Scheduler settings supplied by the stable depot facade."""
|
||||
|
||||
minimum_workers: int
|
||||
|
||||
@property
|
||||
def effective_bandwidth_limit(self) -> int | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def effective_maximum_workers(self) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DownloadTelemetry:
|
||||
"""One internal measurement produced while retrieving a depot file."""
|
||||
|
||||
kind: str
|
||||
file: str
|
||||
server: str
|
||||
attempt: int
|
||||
latency_seconds: float | None = None
|
||||
bytes_written: int = 0
|
||||
write_seconds: float = 0.0
|
||||
reason: str | None = None
|
||||
http_status: int | None = None
|
||||
resumed_bytes: int = 0
|
||||
|
||||
|
||||
DownloadObserver = Callable[[DownloadTelemetry], None]
|
||||
|
||||
|
||||
class BandwidthLimiter:
|
||||
"""Thread-safe aggregate byte-rate limiter with a one-chunk burst allowance."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bytes_per_second: int,
|
||||
*,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
sleeper: Callable[[float], None] = time.sleep,
|
||||
) -> None:
|
||||
if isinstance(bytes_per_second, bool) or bytes_per_second <= 0:
|
||||
raise ValueError("bandwidth limit must be positive")
|
||||
self.bytes_per_second = bytes_per_second
|
||||
self._clock = clock
|
||||
self._sleeper = sleeper
|
||||
self._lock = threading.Lock()
|
||||
self._next_available_at = clock()
|
||||
|
||||
def consume(self, byte_count: int) -> None:
|
||||
"""Reserve aggregate transfer time for a newly received chunk."""
|
||||
|
||||
if byte_count < 0:
|
||||
raise ValueError("byte count must not be negative")
|
||||
if byte_count == 0:
|
||||
return
|
||||
with self._lock:
|
||||
now = self._clock()
|
||||
start = max(now, self._next_available_at)
|
||||
self._next_available_at = start + (byte_count / self.bytes_per_second)
|
||||
delay = start - now
|
||||
if delay > 0:
|
||||
self._sleeper(delay)
|
||||
|
||||
|
||||
def normalized_cpu_load() -> float | None:
|
||||
"""Return one-minute runnable load normalized by the available CPU count."""
|
||||
|
||||
try:
|
||||
load = os.getloadavg()[0]
|
||||
except (AttributeError, OSError):
|
||||
return None
|
||||
processors = os.cpu_count()
|
||||
if processors is None or processors <= 0:
|
||||
return None
|
||||
return max(0.0, load / processors)
|
||||
|
||||
|
||||
def prune_transfer_samples(
|
||||
samples: deque[tuple[float, int]],
|
||||
now: float,
|
||||
*,
|
||||
window_seconds: float,
|
||||
) -> None:
|
||||
"""Discard transfer samples older than the configured rate window."""
|
||||
|
||||
cutoff = now - window_seconds
|
||||
while len(samples) > 2 and samples[1][0] <= cutoff:
|
||||
samples.popleft()
|
||||
|
||||
|
||||
def sample_rate(samples: deque[tuple[float, int]]) -> float | None:
|
||||
"""Calculate bytes per second between the oldest and newest samples."""
|
||||
|
||||
if len(samples) < 2:
|
||||
return None
|
||||
first_time, first_bytes = samples[0]
|
||||
last_time, last_bytes = samples[-1]
|
||||
elapsed = last_time - first_time
|
||||
if elapsed <= 0 or last_bytes < first_bytes:
|
||||
return None
|
||||
return (last_bytes - first_bytes) / elapsed
|
||||
|
||||
|
||||
class AdaptiveConcurrencyController:
|
||||
"""Select a bounded worker target from live network, host, and plan data."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
options: DownloadLimits,
|
||||
*,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
cpu_load: Callable[[], float | None] | None = None,
|
||||
initial_workers: int = 8,
|
||||
evaluation_interval_seconds: float = 1.0,
|
||||
retry_window_seconds: float = 15.0,
|
||||
retry_recovery_seconds: float = 5.0,
|
||||
transfer_rate_window_seconds: float = 5.0,
|
||||
) -> None:
|
||||
self.options = options
|
||||
self._clock = clock
|
||||
self._cpu_load = cpu_load or normalized_cpu_load
|
||||
self._initial_workers = initial_workers
|
||||
self._evaluation_interval_seconds = evaluation_interval_seconds
|
||||
self._retry_window_seconds = retry_window_seconds
|
||||
self._retry_recovery_seconds = retry_recovery_seconds
|
||||
self._transfer_rate_window_seconds = transfer_rate_window_seconds
|
||||
self._lock = threading.Lock()
|
||||
self._target = max(
|
||||
options.minimum_workers,
|
||||
min(initial_workers, options.effective_maximum_workers),
|
||||
)
|
||||
self._last_evaluation = clock()
|
||||
self._transferred_bytes = 0
|
||||
self._transfer_samples: deque[tuple[float, int]] = deque(
|
||||
[(self._last_evaluation, 0)]
|
||||
)
|
||||
self._response_samples: deque[tuple[float, float]] = deque()
|
||||
self._retry_samples: deque[float] = deque()
|
||||
self._retry_count = 0
|
||||
self._handled_retry_count = 0
|
||||
self._last_retry_at: float | None = None
|
||||
self._latency_seconds: float | None = None
|
||||
self._disk_samples: deque[tuple[float, int, float]] = deque()
|
||||
|
||||
def record_response(self, latency_seconds: float) -> None:
|
||||
if latency_seconds < 0:
|
||||
return
|
||||
now = self._clock()
|
||||
with self._lock:
|
||||
self._response_samples.append((now, latency_seconds))
|
||||
self._prune_health_samples(now)
|
||||
if self._latency_seconds is None:
|
||||
self._latency_seconds = latency_seconds
|
||||
else:
|
||||
self._latency_seconds = (
|
||||
(self._latency_seconds * 0.75) + (latency_seconds * 0.25)
|
||||
)
|
||||
|
||||
def record_retry(self) -> None:
|
||||
now = self._clock()
|
||||
with self._lock:
|
||||
self._retry_count += 1
|
||||
self._retry_samples.append(now)
|
||||
self._last_retry_at = now
|
||||
self._prune_health_samples(now)
|
||||
|
||||
def record_chunk(self, byte_count: int, write_seconds: float) -> None:
|
||||
if byte_count <= 0:
|
||||
return
|
||||
now = self._clock()
|
||||
with self._lock:
|
||||
self._transferred_bytes += byte_count
|
||||
self._transfer_samples.append((now, self._transferred_bytes))
|
||||
prune_transfer_samples(
|
||||
self._transfer_samples,
|
||||
now,
|
||||
window_seconds=self._transfer_rate_window_seconds,
|
||||
)
|
||||
self._disk_samples.append((now, byte_count, max(0.0, write_seconds)))
|
||||
self._prune_disk_samples(now)
|
||||
|
||||
def _prune_health_samples(self, now: float) -> None:
|
||||
cutoff = now - self._retry_window_seconds
|
||||
while self._response_samples and self._response_samples[0][0] < cutoff:
|
||||
self._response_samples.popleft()
|
||||
while self._retry_samples and self._retry_samples[0] < cutoff:
|
||||
self._retry_samples.popleft()
|
||||
|
||||
def _prune_disk_samples(self, now: float) -> None:
|
||||
cutoff = now - self._transfer_rate_window_seconds
|
||||
while self._disk_samples and self._disk_samples[0][0] < cutoff:
|
||||
self._disk_samples.popleft()
|
||||
|
||||
def target_workers(self, remaining_files: int, remaining_bytes: int) -> int:
|
||||
"""Return the next target; active workers are allowed to drain naturally."""
|
||||
|
||||
if remaining_files <= 0:
|
||||
return 0
|
||||
maximum = min(self.options.effective_maximum_workers, remaining_files)
|
||||
minimum = min(self.options.minimum_workers, maximum)
|
||||
with self._lock:
|
||||
self._target = min(maximum, max(minimum, self._target))
|
||||
now = self._clock()
|
||||
average_file_size = remaining_bytes / max(1, remaining_files)
|
||||
if now - self._last_evaluation < self._evaluation_interval_seconds:
|
||||
return self._target
|
||||
self._last_evaluation = now
|
||||
prune_transfer_samples(
|
||||
self._transfer_samples,
|
||||
now,
|
||||
window_seconds=self._transfer_rate_window_seconds,
|
||||
)
|
||||
self._prune_health_samples(now)
|
||||
self._prune_disk_samples(now)
|
||||
rate = sample_rate(self._transfer_samples)
|
||||
attempts = len(self._response_samples) + len(self._retry_samples)
|
||||
error_rate = len(self._retry_samples) / attempts if attempts else 0.0
|
||||
latency = self._latency_seconds
|
||||
disk_bytes = sum(sample[1] for sample in self._disk_samples)
|
||||
disk_seconds = sum(sample[2] for sample in self._disk_samples)
|
||||
disk_rate = disk_bytes / disk_seconds if disk_seconds > 0 else None
|
||||
try:
|
||||
cpu_load = self._cpu_load()
|
||||
except (OSError, ValueError):
|
||||
cpu_load = None
|
||||
|
||||
new_retries = self._retry_count - self._handled_retry_count
|
||||
if new_retries:
|
||||
self._handled_retry_count = self._retry_count
|
||||
if error_rate >= 0.12 and self._target > minimum:
|
||||
backoff = min(2, new_retries)
|
||||
self._target = max(minimum, self._target - backoff)
|
||||
return self._target
|
||||
|
||||
resource_constrained = (
|
||||
(cpu_load is not None and cpu_load >= 0.90)
|
||||
or (
|
||||
rate is not None
|
||||
and disk_rate is not None
|
||||
and disk_rate < rate * 0.75
|
||||
)
|
||||
)
|
||||
if resource_constrained and self._target > minimum:
|
||||
self._target -= 1
|
||||
return self._target
|
||||
|
||||
limit = self.options.effective_bandwidth_limit
|
||||
limit_saturated = (
|
||||
limit is not None and rate is not None and rate >= limit * 0.90
|
||||
)
|
||||
healthy = (
|
||||
(cpu_load is None or cpu_load < 0.80)
|
||||
and (rate is None or disk_rate is None or disk_rate >= rate)
|
||||
)
|
||||
retry_recovery_pending = (
|
||||
self._last_retry_at is not None
|
||||
and now - self._last_retry_at < self._retry_recovery_seconds
|
||||
)
|
||||
if (
|
||||
healthy
|
||||
and not retry_recovery_pending
|
||||
and not limit_saturated
|
||||
and self._target < maximum
|
||||
):
|
||||
step = 2 if (
|
||||
(latency is not None and latency >= 0.75)
|
||||
or average_file_size <= 512 * 1024
|
||||
) else 1
|
||||
self._target = min(maximum, self._target + step)
|
||||
return min(maximum, max(minimum, self._target))
|
||||
|
||||
|
||||
class HttpResponse(Protocol):
|
||||
"""The small response surface used by the native depot client."""
|
||||
|
||||
headers: Mapping[str, str]
|
||||
|
||||
def read(self, size: int = -1) -> bytes: ...
|
||||
|
||||
def close(self) -> None: ...
|
||||
|
||||
|
||||
HttpOpener = Callable[..., HttpResponse]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DepotHttpPolicy:
|
||||
"""Facade-owned policy and parsers consumed by the HTTP implementation."""
|
||||
|
||||
configuration_url: Callable[[], str]
|
||||
server_candidates: Callable[
|
||||
[Mapping[str, object] | None, str],
|
||||
tuple[tuple[str, str], ...],
|
||||
]
|
||||
depot_manifest_url: Callable[[str], str]
|
||||
build_manifest_url: Callable[[str, int], str]
|
||||
file_url: Callable[[str, DepotFile], str]
|
||||
ordered_download_servers: Callable[[str, Iterable[str]], tuple[str, ...]]
|
||||
download_partial_path: Callable[[Path], Path]
|
||||
recover_manifest_file: Callable[[Path, Path, DepotFile], bool]
|
||||
parse_snapshot: Callable[
|
||||
[str, str, Mapping[str, object], Mapping[str, object]],
|
||||
DepotSnapshot,
|
||||
]
|
||||
positive_int: Callable[[object, str], int]
|
||||
content_length: Callable[[str | None], int | None]
|
||||
content_range: Callable[[str | None], tuple[int, int, int]]
|
||||
headers: Callable[[], dict[str, str]]
|
||||
read_size: Callable[[], int]
|
||||
hash_read_size: Callable[[], int]
|
||||
clock: Callable[[], float]
|
||||
|
||||
|
||||
class _RetryableDownloadError(DepotError):
|
||||
"""One transient network or content-integrity failure."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
discard_partial: bool = False,
|
||||
http_status: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.discard_partial = discard_partial
|
||||
self.http_status = http_status
|
||||
|
||||
|
||||
class DepotHttpClient:
|
||||
"""HTTP implementation for the publisher's documented public depot."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
policy: DepotHttpPolicy,
|
||||
timeout_seconds: float = 60.0,
|
||||
opener: HttpOpener,
|
||||
retry_delays: tuple[float, ...],
|
||||
sleeper: Callable[[float], None],
|
||||
download_observer: DownloadObserver | None = None,
|
||||
) -> None:
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
if any(delay < 0 for delay in retry_delays):
|
||||
raise ValueError("retry delays must not be negative")
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self._policy = policy
|
||||
self._opener = opener
|
||||
self._retry_delays = retry_delays
|
||||
self._sleeper = sleeper
|
||||
self._candidate_servers: tuple[str, ...] = ()
|
||||
self._download_observer = download_observer
|
||||
self._bandwidth_limiter: BandwidthLimiter | None = None
|
||||
|
||||
@property
|
||||
def download_observer(self) -> DownloadObserver | None:
|
||||
return self._download_observer
|
||||
|
||||
@download_observer.setter
|
||||
def download_observer(self, observer: DownloadObserver | None) -> None:
|
||||
self._download_observer = observer
|
||||
|
||||
@property
|
||||
def bandwidth_limiter(self) -> BandwidthLimiter | None:
|
||||
return self._bandwidth_limiter
|
||||
|
||||
@bandwidth_limiter.setter
|
||||
def bandwidth_limiter(self, limiter: BandwidthLimiter | None) -> None:
|
||||
self._bandwidth_limiter = limiter
|
||||
|
||||
def fetch_snapshot(self, *, preferred_region: str = "Europe") -> DepotSnapshot:
|
||||
"""Fetch the live depot and build manifests from trusted regional servers."""
|
||||
|
||||
configuration: Mapping[str, object] | None = None
|
||||
configuration_error: DepotError | None = None
|
||||
try:
|
||||
configuration = self._get_json(self._policy.configuration_url())
|
||||
except DepotError as exc:
|
||||
configuration_error = exc
|
||||
|
||||
candidates = self._policy.server_candidates(configuration, preferred_region)
|
||||
self._candidate_servers = tuple(server for _region, server in candidates)
|
||||
last_error = configuration_error
|
||||
for region, server in candidates:
|
||||
try:
|
||||
depot_manifest = self._get_json(self._policy.depot_manifest_url(server))
|
||||
build = self._policy.positive_int(
|
||||
depot_manifest.get("Build"),
|
||||
"depot Build",
|
||||
)
|
||||
build_manifest = self._get_json(
|
||||
self._policy.build_manifest_url(server, build)
|
||||
)
|
||||
return self._policy.parse_snapshot(
|
||||
region,
|
||||
server,
|
||||
depot_manifest,
|
||||
build_manifest,
|
||||
)
|
||||
except DepotError as exc:
|
||||
last_error = exc
|
||||
|
||||
if last_error is not None:
|
||||
raise DepotError(f"could not load the PristonTale EU depot: {last_error}")
|
||||
raise DepotError("could not load the PristonTale EU depot")
|
||||
|
||||
def download_file(
|
||||
self,
|
||||
server: str,
|
||||
file: DepotFile,
|
||||
destination: Path,
|
||||
*,
|
||||
progress: Callable[[int], None] | None = None,
|
||||
) -> None:
|
||||
"""Download, resume, and verify one file before replacing its destination."""
|
||||
|
||||
servers = self._policy.ordered_download_servers(
|
||||
server,
|
||||
self._candidate_servers,
|
||||
)
|
||||
temporary = self._policy.download_partial_path(destination)
|
||||
last_error: _RetryableDownloadError | None = None
|
||||
attempts = len(self._retry_delays) + 1
|
||||
for attempt_index in range(attempts):
|
||||
attempt_server = servers[attempt_index % len(servers)]
|
||||
try:
|
||||
self._download_file_once(
|
||||
attempt_server,
|
||||
file,
|
||||
destination,
|
||||
attempt=attempt_index + 1,
|
||||
progress=progress,
|
||||
)
|
||||
return
|
||||
except _RetryableDownloadError as exc:
|
||||
last_error = exc
|
||||
self._notify_download(
|
||||
DownloadTelemetry(
|
||||
kind="retry",
|
||||
file=file.path,
|
||||
server=attempt_server,
|
||||
attempt=attempt_index + 1,
|
||||
reason=str(exc),
|
||||
http_status=exc.http_status,
|
||||
)
|
||||
)
|
||||
if exc.discard_partial:
|
||||
temporary.unlink(missing_ok=True)
|
||||
if attempt_index >= len(self._retry_delays):
|
||||
break
|
||||
self._sleeper(self._retry_delays[attempt_index])
|
||||
|
||||
assert last_error is not None
|
||||
resume_hint = (
|
||||
"; partial data was retained for the next update run"
|
||||
if temporary.is_file() and not temporary.is_symlink()
|
||||
else ""
|
||||
)
|
||||
raise DepotError(
|
||||
f"download failed for {file.path} after {attempts} attempts: "
|
||||
f"{last_error}{resume_hint}"
|
||||
) from last_error
|
||||
|
||||
def _download_file_once(
|
||||
self,
|
||||
server: str,
|
||||
file: DepotFile,
|
||||
destination: Path,
|
||||
*,
|
||||
attempt: int,
|
||||
progress: Callable[[int], None] | None,
|
||||
) -> None:
|
||||
temporary = self._policy.download_partial_path(destination)
|
||||
written = 0
|
||||
digest = hashlib.sha1(usedforsecurity=False)
|
||||
if temporary.is_symlink():
|
||||
raise DepotError(f"refusing to use symbolic-link partial file for {file.path}")
|
||||
if temporary.exists():
|
||||
if not temporary.is_file():
|
||||
raise DepotError(f"partial download is not a file for {file.path}")
|
||||
try:
|
||||
with temporary.open("rb") as partial_handle:
|
||||
while True:
|
||||
chunk = partial_handle.read(self._policy.hash_read_size())
|
||||
if not chunk:
|
||||
break
|
||||
written += len(chunk)
|
||||
digest.update(chunk)
|
||||
except OSError as exc:
|
||||
raise DepotError(
|
||||
f"cannot inspect partial download for {file.path}: {exc}"
|
||||
) from exc
|
||||
|
||||
if written > file.size:
|
||||
temporary.unlink()
|
||||
written = 0
|
||||
digest = hashlib.sha1(usedforsecurity=False)
|
||||
elif written == file.size:
|
||||
if digest.hexdigest().casefold() == file.sha1.casefold():
|
||||
self._notify_download(
|
||||
DownloadTelemetry(
|
||||
kind="resume",
|
||||
file=file.path,
|
||||
server=server,
|
||||
attempt=attempt,
|
||||
resumed_bytes=written,
|
||||
)
|
||||
)
|
||||
temporary.replace(destination)
|
||||
if progress is not None:
|
||||
progress(written)
|
||||
return
|
||||
temporary.unlink()
|
||||
written = 0
|
||||
digest = hashlib.sha1(usedforsecurity=False)
|
||||
elif written and self._policy.recover_manifest_file(
|
||||
temporary,
|
||||
destination,
|
||||
file,
|
||||
):
|
||||
self._notify_download(
|
||||
DownloadTelemetry(
|
||||
kind="resume",
|
||||
file=file.path,
|
||||
server=server,
|
||||
attempt=attempt,
|
||||
resumed_bytes=written,
|
||||
)
|
||||
)
|
||||
temporary.unlink(missing_ok=True)
|
||||
if progress is not None:
|
||||
progress(file.size)
|
||||
return
|
||||
if progress is not None:
|
||||
progress(written)
|
||||
|
||||
requested_offset = written
|
||||
if requested_offset:
|
||||
self._notify_download(
|
||||
DownloadTelemetry(
|
||||
kind="resume",
|
||||
file=file.path,
|
||||
server=server,
|
||||
attempt=attempt,
|
||||
resumed_bytes=requested_offset,
|
||||
)
|
||||
)
|
||||
headers = self._policy.headers()
|
||||
if requested_offset:
|
||||
headers["Range"] = f"bytes={requested_offset}-"
|
||||
url = self._policy.file_url(server, file)
|
||||
request = Request(url, headers=headers, method="GET")
|
||||
self._notify_download(
|
||||
DownloadTelemetry(
|
||||
kind="request",
|
||||
file=file.path,
|
||||
server=server,
|
||||
attempt=attempt,
|
||||
)
|
||||
)
|
||||
request_started_at = self._policy.clock()
|
||||
try:
|
||||
response = self._opener(request, timeout=self.timeout_seconds)
|
||||
except HTTPError as exc:
|
||||
status = exc.code
|
||||
exc.close()
|
||||
if status == 416 and requested_offset:
|
||||
raise _RetryableDownloadError(
|
||||
f"server rejected the saved offset for {file.path}",
|
||||
discard_partial=True,
|
||||
http_status=status,
|
||||
) from exc
|
||||
raise _RetryableDownloadError(
|
||||
f"HTTP {status} from {server} for {file.path}",
|
||||
http_status=status,
|
||||
) from exc
|
||||
except (OSError, HTTPException) as exc:
|
||||
raise _RetryableDownloadError(
|
||||
f"could not connect to {server} for {file.path}: {exc}"
|
||||
) from exc
|
||||
response_status = getattr(response, "status", None)
|
||||
self._notify_download(
|
||||
DownloadTelemetry(
|
||||
kind="response",
|
||||
file=file.path,
|
||||
server=server,
|
||||
attempt=attempt,
|
||||
latency_seconds=max(
|
||||
0.0,
|
||||
self._policy.clock() - request_started_at,
|
||||
),
|
||||
http_status=(
|
||||
response_status if isinstance(response_status, int) else None
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
append = False
|
||||
if requested_offset:
|
||||
range_header = response.headers.get("Content-Range")
|
||||
status = getattr(response, "status", None)
|
||||
if status == 206 or range_header is not None:
|
||||
try:
|
||||
range_start, _range_end, range_total = (
|
||||
self._policy.content_range(range_header)
|
||||
)
|
||||
except DepotError as exc:
|
||||
raise _RetryableDownloadError(
|
||||
f"invalid resume response from {server} for {file.path}: {exc}"
|
||||
) from exc
|
||||
if range_start != requested_offset or range_total != file.size:
|
||||
raise _RetryableDownloadError(
|
||||
f"resume response did not match {file.path}: "
|
||||
f"offset {range_start}/{requested_offset}, "
|
||||
f"size {range_total}/{file.size}"
|
||||
)
|
||||
append = True
|
||||
else:
|
||||
written = 0
|
||||
digest = hashlib.sha1(usedforsecurity=False)
|
||||
if progress is not None:
|
||||
progress(written)
|
||||
|
||||
try:
|
||||
declared_length = self._policy.content_length(
|
||||
response.headers.get("Content-Length")
|
||||
)
|
||||
except DepotError as exc:
|
||||
raise _RetryableDownloadError(
|
||||
f"invalid response length from {server} for {file.path}: {exc}"
|
||||
) from exc
|
||||
remaining = file.size - written
|
||||
if declared_length is not None and declared_length > remaining:
|
||||
raise _RetryableDownloadError(
|
||||
f"download exceeds expected size for {file.path}: "
|
||||
f"server declared {declared_length}, expected at most {remaining}"
|
||||
)
|
||||
|
||||
mode = "ab" if append else "wb"
|
||||
try:
|
||||
with temporary.open(mode) as handle:
|
||||
while True:
|
||||
try:
|
||||
chunk = response.read(self._policy.read_size())
|
||||
except (OSError, HTTPException) as exc:
|
||||
raise _RetryableDownloadError(
|
||||
f"could not read {file.path} from {server}: {exc}"
|
||||
) from exc
|
||||
if not chunk:
|
||||
break
|
||||
written += len(chunk)
|
||||
if written > file.size:
|
||||
raise _RetryableDownloadError(
|
||||
f"download exceeds expected size for {file.path}",
|
||||
discard_partial=True,
|
||||
)
|
||||
if self._bandwidth_limiter is not None:
|
||||
self._bandwidth_limiter.consume(len(chunk))
|
||||
digest.update(chunk)
|
||||
write_started_at = self._policy.clock()
|
||||
handle.write(chunk)
|
||||
self._notify_download(
|
||||
DownloadTelemetry(
|
||||
kind="chunk",
|
||||
file=file.path,
|
||||
server=server,
|
||||
attempt=attempt,
|
||||
bytes_written=len(chunk),
|
||||
write_seconds=max(
|
||||
0.0,
|
||||
self._policy.clock() - write_started_at,
|
||||
),
|
||||
)
|
||||
)
|
||||
if progress is not None:
|
||||
progress(written)
|
||||
except _RetryableDownloadError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise DepotError(
|
||||
f"cannot write partial download for {file.path}: {exc}"
|
||||
) from exc
|
||||
|
||||
if written != file.size or digest.hexdigest().casefold() != file.sha1.casefold():
|
||||
if self._policy.recover_manifest_file(temporary, destination, file):
|
||||
temporary.unlink(missing_ok=True)
|
||||
if progress is not None:
|
||||
progress(file.size)
|
||||
return
|
||||
if written != file.size:
|
||||
raise _RetryableDownloadError(
|
||||
f"incomplete response from {server} for {file.path}: "
|
||||
f"{written}/{file.size} bytes"
|
||||
)
|
||||
raise _RetryableDownloadError(
|
||||
f"SHA-1 mismatch for {file.path} from {server}",
|
||||
discard_partial=True,
|
||||
)
|
||||
temporary.replace(destination)
|
||||
finally:
|
||||
try:
|
||||
response.close()
|
||||
except (OSError, HTTPException):
|
||||
pass
|
||||
|
||||
def _notify_download(self, event: DownloadTelemetry) -> None:
|
||||
observer = self._download_observer
|
||||
if observer is not None:
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
observer(event)
|
||||
except Exception:
|
||||
# Telemetry must never weaken verified download behavior.
|
||||
pass
|
||||
|
||||
def _get_json(self, url: str) -> Mapping[str, object]:
|
||||
request = Request(url, headers=self._policy.headers(), method="GET")
|
||||
try:
|
||||
response = self._opener(request, timeout=self.timeout_seconds)
|
||||
except (OSError, HTTPException) as exc:
|
||||
raise DepotError(f"request failed for {url}: {exc}") from exc
|
||||
payload = b""
|
||||
try:
|
||||
payload = response.read()
|
||||
except (OSError, HTTPException) as exc:
|
||||
raise DepotError(f"could not read {url}: {exc}") from exc
|
||||
finally:
|
||||
try:
|
||||
response.close()
|
||||
except (OSError, HTTPException):
|
||||
pass
|
||||
try:
|
||||
value = json.loads(payload.decode("utf-8-sig"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise DepotError(f"invalid JSON from {url}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise DepotError(f"unexpected JSON root from {url}")
|
||||
return value
|
||||
|
||||
|
||||
def server_candidates(
|
||||
configuration: Mapping[str, object] | None,
|
||||
preferred_region: str,
|
||||
*,
|
||||
fallback_servers: Iterable[str],
|
||||
) -> tuple[tuple[str, str], ...]:
|
||||
"""Return normalized publisher servers in deterministic preference order."""
|
||||
|
||||
entries: list[tuple[str, str]] = []
|
||||
if configuration is not None:
|
||||
regions = configuration.get("Regions")
|
||||
default_region = configuration.get("DefaultRegion")
|
||||
if isinstance(regions, list):
|
||||
ordered = [preferred_region]
|
||||
if isinstance(default_region, str) and default_region not in ordered:
|
||||
ordered.append(default_region)
|
||||
for region_name in ordered:
|
||||
for region in regions:
|
||||
if not isinstance(region, dict) or region.get("Name") != region_name:
|
||||
continue
|
||||
servers = region.get("Servers")
|
||||
if isinstance(servers, list):
|
||||
for server in servers:
|
||||
if not isinstance(server, str):
|
||||
continue
|
||||
try:
|
||||
entries.append((region_name, normalize_server(server)))
|
||||
except DepotError:
|
||||
continue
|
||||
seen: set[str] = set()
|
||||
unique: list[tuple[str, str]] = []
|
||||
for region, server in [
|
||||
*entries,
|
||||
*(("Fallback", value) for value in fallback_servers),
|
||||
]:
|
||||
if server in seen:
|
||||
continue
|
||||
seen.add(server)
|
||||
unique.append((region, server))
|
||||
return tuple(unique)
|
||||
|
||||
|
||||
def normalize_server(value: str) -> str:
|
||||
"""Validate and normalize one trusted CDN host/path value."""
|
||||
|
||||
server = value.strip().strip("/")
|
||||
if not server or "://" in server or any(character.isspace() for character in server):
|
||||
raise DepotError(f"invalid CDN server: {value!r}")
|
||||
parsed = urlparse(f"https://{server}")
|
||||
if not parsed.netloc or parsed.query or parsed.fragment or "@" in parsed.netloc:
|
||||
raise DepotError(f"invalid CDN server: {value!r}")
|
||||
parts = PurePosixPath(parsed.path).parts
|
||||
if any(part in (".", "..") for part in parts):
|
||||
raise DepotError(f"invalid CDN server path: {value!r}")
|
||||
return f"{parsed.netloc}{parsed.path.rstrip('/')}"
|
||||
|
||||
|
||||
def depot_manifest_url(server: str, *, depot_id: int) -> str:
|
||||
return f"https://{server}/depots/{depot_id}/manifest.json"
|
||||
|
||||
|
||||
def build_manifest_url(server: str, build: int, *, depot_id: int) -> str:
|
||||
return f"https://{server}/depots/{depot_id}/builds/{build}.json"
|
||||
|
||||
|
||||
def file_url(server: str, file: DepotFile, *, depot_id: int) -> str:
|
||||
return (
|
||||
f"https://{server}/depots/{depot_id}/builds/{file.build}/"
|
||||
f"{quote(file.path, safe='/')}"
|
||||
)
|
||||
|
||||
|
||||
def ordered_download_servers(
|
||||
primary: str,
|
||||
candidates: Iterable[str],
|
||||
) -> tuple[str, ...]:
|
||||
ordered: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in (primary, *candidates):
|
||||
server = normalize_server(value)
|
||||
if server in seen:
|
||||
continue
|
||||
seen.add(server)
|
||||
ordered.append(server)
|
||||
return tuple(ordered)
|
||||
|
||||
|
||||
def download_partial_path(destination: Path) -> Path:
|
||||
return destination.with_name(f".{destination.name}.ptlinux.partial")
|
||||
|
||||
|
||||
def content_length(value: str | None) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
length = int(value)
|
||||
except ValueError as exc:
|
||||
raise DepotError(f"invalid Content-Length: {value!r}") from exc
|
||||
if length < 0:
|
||||
raise DepotError(f"invalid Content-Length: {value!r}")
|
||||
return length
|
||||
|
||||
|
||||
def content_range(
|
||||
value: str | None,
|
||||
*,
|
||||
pattern: re.Pattern[str],
|
||||
) -> tuple[int, int, int]:
|
||||
if value is None:
|
||||
raise DepotError("206 response omitted Content-Range")
|
||||
match = pattern.fullmatch(value.strip())
|
||||
if match is None:
|
||||
raise DepotError(f"invalid Content-Range: {value!r}")
|
||||
start, end, total = (int(part) for part in match.groups())
|
||||
if start > end or end >= total:
|
||||
raise DepotError(f"invalid Content-Range: {value!r}")
|
||||
return start, end, total
|
||||
|
||||
|
||||
def request_headers(version: str) -> dict[str, str]:
|
||||
return {"User-Agent": f"PT-Linux/{version} (+https://pristontale.eu/)"}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AdaptiveConcurrencyController",
|
||||
"BandwidthLimiter",
|
||||
"DepotHttpClient",
|
||||
"DepotHttpPolicy",
|
||||
"DownloadLimits",
|
||||
"DownloadObserver",
|
||||
"DownloadTelemetry",
|
||||
"HttpOpener",
|
||||
"HttpResponse",
|
||||
"build_manifest_url",
|
||||
"content_length",
|
||||
"content_range",
|
||||
"depot_manifest_url",
|
||||
"download_partial_path",
|
||||
"file_url",
|
||||
"normalize_server",
|
||||
"normalized_cpu_load",
|
||||
"ordered_download_servers",
|
||||
"prune_transfer_samples",
|
||||
"request_headers",
|
||||
"sample_rate",
|
||||
"server_candidates",
|
||||
]
|
||||
151
src/ptlinux/game/ptr_compat.py
Normal file
151
src/ptlinux/game/ptr_compat.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""Realm-specific compatibility files for the PTR game client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
from ..resource_paths import native_resource_path
|
||||
|
||||
|
||||
PTR_BCRYPT_PROXY_FILENAME = "ptr-bcrypt-proxy.dll"
|
||||
PTR_BCRYPT_PROXY_SHA256 = (
|
||||
"cf543a0c8f29231e35123c9d229194ee424b57b073a96da4f4ae6b0ebb5ce16a"
|
||||
)
|
||||
PTR_BCRYPT_RUNTIME_FILENAME = "bcrypt.dll"
|
||||
PTR_BCRYPT_ORIGINAL_RUNTIME_FILENAME = "bcrypt_orig.dll"
|
||||
PTR_BCRYPT_RUNTIME_FILENAMES = frozenset(
|
||||
{PTR_BCRYPT_RUNTIME_FILENAME, PTR_BCRYPT_ORIGINAL_RUNTIME_FILENAME}
|
||||
)
|
||||
_PROTON_BCRYPT_CANDIDATES = (
|
||||
Path("dist/lib/wine/i386-windows/bcrypt.dll"),
|
||||
Path("files/lib/wine/i386-windows/bcrypt.dll"),
|
||||
)
|
||||
|
||||
|
||||
class PtrCompatibilityError(RuntimeError):
|
||||
"""Raised when the verified PTR compatibility overlay cannot be staged."""
|
||||
|
||||
|
||||
def stage_ptr_bcrypt_compatibility(
|
||||
runtime_view: Path,
|
||||
proton_script: str | Path | None,
|
||||
) -> tuple[Path, Path]:
|
||||
"""Stage the verified proxy and Proton 7's original 32-bit BCrypt DLL."""
|
||||
|
||||
if proton_script is None:
|
||||
raise PtrCompatibilityError(
|
||||
"PTR Babel requires the Proton 7 compatibility runtime"
|
||||
)
|
||||
try:
|
||||
proton = Path(proton_script).expanduser().resolve(strict=True)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
raise PtrCompatibilityError(
|
||||
f"could not resolve the Proton 7 script for PTR Babel: {exc}"
|
||||
) from exc
|
||||
if not proton.is_file():
|
||||
raise PtrCompatibilityError(
|
||||
f"the Proton 7 script for PTR Babel is not a file: {proton}"
|
||||
)
|
||||
|
||||
proxy_source = native_resource_path(PTR_BCRYPT_PROXY_FILENAME)
|
||||
proxy_bytes = _read_file(proxy_source, "packaged PTR BCrypt proxy")
|
||||
proxy_hash = _sha256_bytes(proxy_bytes)
|
||||
if proxy_hash != PTR_BCRYPT_PROXY_SHA256:
|
||||
raise PtrCompatibilityError(
|
||||
"the packaged PTR BCrypt proxy failed its integrity check"
|
||||
)
|
||||
|
||||
original_source = _find_proton_bcrypt(proton)
|
||||
original_bytes = _read_file(original_source, "Proton 7 32-bit BCrypt DLL")
|
||||
if not _is_i386_pe(original_bytes):
|
||||
raise PtrCompatibilityError(
|
||||
f"the Proton 7 BCrypt module is not a 32-bit PE DLL: {original_source}"
|
||||
)
|
||||
|
||||
original_target = runtime_view / PTR_BCRYPT_ORIGINAL_RUNTIME_FILENAME
|
||||
proxy_target = runtime_view / PTR_BCRYPT_RUNTIME_FILENAME
|
||||
# Install the forwarding target first, so a partial write can never expose a
|
||||
# proxy whose forwarded exports have nowhere to resolve.
|
||||
_write_runtime_file(original_target, original_bytes)
|
||||
_write_runtime_file(proxy_target, proxy_bytes)
|
||||
return proxy_target, original_target
|
||||
|
||||
|
||||
def ptr_bcrypt_dll_overrides(current: str | None) -> str:
|
||||
"""Force native BCrypt while preserving unrelated Wine DLL overrides."""
|
||||
|
||||
overrides: list[str] = []
|
||||
for entry in (current or "").split(";"):
|
||||
entry = entry.strip()
|
||||
if not entry:
|
||||
continue
|
||||
name, separator, _value = entry.partition("=")
|
||||
if separator and name.strip().lstrip("*").casefold() == "bcrypt":
|
||||
continue
|
||||
overrides.append(entry)
|
||||
overrides.append("bcrypt=n")
|
||||
return ";".join(overrides)
|
||||
|
||||
|
||||
def _find_proton_bcrypt(proton_script: Path) -> Path:
|
||||
for relative in _PROTON_BCRYPT_CANDIDATES:
|
||||
candidate = proton_script.parent / relative
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
locations = ", ".join(str(path) for path in _PROTON_BCRYPT_CANDIDATES)
|
||||
raise PtrCompatibilityError(
|
||||
"the selected Proton 7 installation has no 32-bit BCrypt module "
|
||||
f"({locations})"
|
||||
)
|
||||
|
||||
|
||||
def _read_file(path: Path, description: str) -> bytes:
|
||||
try:
|
||||
return path.read_bytes()
|
||||
except OSError as exc:
|
||||
raise PtrCompatibilityError(f"could not read {description} {path}: {exc}") from exc
|
||||
|
||||
|
||||
def _write_runtime_file(destination: Path, content: bytes) -> None:
|
||||
expected_hash = _sha256_bytes(content)
|
||||
if not destination.is_symlink() and destination.is_file():
|
||||
try:
|
||||
if _sha256_bytes(destination.read_bytes()) == expected_hash:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp")
|
||||
try:
|
||||
temporary.write_bytes(content)
|
||||
temporary.chmod(0o644)
|
||||
temporary.replace(destination)
|
||||
if _sha256_bytes(destination.read_bytes()) != expected_hash:
|
||||
raise OSError("written file did not retain the expected digest")
|
||||
except OSError as exc:
|
||||
raise PtrCompatibilityError(
|
||||
f"could not stage PTR compatibility file {destination}: {exc}"
|
||||
) from exc
|
||||
finally:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _is_i386_pe(content: bytes) -> bool:
|
||||
if len(content) < 0x40 or content[:2] != b"MZ":
|
||||
return False
|
||||
pe_offset = int.from_bytes(content[0x3C:0x40], "little")
|
||||
if pe_offset < 0x40 or pe_offset + 26 > len(content):
|
||||
return False
|
||||
return (
|
||||
content[pe_offset : pe_offset + 4] == b"PE\0\0"
|
||||
and int.from_bytes(content[pe_offset + 4 : pe_offset + 6], "little") == 0x14C
|
||||
and int.from_bytes(content[pe_offset + 24 : pe_offset + 26], "little") == 0x10B
|
||||
)
|
||||
|
||||
|
||||
def _sha256_bytes(content: bytes) -> str:
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
1062
src/ptlinux/game/runtime.py
Normal file
1062
src/ptlinux/game/runtime.py
Normal file
File diff suppressed because it is too large
Load Diff
1218
src/ptlinux/game/runtime_view.py
Normal file
1218
src/ptlinux/game/runtime_view.py
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user