Init
This commit is contained in:
28
ptdiag/__init__.py
Normal file
28
ptdiag/__init__.py
Normal file
@@ -0,0 +1,28 @@
|
||||
"""PristonTale EU Linux diagnostic harness and shared binary helpers."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .config import AppConfig, ConfigError, load_config
|
||||
|
||||
|
||||
__all__ = ["AppConfig", "ConfigError", "load_config"]
|
||||
__version__ = "0.1.0"
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""Load diagnostic configuration helpers only when the full harness needs them.
|
||||
|
||||
The production launcher ships only :mod:`ptdiag.assbin`, which it uses for
|
||||
verified scene handling. Deferring the larger diagnostic-only imports keeps
|
||||
the compact package importable without shipping the investigation harness.
|
||||
"""
|
||||
if name in {"AppConfig", "ConfigError", "load_config"}:
|
||||
from .config import AppConfig, ConfigError, load_config
|
||||
|
||||
return {
|
||||
"AppConfig": AppConfig,
|
||||
"ConfigError": ConfigError,
|
||||
"load_config": load_config,
|
||||
}[name]
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
6
ptdiag/__main__.py
Normal file
6
ptdiag/__main__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
from .cli import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
1081
ptdiag/assbin.py
Normal file
1081
ptdiag/assbin.py
Normal file
File diff suppressed because it is too large
Load Diff
203
ptdiag/cli.py
Normal file
203
ptdiag/cli.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""Command-line interface for diagnostics and controlled runtime experiments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Sequence
|
||||
|
||||
from .assbin import (
|
||||
AssbinError,
|
||||
extract_embedded_texture,
|
||||
inspect_wrapped_assbin,
|
||||
normalize_wrapped_assbin_textures,
|
||||
)
|
||||
from .config import ConfigError, load_config
|
||||
from .experiment import run_experiment
|
||||
from .filesystem import verify_known_game_hashes
|
||||
from .ini import IniError, override_ini_value
|
||||
from .logs import parse_native_log
|
||||
from .process import collect_environment
|
||||
from .testcase import TestcaseError, load_testcase
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="ptdiag")
|
||||
parser.add_argument("--config", type=Path, help="path to local.toml")
|
||||
subcommands = parser.add_subparsers(dest="command", required=True)
|
||||
subcommands.add_parser("env", help="inspect host and configured paths")
|
||||
subcommands.add_parser("verify", help="verify environment and known core hashes")
|
||||
|
||||
log_parser = subcommands.add_parser("log", help="native game-log operations")
|
||||
log_commands = log_parser.add_subparsers(dest="log_command", required=True)
|
||||
analyze = log_commands.add_parser("analyze", help="parse a native game log")
|
||||
analyze.add_argument("path", type=Path)
|
||||
|
||||
scene_parser = subcommands.add_parser("scene", help="wrapped scene-cache operations")
|
||||
scene_commands = scene_parser.add_subparsers(dest="scene_command", required=True)
|
||||
inspect = scene_commands.add_parser(
|
||||
"inspect",
|
||||
help="inspect an XOR-wrapped Assimp .rovug cache",
|
||||
)
|
||||
inspect.add_argument("path", type=Path)
|
||||
inspect.add_argument("--output", type=Path, help="also write the complete JSON result")
|
||||
extract = scene_commands.add_parser(
|
||||
"extract-texture",
|
||||
help="extract and verify one embedded texture from a .rovug cache",
|
||||
)
|
||||
extract.add_argument("path", type=Path)
|
||||
extract.add_argument("index", type=int)
|
||||
extract.add_argument("output", type=Path)
|
||||
normalize = scene_commands.add_parser(
|
||||
"normalize-textures",
|
||||
help="write a cache with selected 16-bit RGB PNGs normalized to 8-bit",
|
||||
)
|
||||
normalize.add_argument("path", type=Path)
|
||||
normalize.add_argument("output", type=Path)
|
||||
normalize.add_argument("indices", type=int, nargs="+")
|
||||
|
||||
ini_parser = subcommands.add_parser("ini", help="verified INI operations")
|
||||
ini_commands = ini_parser.add_subparsers(dest="ini_command", required=True)
|
||||
ini_override = ini_commands.add_parser(
|
||||
"override",
|
||||
help="replace one exact INI value after checking its expected value",
|
||||
)
|
||||
ini_override.add_argument("path", type=Path)
|
||||
ini_override.add_argument("section")
|
||||
ini_override.add_argument("key")
|
||||
ini_override.add_argument("expected")
|
||||
ini_override.add_argument("value")
|
||||
|
||||
run_parser = subcommands.add_parser("run", help="run a controlled testcase")
|
||||
run_parser.add_argument("testcase", help="testcase name or TOML path")
|
||||
run_parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="write and validate the complete plan without launching Game.exe",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--results-root",
|
||||
type=Path,
|
||||
help="override the result-directory root",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> None:
|
||||
raise SystemExit(run(argv))
|
||||
|
||||
|
||||
def run(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
if args.command == "log" and args.log_command == "analyze":
|
||||
try:
|
||||
analysis = parse_native_log(args.path)
|
||||
except OSError as exc:
|
||||
_print_json({"ok": False, "error": str(exc)})
|
||||
return 2
|
||||
_print_json({"ok": True, **analysis.to_summary()})
|
||||
return 0
|
||||
if args.command == "scene" and args.scene_command == "inspect":
|
||||
try:
|
||||
inspection = inspect_wrapped_assbin(args.path)
|
||||
except (AssbinError, OSError) as exc:
|
||||
_print_json({"ok": False, "error": str(exc)})
|
||||
return 2
|
||||
payload = {"ok": True, **inspection.to_dict()}
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output.open("w", encoding="utf-8") as stream:
|
||||
json.dump(payload, stream, indent=2, sort_keys=True)
|
||||
stream.write("\n")
|
||||
_print_json(payload)
|
||||
return 0
|
||||
if args.command == "scene" and args.scene_command == "extract-texture":
|
||||
try:
|
||||
inspection = inspect_wrapped_assbin(args.path)
|
||||
extraction = extract_embedded_texture(
|
||||
args.path,
|
||||
inspection,
|
||||
args.index,
|
||||
args.output,
|
||||
)
|
||||
except (AssbinError, OSError) as exc:
|
||||
_print_json({"ok": False, "error": str(exc)})
|
||||
return 2
|
||||
_print_json({"ok": True, **extraction})
|
||||
return 0
|
||||
if args.command == "scene" and args.scene_command == "normalize-textures":
|
||||
try:
|
||||
normalization = normalize_wrapped_assbin_textures(
|
||||
args.path,
|
||||
args.output,
|
||||
tuple(args.indices),
|
||||
)
|
||||
except (AssbinError, OSError) as exc:
|
||||
_print_json({"ok": False, "error": str(exc)})
|
||||
return 2
|
||||
_print_json({"ok": True, **normalization})
|
||||
return 0
|
||||
if args.command == "ini" and args.ini_command == "override":
|
||||
try:
|
||||
result = override_ini_value(
|
||||
args.path,
|
||||
args.section,
|
||||
args.key,
|
||||
args.expected,
|
||||
args.value,
|
||||
)
|
||||
except (IniError, OSError) as exc:
|
||||
_print_json({"ok": False, "error": str(exc)})
|
||||
return 2
|
||||
_print_json(result)
|
||||
return 0
|
||||
|
||||
try:
|
||||
config = load_config(args.config)
|
||||
except ConfigError as exc:
|
||||
_print_json({"ok": False, "error": str(exc)})
|
||||
return 2
|
||||
|
||||
if args.command == "run":
|
||||
try:
|
||||
testcase = load_testcase(args.testcase)
|
||||
outcome = run_experiment(
|
||||
config,
|
||||
testcase,
|
||||
dry_run=args.dry_run,
|
||||
results_root=args.results_root,
|
||||
)
|
||||
except (TestcaseError, OSError, ValueError) as exc:
|
||||
_print_json({"ok": False, "error": str(exc)})
|
||||
return 2
|
||||
_print_json(
|
||||
{
|
||||
"ok": outcome.summary.get("ok", False),
|
||||
"status": outcome.summary.get("status"),
|
||||
"result_directory": str(outcome.result_directory),
|
||||
}
|
||||
)
|
||||
return outcome.exit_code
|
||||
|
||||
environment = collect_environment(config)
|
||||
if args.command == "env":
|
||||
_print_json(environment)
|
||||
return 0 if environment["ok"] else 1
|
||||
|
||||
hashes = verify_known_game_hashes(config.paths.live_game)
|
||||
hashes_ok = all(item.ok for item in hashes)
|
||||
payload = {
|
||||
"ok": bool(environment["ok"]) and hashes_ok,
|
||||
"environment": environment,
|
||||
"hashes": [item.to_dict() for item in hashes],
|
||||
}
|
||||
_print_json(payload)
|
||||
return 0 if payload["ok"] else 1
|
||||
|
||||
|
||||
def _print_json(value: object) -> None:
|
||||
json.dump(value, sys.stdout, indent=2, sort_keys=True)
|
||||
sys.stdout.write("\n")
|
||||
162
ptdiag/config.py
Normal file
162
ptdiag/config.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""Typed TOML configuration loading."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
import re
|
||||
import tomllib
|
||||
|
||||
|
||||
DEFAULT_CONFIG_PATH = Path("config/local.toml")
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""Raised when local configuration is missing or invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PathsConfig:
|
||||
app_root: Path
|
||||
live_prefix: Path
|
||||
live_game: Path
|
||||
logs: Path
|
||||
native_logs: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Proton7Config:
|
||||
proton: Path
|
||||
steam_root: Path
|
||||
compat: Path
|
||||
prefix: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GameConfig:
|
||||
exe: str
|
||||
args: tuple[str, ...]
|
||||
virtual_desktop: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SafetyConfig:
|
||||
allow_live_game_modification: bool
|
||||
allow_prefix_modification: bool
|
||||
allow_binary_patch: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppConfig:
|
||||
paths: PathsConfig
|
||||
proton7: Proton7Config
|
||||
game: GameConfig
|
||||
safety: SafetyConfig
|
||||
source: Path
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = asdict(self)
|
||||
return _paths_to_strings(result)
|
||||
|
||||
|
||||
def project_root() -> Path:
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def load_config(path: str | Path | None = None) -> AppConfig:
|
||||
config_path = Path(path) if path is not None else project_root() / DEFAULT_CONFIG_PATH
|
||||
config_path = config_path.expanduser().absolute()
|
||||
try:
|
||||
with config_path.open("rb") as handle:
|
||||
data = tomllib.load(handle)
|
||||
except FileNotFoundError as exc:
|
||||
raise ConfigError(
|
||||
f"Configuration not found: {config_path}. "
|
||||
"Copy config/local.example.toml to config/local.toml."
|
||||
) from exc
|
||||
except tomllib.TOMLDecodeError as exc:
|
||||
raise ConfigError(f"Invalid TOML in {config_path}: {exc}") from exc
|
||||
|
||||
paths = _section(data, "paths")
|
||||
proton7 = _section(data, "proton7")
|
||||
game = _section(data, "game")
|
||||
safety = _section(data, "safety")
|
||||
virtual_desktop = _required_string(game, "virtual_desktop")
|
||||
if re.fullmatch(r"[1-9][0-9]*x[1-9][0-9]*", virtual_desktop) is None:
|
||||
raise ConfigError("game.virtual_desktop must look like 1600x900")
|
||||
|
||||
return AppConfig(
|
||||
paths=PathsConfig(
|
||||
app_root=_required_path(paths, "app_root"),
|
||||
live_prefix=_required_path(paths, "live_prefix"),
|
||||
live_game=_required_path(paths, "live_game"),
|
||||
logs=_required_path(paths, "logs"),
|
||||
native_logs=_required_path(paths, "native_logs"),
|
||||
),
|
||||
proton7=Proton7Config(
|
||||
proton=_required_path(proton7, "proton"),
|
||||
steam_root=_required_path(proton7, "steam_root"),
|
||||
compat=_required_path(proton7, "compat"),
|
||||
prefix=_required_path(proton7, "prefix"),
|
||||
),
|
||||
game=GameConfig(
|
||||
exe=_required_string(game, "exe"),
|
||||
args=_required_string_tuple(game, "args"),
|
||||
virtual_desktop=virtual_desktop,
|
||||
),
|
||||
safety=SafetyConfig(
|
||||
allow_live_game_modification=_required_bool(
|
||||
safety, "allow_live_game_modification"
|
||||
),
|
||||
allow_prefix_modification=_required_bool(
|
||||
safety, "allow_prefix_modification"
|
||||
),
|
||||
allow_binary_patch=_required_bool(safety, "allow_binary_patch"),
|
||||
),
|
||||
source=config_path,
|
||||
)
|
||||
|
||||
|
||||
def _section(data: Mapping[str, Any], name: str) -> Mapping[str, Any]:
|
||||
value = data.get(name)
|
||||
if not isinstance(value, Mapping):
|
||||
raise ConfigError(f"Missing or invalid [{name}] section")
|
||||
return value
|
||||
|
||||
|
||||
def _required_string(section: Mapping[str, Any], key: str) -> str:
|
||||
value = section.get(key)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ConfigError(f"{key} must be a non-empty string")
|
||||
return value
|
||||
|
||||
|
||||
def _required_path(section: Mapping[str, Any], key: str) -> Path:
|
||||
return Path(_required_string(section, key)).expanduser().absolute()
|
||||
|
||||
|
||||
def _required_string_tuple(section: Mapping[str, Any], key: str) -> tuple[str, ...]:
|
||||
value = section.get(key)
|
||||
if not isinstance(value, list) or not value or not all(
|
||||
isinstance(item, str) and item for item in value
|
||||
):
|
||||
raise ConfigError(f"{key} must be a non-empty array of strings")
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _required_bool(section: Mapping[str, Any], key: str) -> bool:
|
||||
value = section.get(key)
|
||||
if not isinstance(value, bool):
|
||||
raise ConfigError(f"{key} must be true or false")
|
||||
return value
|
||||
|
||||
|
||||
def _paths_to_strings(value: Any) -> Any:
|
||||
if isinstance(value, Path):
|
||||
return str(value)
|
||||
if isinstance(value, dict):
|
||||
return {key: _paths_to_strings(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_paths_to_strings(item) for item in value]
|
||||
return value
|
||||
1812
ptdiag/experiment.py
Normal file
1812
ptdiag/experiment.py
Normal file
File diff suppressed because it is too large
Load Diff
59
ptdiag/filesystem.py
Normal file
59
ptdiag/filesystem.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Read-only filesystem and hash verification helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Mapping
|
||||
|
||||
|
||||
KNOWN_GAME_HASHES: dict[str, str] = {
|
||||
"Game.exe": "3cc0980aac0f65b379ff340ec61750498fb9f620fb6c88aa66281e8ab27023a5",
|
||||
"game.dll": "7ea85b19f452962a39c91f967cad22495f066b2d52e8ac37f4e234463d5c4f09",
|
||||
"gamese.exe": "c3443e50a93c6c778eff456122c351e5c7d2c1ceaea50e5a650bffe846a05c87",
|
||||
"gamese.dll": "0ae6822f5f347f04bd2b0dc947f0286089c711c84703e110f6f8fc6e0d626131",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HashVerification:
|
||||
path: Path
|
||||
expected_sha256: str
|
||||
actual_sha256: str | None
|
||||
ok: bool
|
||||
error: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, str | bool | None]:
|
||||
data = asdict(self)
|
||||
data["path"] = str(self.path)
|
||||
return data
|
||||
|
||||
|
||||
def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str:
|
||||
if chunk_size <= 0:
|
||||
raise ValueError("chunk_size must be positive")
|
||||
digest = sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(chunk_size), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def verify_file_hash(path: Path, expected_sha256: str) -> HashVerification:
|
||||
expected = expected_sha256.casefold()
|
||||
try:
|
||||
actual = sha256_file(path)
|
||||
except (OSError, ValueError) as exc:
|
||||
return HashVerification(path, expected, None, False, str(exc))
|
||||
return HashVerification(path, expected, actual, actual == expected)
|
||||
|
||||
|
||||
def verify_hashes(expected: Mapping[Path, str]) -> list[HashVerification]:
|
||||
return [verify_file_hash(path, digest) for path, digest in expected.items()]
|
||||
|
||||
|
||||
def verify_known_game_hashes(live_game: Path) -> list[HashVerification]:
|
||||
expected = {live_game / name: digest for name, digest in KNOWN_GAME_HASHES.items()}
|
||||
return verify_hashes(expected)
|
||||
|
||||
112
ptdiag/ini.py
Normal file
112
ptdiag/ini.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""Exact, verified INI value inspection and replacement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from typing import Any
|
||||
|
||||
from .filesystem import sha256_file
|
||||
|
||||
|
||||
class IniError(ValueError):
|
||||
"""Raised when an INI override cannot be applied exactly."""
|
||||
|
||||
|
||||
def inspect_ini_value(path: Path, section: str, key: str) -> dict[str, Any]:
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise IniError(f"INI path is missing or unsafe: {path}")
|
||||
text = _read_ini_text(path)
|
||||
_, value = _find_ini_value(text, section, key)
|
||||
state = path.stat()
|
||||
return {
|
||||
"path": str(path),
|
||||
"section": section,
|
||||
"key": key,
|
||||
"value": value,
|
||||
"sha256": sha256_file(path),
|
||||
"size": state.st_size,
|
||||
"mtime_ns": state.st_mtime_ns,
|
||||
}
|
||||
|
||||
|
||||
def override_ini_value(
|
||||
path: Path,
|
||||
section: str,
|
||||
key: str,
|
||||
expected: str,
|
||||
value: str,
|
||||
) -> dict[str, Any]:
|
||||
before = inspect_ini_value(path, section, key)
|
||||
if before["value"] != expected:
|
||||
raise IniError(
|
||||
f"INI value mismatch for [{section}] {key}: "
|
||||
f"expected {expected!r}, found {before['value']!r}"
|
||||
)
|
||||
text = _read_ini_text(path)
|
||||
line_index, _ = _find_ini_value(text, section, key)
|
||||
lines = text.splitlines(keepends=True)
|
||||
body = lines[line_index].rstrip("\r\n")
|
||||
newline = lines[line_index][len(body) :]
|
||||
left, separator, right = body.partition("=")
|
||||
if not separator:
|
||||
raise IniError(f"INI key has no equals separator: [{section}] {key}")
|
||||
leading = right[: len(right) - len(right.lstrip())]
|
||||
trailing = right[len(right.rstrip()) :]
|
||||
lines[line_index] = f"{left}={leading}{value}{trailing}{newline}"
|
||||
updated = "".join(lines)
|
||||
if updated == text:
|
||||
raise IniError(f"INI override made no change: [{section}] {key}")
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
if temporary.exists() or temporary.is_symlink():
|
||||
raise IniError(f"INI temporary path already exists: {temporary}")
|
||||
try:
|
||||
with temporary.open("w", encoding="utf-8", newline="") as stream:
|
||||
stream.write(updated)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
shutil.copystat(path, temporary, follow_symlinks=False)
|
||||
temporary.replace(path)
|
||||
finally:
|
||||
if temporary.exists() or temporary.is_symlink():
|
||||
temporary.unlink()
|
||||
after = inspect_ini_value(path, section, key)
|
||||
if after["value"] != value:
|
||||
raise IniError(f"INI override verification failed: [{section}] {key}")
|
||||
if after["sha256"] == before["sha256"]:
|
||||
raise IniError(f"INI hash did not change: [{section}] {key}")
|
||||
return {
|
||||
"ok": True,
|
||||
"before": before,
|
||||
"after": after,
|
||||
"expected": expected,
|
||||
"value": value,
|
||||
}
|
||||
|
||||
|
||||
def _read_ini_text(path: Path) -> str:
|
||||
with path.open("r", encoding="utf-8", newline="") as stream:
|
||||
return stream.read()
|
||||
|
||||
|
||||
def _find_ini_value(text: str, section: str, key: str) -> tuple[int, str]:
|
||||
current_section: str | None = None
|
||||
matches: list[tuple[int, str]] = []
|
||||
for index, line in enumerate(text.splitlines(keepends=True)):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("[") and stripped.endswith("]"):
|
||||
current_section = stripped[1:-1].strip()
|
||||
continue
|
||||
if current_section is None or current_section.casefold() != section.casefold():
|
||||
continue
|
||||
if not stripped or stripped.startswith((";", "#")):
|
||||
continue
|
||||
left, separator, right = line.rstrip("\r\n").partition("=")
|
||||
if separator and left.strip().casefold() == key.casefold():
|
||||
matches.append((index, right.strip()))
|
||||
if not matches:
|
||||
raise IniError(f"INI key not found: [{section}] {key}")
|
||||
if len(matches) != 1:
|
||||
raise IniError(f"INI key is duplicated: [{section}] {key}")
|
||||
return matches[0]
|
||||
288
ptdiag/logs.py
Normal file
288
ptdiag/logs.py
Normal file
@@ -0,0 +1,288 @@
|
||||
"""Native game-log discovery, freshness validation, and parsing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Iterable, Mapping
|
||||
|
||||
|
||||
DEFAULT_NATIVE_LOG_PATTERNS = ("log_*.txt", "*.log")
|
||||
|
||||
_SCENE_RE = re.compile(
|
||||
r"\[S1\]\s+after import\s+(?P<scene>[^:]+):\s*"
|
||||
r"materials=(?P<materials>\d+)\s+textures=(?P<textures>\d+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FAILED_FILE_RE = re.compile(
|
||||
r"Failed to (?P<operation>find|load|read) file\s*:\s*(?P<path>.*)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FAILED_IMPORT_RE = re.compile(
|
||||
r"Failed to import scene:\s*(?P<scene>.*)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TIMESTAMP_RE = re.compile(
|
||||
r"^\s*(?:\[)?(?P<timestamp>"
|
||||
r"\d{4}-\d{2}-\d{2}[T ][0-9:.+Z-]+|\d{2}:\d{2}:\d{2}(?:\.\d+)?)"
|
||||
r"(?:\])?"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LogFileState:
|
||||
path: Path
|
||||
mtime_ns: int
|
||||
size: int
|
||||
inode: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LogEvent:
|
||||
kind: str
|
||||
line_number: int
|
||||
raw: str
|
||||
timestamp_text: str | None = None
|
||||
value: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, str | int | None]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SceneImport:
|
||||
name: str
|
||||
materials: int
|
||||
textures: int
|
||||
line_number: int
|
||||
timestamp_text: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, str | int | None]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NativeLogAnalysis:
|
||||
scenes: dict[str, list[SceneImport]] = field(default_factory=dict)
|
||||
events: list[LogEvent] = field(default_factory=list)
|
||||
failed_files: list[LogEvent] = field(default_factory=list)
|
||||
failed_import_scenes: list[str] = field(default_factory=list)
|
||||
load_texture_to_material_failed: bool = False
|
||||
import_scene_threw: bool = False
|
||||
stage_without_new_stage: bool = False
|
||||
tex_reclaim_count: int = 0
|
||||
|
||||
def latest_scene(self, name: str) -> SceneImport | None:
|
||||
wanted = _scene_key(name)
|
||||
for scene_name, imports in self.scenes.items():
|
||||
if _scene_key(scene_name) == wanted and imports:
|
||||
return imports[-1]
|
||||
return None
|
||||
|
||||
def first_event(self, kind: str) -> LogEvent | None:
|
||||
return next((event for event in self.events if event.kind == kind), None)
|
||||
|
||||
def to_summary(self) -> dict[str, object]:
|
||||
summary: dict[str, object] = {
|
||||
"scenes": {
|
||||
name: [item.to_dict() for item in imports]
|
||||
for name, imports in sorted(self.scenes.items())
|
||||
},
|
||||
"load_texture_to_material_failed": self.load_texture_to_material_failed,
|
||||
"import_scene_threw": self.import_scene_threw,
|
||||
"failed_import_scenes": self.failed_import_scenes,
|
||||
"failed_files": [event.to_dict() for event in self.failed_files],
|
||||
"stage_without_new_stage": self.stage_without_new_stage,
|
||||
"tex_reclaim_count": self.tex_reclaim_count,
|
||||
}
|
||||
for output_key, scene_name in (
|
||||
("whispering_vale", "WhisperingVale"),
|
||||
("ricarten", "Ricarten"),
|
||||
):
|
||||
scene = self.latest_scene(scene_name)
|
||||
if scene is not None:
|
||||
summary[output_key] = {
|
||||
"materials": scene.materials,
|
||||
"textures": scene.textures,
|
||||
}
|
||||
if "whispering_vale" in summary:
|
||||
whispering_vale = summary["whispering_vale"]
|
||||
assert isinstance(whispering_vale, dict)
|
||||
whispering_vale["load_texture_to_material_failed"] = (
|
||||
self.load_texture_to_material_failed
|
||||
)
|
||||
whispering_vale["import_scene_failed"] = self.import_scene_threw or any(
|
||||
"whisperingvale" in _scene_key(item)
|
||||
for item in self.failed_import_scenes
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
def snapshot_native_logs(
|
||||
log_directory: Path,
|
||||
patterns: Iterable[str] = DEFAULT_NATIVE_LOG_PATTERNS,
|
||||
) -> dict[str, LogFileState]:
|
||||
states: dict[str, LogFileState] = {}
|
||||
if not log_directory.is_dir():
|
||||
return states
|
||||
for pattern in patterns:
|
||||
for path in log_directory.glob(pattern):
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
state = _state(path)
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
states[_path_key(path)] = state
|
||||
return states
|
||||
|
||||
|
||||
def is_current_run_log(
|
||||
state: LogFileState,
|
||||
baseline: Mapping[str, LogFileState],
|
||||
test_start_ns: int,
|
||||
) -> bool:
|
||||
"""Return true only for a new log or a changed post-start existing log.
|
||||
|
||||
A path absent from the pre-launch baseline is new for this run. An existing
|
||||
path must both change and have an mtime at or after the recorded test start.
|
||||
The additional change check prevents a stale future-dated file from passing.
|
||||
"""
|
||||
previous = baseline.get(_path_key(state.path))
|
||||
if previous is None:
|
||||
return True
|
||||
changed = (
|
||||
state.mtime_ns != previous.mtime_ns
|
||||
or state.size != previous.size
|
||||
or state.inode != previous.inode
|
||||
)
|
||||
return changed and state.mtime_ns >= test_start_ns
|
||||
|
||||
|
||||
def discover_current_native_logs(
|
||||
log_directory: Path,
|
||||
baseline: Mapping[str, LogFileState],
|
||||
test_start_ns: int,
|
||||
patterns: Iterable[str] = DEFAULT_NATIVE_LOG_PATTERNS,
|
||||
) -> list[LogFileState]:
|
||||
current = snapshot_native_logs(log_directory, patterns)
|
||||
eligible = [
|
||||
state
|
||||
for state in current.values()
|
||||
if is_current_run_log(state, baseline, test_start_ns)
|
||||
]
|
||||
return sorted(eligible, key=lambda item: (item.mtime_ns, str(item.path)))
|
||||
|
||||
|
||||
def newest_current_native_log(
|
||||
log_directory: Path,
|
||||
baseline: Mapping[str, LogFileState],
|
||||
test_start_ns: int,
|
||||
patterns: Iterable[str] = DEFAULT_NATIVE_LOG_PATTERNS,
|
||||
) -> LogFileState | None:
|
||||
logs = discover_current_native_logs(log_directory, baseline, test_start_ns, patterns)
|
||||
return logs[-1] if logs else None
|
||||
|
||||
|
||||
def parse_native_log(path: Path) -> NativeLogAnalysis:
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
return parse_native_log_lines(handle)
|
||||
|
||||
|
||||
def read_current_run_log_text(
|
||||
state: LogFileState,
|
||||
baseline: Mapping[str, LogFileState],
|
||||
) -> str:
|
||||
"""Read only bytes attributable to this run when a log path is reused."""
|
||||
previous = baseline.get(_path_key(state.path))
|
||||
offset = 0
|
||||
if (
|
||||
previous is not None
|
||||
and state.inode == previous.inode
|
||||
and state.size >= previous.size
|
||||
):
|
||||
offset = previous.size
|
||||
with state.path.open("rb") as handle:
|
||||
handle.seek(offset)
|
||||
return handle.read().decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def parse_current_run_log(
|
||||
state: LogFileState,
|
||||
baseline: Mapping[str, LogFileState],
|
||||
) -> NativeLogAnalysis:
|
||||
return parse_native_log_lines(read_current_run_log_text(state, baseline).splitlines())
|
||||
|
||||
|
||||
def parse_native_log_lines(lines: Iterable[str]) -> NativeLogAnalysis:
|
||||
analysis = NativeLogAnalysis()
|
||||
for line_number, raw_line in enumerate(lines, start=1):
|
||||
raw = raw_line.rstrip("\r\n")
|
||||
timestamp = _timestamp(raw)
|
||||
scene_match = _SCENE_RE.search(raw)
|
||||
if scene_match is not None:
|
||||
scene = SceneImport(
|
||||
name=scene_match.group("scene").strip(),
|
||||
materials=int(scene_match.group("materials")),
|
||||
textures=int(scene_match.group("textures")),
|
||||
line_number=line_number,
|
||||
timestamp_text=timestamp,
|
||||
)
|
||||
analysis.scenes.setdefault(scene.name, []).append(scene)
|
||||
analysis.events.append(
|
||||
LogEvent("scene_import", line_number, raw, timestamp, scene.name)
|
||||
)
|
||||
|
||||
if "GRX::SurfaceManager::LoadTextureToMaterial" in raw:
|
||||
analysis.load_texture_to_material_failed = True
|
||||
analysis.events.append(
|
||||
LogEvent("load_texture_to_material", line_number, raw, timestamp)
|
||||
)
|
||||
if "ImportScene threw" in raw:
|
||||
analysis.import_scene_threw = True
|
||||
analysis.events.append(LogEvent("import_scene_threw", line_number, raw, timestamp))
|
||||
|
||||
failed_import = _FAILED_IMPORT_RE.search(raw)
|
||||
if failed_import is not None:
|
||||
value = failed_import.group("scene").strip()
|
||||
analysis.failed_import_scenes.append(value)
|
||||
analysis.events.append(
|
||||
LogEvent("failed_import_scene", line_number, raw, timestamp, value)
|
||||
)
|
||||
|
||||
failed_file = _FAILED_FILE_RE.search(raw)
|
||||
if failed_file is not None:
|
||||
operation = failed_file.group("operation").casefold()
|
||||
value = failed_file.group("path").strip()
|
||||
event = LogEvent(f"failed_{operation}_file", line_number, raw, timestamp, value)
|
||||
analysis.failed_files.append(event)
|
||||
analysis.events.append(event)
|
||||
|
||||
if "Stage without new stage" in raw:
|
||||
analysis.stage_without_new_stage = True
|
||||
analysis.events.append(
|
||||
LogEvent("stage_without_new_stage", line_number, raw, timestamp)
|
||||
)
|
||||
if "TexReclaim" in raw:
|
||||
analysis.tex_reclaim_count += 1
|
||||
analysis.events.append(LogEvent("tex_reclaim", line_number, raw, timestamp))
|
||||
return analysis
|
||||
|
||||
|
||||
def _state(path: Path) -> LogFileState:
|
||||
stat = path.stat()
|
||||
return LogFileState(path.absolute(), stat.st_mtime_ns, stat.st_size, stat.st_ino)
|
||||
|
||||
|
||||
def _path_key(path: Path) -> str:
|
||||
return str(path.absolute())
|
||||
|
||||
|
||||
def _scene_key(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]", "", value.casefold())
|
||||
|
||||
|
||||
def _timestamp(line: str) -> str | None:
|
||||
match = _TIMESTAMP_RE.match(line)
|
||||
return match.group("timestamp") if match is not None else None
|
||||
278
ptdiag/performance.py
Normal file
278
ptdiag/performance.py
Normal file
@@ -0,0 +1,278 @@
|
||||
"""Low-overhead process and device sampling for performance experiments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
from typing import Any, TextIO
|
||||
|
||||
|
||||
_SAMPLE_FIELDS = (
|
||||
"epoch",
|
||||
"elapsed_seconds",
|
||||
"process_cpu_percent",
|
||||
"system_cpu_percent",
|
||||
"rss_mib",
|
||||
"read_mib_per_second",
|
||||
"write_mib_per_second",
|
||||
"voluntary_context_switches_per_second",
|
||||
"nonvoluntary_context_switches_per_second",
|
||||
"gpu_busy_percent",
|
||||
"vram_busy_percent",
|
||||
"gpu_sclk_mhz",
|
||||
"gpu_mclk_mhz",
|
||||
"cpu_average_mhz",
|
||||
"cpu_minimum_mhz",
|
||||
"cpu_maximum_mhz",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Counters:
|
||||
process_ticks: int
|
||||
rss_bytes: int
|
||||
read_bytes: int
|
||||
write_bytes: int
|
||||
voluntary_context_switches: int
|
||||
nonvoluntary_context_switches: int
|
||||
system_total_ticks: int
|
||||
system_idle_ticks: int
|
||||
|
||||
|
||||
class PerformanceSampler:
|
||||
"""Sample one game process and the active AMD GPU at a fixed interval."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pid: int,
|
||||
output: Path,
|
||||
*,
|
||||
interval_seconds: float = 0.5,
|
||||
) -> None:
|
||||
if pid <= 0:
|
||||
raise ValueError("pid must be positive")
|
||||
if interval_seconds <= 0:
|
||||
raise ValueError("interval_seconds must be positive")
|
||||
self.pid = pid
|
||||
self.output = output
|
||||
self.interval_seconds = interval_seconds
|
||||
self._interval_ns = int(interval_seconds * 1_000_000_000)
|
||||
self._started_ns = time.time_ns()
|
||||
self._last_sample_ns: int | None = None
|
||||
self._previous: _Counters | None = None
|
||||
self._handle: TextIO = output.open("w", encoding="utf-8", newline="")
|
||||
self._writer = csv.DictWriter(self._handle, fieldnames=_SAMPLE_FIELDS)
|
||||
self._writer.writeheader()
|
||||
self._gpu_device = _discover_gpu_device()
|
||||
self._sample_count = 0
|
||||
self._peaks: dict[str, float] = {}
|
||||
self._first_counters: _Counters | None = None
|
||||
self._last_counters: _Counters | None = None
|
||||
|
||||
def maybe_sample(self, now_ns: int | None = None) -> bool:
|
||||
now_ns = now_ns if now_ns is not None else time.time_ns()
|
||||
previous_sample_ns = self._last_sample_ns
|
||||
if (
|
||||
previous_sample_ns is not None
|
||||
and now_ns - previous_sample_ns < self._interval_ns
|
||||
):
|
||||
return False
|
||||
self._last_sample_ns = now_ns
|
||||
try:
|
||||
counters = _read_counters(self.pid)
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
if self._first_counters is None:
|
||||
self._first_counters = counters
|
||||
self._last_counters = counters
|
||||
row = self._sample_row(now_ns, previous_sample_ns, counters)
|
||||
self._writer.writerow(row)
|
||||
self._handle.flush()
|
||||
self._sample_count += 1
|
||||
for key, value in row.items():
|
||||
if key in {"epoch", "elapsed_seconds"} or not isinstance(value, (int, float)):
|
||||
continue
|
||||
self._peaks[key] = max(float(value), self._peaks.get(key, float("-inf")))
|
||||
self._previous = counters
|
||||
return True
|
||||
|
||||
def close(self) -> dict[str, Any]:
|
||||
if not self._handle.closed:
|
||||
self._handle.close()
|
||||
first = self._first_counters
|
||||
last = self._last_counters
|
||||
return {
|
||||
"pid": self.pid,
|
||||
"output": str(self.output),
|
||||
"interval_seconds": self.interval_seconds,
|
||||
"sample_count": self._sample_count,
|
||||
"gpu_device": str(self._gpu_device) if self._gpu_device else None,
|
||||
"peaks": dict(sorted(self._peaks.items())),
|
||||
"process_io_bytes": {
|
||||
"read": max(0, last.read_bytes - first.read_bytes) if first and last else None,
|
||||
"write": max(0, last.write_bytes - first.write_bytes) if first and last else None,
|
||||
},
|
||||
}
|
||||
|
||||
def _sample_row(
|
||||
self,
|
||||
now_ns: int,
|
||||
previous_sample_ns: int | None,
|
||||
counters: _Counters,
|
||||
) -> dict[str, float | str | None]:
|
||||
elapsed_seconds = (now_ns - self._started_ns) / 1_000_000_000
|
||||
row: dict[str, float | str | None] = {
|
||||
"epoch": now_ns / 1_000_000_000,
|
||||
"elapsed_seconds": elapsed_seconds,
|
||||
"process_cpu_percent": None,
|
||||
"system_cpu_percent": None,
|
||||
"rss_mib": counters.rss_bytes / 1024**2,
|
||||
"read_mib_per_second": None,
|
||||
"write_mib_per_second": None,
|
||||
"voluntary_context_switches_per_second": None,
|
||||
"nonvoluntary_context_switches_per_second": None,
|
||||
"gpu_busy_percent": _read_float(self._gpu_device, "gpu_busy_percent"),
|
||||
"vram_busy_percent": _read_float(self._gpu_device, "mem_busy_percent"),
|
||||
"gpu_sclk_mhz": _read_active_mhz(self._gpu_device, "pp_dpm_sclk"),
|
||||
"gpu_mclk_mhz": _read_active_mhz(self._gpu_device, "pp_dpm_mclk"),
|
||||
"cpu_average_mhz": None,
|
||||
"cpu_minimum_mhz": None,
|
||||
"cpu_maximum_mhz": None,
|
||||
}
|
||||
frequencies = _read_cpu_frequencies_mhz()
|
||||
if frequencies:
|
||||
row["cpu_average_mhz"] = sum(frequencies) / len(frequencies)
|
||||
row["cpu_minimum_mhz"] = min(frequencies)
|
||||
row["cpu_maximum_mhz"] = max(frequencies)
|
||||
previous = self._previous
|
||||
if previous is None or previous_sample_ns is None:
|
||||
return row
|
||||
delta_seconds = max((now_ns - previous_sample_ns) / 1_000_000_000, 0.001)
|
||||
clock_ticks = os.sysconf("SC_CLK_TCK")
|
||||
row["process_cpu_percent"] = (
|
||||
max(0, counters.process_ticks - previous.process_ticks)
|
||||
/ clock_ticks
|
||||
/ delta_seconds
|
||||
* 100
|
||||
)
|
||||
system_total = counters.system_total_ticks - previous.system_total_ticks
|
||||
system_idle = counters.system_idle_ticks - previous.system_idle_ticks
|
||||
if system_total > 0:
|
||||
row["system_cpu_percent"] = max(0, system_total - system_idle) / system_total * 100
|
||||
row["read_mib_per_second"] = (
|
||||
max(0, counters.read_bytes - previous.read_bytes) / 1024**2 / delta_seconds
|
||||
)
|
||||
row["write_mib_per_second"] = (
|
||||
max(0, counters.write_bytes - previous.write_bytes) / 1024**2 / delta_seconds
|
||||
)
|
||||
row["voluntary_context_switches_per_second"] = (
|
||||
max(
|
||||
0,
|
||||
counters.voluntary_context_switches - previous.voluntary_context_switches,
|
||||
)
|
||||
/ delta_seconds
|
||||
)
|
||||
row["nonvoluntary_context_switches_per_second"] = (
|
||||
max(
|
||||
0,
|
||||
counters.nonvoluntary_context_switches
|
||||
- previous.nonvoluntary_context_switches,
|
||||
)
|
||||
/ delta_seconds
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def _read_counters(pid: int) -> _Counters:
|
||||
root = Path("/proc") / str(pid)
|
||||
process_ticks, rss_bytes = _parse_proc_stat((root / "stat").read_text(encoding="utf-8"))
|
||||
io = _parse_key_values((root / "io").read_text(encoding="utf-8"))
|
||||
status = _parse_key_values((root / "status").read_text(encoding="utf-8"))
|
||||
system_total, system_idle = _parse_system_cpu(Path("/proc/stat").read_text(encoding="utf-8"))
|
||||
return _Counters(
|
||||
process_ticks=process_ticks,
|
||||
rss_bytes=rss_bytes,
|
||||
read_bytes=io.get("read_bytes", 0),
|
||||
write_bytes=io.get("write_bytes", 0),
|
||||
voluntary_context_switches=status.get("voluntary_ctxt_switches", 0),
|
||||
nonvoluntary_context_switches=status.get("nonvoluntary_ctxt_switches", 0),
|
||||
system_total_ticks=system_total,
|
||||
system_idle_ticks=system_idle,
|
||||
)
|
||||
|
||||
|
||||
def _parse_proc_stat(value: str) -> tuple[int, int]:
|
||||
closing = value.rfind(")")
|
||||
if closing < 0:
|
||||
raise ValueError("Malformed /proc process stat")
|
||||
fields = value[closing + 2 :].split()
|
||||
if len(fields) < 22:
|
||||
raise ValueError("Truncated /proc process stat")
|
||||
process_ticks = int(fields[11]) + int(fields[12])
|
||||
rss_bytes = int(fields[21]) * os.sysconf("SC_PAGE_SIZE")
|
||||
return process_ticks, rss_bytes
|
||||
|
||||
|
||||
def _parse_key_values(value: str) -> dict[str, int]:
|
||||
result: dict[str, int] = {}
|
||||
for line in value.splitlines():
|
||||
key, separator, raw = line.partition(":")
|
||||
if not separator:
|
||||
continue
|
||||
match = re.match(r"\s*([0-9]+)", raw)
|
||||
if match:
|
||||
result[key] = int(match.group(1))
|
||||
return result
|
||||
|
||||
|
||||
def _parse_system_cpu(value: str) -> tuple[int, int]:
|
||||
first = value.splitlines()[0].split()
|
||||
if not first or first[0] != "cpu" or len(first) < 5:
|
||||
raise ValueError("Malformed /proc/stat CPU line")
|
||||
values = [int(item) for item in first[1:]]
|
||||
return sum(values), values[3] + (values[4] if len(values) > 4 else 0)
|
||||
|
||||
|
||||
def _discover_gpu_device() -> Path | None:
|
||||
for path in sorted(Path("/sys/class/drm").glob("card*/device")):
|
||||
if (path / "gpu_busy_percent").is_file():
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def _read_float(root: Path | None, name: str) -> float | None:
|
||||
if root is None:
|
||||
return None
|
||||
try:
|
||||
return float((root / name).read_text(encoding="utf-8").strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _read_active_mhz(root: Path | None, name: str) -> float | None:
|
||||
if root is None:
|
||||
return None
|
||||
try:
|
||||
value = (root / name).read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return None
|
||||
return _parse_active_mhz(value)
|
||||
|
||||
|
||||
def _parse_active_mhz(value: str) -> float | None:
|
||||
match = re.search(r"([0-9]+)\s*Mhz\s*\*", value, flags=re.IGNORECASE)
|
||||
return float(match.group(1)) if match else None
|
||||
|
||||
|
||||
def _read_cpu_frequencies_mhz() -> list[float]:
|
||||
result: list[float] = []
|
||||
for path in sorted(Path("/sys/devices/system/cpu").glob("cpu[0-9]*/cpufreq/scaling_cur_freq")):
|
||||
try:
|
||||
result.append(float(path.read_text(encoding="utf-8").strip()) / 1000)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return result
|
||||
65
ptdiag/prefix.py
Normal file
65
ptdiag/prefix.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""Guarded helpers for the dedicated Proton test prefix."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .config import AppConfig
|
||||
from .runner import CommandLogger, CommandResult
|
||||
|
||||
|
||||
class PrefixSafetyError(RuntimeError):
|
||||
"""Raised when a cleanup target is not provably the dedicated test prefix."""
|
||||
|
||||
|
||||
def validate_dedicated_prefix(config: AppConfig) -> Path:
|
||||
prefix = config.proton7.prefix.resolve(strict=False)
|
||||
compat = config.proton7.compat.resolve(strict=False)
|
||||
live_prefix = config.paths.live_prefix.resolve(strict=False)
|
||||
if prefix == live_prefix:
|
||||
raise PrefixSafetyError("Dedicated prefix resolves to the live launcher prefix")
|
||||
try:
|
||||
prefix.relative_to(compat)
|
||||
except ValueError as exc:
|
||||
raise PrefixSafetyError(
|
||||
f"Dedicated prefix {prefix} is not inside compat directory {compat}"
|
||||
) from exc
|
||||
if prefix == compat:
|
||||
raise PrefixSafetyError("Dedicated prefix must be a child of the compat directory")
|
||||
return prefix
|
||||
|
||||
|
||||
def resolve_proton_wineserver(config: AppConfig) -> Path:
|
||||
candidates = proton_wineserver_candidates(config)
|
||||
for candidate in candidates:
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
"Proton wineserver was not found in dist/bin or files/bin under "
|
||||
f"{config.proton7.proton.parent}"
|
||||
)
|
||||
|
||||
|
||||
def proton_wineserver_candidates(config: AppConfig) -> tuple[Path, Path]:
|
||||
proton_directory = config.proton7.proton.parent
|
||||
return (
|
||||
proton_directory / "dist/bin/wineserver",
|
||||
proton_directory / "files/bin/wineserver",
|
||||
)
|
||||
|
||||
|
||||
def cleanup_dedicated_wineserver(
|
||||
config: AppConfig,
|
||||
command_logger: CommandLogger,
|
||||
*,
|
||||
timeout_seconds: float = 15.0,
|
||||
dry_run: bool = False,
|
||||
) -> CommandResult:
|
||||
prefix = validate_dedicated_prefix(config)
|
||||
wineserver = resolve_proton_wineserver(config)
|
||||
return command_logger.run(
|
||||
[str(wineserver), "-k"],
|
||||
env_overrides={"WINEPREFIX": str(prefix), "WINEDEBUG": "-all"},
|
||||
timeout_seconds=timeout_seconds,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
105
ptdiag/process.py
Normal file
105
ptdiag/process.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""Read-only host and configured environment inspection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from .config import AppConfig
|
||||
from .prefix import proton_wineserver_candidates
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EnvironmentCheck:
|
||||
name: str
|
||||
ok: bool
|
||||
value: str
|
||||
required: bool = True
|
||||
|
||||
def to_dict(self) -> dict[str, str | bool]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def collect_environment(config: AppConfig) -> dict[str, Any]:
|
||||
checks = environment_checks(config)
|
||||
return {
|
||||
"collected_at": datetime.now().astimezone().isoformat(),
|
||||
"host": {
|
||||
"platform": platform.platform(),
|
||||
"machine": platform.machine(),
|
||||
"python": sys.version,
|
||||
"executable": sys.executable,
|
||||
"cwd": os.getcwd(),
|
||||
},
|
||||
"config": config.to_dict(),
|
||||
"checks": [check.to_dict() for check in checks],
|
||||
"ok": all(check.ok for check in checks if check.required),
|
||||
}
|
||||
|
||||
|
||||
def environment_checks(config: AppConfig) -> list[EnvironmentCheck]:
|
||||
checks = [
|
||||
_path_check("app_root", config.paths.app_root, "directory"),
|
||||
_path_check("live_prefix", config.paths.live_prefix, "directory"),
|
||||
_path_check("live_game", config.paths.live_game, "directory"),
|
||||
_path_check("logs", config.paths.logs, "directory"),
|
||||
_path_check("native_logs", config.paths.native_logs, "directory"),
|
||||
_path_check("proton7", config.proton7.proton, "file"),
|
||||
_any_path_check("proton7_wineserver", proton_wineserver_candidates(config)),
|
||||
_path_check("steam_root", config.proton7.steam_root, "directory"),
|
||||
_path_check("compat", config.proton7.compat, "directory"),
|
||||
_path_check("dedicated_prefix", config.proton7.prefix, "directory"),
|
||||
_path_check("game_exe", config.paths.live_game / config.game.exe, "file"),
|
||||
EnvironmentCheck(
|
||||
"game_argument",
|
||||
config.game.args == ("PTEU_REBORN",),
|
||||
" ".join(config.game.args),
|
||||
),
|
||||
EnvironmentCheck(
|
||||
"virtual_desktop",
|
||||
config.game.virtual_desktop == "1600x900",
|
||||
config.game.virtual_desktop,
|
||||
),
|
||||
_tool_check("strace", required=False),
|
||||
_tool_check("gdb", required=False),
|
||||
_i386_vulkan_check(),
|
||||
]
|
||||
return checks
|
||||
|
||||
|
||||
def _path_check(name: str, path: Path, kind: str) -> EnvironmentCheck:
|
||||
ok = path.is_file() if kind == "file" else path.is_dir()
|
||||
return EnvironmentCheck(name, ok, str(path))
|
||||
|
||||
|
||||
def _tool_check(name: str, required: bool) -> EnvironmentCheck:
|
||||
path = shutil.which(name)
|
||||
return EnvironmentCheck(name, path is not None, path or "not found", required)
|
||||
|
||||
|
||||
def _any_path_check(name: str, candidates: tuple[Path, ...]) -> EnvironmentCheck:
|
||||
found = next((path for path in candidates if path.is_file()), None)
|
||||
return EnvironmentCheck(
|
||||
name,
|
||||
found is not None,
|
||||
str(found) if found is not None else "not found: " + ", ".join(map(str, candidates)),
|
||||
)
|
||||
|
||||
|
||||
def _i386_vulkan_check() -> EnvironmentCheck:
|
||||
candidates = (
|
||||
Path("/usr/lib/i386-linux-gnu/libvulkan.so.1"),
|
||||
Path("/usr/lib/i386-linux-gnu/libvulkan.so"),
|
||||
)
|
||||
found = next((path for path in candidates if path.exists()), None)
|
||||
return EnvironmentCheck(
|
||||
"i386_vulkan_loader",
|
||||
found is not None,
|
||||
str(found) if found is not None else "not found",
|
||||
)
|
||||
32
ptdiag/proton_compat.py
Normal file
32
ptdiag/proton_compat.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Run the retained Proton 7 script safely on Python 3.14.
|
||||
|
||||
Proton 7 predates Python's tar extraction-filter API. The known-good v41
|
||||
runtime used this compatibility shim; keeping it in the harness makes the
|
||||
exact launch recipe inspectable and reproducible.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import runpy
|
||||
import sys
|
||||
import tarfile
|
||||
from typing import Sequence
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> None:
|
||||
arguments = list(sys.argv[1:] if argv is None else argv)
|
||||
if not arguments:
|
||||
raise SystemExit("usage: proton_compat.py /path/to/proton [arguments...]")
|
||||
proton = Path(arguments[0]).expanduser().resolve(strict=True)
|
||||
proton_directory = str(proton.parent)
|
||||
if proton_directory not in sys.path:
|
||||
sys.path.insert(0, proton_directory)
|
||||
if hasattr(tarfile, "fully_trusted_filter"):
|
||||
tarfile.TarFile.extraction_filter = staticmethod(tarfile.fully_trusted_filter)
|
||||
sys.argv = [str(proton), *arguments[1:]]
|
||||
runpy.run_path(str(proton), run_name="__main__")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
100
ptdiag/registry.py
Normal file
100
ptdiag/registry.py
Normal file
@@ -0,0 +1,100 @@
|
||||
"""Read-only Wine registry snapshots for reversible desktop configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from .filesystem import sha256_file
|
||||
|
||||
|
||||
_VALUE_RE = re.compile(r'^(?P<name>"(?:\\.|[^"\\])*")=(?P<value>.*)$')
|
||||
_SECTION_RE = re.compile(r"^\[(?P<name>.*)\](?:\s+[0-9]+)?$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RegistryValue:
|
||||
exists: bool
|
||||
value: str | None
|
||||
|
||||
def to_dict(self) -> dict[str, str | bool | None]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DesktopRegistrySnapshot:
|
||||
user_reg: Path
|
||||
sha256: str
|
||||
desktop_name: str
|
||||
selected_desktop: RegistryValue
|
||||
desktop_size: RegistryValue
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"user_reg": str(self.user_reg),
|
||||
"sha256": self.sha256,
|
||||
"desktop_name": self.desktop_name,
|
||||
"selected_desktop": self.selected_desktop.to_dict(),
|
||||
"desktop_size": self.desktop_size.to_dict(),
|
||||
}
|
||||
|
||||
|
||||
def snapshot_desktop_registry(prefix: Path, desktop_name: str) -> DesktopRegistrySnapshot:
|
||||
user_reg = prefix / "user.reg"
|
||||
sections = read_registry_string_values(user_reg)
|
||||
selected = sections.get(r"Software\Wine\Explorer", {}).get("Desktop")
|
||||
size = sections.get(r"Software\Wine\Explorer\Desktops", {}).get(desktop_name)
|
||||
return DesktopRegistrySnapshot(
|
||||
user_reg=user_reg,
|
||||
sha256=sha256_file(user_reg),
|
||||
desktop_name=desktop_name,
|
||||
selected_desktop=RegistryValue(selected is not None, selected),
|
||||
desktop_size=RegistryValue(size is not None, size),
|
||||
)
|
||||
|
||||
|
||||
def desktop_registry_restored(
|
||||
before: DesktopRegistrySnapshot,
|
||||
after: DesktopRegistrySnapshot,
|
||||
) -> bool:
|
||||
return (
|
||||
before.desktop_name == after.desktop_name
|
||||
and before.selected_desktop == after.selected_desktop
|
||||
and before.desktop_size == after.desktop_size
|
||||
)
|
||||
|
||||
|
||||
def read_registry_string_values(path: Path) -> dict[str, dict[str, str]]:
|
||||
sections: dict[str, dict[str, str]] = {}
|
||||
current: dict[str, str] | None = None
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
for raw in handle:
|
||||
line = raw.rstrip("\r\n")
|
||||
section = _SECTION_RE.match(line)
|
||||
if section is not None:
|
||||
name = section.group("name").replace(r"\\", "\\")
|
||||
current = sections.setdefault(name, {})
|
||||
continue
|
||||
if current is None:
|
||||
continue
|
||||
match = _VALUE_RE.match(line)
|
||||
if match is None:
|
||||
continue
|
||||
name = _decode_string(match.group("name"))
|
||||
value_text = match.group("value")
|
||||
if name is None or not value_text.startswith('"'):
|
||||
continue
|
||||
value = _decode_string(value_text)
|
||||
if value is not None:
|
||||
current[name] = value
|
||||
return sections
|
||||
|
||||
|
||||
def _decode_string(value: str) -> str | None:
|
||||
try:
|
||||
result = ast.literal_eval(value)
|
||||
except (SyntaxError, ValueError):
|
||||
return None
|
||||
return result if isinstance(result, str) else None
|
||||
71
ptdiag/result.py
Normal file
71
ptdiag/result.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""Unique result-directory creation and JSON output."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResultDirectory:
|
||||
path: Path
|
||||
metadata_path: Path
|
||||
commands_path: Path
|
||||
|
||||
|
||||
def create_result_directory(
|
||||
root: Path,
|
||||
test_name: str,
|
||||
started_at: datetime | None = None,
|
||||
) -> ResultDirectory:
|
||||
name = _slug(test_name)
|
||||
started = started_at or datetime.now().astimezone()
|
||||
base_name = f"{started:%Y%m%d-%H%M%S}-{name}"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
path = _mkdir_unique(root, base_name)
|
||||
metadata_path = path / "metadata.json"
|
||||
commands_path = path / "commands.log"
|
||||
commands_path.touch(exist_ok=False)
|
||||
write_json(
|
||||
metadata_path,
|
||||
{
|
||||
"test": name,
|
||||
"started_at": started.isoformat(),
|
||||
"result_directory": str(path),
|
||||
},
|
||||
)
|
||||
return ResultDirectory(path, metadata_path, commands_path)
|
||||
|
||||
|
||||
def write_json(path: Path, value: Mapping[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
with temporary.open("w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def _slug(value: str) -> str:
|
||||
result = re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-")
|
||||
if not result:
|
||||
raise ValueError("test_name must contain a letter or digit")
|
||||
return result
|
||||
|
||||
|
||||
def _mkdir_unique(root: Path, base_name: str) -> Path:
|
||||
for index in range(1000):
|
||||
suffix = "" if index == 0 else f"-{index:02d}"
|
||||
candidate = root / f"{base_name}{suffix}"
|
||||
try:
|
||||
candidate.mkdir()
|
||||
except FileExistsError:
|
||||
continue
|
||||
return candidate
|
||||
raise FileExistsError(f"Could not allocate a unique result directory for {base_name}")
|
||||
|
||||
164
ptdiag/runner.py
Normal file
164
ptdiag/runner.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""Structured command logging with argument-array subprocess execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from typing import Mapping, Sequence
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CommandResult:
|
||||
argv: tuple[str, ...]
|
||||
returncode: int | None
|
||||
stdout: str
|
||||
stderr: str
|
||||
timed_out: bool
|
||||
dry_run: bool
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class CommandLogger:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def run(
|
||||
self,
|
||||
argv: Sequence[str],
|
||||
*,
|
||||
cwd: Path | None = None,
|
||||
env_overrides: Mapping[str, str] | None = None,
|
||||
timeout_seconds: float,
|
||||
dry_run: bool = False,
|
||||
) -> CommandResult:
|
||||
arguments = _validate_argv(argv)
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
overrides = dict(env_overrides or {})
|
||||
started_at = datetime.now().astimezone()
|
||||
if dry_run:
|
||||
result = CommandResult(arguments, None, "", "", False, True)
|
||||
self._append(started_at, cwd, overrides, timeout_seconds, result)
|
||||
return result
|
||||
|
||||
environment = os.environ.copy()
|
||||
environment.update(overrides)
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
list(arguments),
|
||||
cwd=cwd,
|
||||
env=environment,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout_seconds,
|
||||
check=False,
|
||||
)
|
||||
result = CommandResult(
|
||||
arguments,
|
||||
completed.returncode,
|
||||
completed.stdout,
|
||||
completed.stderr,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
result = CommandResult(
|
||||
arguments,
|
||||
None,
|
||||
_decode_timeout_stream(exc.stdout),
|
||||
_decode_timeout_stream(exc.stderr),
|
||||
True,
|
||||
False,
|
||||
)
|
||||
except OSError as exc:
|
||||
result = CommandResult(
|
||||
arguments,
|
||||
None,
|
||||
"",
|
||||
"",
|
||||
False,
|
||||
False,
|
||||
str(exc),
|
||||
)
|
||||
self._append(started_at, cwd, overrides, timeout_seconds, result)
|
||||
return result
|
||||
|
||||
def record_lifecycle(
|
||||
self,
|
||||
argv: Sequence[str],
|
||||
*,
|
||||
state: str,
|
||||
cwd: Path | None,
|
||||
env_overrides: Mapping[str, str],
|
||||
timeout_seconds: float,
|
||||
started_at: datetime,
|
||||
pid: int | None = None,
|
||||
returncode: int | None = None,
|
||||
timed_out: bool = False,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
arguments = _validate_argv(argv)
|
||||
record = {
|
||||
"argv": list(arguments),
|
||||
"cwd": str(cwd) if cwd is not None else None,
|
||||
"env_overrides": dict(sorted(env_overrides.items())),
|
||||
"error": error,
|
||||
"observed_at": datetime.now().astimezone().isoformat(),
|
||||
"pid": pid,
|
||||
"returncode": returncode,
|
||||
"started_at": started_at.isoformat(),
|
||||
"state": state,
|
||||
"timed_out": timed_out,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
}
|
||||
with self.path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(record, sort_keys=True))
|
||||
handle.write("\n")
|
||||
|
||||
def _append(
|
||||
self,
|
||||
started_at: datetime,
|
||||
cwd: Path | None,
|
||||
env_overrides: Mapping[str, str],
|
||||
timeout_seconds: float,
|
||||
result: CommandResult,
|
||||
) -> None:
|
||||
record = {
|
||||
"argv": list(result.argv),
|
||||
"cwd": str(cwd) if cwd is not None else None,
|
||||
"dry_run": result.dry_run,
|
||||
"env_overrides": dict(sorted(env_overrides.items())),
|
||||
"error": result.error,
|
||||
"finished_at": datetime.now().astimezone().isoformat(),
|
||||
"returncode": result.returncode,
|
||||
"started_at": started_at.isoformat(),
|
||||
"stderr": result.stderr,
|
||||
"stdout": result.stdout,
|
||||
"timed_out": result.timed_out,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
}
|
||||
with self.path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(record, sort_keys=True))
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def _validate_argv(argv: Sequence[str]) -> tuple[str, ...]:
|
||||
if isinstance(argv, (str, bytes)):
|
||||
raise TypeError("argv must be a sequence of strings, not a shell command")
|
||||
arguments = tuple(argv)
|
||||
if not arguments or not all(isinstance(argument, str) and argument for argument in arguments):
|
||||
raise ValueError("argv must contain non-empty strings")
|
||||
return arguments
|
||||
|
||||
|
||||
def _decode_timeout_stream(value: str | bytes | None) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="replace")
|
||||
return value
|
||||
331
ptdiag/testcase.py
Normal file
331
ptdiag/testcase.py
Normal file
@@ -0,0 +1,331 @@
|
||||
"""Typed testcase loading with recursive TOML inheritance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
import tomllib
|
||||
|
||||
from .config import project_root
|
||||
|
||||
|
||||
class TestcaseError(ValueError):
|
||||
"""Raised when an experiment testcase is missing or invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestGame:
|
||||
args: tuple[str, ...]
|
||||
cwd: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestDisplay:
|
||||
virtual_desktop: bool
|
||||
size: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestWatch:
|
||||
require_new_native_log: bool
|
||||
patterns: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestCapture:
|
||||
process_maps: bool
|
||||
cwd: bool
|
||||
environment: bool
|
||||
dll_overrides: bool
|
||||
hashes: bool
|
||||
dxvk_log: bool
|
||||
native_log: bool
|
||||
strace_files: bool
|
||||
performance_stats: bool
|
||||
focus_before_seconds: float
|
||||
focus_after_seconds: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestIniOverride:
|
||||
path: str
|
||||
section: str
|
||||
key: str
|
||||
expected: str
|
||||
value: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestModification:
|
||||
kind: str
|
||||
source: str
|
||||
destination: str
|
||||
texture_indices: tuple[int, ...]
|
||||
ini_overrides: tuple[TestIniOverride, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Testcase:
|
||||
name: str
|
||||
description: str
|
||||
runner: str
|
||||
timeout_seconds: float
|
||||
stop_after_failure_seconds: float
|
||||
game: TestGame
|
||||
display: TestDisplay
|
||||
env: dict[str, str]
|
||||
watch: TestWatch
|
||||
capture: TestCapture
|
||||
modification: TestModification | None
|
||||
source: Path
|
||||
sources: tuple[Path, ...]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = asdict(self)
|
||||
result["source"] = str(self.source)
|
||||
result["sources"] = [str(path) for path in self.sources]
|
||||
return result
|
||||
|
||||
|
||||
def resolve_testcase_path(value: str | Path) -> Path:
|
||||
path = Path(value)
|
||||
if path.suffix or path.parent != Path("."):
|
||||
return path.expanduser().absolute()
|
||||
return (project_root() / "testcases" / f"{path.name}.toml").absolute()
|
||||
|
||||
|
||||
def load_testcase(value: str | Path) -> Testcase:
|
||||
path = resolve_testcase_path(value)
|
||||
data, sources = _load_inherited(path, ())
|
||||
game = _section(data, "game")
|
||||
display = _section(data, "display")
|
||||
environment = _section(data, "env")
|
||||
watch = _section(data, "watch")
|
||||
capture = _section(data, "capture")
|
||||
modification = _optional_section(data, "modification")
|
||||
return Testcase(
|
||||
name=_string(data, "name"),
|
||||
description=_string(data, "description"),
|
||||
runner=_string(data, "runner"),
|
||||
timeout_seconds=_positive_number(data, "timeout_seconds"),
|
||||
stop_after_failure_seconds=_nonnegative_number(
|
||||
data, "stop_after_failure_seconds"
|
||||
),
|
||||
game=TestGame(
|
||||
args=_string_tuple(game, "args"),
|
||||
cwd=_string(game, "cwd"),
|
||||
),
|
||||
display=TestDisplay(
|
||||
virtual_desktop=_boolean(display, "virtual_desktop"),
|
||||
size=_string(display, "size"),
|
||||
),
|
||||
env=_string_mapping(environment, "env"),
|
||||
watch=TestWatch(
|
||||
require_new_native_log=_boolean(watch, "require_new_native_log"),
|
||||
patterns=_string_tuple(watch, "patterns"),
|
||||
),
|
||||
capture=TestCapture(
|
||||
process_maps=_boolean(capture, "process_maps"),
|
||||
cwd=_boolean(capture, "cwd"),
|
||||
environment=_boolean(capture, "environment"),
|
||||
dll_overrides=_boolean(capture, "dll_overrides"),
|
||||
hashes=_boolean(capture, "hashes"),
|
||||
dxvk_log=_boolean(capture, "dxvk_log"),
|
||||
native_log=_boolean(capture, "native_log"),
|
||||
strace_files=_boolean(capture, "strace_files"),
|
||||
performance_stats=_optional_boolean(capture, "performance_stats", False),
|
||||
focus_before_seconds=_optional_nonnegative_number(
|
||||
capture, "focus_before_seconds", 4.0
|
||||
),
|
||||
focus_after_seconds=_optional_nonnegative_number(
|
||||
capture, "focus_after_seconds", 1.0
|
||||
),
|
||||
),
|
||||
modification=(
|
||||
TestModification(
|
||||
kind=_string(modification, "kind"),
|
||||
source=_string(modification, "source"),
|
||||
destination=_string(modification, "destination"),
|
||||
texture_indices=_optional_integer_tuple(
|
||||
modification,
|
||||
"texture_indices",
|
||||
),
|
||||
ini_overrides=_optional_ini_overrides(modification),
|
||||
)
|
||||
if modification is not None
|
||||
else None
|
||||
),
|
||||
source=path,
|
||||
sources=sources,
|
||||
)
|
||||
|
||||
|
||||
def _load_inherited(
|
||||
path: Path,
|
||||
stack: tuple[Path, ...],
|
||||
) -> tuple[dict[str, Any], tuple[Path, ...]]:
|
||||
resolved = path.resolve(strict=False)
|
||||
if resolved in stack:
|
||||
chain = " -> ".join(str(item) for item in (*stack, resolved))
|
||||
raise TestcaseError(f"Testcase inheritance cycle: {chain}")
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
data = tomllib.load(handle)
|
||||
except FileNotFoundError as exc:
|
||||
raise TestcaseError(f"Testcase not found: {path}") from exc
|
||||
except tomllib.TOMLDecodeError as exc:
|
||||
raise TestcaseError(f"Invalid TOML in {path}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise TestcaseError(f"Testcase root must be a table: {path}")
|
||||
|
||||
parent_name = data.pop("extends", None)
|
||||
if parent_name is None:
|
||||
return data, (resolved,)
|
||||
if not isinstance(parent_name, str) or not parent_name.strip():
|
||||
raise TestcaseError(f"extends must be a non-empty string in {path}")
|
||||
parent_file = Path(parent_name)
|
||||
if parent_file.suffix == "":
|
||||
parent_file = parent_file.with_suffix(".toml")
|
||||
if not parent_file.is_absolute():
|
||||
parent_file = path.parent / parent_file
|
||||
parent, sources = _load_inherited(parent_file, (*stack, resolved))
|
||||
return _merge(parent, data), (*sources, resolved)
|
||||
|
||||
|
||||
def _merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = dict(base)
|
||||
for key, value in override.items():
|
||||
current = result.get(key)
|
||||
if isinstance(current, Mapping) and isinstance(value, Mapping):
|
||||
result[key] = _merge(current, value)
|
||||
else:
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _section(data: Mapping[str, Any], name: str) -> Mapping[str, Any]:
|
||||
value = data.get(name)
|
||||
if not isinstance(value, Mapping):
|
||||
raise TestcaseError(f"Missing or invalid [{name}] section")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_section(data: Mapping[str, Any], name: str) -> Mapping[str, Any] | None:
|
||||
value = data.get(name)
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, Mapping):
|
||||
raise TestcaseError(f"Invalid [{name}] section")
|
||||
return value
|
||||
|
||||
|
||||
def _string(data: Mapping[str, Any], key: str) -> str:
|
||||
value = data.get(key)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise TestcaseError(f"{key} must be a non-empty string")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _boolean(data: Mapping[str, Any], key: str) -> bool:
|
||||
value = data.get(key)
|
||||
if not isinstance(value, bool):
|
||||
raise TestcaseError(f"{key} must be true or false")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_boolean(data: Mapping[str, Any], key: str, default: bool) -> bool:
|
||||
if key not in data:
|
||||
return default
|
||||
return _boolean(data, key)
|
||||
|
||||
|
||||
def _string_tuple(data: Mapping[str, Any], key: str) -> tuple[str, ...]:
|
||||
value = data.get(key)
|
||||
if not isinstance(value, list) or not value or not all(
|
||||
isinstance(item, str) and item for item in value
|
||||
):
|
||||
raise TestcaseError(f"{key} must be a non-empty array of strings")
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _optional_integer_tuple(data: Mapping[str, Any], key: str) -> tuple[int, ...]:
|
||||
value = data.get(key, [])
|
||||
if not isinstance(value, list) or not all(
|
||||
isinstance(item, int) and not isinstance(item, bool) and item >= 0
|
||||
for item in value
|
||||
):
|
||||
raise TestcaseError(f"{key} must be an array of nonnegative integers")
|
||||
if len(value) != len(set(value)):
|
||||
raise TestcaseError(f"{key} must not contain duplicates")
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _optional_ini_overrides(data: Mapping[str, Any]) -> tuple[TestIniOverride, ...]:
|
||||
value = data.get("ini_overrides", [])
|
||||
if not isinstance(value, list):
|
||||
raise TestcaseError("ini_overrides must be an array of tables")
|
||||
result: list[TestIniOverride] = []
|
||||
identities: set[tuple[str, str, str]] = set()
|
||||
for item in value:
|
||||
if not isinstance(item, Mapping):
|
||||
raise TestcaseError("ini_overrides must contain only tables")
|
||||
override = TestIniOverride(
|
||||
path=_string(item, "path"),
|
||||
section=_string(item, "section"),
|
||||
key=_string(item, "key"),
|
||||
expected=_string(item, "expected"),
|
||||
value=_string(item, "value"),
|
||||
)
|
||||
identity = (
|
||||
override.path.casefold(),
|
||||
override.section.casefold(),
|
||||
override.key.casefold(),
|
||||
)
|
||||
if identity in identities:
|
||||
raise TestcaseError("ini_overrides must not contain duplicate keys")
|
||||
if override.expected == override.value:
|
||||
raise TestcaseError("INI expected and replacement values must differ")
|
||||
identities.add(identity)
|
||||
result.append(override)
|
||||
return tuple(result)
|
||||
|
||||
|
||||
def _string_mapping(data: Mapping[str, Any], name: str) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
for key, value in data.items():
|
||||
if not isinstance(key, str) or not isinstance(value, str):
|
||||
raise TestcaseError(f"[{name}] must contain only string values")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _positive_number(data: Mapping[str, Any], key: str) -> float:
|
||||
value = _number(data, key)
|
||||
if value <= 0:
|
||||
raise TestcaseError(f"{key} must be positive")
|
||||
return value
|
||||
|
||||
|
||||
def _nonnegative_number(data: Mapping[str, Any], key: str) -> float:
|
||||
value = _number(data, key)
|
||||
if value < 0:
|
||||
raise TestcaseError(f"{key} must be nonnegative")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_nonnegative_number(
|
||||
data: Mapping[str, Any],
|
||||
key: str,
|
||||
default: float,
|
||||
) -> float:
|
||||
if key not in data:
|
||||
return default
|
||||
return _nonnegative_number(data, key)
|
||||
|
||||
|
||||
def _number(data: Mapping[str, Any], key: str) -> float:
|
||||
value = data.get(key)
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise TestcaseError(f"{key} must be a number")
|
||||
return float(value)
|
||||
242
ptdiag/trace.py
Normal file
242
ptdiag/trace.py
Normal file
@@ -0,0 +1,242 @@
|
||||
"""Streaming merge and normalization for per-PID strace file traces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from collections import Counter
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import asdict, dataclass
|
||||
import heapq
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Iterator, TextIO
|
||||
|
||||
|
||||
_TIMESTAMP_RE = re.compile(r"^(?P<timestamp>[0-9]+(?:\.[0-9]+)?)\s+(?P<body>.*)$")
|
||||
_SYSCALL_RE = re.compile(r"^(?P<syscall>[A-Za-z_][A-Za-z0-9_]*)\(")
|
||||
_ERRNO_RE = re.compile(
|
||||
r"=\s+-1\s+(?P<errno>E[A-Z0-9]+)\s+\((?P<description>[^)]*)\)"
|
||||
)
|
||||
_QUOTED_RE = re.compile(r'"(?:\\.|[^"\\])*"')
|
||||
|
||||
_ERROR_CLASSIFICATIONS = {
|
||||
"ENOENT": "missing",
|
||||
"ENOTDIR": "path-component-not-directory",
|
||||
"EACCES": "permission-denied",
|
||||
"EPERM": "operation-not-permitted",
|
||||
"ELOOP": "symlink-loop",
|
||||
"ENAMETOOLONG": "name-too-long",
|
||||
"EROFS": "read-only-filesystem",
|
||||
"EIO": "io-error",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TraceSummary:
|
||||
shards: int
|
||||
lines: int
|
||||
normalized_records: int
|
||||
focused_lines: int
|
||||
error_counts: dict[str, int]
|
||||
top_errors: list[dict[str, object]]
|
||||
merged_path: Path
|
||||
normalized_path: Path
|
||||
focused_path: Path
|
||||
focused_json_path: Path
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
result = asdict(self)
|
||||
for key in (
|
||||
"merged_path",
|
||||
"normalized_path",
|
||||
"focused_path",
|
||||
"focused_json_path",
|
||||
):
|
||||
result[key] = str(result[key])
|
||||
return result
|
||||
|
||||
|
||||
def process_trace_shards(
|
||||
raw_directory: Path,
|
||||
result_directory: Path,
|
||||
*,
|
||||
trigger_epoch: float,
|
||||
focus_before_seconds: float,
|
||||
focus_after_seconds: float,
|
||||
live_game: Path,
|
||||
) -> TraceSummary:
|
||||
shards = sorted(
|
||||
(path for path in raw_directory.glob("trace.*") if path.is_file()),
|
||||
key=_shard_pid,
|
||||
)
|
||||
merged_path = result_directory / "merged.txt"
|
||||
normalized_path = result_directory / "filesystem.jsonl"
|
||||
focused_path = result_directory / "focused.txt"
|
||||
focused_json_path = result_directory / "focused.jsonl"
|
||||
error_counts: Counter[str] = Counter()
|
||||
error_paths: Counter[tuple[str, str, str, str | None]] = Counter()
|
||||
line_count = 0
|
||||
normalized_count = 0
|
||||
focused_count = 0
|
||||
focus_start = trigger_epoch - focus_before_seconds
|
||||
focus_end = trigger_epoch + focus_after_seconds
|
||||
|
||||
with (
|
||||
merged_path.open("w", encoding="utf-8") as merged,
|
||||
normalized_path.open("w", encoding="utf-8") as normalized,
|
||||
focused_path.open("w", encoding="utf-8") as focused,
|
||||
focused_json_path.open("w", encoding="utf-8") as focused_json,
|
||||
):
|
||||
for timestamp, pid, raw in _merge_shards(shards):
|
||||
line_count += 1
|
||||
body = _body(raw)
|
||||
merged_line = f"{timestamp:.6f} [{pid}] {body}\n"
|
||||
merged.write(merged_line)
|
||||
record = parse_trace_line(pid, raw, live_game)
|
||||
if record is not None:
|
||||
normalized_count += 1
|
||||
normalized.write(json.dumps(record, sort_keys=True))
|
||||
normalized.write("\n")
|
||||
errno = record.get("errno")
|
||||
classification = record.get("error_classification")
|
||||
if isinstance(errno, str) and isinstance(classification, str):
|
||||
error_counts[classification] += 1
|
||||
paths = record.get("paths")
|
||||
path = paths[0] if isinstance(paths, list) and paths else ""
|
||||
game_relative = record.get("game_relative")
|
||||
error_paths[
|
||||
(
|
||||
classification,
|
||||
errno,
|
||||
str(path),
|
||||
str(game_relative) if game_relative is not None else None,
|
||||
)
|
||||
] += 1
|
||||
if focus_start <= timestamp <= focus_end:
|
||||
focused_count += 1
|
||||
focused.write(merged_line)
|
||||
if record is not None:
|
||||
focused_json.write(json.dumps(record, sort_keys=True))
|
||||
focused_json.write("\n")
|
||||
|
||||
top_errors = [
|
||||
{
|
||||
"classification": key[0],
|
||||
"errno": key[1],
|
||||
"path": key[2],
|
||||
"game_relative": key[3],
|
||||
"count": count,
|
||||
}
|
||||
for key, count in error_paths.most_common(200)
|
||||
]
|
||||
return TraceSummary(
|
||||
shards=len(shards),
|
||||
lines=line_count,
|
||||
normalized_records=normalized_count,
|
||||
focused_lines=focused_count,
|
||||
error_counts=dict(sorted(error_counts.items())),
|
||||
top_errors=top_errors,
|
||||
merged_path=merged_path,
|
||||
normalized_path=normalized_path,
|
||||
focused_path=focused_path,
|
||||
focused_json_path=focused_json_path,
|
||||
)
|
||||
|
||||
|
||||
def parse_trace_line(pid: int, line: str, live_game: Path) -> dict[str, object] | None:
|
||||
match = _TIMESTAMP_RE.match(line.rstrip("\n"))
|
||||
if match is None:
|
||||
return None
|
||||
timestamp = float(match.group("timestamp"))
|
||||
body = match.group("body")
|
||||
syscall_match = _SYSCALL_RE.match(body)
|
||||
if syscall_match is None:
|
||||
return None
|
||||
paths = [_decode_quoted(item) for item in _QUOTED_RE.findall(body)]
|
||||
paths = [item for item in paths if item is not None]
|
||||
errno_match = _ERRNO_RE.search(body)
|
||||
errno = errno_match.group("errno") if errno_match is not None else None
|
||||
classification = (
|
||||
_ERROR_CLASSIFICATIONS.get(errno, "other-error") if errno is not None else None
|
||||
)
|
||||
game_relative = next(
|
||||
(
|
||||
_relative_game_path(path, live_game)
|
||||
for path in paths
|
||||
if _relative_game_path(path, live_game) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
return {
|
||||
"timestamp": timestamp,
|
||||
"pid": pid,
|
||||
"syscall": syscall_match.group("syscall"),
|
||||
"paths": paths,
|
||||
"errno": errno,
|
||||
"error_classification": classification,
|
||||
"game_relative": game_relative,
|
||||
"raw": body,
|
||||
}
|
||||
|
||||
|
||||
def _merge_shards(paths: list[Path]) -> Iterator[tuple[float, int, str]]:
|
||||
with ExitStack() as stack:
|
||||
iterators: list[Iterator[tuple[float, int, str]]] = []
|
||||
for path in paths:
|
||||
handle = stack.enter_context(path.open("r", encoding="utf-8", errors="replace"))
|
||||
iterators.append(_iter_shard(handle, _shard_pid(path)))
|
||||
heap: list[tuple[float, int, int, str]] = []
|
||||
for index, iterator in enumerate(iterators):
|
||||
try:
|
||||
timestamp, pid, raw = next(iterator)
|
||||
except StopIteration:
|
||||
continue
|
||||
heapq.heappush(heap, (timestamp, pid, index, raw))
|
||||
while heap:
|
||||
timestamp, pid, index, raw = heapq.heappop(heap)
|
||||
yield timestamp, pid, raw
|
||||
try:
|
||||
next_timestamp, next_pid, next_raw = next(iterators[index])
|
||||
except StopIteration:
|
||||
continue
|
||||
heapq.heappush(heap, (next_timestamp, next_pid, index, next_raw))
|
||||
|
||||
|
||||
def _iter_shard(handle: TextIO, pid: int) -> Iterator[tuple[float, int, str]]:
|
||||
last_timestamp = 0.0
|
||||
for raw in handle:
|
||||
match = _TIMESTAMP_RE.match(raw)
|
||||
if match is not None:
|
||||
last_timestamp = float(match.group("timestamp"))
|
||||
yield last_timestamp, pid, raw
|
||||
|
||||
|
||||
def _shard_pid(path: Path) -> int:
|
||||
try:
|
||||
return int(path.name.rsplit(".", 1)[1])
|
||||
except (IndexError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _body(raw: str) -> str:
|
||||
match = _TIMESTAMP_RE.match(raw.rstrip("\n"))
|
||||
return match.group("body") if match is not None else raw.rstrip("\n")
|
||||
|
||||
|
||||
def _decode_quoted(value: str) -> str | None:
|
||||
try:
|
||||
decoded = ast.literal_eval(value)
|
||||
except (SyntaxError, ValueError):
|
||||
return None
|
||||
return decoded if isinstance(decoded, str) else None
|
||||
|
||||
|
||||
def _relative_game_path(value: str, live_game: Path) -> str | None:
|
||||
normalized = value.replace("\\", "/")
|
||||
root = str(live_game).replace("\\", "/").rstrip("/")
|
||||
index = normalized.casefold().find(root.casefold())
|
||||
if index < 0:
|
||||
return None
|
||||
relative = normalized[index + len(root) :].lstrip("/")
|
||||
return relative or "."
|
||||
Reference in New Issue
Block a user