Added download settings and optimized the downloader system with concurrency.

ptlinux/depot.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from base64 import b64decode
 from binascii import Error as BinAsciiError
+from collections import deque
 from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
 from dataclasses import asdict, dataclass
 from datetime import datetime
@@ -35,8 +36,15 @@ FALLBACK_SERVERS = (
 READ_SIZE = 128 * 1024
 HASH_READ_SIZE = 1024 * 1024
 DOWNLOAD_WORKERS = 8
+DOWNLOAD_INITIAL_WORKERS = DOWNLOAD_WORKERS
+LOW_BANDWIDTH_WORKERS = 2
+LOW_BANDWIDTH_BYTES_PER_SECOND = 5_000_000
 DOWNLOAD_RETRY_DELAYS_SECONDS = (1.0, 2.0, 4.0, 8.0, 16.0)
 PROGRESS_UPDATE_INTERVAL_SECONDS = 0.075
+ADAPTIVE_EVALUATION_INTERVAL_SECONDS = 1.0
+ADAPTIVE_RETRY_WINDOW_SECONDS = 15.0
+ADAPTIVE_RETRY_RECOVERY_SECONDS = 5.0
+TRANSFER_RATE_WINDOW_SECONDS = 5.0
 MAX_CREATE_IF_MISSING_BYTES = 16 * 1024 * 1024
 MAX_LINE_ENDING_RECOVERY_BYTES = 1024 * 1024
 UNMANAGED_EXECUTABLE_SUFFIXES = frozenset(
@@ -88,6 +96,275 @@ class _RetryableDownloadError(DepotError):
 		self.discard_partial = discard_partial
 
 
+@dataclass(frozen=True)
+class DownloadOptions:
+	"""User-selected limits around the adaptive depot scheduler."""
+
+	bandwidth_limit_bytes_per_second: int | None = None
+	low_bandwidth: bool = False
+	minimum_workers: int = 1
+	maximum_workers: int | None = None
+
+	def __post_init__(self) -> None:
+		if not isinstance(self.low_bandwidth, bool):
+			raise ValueError("low_bandwidth must be a boolean")
+		if (
+			self.bandwidth_limit_bytes_per_second is not None
+			and (
+				isinstance(self.bandwidth_limit_bytes_per_second, bool)
+				or self.bandwidth_limit_bytes_per_second <= 0
+			)
+		):
+			raise ValueError("bandwidth limit must be positive or None")
+		if isinstance(self.minimum_workers, bool) or self.minimum_workers <= 0:
+			raise ValueError("minimum download workers must be positive")
+		if self.maximum_workers is not None and (
+			isinstance(self.maximum_workers, bool) or self.maximum_workers <= 0
+		):
+			raise ValueError("maximum download workers must be positive or None")
+		if (
+			self.maximum_workers is not None
+			and self.minimum_workers > self.maximum_workers
+		):
+			raise ValueError("minimum download workers exceed the maximum")
+		if self.low_bandwidth and self.minimum_workers > LOW_BANDWIDTH_WORKERS:
+			raise ValueError("low-bandwidth minimum workers exceed its worker cap")
+
+	@property
+	def effective_bandwidth_limit(self) -> int | None:
+		configured = self.bandwidth_limit_bytes_per_second
+		if not self.low_bandwidth:
+			return configured
+		return min(
+			configured if configured is not None else LOW_BANDWIDTH_BYTES_PER_SECOND,
+			LOW_BANDWIDTH_BYTES_PER_SECOND,
+		)
+
+	@property
+	def effective_maximum_workers(self) -> int:
+		configured = self.maximum_workers or DOWNLOAD_WORKERS
+		if self.low_bandwidth:
+			configured = min(configured, LOW_BANDWIDTH_WORKERS)
+		return max(self.minimum_workers, configured)
+
+
+@dataclass(frozen=True)
+class DownloadTelemetry:
+	"""One internal measurement produced while retrieving a depot file."""
+
+	kind: str
+	file: str
+	server: str
+	attempt: int
+	latency_seconds: float | None = None
+	bytes_written: int = 0
+	write_seconds: float = 0.0
+	reason: str | None = None
+
+
+DownloadObserver = Callable[[DownloadTelemetry], None]
+
+
+class BandwidthLimiter:
+	"""Thread-safe aggregate byte-rate limiter with a one-chunk burst allowance."""
+
+	def __init__(
+		self,
+		bytes_per_second: int,
+		*,
+		clock: Callable[[], float] = time.monotonic,
+		sleeper: Callable[[float], None] = time.sleep,
+	) -> None:
+		if isinstance(bytes_per_second, bool) or bytes_per_second <= 0:
+			raise ValueError("bandwidth limit must be positive")
+		self.bytes_per_second = bytes_per_second
+		self._clock = clock
+		self._sleeper = sleeper
+		self._lock = threading.Lock()
+		self._next_available_at = clock()
+
+	def consume(self, byte_count: int) -> None:
+		"""Wait until this newly received chunk's aggregate reservation begins.
+
+		The network read happens before this call. Reserving the chunk's *start*
+		keeps all workers under one aggregate rate without charging again for time
+		already spent receiving it. The first chunk is the bucket's burst allowance.
+		"""
+
+		if byte_count < 0:
+			raise ValueError("byte count must not be negative")
+		if byte_count == 0:
+			return
+		with self._lock:
+			now = self._clock()
+			start = max(now, self._next_available_at)
+			self._next_available_at = start + (byte_count / self.bytes_per_second)
+			delay = start - now
+		if delay > 0:
+			self._sleeper(delay)
+
+
+class AdaptiveConcurrencyController:
+	"""Select a bounded worker target from live network, host, and plan data."""
+
+	def __init__(
+		self,
+		options: DownloadOptions,
+		*,
+		clock: Callable[[], float] = time.monotonic,
+		cpu_load: Callable[[], float | None] | None = None,
+	) -> None:
+		self.options = options
+		self._clock = clock
+		self._cpu_load = cpu_load or _normalized_cpu_load
+		self._lock = threading.Lock()
+		self._target = max(
+			options.minimum_workers,
+			min(DOWNLOAD_INITIAL_WORKERS, options.effective_maximum_workers),
+		)
+		self._last_evaluation = clock()
+		self._transferred_bytes = 0
+		self._transfer_samples: deque[tuple[float, int]] = deque(
+			[(self._last_evaluation, 0)]
+		)
+		self._response_samples: deque[tuple[float, float]] = deque()
+		self._retry_samples: deque[float] = deque()
+		self._retry_count = 0
+		self._handled_retry_count = 0
+		self._last_retry_at: float | None = None
+		self._latency_seconds: float | None = None
+		self._disk_samples: deque[tuple[float, int, float]] = deque()
+
+	def record_response(self, latency_seconds: float) -> None:
+		if latency_seconds < 0:
+			return
+		now = self._clock()
+		with self._lock:
+			self._response_samples.append((now, latency_seconds))
+			self._prune_health_samples(now)
+			if self._latency_seconds is None:
+				self._latency_seconds = latency_seconds
+			else:
+				self._latency_seconds = (
+					(self._latency_seconds * 0.75) + (latency_seconds * 0.25)
+				)
+
+	def record_retry(self) -> None:
+		now = self._clock()
+		with self._lock:
+			self._retry_count += 1
+			self._retry_samples.append(now)
+			self._last_retry_at = now
+			self._prune_health_samples(now)
+
+	def record_chunk(self, byte_count: int, write_seconds: float) -> None:
+		if byte_count <= 0:
+			return
+		now = self._clock()
+		with self._lock:
+			self._transferred_bytes += byte_count
+			self._transfer_samples.append((now, self._transferred_bytes))
+			_prune_transfer_samples(self._transfer_samples, now)
+			self._disk_samples.append((now, byte_count, max(0.0, write_seconds)))
+			self._prune_disk_samples(now)
+
+	def _prune_health_samples(self, now: float) -> None:
+		cutoff = now - ADAPTIVE_RETRY_WINDOW_SECONDS
+		while self._response_samples and self._response_samples[0][0] < cutoff:
+			self._response_samples.popleft()
+		while self._retry_samples and self._retry_samples[0] < cutoff:
+			self._retry_samples.popleft()
+
+	def _prune_disk_samples(self, now: float) -> None:
+		cutoff = now - TRANSFER_RATE_WINDOW_SECONDS
+		while self._disk_samples and self._disk_samples[0][0] < cutoff:
+			self._disk_samples.popleft()
+
+	def target_workers(self, remaining_files: int, remaining_bytes: int) -> int:
+		"""Return the next target; active workers are allowed to drain naturally."""
+
+		if remaining_files <= 0:
+			return 0
+		maximum = min(self.options.effective_maximum_workers, remaining_files)
+		minimum = min(self.options.minimum_workers, maximum)
+		with self._lock:
+			self._target = min(maximum, max(minimum, self._target))
+			now = self._clock()
+			average_file_size = remaining_bytes / max(1, remaining_files)
+			if now - self._last_evaluation < ADAPTIVE_EVALUATION_INTERVAL_SECONDS:
+				return self._target
+			self._last_evaluation = now
+			_prune_transfer_samples(self._transfer_samples, now)
+			self._prune_health_samples(now)
+			self._prune_disk_samples(now)
+			rate = _sample_rate(self._transfer_samples)
+			attempts = len(self._response_samples) + len(self._retry_samples)
+			error_rate = len(self._retry_samples) / attempts if attempts else 0.0
+			latency = self._latency_seconds
+			disk_bytes = sum(sample[1] for sample in self._disk_samples)
+			disk_seconds = sum(sample[2] for sample in self._disk_samples)
+			disk_rate = (
+				disk_bytes / disk_seconds
+				if disk_seconds > 0
+				else None
+			)
+			try:
+				cpu_load = self._cpu_load()
+			except (OSError, ValueError):
+				cpu_load = None
+
+			new_retries = self._retry_count - self._handled_retry_count
+			if new_retries:
+				self._handled_retry_count = self._retry_count
+				if error_rate >= 0.12 and self._target > minimum:
+					backoff = min(2, new_retries)
+					self._target = max(minimum, self._target - backoff)
+				return self._target
+
+			resource_constrained = (
+				(cpu_load is not None and cpu_load >= 0.90)
+				or (
+					rate is not None
+					and disk_rate is not None
+					and disk_rate < rate * 0.75
+				)
+			)
+			if resource_constrained and self._target > minimum:
+				self._target -= 1
+				return self._target
+
+			limit = self.options.effective_bandwidth_limit
+			limit_saturated = (
+				limit is not None and rate is not None and rate >= limit * 0.90
+			)
+			healthy = (
+				(cpu_load is None or cpu_load < 0.80)
+				and (
+					rate is None
+					or disk_rate is None
+					or disk_rate >= rate
+				)
+			)
+			retry_recovery_pending = (
+				self._last_retry_at is not None
+				and now - self._last_retry_at < ADAPTIVE_RETRY_RECOVERY_SECONDS
+			)
+			if (
+				healthy
+				and not retry_recovery_pending
+				and not limit_saturated
+				and self._target < maximum
+			):
+				# Parallel requests hide CDN round-trip time and are especially useful
+				# for this depot's large population of small files.
+				step = 2 if (
+					(latency is not None and latency >= 0.75)
+					or average_file_size <= 512 * 1024
+				) else 1
+				self._target = min(maximum, self._target + step)
+			return min(maximum, max(minimum, self._target))
+
+
 @dataclass(frozen=True)
 class DepotFile:
 	"""One exact file in the published depot build."""
@@ -258,6 +535,14 @@ class SyncProgress:
 	bytes_total: int
 	current_file: str | None
 	bytes_transferred: int = 0
+	bytes_remaining: int = 0
+	transfer_rate_bytes_per_second: float | None = None
+	eta_seconds: float | None = None
+	current_server: str | None = None
+	retry_count: int = 0
+	retry_reason: str | None = None
+	active_workers: int = 0
+	target_workers: int = 0
 
 
 @dataclass(frozen=True)
@@ -315,6 +600,7 @@ class DepotClient:
 		opener: HttpOpener = urlopen,
 		retry_delays: tuple[float, ...] = DOWNLOAD_RETRY_DELAYS_SECONDS,
 		sleeper: Callable[[float], None] = time.sleep,
+		download_observer: DownloadObserver | None = None,
 	) -> None:
 		if timeout_seconds <= 0:
 			raise ValueError("timeout_seconds must be positive")
@@ -325,6 +611,24 @@ class DepotClient:
 		self._retry_delays = retry_delays
 		self._sleeper = sleeper
 		self._candidate_servers: tuple[str, ...] = ()
+		self._download_observer = download_observer
+		self._bandwidth_limiter: BandwidthLimiter | None = None
+
+	@property
+	def download_observer(self) -> DownloadObserver | None:
+		return self._download_observer
+
+	@download_observer.setter
+	def download_observer(self, observer: DownloadObserver | None) -> None:
+		self._download_observer = observer
+
+	@property
+	def bandwidth_limiter(self) -> BandwidthLimiter | None:
+		return self._bandwidth_limiter
+
+	@bandwidth_limiter.setter
+	def bandwidth_limiter(self, limiter: BandwidthLimiter | None) -> None:
+		self._bandwidth_limiter = limiter
 
 	def fetch_snapshot(self, *, preferred_region: str = "Europe") -> DepotSnapshot:
 		"""Fetch the live depot and build manifests, trying trusted regional fallbacks."""
@@ -379,11 +683,21 @@ class DepotClient:
 					attempt_server,
 					file,
 					destination,
+					attempt=attempt_index + 1,
 					progress=progress,
 				)
 				return
 			except _RetryableDownloadError as exc:
 				last_error = exc
+				self._notify_download(
+					DownloadTelemetry(
+						kind="retry",
+						file=file.path,
+						server=attempt_server,
+						attempt=attempt_index + 1,
+						reason=str(exc),
+					)
+				)
 				if exc.discard_partial:
 					temporary.unlink(missing_ok=True)
 				if attempt_index >= len(self._retry_delays):
@@ -407,6 +721,7 @@ class DepotClient:
 		file: DepotFile,
 		destination: Path,
 		*,
+		attempt: int,
 		progress: Callable[[int], None] | None,
 	) -> None:
 		temporary = _download_partial_path(destination)
@@ -455,6 +770,15 @@ class DepotClient:
 			headers["Range"] = f"bytes={requested_offset}-"
 		url = _file_url(server, file)
 		request = Request(url, headers=headers, method="GET")
+		self._notify_download(
+			DownloadTelemetry(
+				kind="request",
+				file=file.path,
+				server=server,
+				attempt=attempt,
+			)
+		)
+		request_started_at = time.monotonic()
 		try:
 			response = self._opener(request, timeout=self.timeout_seconds)
 		except HTTPError as exc:
@@ -470,6 +794,15 @@ class DepotClient:
 			raise _RetryableDownloadError(
 				f"could not connect to {server} for {file.path}: {exc}"
 			) from exc
+		self._notify_download(
+			DownloadTelemetry(
+				kind="response",
+				file=file.path,
+				server=server,
+				attempt=attempt,
+				latency_seconds=max(0.0, time.monotonic() - request_started_at),
+			)
+		)
 
 		try:
 			append = False
@@ -528,8 +861,24 @@ class DepotClient:
 								f"download exceeds expected size for {file.path}",
 								discard_partial=True,
 							)
+						if self._bandwidth_limiter is not None:
+							self._bandwidth_limiter.consume(len(chunk))
 						digest.update(chunk)
+						write_started_at = time.monotonic()
 						handle.write(chunk)
+						self._notify_download(
+							DownloadTelemetry(
+								kind="chunk",
+								file=file.path,
+								server=server,
+								attempt=attempt,
+								bytes_written=len(chunk),
+								write_seconds=max(
+									0.0,
+									time.monotonic() - write_started_at,
+								),
+							)
+						)
 						if progress is not None:
 							progress(written)
 			except _RetryableDownloadError:
@@ -559,6 +908,16 @@ class DepotClient:
 			except (OSError, HTTPException):
 				pass
 
+	def _notify_download(self, event: DownloadTelemetry) -> None:
+		observer = self._download_observer
+		if observer is not None:
+			# noinspection PyBroadException
+			try:
+				observer(event)
+			except Exception:
+				# Telemetry must never weaken verified download behavior.
+				pass
+
 	def _get_json(self, url: str) -> Mapping[str, object]:
 		request = Request(url, headers=_headers(), method="GET")
 		try:
@@ -642,6 +1001,7 @@ def sync_managed_game(
 	dry_run: bool = False,
 	progress: ProgressCallback | None = None,
 	client: DepotClient | None = None,
+	download_options: DownloadOptions | None = None,
 ) -> DepotSyncResult:
 	"""Synchronize the managed game directory using only official public data."""
 
@@ -676,7 +1036,14 @@ def sync_managed_game(
 		)
 
 	root = _prepare_game_root(paths.source_game, create=True)
-	_download_plan(depot_client, snapshot, root, plan, progress)
+	_download_plan(
+		depot_client,
+		snapshot,
+		root,
+		plan,
+		progress,
+		download_options or DownloadOptions(),
+	)
 	deleted = _apply_deletions(root, plan.deletes)
 	created = _apply_create_if_missing(root, snapshot.create_if_missing)
 	result = _sync_result(
@@ -814,6 +1181,7 @@ def _download_plan(
 	root: Path,
 	plan: UpdatePlan,
 	progress: ProgressCallback | None,
+	options: DownloadOptions,
 ) -> None:
 	if not plan.downloads:
 		return
@@ -821,10 +1189,22 @@ def _download_plan(
 	lock = threading.Lock()
 	emit_lock = threading.Lock()
 	current_bytes: dict[str, int] = {}
+	current_servers: dict[str, str] = {}
 	completed_files = 0
 	completed_bytes = 0
 	transferred_bytes = 0
+	retry_count = 0
+	retry_reason: str | None = None
+	active_workers = 0
+	target_workers = 0
 	last_progress_emitted_at = 0.0
+	started_at = time.monotonic()
+	transfer_samples: deque[tuple[float, int]] = deque([(started_at, 0)])
+	controller = AdaptiveConcurrencyController(options)
+	bandwidth_limit = options.effective_bandwidth_limit
+	limiter = BandwidthLimiter(bandwidth_limit) if bandwidth_limit is not None else None
+	previous_observer = client.download_observer
+	previous_limiter = client.bandwidth_limiter
 
 	def emit_download_progress(
 		current_file: str,
@@ -847,27 +1227,72 @@ def _download_plan(
 				return
 			last_progress_emitted_at = now
 			with lock:
+				bytes_completed = min(
+					plan.bytes_total,
+					completed_bytes + sum(current_bytes.values()),
+				)
+				_prune_transfer_samples(transfer_samples, now)
+				transfer_rate = _sample_rate(transfer_samples)
+				bytes_remaining = max(0, plan.bytes_total - bytes_completed)
+				eta_seconds = (
+					bytes_remaining / transfer_rate
+					if transfer_rate is not None and transfer_rate > 0
+					else None
+				)
 				event = SyncProgress(
 					"downloading",
 					completed_files,
 					len(plan.downloads),
-					completed_bytes + sum(current_bytes.values()),
+					bytes_completed,
 					plan.bytes_total,
 					current_file,
 					transferred_bytes,
+					bytes_remaining,
+					transfer_rate,
+					eta_seconds,
+					current_servers.get(
+						current_file,
+						_server_display_name(depot_snapshot.server),
+					),
+					retry_count,
+					retry_reason,
+					active_workers,
+					target_workers,
 				)
 			progress(event)
 
+	def observe_download(event: DownloadTelemetry) -> None:
+		nonlocal transferred_bytes, retry_count, retry_reason
+		if event.kind == "response" and event.latency_seconds is not None:
+			controller.record_response(event.latency_seconds)
+		elif event.kind == "retry":
+			controller.record_retry()
+		elif event.kind == "chunk":
+			controller.record_chunk(event.bytes_written, event.write_seconds)
+
+		force = event.kind == "retry"
+		if event.kind in {"request", "response", "retry", "chunk"}:
+			with lock:
+				current_servers[event.file] = _server_display_name(event.server)
+				if event.kind == "chunk":
+					transferred_bytes += event.bytes_written
+					now = time.monotonic()
+					transfer_samples.append((now, transferred_bytes))
+					_prune_transfer_samples(transfer_samples, now)
+				if event.kind == "retry":
+					retry_count += 1
+					retry_reason = event.reason
+		if previous_observer is not None:
+			previous_observer(event)
+		if event.kind != "chunk":
+			emit_download_progress(event.file, force=force)
+
 	def download(planned_download: PlannedDownload) -> None:
 		target = _safe_target(root, planned_download.file.path)
 		_ensure_target_parent(root, target)
 
 		def report_file_bytes(written: int) -> None:
-			nonlocal transferred_bytes
 			with lock:
-				previous = current_bytes.get(planned_download.file.path)
-				if previous is not None and written > previous:
-					transferred_bytes += written - previous
 				current_bytes[planned_download.file.path] = written
 			emit_download_progress(planned_download.file.path)
 
@@ -878,49 +1303,84 @@ def _download_plan(
 			progress=report_file_bytes if progress is not None else None,
 		)
 
-	worker_count = min(DOWNLOAD_WORKERS, len(plan.downloads))
-	remaining_items = iter(plan.downloads)
-	with ThreadPoolExecutor(max_workers=worker_count) as executor:
-		future_items: dict[Future[None], PlannedDownload] = {}
+	maximum_workers = min(options.effective_maximum_workers, len(plan.downloads))
+	# Start large transfers first so they overlap the small-file request workload
+	# instead of becoming a single-connection tail at the end of an install.
+	remaining_items: deque[PlannedDownload] = deque(
+		sorted(plan.downloads, key=lambda item: item.file.size, reverse=True)
+	)
+	client.download_observer = observe_download
+	client.bandwidth_limiter = limiter
+	try:
+		with ThreadPoolExecutor(max_workers=maximum_workers) as executor:
+			future_items: dict[Future[None], PlannedDownload] = {}
 
-		def submit_next() -> bool:
-			try:
-				next_item = next(remaining_items)
-			except StopIteration:
-				return False
-			future_items[executor.submit(download, next_item)] = next_item
-			return True
-
-		try:
-			for _worker in range(worker_count):
-				if not submit_next():
-					break
-			while future_items:
-				done, _pending = wait(
-					tuple(future_items),
-					return_when=FIRST_COMPLETED,
-				)
-				completed_items: list[PlannedDownload] = []
-				for future in done:
-					completed_item = future_items.pop(future)
-					future.result()
-					completed_items.append(completed_item)
-				for completed_item in completed_items:
+			def submit_until(worker_target: int) -> None:
+				nonlocal active_workers, target_workers
+				with lock:
+					target_workers = worker_target
+				while remaining_items and len(future_items) < worker_target:
+					next_item = remaining_items.popleft()
 					with lock:
-						completed_files += 1
-						completed_bytes += completed_item.file.size
-						current_bytes.pop(completed_item.file.path, None)
-						is_final_download = completed_files == len(plan.downloads)
-					emit_download_progress(
-						completed_item.file.path,
-						force=is_final_download,
+						active_workers += 1
+					try:
+						submitted_future = executor.submit(download, next_item)
+					except BaseException:
+						with lock:
+							active_workers -= 1
+						remaining_items.appendleft(next_item)
+						raise
+					future_items[submitted_future] = next_item
+
+			try:
+				initial_target = controller.target_workers(
+					len(remaining_items),
+					plan.bytes_total,
+				)
+				submit_until(initial_target)
+				while future_items:
+					done, _pending = wait(
+						tuple(future_items),
+						timeout=ADAPTIVE_EVALUATION_INTERVAL_SECONDS,
+						return_when=FIRST_COMPLETED,
 					)
-				for _item in completed_items:
-					submit_next()
-		except BaseException:
-			for future in future_items:
-				future.cancel()
-			raise
+					completed_items: list[PlannedDownload] = []
+					for future in done:
+						completed_item = future_items.pop(future)
+						future.result()
+						completed_items.append(completed_item)
+					for completed_item in completed_items:
+						with lock:
+							active_workers -= 1
+							completed_files += 1
+							completed_bytes += completed_item.file.size
+							current_bytes.pop(completed_item.file.path, None)
+							is_final_download = completed_files == len(plan.downloads)
+						emit_download_progress(
+							completed_item.file.path,
+							force=is_final_download,
+						)
+						with lock:
+							current_servers.pop(completed_item.file.path, None)
+					with lock:
+						in_progress_bytes = sum(current_bytes.values())
+					remaining_file_count = len(remaining_items) + len(future_items)
+					remaining_byte_count = max(
+						0,
+						plan.bytes_total - completed_bytes - in_progress_bytes,
+					)
+					next_target = controller.target_workers(
+						remaining_file_count,
+						remaining_byte_count,
+					)
+					submit_until(next_target)
+			except BaseException:
+				for future in future_items:
+					future.cancel()
+				raise
+	finally:
+		client.download_observer = previous_observer
+		client.bandwidth_limiter = previous_limiter
 
 
 def _apply_deletions(root: Path, relative_paths: Iterable[str]) -> int:
@@ -1415,6 +1875,44 @@ def _content_range(value: str | None) -> tuple[int, int, int]:
 	return start, end, total
 
 
+def _normalized_cpu_load() -> float | None:
+	"""Return one-minute runnable load normalized by the available CPU count."""
+
+	try:
+		load = os.getloadavg()[0]
+	except (AttributeError, OSError):
+		return None
+	processors = os.cpu_count()
+	if processors is None or processors <= 0:
+		return None
+	return max(0.0, load / processors)
+
+
+def _prune_transfer_samples(
+	samples: deque[tuple[float, int]],
+	now: float,
+) -> None:
+	cutoff = now - TRANSFER_RATE_WINDOW_SECONDS
+	while len(samples) > 2 and samples[1][0] <= cutoff:
+		samples.popleft()
+
+
+def _sample_rate(samples: deque[tuple[float, int]]) -> float | None:
+	if len(samples) < 2:
+		return None
+	first_time, first_bytes = samples[0]
+	last_time, last_bytes = samples[-1]
+	elapsed = last_time - first_time
+	if elapsed <= 0 or last_bytes < first_bytes:
+		return None
+	return (last_bytes - first_bytes) / elapsed
+
+
+def _server_display_name(server: str) -> str:
+	parsed = urlparse(server if "://" in server else f"//{server}")
+	return parsed.hostname or server.split("/", 1)[0]
+
+
 def _headers() -> dict[str, str]:
 	return {"User-Agent": f"PT-Linux/{__version__} (+https://pristontale.eu/)"}
 
@@ -1451,5 +1949,6 @@ def _emit_progress(
 				bytes_completed,
 				bytes_total,
 				current_file,
+				bytes_remaining=max(0, bytes_total - bytes_completed),
 			)
 		)