Reworked the gamescope to work better with scaling and maximize window.

ptlinux/gamebox_runtime.py

@@ -46,9 +46,12 @@ _X_SUBSTRUCTURE_REDIRECT_MASK = 1 << 20
 _X_WM_STATE_ICONIC = 3
 _NET_WM_STATE_REMOVE = 0
 _NET_WM_STATE_SOURCE_APPLICATION = 1
+_NET_WM_BYPASS_COMPOSITOR_PREFER_COMPOSITING = 2
 WINDOW_FOCUS_REPAIR_INTERVAL_SECONDS = 1.0
 _P_MIN_SIZE = 1 << 4
 _P_MAX_SIZE = 1 << 5
+_P_RESIZE_INC = 1 << 6
+_P_BASE_SIZE = 1 << 8
 _MOTIF_HINTS_FUNCTIONS = 1 << 0
 _MOTIF_HINTS_DECORATIONS = 1 << 1
 _MOTIF_ALL = 1 << 0
@@ -148,6 +151,26 @@ def _normal_hints_without_fixed_size(
 	return tuple(updated)
 
 
+def _normal_hints_with_resize_increments(
+	hints: Sequence[int],
+	current_size: tuple[int, int],
+	increments: tuple[int, int],
+) -> tuple[int, ...]:
+	"""Add resize increments while preserving every unrelated size hint."""
+
+	updated = list(hints)
+	if len(updated) < 18:
+		updated.extend(0 for _ in range(18 - len(updated)))
+	width, height = current_size
+	width_increment, height_increment = increments
+	updated[0] |= _P_RESIZE_INC | _P_BASE_SIZE
+	updated[9] = width_increment
+	updated[10] = height_increment
+	updated[15] = width % width_increment
+	updated[16] = height % height_increment
+	return tuple(updated)
+
+
 class GameboxRuntimeError(RuntimeError):
 	"""Raised when the game command or X11 mapper cannot be started."""
 
@@ -258,6 +281,69 @@ class _X11:
 		window, title = selected
 		return self._client_window(window), title
 
+	def find_window_by_pid(self, expected_pid: int) -> int | None:
+		"""Find a top-level host window created by one native process."""
+
+		queue = self._root_windows()
+		visited: set[int] = set()
+		while queue and len(visited) < 4096:
+			window = queue.pop(0)
+			if window in visited:
+				continue
+			visited.add(window)
+			pid_values = self._long_property(window, b"_NET_WM_PID", 1)
+			if pid_values and pid_values[0] == expected_pid:
+				return window
+			queue.extend(self._child_windows(window))
+		return None
+
+	def set_resize_increments(
+		self,
+		window: int,
+		current_size: tuple[int, int],
+		increments: tuple[int, int],
+	) -> bool:
+		"""Quantize host resize requests without replacing other WM hints."""
+
+		if any(value <= 0 for value in (*current_size, *increments)):
+			raise ValueError("window sizes and resize increments must be positive")
+		hints = self._long_property(window, b"WM_NORMAL_HINTS", 18) or ()
+		updated = _normal_hints_with_resize_increments(
+			hints,
+			current_size,
+			increments,
+		)
+		self._replace_long_property(
+			window,
+			b"WM_NORMAL_HINTS",
+			b"WM_SIZE_HINTS",
+			updated,
+		)
+		self.library.XSync(self.display, 0)
+		confirmed = self._long_property(window, b"WM_NORMAL_HINTS", 18)
+		return bool(
+			confirmed
+			and len(confirmed) >= 17
+			and confirmed[0] & _P_RESIZE_INC
+			and confirmed[9:11] == increments
+		)
+
+	def prefer_composited_presentation(self, window: int) -> bool:
+		"""Keep the host compositor active for this window at every size."""
+
+		self._replace_long_property(
+			window,
+			b"_NET_WM_BYPASS_COMPOSITOR",
+			b"CARDINAL",
+			(_NET_WM_BYPASS_COMPOSITOR_PREFER_COMPOSITING,),
+		)
+		self.library.XSync(self.display, 0)
+		return self._long_property(
+			window,
+			b"_NET_WM_BYPASS_COMPOSITOR",
+			1,
+		) == (_NET_WM_BYPASS_COMPOSITOR_PREFER_COMPOSITING,)
+
 	def _root_windows(self) -> list[int]:
 		root = self.library.XDefaultRootWindow(self.display)
 		return self._child_windows(root)
@@ -1446,6 +1532,30 @@ def _maintain_game_window_scaling(
 	return int(process.wait())
 
 
+def _maintain_game_window_focus(
+	mapper: _X11,
+	process: subprocess.Popen[bytes],
+	window: int,
+) -> int:
+	"""Keep the keyboard-focus repair while retaining Wine's fixed-size hints."""
+
+	print(
+		"Game window resizing is disabled to prevent repeated Direct3D "
+		"device resets.",
+		file=sys.stderr,
+		flush=True,
+	)
+	next_focus_repair_at = time.monotonic() + WINDOW_FOCUS_REPAIR_INTERVAL_SECONDS
+	while process.poll() is None:
+		time.sleep(WINDOW_SCALING_EVENT_TIMEOUT_SECONDS)
+		now = time.monotonic()
+		if now < next_focus_repair_at:
+			continue
+		_repair_game_window_focus(mapper, window)
+		next_focus_repair_at = time.monotonic() + WINDOW_FOCUS_REPAIR_INTERVAL_SECONDS
+	return int(process.wait())
+
+
 def _guard_game_window_startup(
 	mapper: _X11,
 	process: subprocess.Popen[bytes],
@@ -1552,6 +1662,8 @@ def run_game_window(
 	window_title: str = WINDOW_TITLE,
 	*,
 	borderless: bool = False,
+	allow_window_resizing: bool = False,
+	gamescope_scaling: bool = False,
 	suppress_wm_ping: bool = False,
 ) -> int:
 	if not command:
@@ -1604,6 +1716,23 @@ def run_game_window(
 				and game_window_created_at is not None
 				and now - game_window_created_at >= WINDOW_READY_DELAY_SECONDS
 			):
+				if gamescope_scaling:
+					# Gamescope owns the nested child's fullscreen/maximized state and
+					# surface geometry. Trying to restore or nudge that window makes the
+					# two window managers fight and can produce visible frame flicker.
+					print(
+						f"Game window ready: {game_title or window_title}",
+						file=sys.stderr,
+						flush=True,
+					)
+					_focus_game_window(mapper, game_window)
+					print(
+						"Game presentation is isolated; Gamescope handles "
+						"aspect-preserving outer-window resizing.",
+						file=sys.stderr,
+						flush=True,
+					)
+					return int(process.wait())
 				_prepare_game_window(
 					mapper,
 					game_window,
@@ -1622,12 +1751,30 @@ def run_game_window(
 					game_window,
 					borderless=borderless,
 				)
-				if not borderless:
+				if (
+					allow_window_resizing
+					and not gamescope_scaling
+					and startup_stable
+					and not borderless
+				):
 					return _maintain_game_window_scaling(
 						mapper,
 						process,
 						game_window,
 					)
+				if not borderless:
+					if allow_window_resizing and not startup_stable:
+						print(
+							"Warning: live window resizing was not enabled because the "
+							"startup window did not stabilize.",
+							file=sys.stderr,
+							flush=True,
+						)
+					return _maintain_game_window_focus(
+						mapper,
+						process,
+						game_window,
+					)
 				return int(process.wait())
 			time.sleep(0.1)
 		if process.poll() is None:
@@ -1636,7 +1783,7 @@ def run_game_window(
 				file=sys.stderr,
 				flush=True,
 			)
-			if game_window is not None:
+			if game_window is not None and not gamescope_scaling:
 				_prepare_game_window(
 					mapper,
 					game_window,
@@ -1652,6 +1799,12 @@ def main(argv: Sequence[str] | None = None) -> int:
 	parser = argparse.ArgumentParser()
 	parser.add_argument("--window-title", default=WINDOW_TITLE)
 	parser.add_argument("--borderless", action="store_true")
+	parser.add_argument(
+		"--allow-window-resizing",
+		action="store_true",
+		help="enable unsafe native client resizing for diagnostics",
+	)
+	parser.add_argument("--gamescope-scaling", action="store_true")
 	parser.add_argument("--suppress-wm-ping", action="store_true")
 	parser.add_argument("command", nargs=argparse.REMAINDER)
 	arguments = parser.parse_args(argv)
@@ -1663,6 +1816,8 @@ def main(argv: Sequence[str] | None = None) -> int:
 			command,
 			arguments.window_title,
 			borderless=arguments.borderless,
+			allow_window_resizing=arguments.allow_window_resizing,
+			gamescope_scaling=arguments.gamescope_scaling,
 			suppress_wm_ping=arguments.suppress_wm_ping,
 		)
 	except (GameboxRuntimeError, OSError) as exc: