fix: apply video driver settings before the Qt app is created (#5701)

## Linked issue (required)

Fixes #5700

## Summary / motivation (required)

Since #5637, `setupGL()` runs after `AnkiApp` is constructed. Some of
what it sets is only read by Qt at startup (`AA_UseSoftwareOpenGL` on
macOS, `QT_OPENGL` on Windows), so safe mode and the Software driver are
no longer fully applied.

This calls `setupGL()` before creating the app again, where it was
before #5637. Safe mode is decided at that point, from `--safemode` or
from Shift. Since Qt can only detect Shift once the app exists,
`shift_held_before_app()` checks it with platform APIs:
`CGEventSourceFlagsState()` on macOS and `GetAsyncKeyState(VK_SHIFT)` on
Windows.

If Qt still detects Shift after the app is created but the check above
didn't (e.g. on Linux, where it isn't implemented), `setupGL()` is
called again with the Software driver. On Linux the settings it makes
still take effect at that point, so that path keeps working as it does
today. Happy to drop that fallback if you'd prefer.

## Steps to reproduce (required, use N/A if not applicable)

1. On macOS, run `/Applications/Anki.app/Contents/MacOS/anki --safemode`
from a terminal
2. The console shows `Qt warning: Attribute Qt::AA_UseSoftwareOpenGL
must be set before QCoreApplication is created.`

## How to test (required)

### Checklist (minimum)

- [ ] I ran `./ninja check` or an equivalent relevant check locally.
- [x] I added or updated tests when the change is non-trivial or
behavior changed.

### Details

- `qt/tests/test_startup_video_driver.py` runs `_run()` with a stand-in
for `AnkiApp` and checks that `setupGL()` is called with the expected
driver (profile driver, or Software with `--safemode` or Shift held)
before the app is constructed. Fails on main, passes with this change.
- Launched the source tree on macOS with a temporary base folder, with
`--safemode`: the warning above appears on main and is gone with this
change. With Shift held, `shift_held_before_app()` returns True and the
warning doesn't appear either.
- On Windows 11, checked the `GetAsyncKeyState(VK_SHIFT)` call used
here: it returns False before Shift is pressed and True while it's held.
I haven't tested a full Anki launch on Windows.
- ruff check/format pass on the changed files. I haven't been able to
run the full `just check` locally yet, so I'm relying on CI for the
rest.

## Before / after behavior (optional)

Before: `--safemode` and the Software driver don't set
`AA_UseSoftwareOpenGL` (macOS) / `QT_OPENGL` (Windows) in time. After:
they're applied before the app is created.

## Risk / compatibility / migration (optional)

This touches the same startup block as #5686, so whichever lands second
will need a small rebase. I'll take care of it if this one goes second.

## UI evidence (required for visual changes; otherwise N/A)

N/A

## Scope

- [x] This PR is focused on one change (no unrelated edits).

---------

Co-authored-by: Abdo <abdo@abdnh.net>

CONTRIBUTORS

@@ -311,6 +311,7 @@ wofiporia <1029687661@qq.com>
 Zain <https://github.com/ZainnQureshii>
 mslxl <i@mslxl.com>
 Artem Lytkin <146867384+4RH1T3CT0R7@users.noreply.github.com>
+Louis Vicat <https://github.com/LouisVct>
 ********************
 
 The text of the 3 clause BSD license follows:

qt/aqt/__init__.py

@@ -338,12 +338,12 @@ class AnkiApp(QApplication):
     )
     TMOUT = 30000
 
-    def __init__(self, argv: list[str]) -> None:
+    def __init__(self, argv: list[str], force_safemode: bool = False) -> None:
         QApplication.__init__(self, argv)
         self.installEventFilter(self)
         self._argv = argv
         self._native_event_filter = NativeEventFilter()
-        self.safeMode = (
+        self.safeMode = force_safemode or (
             bool(self.queryKeyboardModifiers() & Qt.KeyboardModifier.ShiftModifier)
             or "--safemode" in argv
         )
@@ -504,6 +504,37 @@ def parseArgs(argv: list[str]) -> tuple[argparse.Namespace, list[str]]:
     return parser.parse_known_args(argv[1:])
 
 
+def shift_held_before_app() -> bool:
+    """Check whether Shift is held, without needing a QApplication.
+
+    Some video driver settings must be applied before the app is created, but
+    QGuiApplication.queryKeyboardModifiers() only works once it exists. Returns
+    False on platforms where this isn't implemented, or if the check fails.
+    """
+    try:
+        if is_mac:
+            import ctypes
+
+            core_graphics = ctypes.cdll.LoadLibrary(
+                "/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics"
+            )
+            core_graphics.CGEventSourceFlagsState.argtypes = [ctypes.c_int32]
+            core_graphics.CGEventSourceFlagsState.restype = ctypes.c_uint64
+            # kCGEventSourceStateCombinedSessionState, kCGEventFlagMaskShift
+            return bool(core_graphics.CGEventSourceFlagsState(0) & 0x20000)
+        if is_win:
+            import ctypes
+
+            user32 = ctypes.windll.user32  # type: ignore
+            user32.GetAsyncKeyState.argtypes = [ctypes.c_int]
+            user32.GetAsyncKeyState.restype = ctypes.c_short
+            # VK_SHIFT; the most significant bit is set while the key is down
+            return bool(user32.GetAsyncKeyState(0x10) & 0x8000)
+    except Exception:
+        traceback.print_exc()
+    return False
+
+
 def setupGL(pm: aqt.profiles.ProfileManager, driver: VideoDriver | None = None) -> None:
     if driver is None:
         driver = pm.video_driver()
@@ -694,7 +725,14 @@ def _run(argv: list[str] | None = None, exec: bool = True) -> AnkiApp | None:
         traceback.print_exc()
         pm = None
 
+    # Safe mode forces software rendering, which must be set up before the app is
+    # created, so Shift has to be checked without relying on Qt
+    safe_mode = opts.safemode or shift_held_before_app()
+    driver = None
     if pm:
+        driver = VideoDriver.Software if safe_mode else pm.video_driver()
+        # gl workarounds; some of them only take effect before the app is created
+        setupGL(pm, driver)
         # apply user-provided scale factor
         os.environ["QT_SCALE_FACTOR"] = str(pm.uiScale())
 
@@ -718,16 +756,17 @@ def _run(argv: list[str] | None = None, exec: bool = True) -> AnkiApp | None:
     # create the app
     QCoreApplication.setApplicationName("Anki")
     QGuiApplication.setDesktopFileName("anki")
-    app = AnkiApp(argv)
+    app = AnkiApp(argv, safe_mode)
     if app.secondInstance():
         # we've signaled the primary instance, so we should close
         return None
 
-    driver = None
     if pm:
-        driver = pm.video_driver() if not app.safeMode else VideoDriver.Software
-        # gl workarounds
-        setupGL(pm, driver)
+        if app.safeMode and driver != VideoDriver.Software:
+            # Shift was only detected by Qt (e.g. on Linux, where the settings
+            # setupGL() makes still take effect after the app is created)
+            driver = VideoDriver.Software
+            setupGL(pm, driver)
     else:
         if i18n_setup:
             QMessageBox.critical(

qt/tests/test_startup.py

@@ -5,12 +5,27 @@ from __future__ import annotations
 
 import os
 from pathlib import Path
-from unittest.mock import patch
+from typing import Any, Callable
+from unittest.mock import MagicMock, patch
 
 import pytest
 
 import aqt
-from aqt.profiles import ProfileManager
+from aqt.profiles import ProfileManager, VideoDriver
+
+
+class FakeApp:
+    def secondInstance(self) -> bool:
+        # makes _run() return right after creating the app
+        return True
+
+
+def get_fake_app_class(on_init: Callable) -> type[FakeApp]:
+    class FakeAppSub(FakeApp):
+        def __init__(self, *args: Any, **kwargs: Any) -> None:
+            on_init()
+
+    return FakeAppSub
 
 
 @patch.dict(os.environ)
@@ -21,17 +36,45 @@ def test_ui_scale_is_applied_before_app_is_created(
     # afterwards has no effect (#5676)
     monkeypatch.setattr(ProfileManager, "uiScale", lambda self: 2.0)
     seen: list[str | None] = []
-
-    class FakeApp:
-        def __init__(self, argv: list[str]) -> None:
-            seen.append(os.environ.get("QT_SCALE_FACTOR"))
-
-        def secondInstance(self) -> bool:
-            # makes _run() return right after creating the app
-            return True
-
-    monkeypatch.setattr(aqt, "AnkiApp", FakeApp)
+    fake_app_class = get_fake_app_class(
+        lambda: seen.append(os.environ.get("QT_SCALE_FACTOR"))
+    )
+    monkeypatch.setattr(aqt, "AnkiApp", fake_app_class)
+    monkeypatch.setattr(aqt, "setupGL", MagicMock())
 
     aqt._run(["anki", "-b", str(tmp_path)], exec=False)
 
     assert seen == ["2.0"]
+
+
+@pytest.mark.parametrize(
+    ("extra_args", "shift_held", "expected_driver"),
+    [
+        ([], False, VideoDriver.OpenGL),
+        (["--safemode"], False, VideoDriver.Software),
+        ([], True, VideoDriver.Software),
+    ],
+)
+def test_run_applies_video_driver_before_app_is_created(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    extra_args: list[str],
+    shift_held: bool,
+    expected_driver: VideoDriver,
+) -> None:
+    # Some of setupGL()'s settings, such as AA_UseSoftwareOpenGL on macOS and
+    # QT_OPENGL on Windows, are ignored once the app exists
+    monkeypatch.setattr(ProfileManager, "video_driver", lambda self: VideoDriver.OpenGL)
+    monkeypatch.setattr(aqt, "shift_held_before_app", lambda: shift_held)
+    events: list[tuple[str, VideoDriver | None]] = []
+
+    def fake_setup_gl(pm: Any, driver: VideoDriver | None = None) -> None:
+        events.append(("setupGL", driver))
+
+    fake_app_class = get_fake_app_class(lambda: events.append(("app created", None)))
+    monkeypatch.setattr(aqt, "AnkiApp", fake_app_class)
+    monkeypatch.setattr(aqt, "setupGL", fake_setup_gl)
+
+    aqt._run(["anki", "-b", str(tmp_path), *extra_args], exec=False)
+
+    assert events == [("setupGL", expected_driver), ("app created", None)]