macos: Patch QMetalLayer.setNeedsDisplayInRect at runtime to avoid freezing on recent Qt

src/citra_meta/CMakeLists.txt

@@ -60,6 +60,10 @@ if (ENABLE_QT AND UNIX AND NOT APPLE)
     target_link_libraries(citra_meta PRIVATE Qt6::DBus gamemode)
 endif()
 
+if (ENABLE_QT AND APPLE)
+    target_link_libraries(citra_meta PRIVATE Qt6::GuiPrivate)
+endif()
+
 if (ENABLE_QT AND USE_DISCORD_PRESENCE)
     target_link_libraries(citra_meta PRIVATE discord-rpc)
 endif()

src/citra_qt/CMakeLists.txt

@@ -172,12 +172,12 @@ add_library(citra_qt STATIC EXCLUDE_FROM_ALL
     multiplayer/state.h
     multiplayer/validation.h
     precompiled_headers.h
+    qt_image_interface.cpp
+    qt_image_interface.h
     uisettings.cpp
     uisettings.h
     user_data_migration.cpp
     user_data_migration.h
-    qt_image_interface.cpp
-    qt_image_interface.h
     util/clickable_label.cpp
     util/clickable_label.h
     util/graphics_device_info.cpp
@@ -190,6 +190,13 @@ add_library(citra_qt STATIC EXCLUDE_FROM_ALL
     util/util.h
 )
 
+if (APPLE)
+  target_sources(citra_qt PUBLIC
+    qt_swizzle.h
+    qt_swizzle.mm
+  )
+endif()
+
 file(GLOB COMPAT_LIST
      ${PROJECT_BINARY_DIR}/dist/compatibility_list/compatibility_list.qrc
      ${PROJECT_BINARY_DIR}/dist/compatibility_list/compatibility_list.json)
@@ -275,6 +282,10 @@ if (NOT WIN32)
     target_include_directories(citra_qt PRIVATE ${Qt6Gui_PRIVATE_INCLUDE_DIRS})
 endif()
 
+if (APPLE)
+    target_link_libraries(citra_qt PRIVATE Qt6::GuiPrivate)
+endif()
+
 if (UNIX AND NOT APPLE)
     target_link_libraries(citra_qt PRIVATE Qt6::DBus gamemode)
 endif()

src/citra_qt/citra_qt.cpp

@@ -69,6 +69,7 @@
 #include "citra_qt/movie/movie_record_dialog.h"
 #include "citra_qt/multiplayer/state.h"
 #include "citra_qt/qt_image_interface.h"
+#include "citra_qt/qt_swizzle.h"
 #include "citra_qt/uisettings.h"
 #include "common/play_time_manager.h"
 #ifdef ENABLE_QT_UPDATE_CHECKER
@@ -4112,6 +4113,11 @@ static Qt::HighDpiScaleFactorRoundingPolicy GetHighDpiRoundingPolicy() {
 }
 
 void LaunchQtFrontend(int argc, char* argv[]) {
+#ifdef __APPLE__
+    // Ensure that the linker doesn't optimize qt_swizzle.mm out of existence.
+    QtSwizzle::Dummy();
+#endif
+
     Common::DetachedTasks detached_tasks;
 
 #if MICROPROFILE_ENABLED

src/citra_qt/qt_swizzle.h

@@ -0,0 +1,9 @@
+// Copyright Citra Emulator Project / Azahar Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+namespace QtSwizzle {
+
+void Dummy();
+
+} // namespace QtSwizzle

src/citra_qt/qt_swizzle.mm

@@ -0,0 +1,48 @@
+// Copyright Citra Emulator Project / Azahar Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#import <QtGui/private/qmetallayer_p.h>
+#import <objc/runtime.h>
+
+namespace QtSwizzle {
+
+void Dummy() {
+    // Call this anywhere to make sure that qt_swizzle.mm is linked.
+    // noop
+}
+
+} // namespace QtSwizzle
+
+@implementation QMetalLayer (AzaharPatch)
+
++ (void)load {
+    static dispatch_once_t onceToken;
+    dispatch_once(&onceToken, ^{
+      Class targetClass = [self class];
+
+      // Get the original and swizzled methods
+      Method originalMethod =
+          class_getInstanceMethod(targetClass, @selector(setNeedsDisplayInRect:));
+      Method swizzledMethod =
+          class_getInstanceMethod(targetClass, @selector(swizzled_setNeedsDisplayInRect:));
+
+      // Swap the implementations
+      method_exchangeImplementations(originalMethod, swizzledMethod);
+    });
+}
+
+- (void)swizzled_setNeedsDisplayInRect:(CGRect)rect {
+    constexpr auto tooBig = 1e10; // Arbitrary large number
+
+    // Check for problematic huge rectangles
+    if ((!self.needsDisplay) && (rect.size.width > tooBig || rect.size.height > tooBig ||
+                                 rect.origin.x < -tooBig || rect.origin.y < -tooBig)) {
+        return;
+    }
+
+    // Call the original implementation
+    [self swizzled_setNeedsDisplayInRect:rect];
+}
+
+@end