diff --git a/src/audio_core/hle/source.h b/src/audio_core/hle/source.h
index d9abf4ed2..2073e9033 100644
--- a/src/audio_core/hle/source.h
+++ b/src/audio_core/hle/source.h
@@ -66,7 +66,7 @@ public:
 
 private:
     const std::size_t source_id;
-    const Memory::MemorySystem* memory_system{};
+    Memory::MemorySystem* memory_system{};
     StereoFrame16 current_frame;
 
     using Format = SourceConfiguration::Configuration::Format;
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 5bb8abdfc..ab4eff6aa 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -586,7 +586,7 @@ bool MemorySystem::IsValidVirtualAddress(const Kernel::Process& process, const V
     return false;
 }
 
-bool MemorySystem::IsValidPhysicalAddress(const PAddr paddr) const {
+bool MemorySystem::IsValidPhysicalAddress(const PAddr paddr) {
     return GetPhysicalRef(paddr);
 }
 
@@ -637,11 +637,15 @@ std::string MemorySystem::ReadCString(VAddr vaddr, std::size_t max_length) {
     return string;
 }
 
-u8* MemorySystem::GetPhysicalPointer(PAddr address) const {
+u8* MemorySystem::GetPhysicalPointer(PAddr address) {
     return GetPhysicalRef(address);
 }
 
-MemoryRef MemorySystem::GetPhysicalRef(PAddr address) const {
+MemoryRef MemorySystem::GetPhysicalRef(PAddr address) {
+    if (address == physical_ptr_cache.first) {
+        return physical_ptr_cache.second;
+    }
+
     constexpr std::array memory_areas = {
         std::make_pair(VRAM_PADDR, VRAM_SIZE),
         std::make_pair(DSP_RAM_PADDR, DSP_RAM_SIZE),
@@ -655,10 +659,11 @@ MemoryRef MemorySystem::GetPhysicalRef(PAddr address) const {
         return address >= area.first && address <= area.first + area.second;
     });
 
-    if (area == memory_areas.end()) {
+    if (area == memory_areas.end()) [[unlikely]] {
         LOG_ERROR(HW_Memory, "Unknown GetPhysicalPointer @ {:#08X} at PC {:#08X}", address,
                   impl->GetPC());
-        return nullptr;
+        physical_ptr_cache = {address, {nullptr}};
+        return physical_ptr_cache.second;
     }
 
     u32 offset_into_region = address - area->first;
@@ -680,11 +685,13 @@ MemoryRef MemorySystem::GetPhysicalRef(PAddr address) const {
     default:
         UNREACHABLE();
     }
-    if (offset_into_region > target_mem->GetSize()) {
-        return {nullptr};
+    if (offset_into_region > target_mem->GetSize()) [[unlikely]] {
+        physical_ptr_cache = {address, {nullptr}};
+        return physical_ptr_cache.second;
     }
 
-    return {target_mem, offset_into_region};
+    physical_ptr_cache = {address, {target_mem, offset_into_region}};
+    return physical_ptr_cache.second;
 }
 
 std::vector<VAddr> MemorySystem::PhysicalToVirtualAddressForRasterizer(PAddr addr) {
diff --git a/src/core/memory.h b/src/core/memory.h
index 5bb98fb18..45bf54814 100644
--- a/src/core/memory.h
+++ b/src/core/memory.h
@@ -520,16 +520,16 @@ public:
     std::vector<VAddr> PhysicalToVirtualAddressForRasterizer(PAddr addr);
 
     /// Gets a pointer to the memory region beginning at the specified physical address.
-    u8* GetPhysicalPointer(PAddr address) const;
+    u8* GetPhysicalPointer(PAddr address);
 
     /// Returns a reference to the memory region beginning at the specified physical address
-    MemoryRef GetPhysicalRef(PAddr address) const;
+    MemoryRef GetPhysicalRef(PAddr address);
 
     /// Determines if the given VAddr is valid for the specified process.
     bool IsValidVirtualAddress(const Kernel::Process& process, VAddr vaddr);
 
     /// Returns true if the address refers to a valid memory region
-    bool IsValidPhysicalAddress(PAddr paddr) const;
+    bool IsValidPhysicalAddress(PAddr paddr);
 
     /// Gets offset in FCRAM from a pointer inside FCRAM range
     u32 GetFCRAMOffset(const u8* pointer) const;
@@ -573,6 +573,8 @@ private:
 
     void MapPages(PageTable& page_table, u32 base, u32 size, MemoryRef memory, PageType type);
 
+    std::pair<PAddr, MemoryRef> physical_ptr_cache;
+
 private:
     class Impl;
     std::unique_ptr<Impl> impl;