diff --git a/src/audio_core/hle/hle.cpp b/src/audio_core/hle/hle.cpp
index 56ca69749..2ad86ca73 100644
--- a/src/audio_core/hle/hle.cpp
+++ b/src/audio_core/hle/hle.cpp
@@ -419,8 +419,8 @@ void DspHle::Impl::AudioTickCallback(s64 cycles_late) {
     // Reschedule recurrent event
     const double time_scale =
         Settings::values.enable_realtime_audio
-            ? std::clamp(Core::System::GetInstance().GetStableFrameTimeScale(),
-                         100. / Settings::values.frame_limit.GetValue(), 3.0)
+            ? std::max(0.01, // Arbitrary small value to prevent time_scale from approaching zero
+                       Core::System::GetInstance().GetStableFrameTimeScale())
             : 1.0;
     s64 adjusted_ticks = static_cast<s64>(audio_frame_ticks / time_scale - cycles_late);
     core_timing.ScheduleEvent(adjusted_ticks, tick_event);
diff --git a/src/core/perf_stats.cpp b/src/core/perf_stats.cpp
index ea7f01ca6..5da7a55f0 100644
--- a/src/core/perf_stats.cpp
+++ b/src/core/perf_stats.cpp
@@ -22,6 +22,7 @@ using DoubleSecs = std::chrono::duration<double, std::chrono::seconds::period>;
 using std::chrono::duration_cast;
 using std::chrono::microseconds;
 
+constexpr double FRAME_LENGTH = 1.0 / SCREEN_REFRESH_RATE;
 // Purposefully ignore the first five frames, as there's a significant amount of overhead in
 // booting that we shouldn't account for
 constexpr std::size_t IgnoreFrames = 5;
@@ -65,6 +66,9 @@ void PerfStats::EndSystemFrame() {
     accumulated_frametime += frame_time;
     system_frames += 1;
 
+    // TODO: Track previous frame times in a less stupid way. -OS
+    previous_previous_frame_length = previous_frame_length;
+
     previous_frame_length = frame_end - previous_frame_end;
     previous_frame_end = frame_end;
 }
@@ -125,19 +129,17 @@ PerfStats::Results PerfStats::GetLastStats() {
 double PerfStats::GetLastFrameTimeScale() const {
     std::scoped_lock lock{object_mutex};
 
-    constexpr double FRAME_LENGTH = 1.0 / SCREEN_REFRESH_RATE;
     return duration_cast<DoubleSecs>(previous_frame_length).count() / FRAME_LENGTH;
 }
 
 double PerfStats::GetStableFrameTimeScale() const {
     std::scoped_lock lock{object_mutex};
 
-    constexpr double FRAME_LENGTH_MILLIS = (1.0 / SCREEN_REFRESH_RATE) * 1000;
-    const size_t num_frames = std::min<size_t>(50UL, current_index + 1);
-    const double sum = std::accumulate(perf_history.begin() + current_index - num_frames,
-                                       perf_history.begin() + current_index, 0.0);
-    const double stable_frame_length = sum / num_frames;
-    return stable_frame_length / FRAME_LENGTH_MILLIS;
+    const double stable_previous_frame_length =
+        (duration_cast<DoubleSecs>(previous_frame_length).count() +
+         duration_cast<DoubleSecs>(previous_previous_frame_length).count()) /
+        2;
+    return stable_previous_frame_length / FRAME_LENGTH;
 }
 
 void FrameLimiter::WaitOnce() {
diff --git a/src/core/perf_stats.h b/src/core/perf_stats.h
index ce289592f..8c4917783 100644
--- a/src/core/perf_stats.h
+++ b/src/core/perf_stats.h
@@ -85,7 +85,7 @@ public:
 
     /**
      * Has the same functionality as GetLastFrameTimeScale, but uses the mean frame time over the
-     * last 50 frames rather than only the frame time of the previous frame.
+     * last 2 frames rather than only the frame time of the previous frame.
      */
     double GetStableFrameTimeScale() const;
 
@@ -137,6 +137,8 @@ private:
     Clock::time_point frame_begin = reset_point;
     /// Total visible duration (including frame-limiting, etc.) of the previous system frame
     Clock::duration previous_frame_length = Clock::duration::zero();
+    /// Visible duration for the frame prior to previous_frame_length
+    Clock::duration previous_previous_frame_length = Clock::duration::zero();
 
     /// Last recorded performance statistics.
     Results last_stats;