Fix aac_decoder memory leak (#1167)

* Fix aac_decoder memory leak

Only call NeAACDecInit on the first AAC frame and create new NeAACDecoder on DecoderCommand::Init request

* update license headers

* fix oversight

* reorganized code

-put open new FAAD2 AAC decoder code into a separate function
-removed LOG_INFO for open/close FAAD2 AAC decoder
-added LOG_ERROR when no decoder is created to handle decode request, either decoder creation failed or DecoderCommand::Init command not received

* Update aac_decoder.cpp

fix clang coding style check

* fix load savestate

Loading a savestate creates a situation where decode requests aren't preceded by an init request, so we open a new decoder by default in the constructor. A new AACDecoder instance is always constructed on load savestate.

src/audio_core/hle/aac_decoder.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -8,31 +8,13 @@
 namespace AudioCore::HLE {
 
 AACDecoder::AACDecoder(Memory::MemorySystem& memory) : memory(memory) {
-    decoder = NeAACDecOpen();
+    OpenNewDecoder();
-    if (decoder == nullptr) {
-        LOG_CRITICAL(Audio_DSP, "Could not open FAAD2 decoder.");
-        return;
-    }
-
-    auto config = NeAACDecGetCurrentConfiguration(decoder);
-    config->defObjectType = LC;
-    config->outputFormat = FAAD_FMT_16BIT;
-    if (!NeAACDecSetConfiguration(decoder, config)) {
-        LOG_CRITICAL(Audio_DSP, "Could not configure FAAD2 decoder.");
-        NeAACDecClose(decoder);
-        decoder = nullptr;
-        return;
-    }
-
-    LOG_INFO(Audio_DSP, "Created FAAD2 AAC decoder.");
 }
 
 AACDecoder::~AACDecoder() {
     if (decoder) {
         NeAACDecClose(decoder);
         decoder = nullptr;
-
-        LOG_INFO(Audio_DSP, "Destroyed FAAD2 AAC decoder.");
     }
 }
 
@@ -52,6 +34,9 @@ BinaryMessage AACDecoder::ProcessRequest(const BinaryMessage& request) {
     case DecoderCommand::Init: {
         BinaryMessage response = request;
         response.header.result = ResultStatus::Success;
+        if (!OpenNewDecoder()) {
+            response.header.result = ResultStatus::Error;
+        }
         return response;
     }
     case DecoderCommand::EncodeDecode: {
@@ -89,6 +74,7 @@ BinaryMessage AACDecoder::Decode(const BinaryMessage& request) {
     response.decode_aac_response.num_samples = 1024;
 
     if (decoder == nullptr) {
+        LOG_ERROR(Audio_DSP, "Failed to handle decode request: FAAD2 AAC decoder not open.");
         return response;
     }
 
@@ -102,17 +88,22 @@ BinaryMessage AACDecoder::Decode(const BinaryMessage& request) {
     u8* data = memory.GetFCRAMPointer(request.decode_aac_request.src_addr - Memory::FCRAM_PADDR);
     u32 data_len = request.decode_aac_request.size;
 
-    unsigned long sample_rate;
+    if (!decoder_initialized) {
-    u8 num_channels;
+        unsigned long sample_rate;
-    auto init_result = NeAACDecInit(decoder, data, data_len, &sample_rate, &num_channels);
+        u8 num_channels;
-    if (init_result < 0) {
+        auto init_result = NeAACDecInit(decoder, data, data_len, &sample_rate, &num_channels);
-        LOG_ERROR(Audio_DSP, "Could not initialize FAAD2 AAC decoder for request: {}", init_result);
+        if (init_result < 0) {
-        return response;
+            LOG_ERROR(Audio_DSP, "Could not initialize FAAD2 AAC decoder for request: {}",
-    }
+                      init_result);
+            return response;
+        }
 
-    // Advance past the frame header if needed.
+        decoder_initialized = true;
-    data += init_result;
+
-    data_len -= init_result;
+        // Advance past the frame header if needed.
+        data += init_result;
+        data_len -= init_result;
+    }
 
     std::array<std::vector<s16>, 2> out_streams;
 
@@ -125,6 +116,10 @@ BinaryMessage AACDecoder::Decode(const BinaryMessage& request) {
             return response;
         }
 
+        // Set the output frame info.
+        response.decode_aac_response.sample_rate = GetSampleRateEnum(frame_info.samplerate);
+        response.decode_aac_response.num_channels = frame_info.channels;
+
         // Split the decode result into channels.
         u32 num_samples = frame_info.samples / frame_info.channels;
         for (u32 sample = 0; sample < num_samples; sample++) {
@@ -155,11 +150,34 @@ BinaryMessage AACDecoder::Decode(const BinaryMessage& request) {
     }
 
     // Set the output frame info.
-    response.decode_aac_response.sample_rate = GetSampleRateEnum(sample_rate);
-    response.decode_aac_response.num_channels = num_channels;
     response.decode_aac_response.num_samples = static_cast<u32_le>(out_streams[0].size());
 
     return response;
 }
 
+bool AACDecoder::OpenNewDecoder() {
+    if (decoder) {
+        NeAACDecClose(decoder);
+    }
+    decoder_initialized = false;
+
+    decoder = NeAACDecOpen();
+    if (decoder == nullptr) {
+        LOG_CRITICAL(Audio_DSP, "Could not open FAAD2 decoder.");
+        return false;
+    }
+
+    auto config = NeAACDecGetCurrentConfiguration(decoder);
+    config->defObjectType = LC;
+    config->outputFormat = FAAD_FMT_16BIT;
+    if (!NeAACDecSetConfiguration(decoder, config)) {
+        LOG_CRITICAL(Audio_DSP, "Could not configure FAAD2 decoder.");
+        NeAACDecClose(decoder);
+        decoder = nullptr;
+        return false;
+    }
+
+    return true;
+}
+
 } // namespace AudioCore::HLE

src/audio_core/hle/aac_decoder.h

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -18,9 +18,11 @@ public:
 
 private:
     BinaryMessage Decode(const BinaryMessage& request);
+    bool OpenNewDecoder();
 
     Memory::MemorySystem& memory;
     NeAACDecHandle decoder = nullptr;
+    bool decoder_initialized = false;
 };
 
 } // namespace AudioCore::HLE