From d8e74a9ff4ea88e1b8810b86ea1d20637cba29cc Mon Sep 17 00:00:00 2001 From: Steveice10 <1269164+Steveice10@users.noreply.github.com> Date: Tue, 9 May 2023 16:35:49 -0700 Subject: [PATCH] audio_core: Implement Apple AudioToolbox AAC decoder. (#6510) --- .ci/macos/build.sh | 1 - CMakeLists.txt | 1 + src/audio_core/CMakeLists.txt | 8 + src/audio_core/hle/adts.h | 1 + src/audio_core/hle/adts_reader.cpp | 2 + src/audio_core/hle/audiotoolbox_decoder.cpp | 255 ++++++++++++++++++++ src/audio_core/hle/audiotoolbox_decoder.h | 23 ++ src/audio_core/hle/hle.cpp | 4 + src/audio_core/hle/mediandk_decoder.cpp | 7 +- 9 files changed, 298 insertions(+), 4 deletions(-) create mode 100644 src/audio_core/hle/audiotoolbox_decoder.cpp create mode 100644 src/audio_core/hle/audiotoolbox_decoder.h diff --git a/.ci/macos/build.sh b/.ci/macos/build.sh index b6a2bb54a..94fd523af 100755 --- a/.ci/macos/build.sh +++ b/.ci/macos/build.sh @@ -22,7 +22,6 @@ cmake .. -DCMAKE_BUILD_TYPE=Release \ -DCITRA_ENABLE_COMPATIBILITY_REPORTING=${ENABLE_COMPATIBILITY_REPORTING:-"OFF"} \ -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON \ -DUSE_DISCORD_PRESENCE=ON \ - -DENABLE_FFMPEG_AUDIO_DECODER=ON \ -DENABLE_FFMPEG_VIDEO_DUMPER=ON \ -DENABLE_ASM=OFF \ -GNinja diff --git a/CMakeLists.txt b/CMakeLists.txt index 83b20193e..576cc57e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,7 @@ option(USE_DISCORD_PRESENCE "Enables Discord Rich Presence" OFF) option(CITRA_USE_PRECOMPILED_HEADERS "Use precompiled headers" ON) CMAKE_DEPENDENT_OPTION(ENABLE_MF "Use Media Foundation decoder (preferred over FFmpeg)" ON "WIN32" OFF) +CMAKE_DEPENDENT_OPTION(ENABLE_AUDIOTOOLBOX "Use AudioToolbox decoder (preferred over FFmpeg)" ON "APPLE" OFF) CMAKE_DEPENDENT_OPTION(COMPILE_WITH_DWARF "Add DWARF debugging information" ON "MINGW" OFF) diff --git a/src/audio_core/CMakeLists.txt b/src/audio_core/CMakeLists.txt index 76c930496..32fb85a13 100644 --- a/src/audio_core/CMakeLists.txt +++ b/src/audio_core/CMakeLists.txt @@ -59,6 +59,14 @@ if(ENABLE_MF) # just a static library of GUIDS so include that one directly. target_link_libraries(audio_core PRIVATE mfuuid.lib) target_compile_definitions(audio_core PUBLIC HAVE_MF) +elseif(ENABLE_AUDIOTOOLBOX) + target_sources(audio_core PRIVATE + hle/audiotoolbox_decoder.cpp + hle/audiotoolbox_decoder.h + ) + find_library(AUDIOTOOLBOX AudioToolbox) + target_link_libraries(audio_core PRIVATE ${AUDIOTOOLBOX}) + target_compile_definitions(audio_core PUBLIC HAVE_AUDIOTOOLBOX) elseif(ENABLE_FFMPEG_AUDIO_DECODER) target_sources(audio_core PRIVATE hle/ffmpeg_decoder.cpp diff --git a/src/audio_core/hle/adts.h b/src/audio_core/hle/adts.h index 9aba09fc3..cc602e125 100644 --- a/src/audio_core/hle/adts.h +++ b/src/audio_core/hle/adts.h @@ -6,6 +6,7 @@ #include "common/common_types.h" struct ADTSData { + u8 header_length; bool MPEG2; u8 profile; u8 channels; diff --git a/src/audio_core/hle/adts_reader.cpp b/src/audio_core/hle/adts_reader.cpp index e70c32a3b..417e02176 100644 --- a/src/audio_core/hle/adts_reader.cpp +++ b/src/audio_core/hle/adts_reader.cpp @@ -18,6 +18,8 @@ ADTSData ParseADTS(const char* buffer) { out.length = 0; return out; } + // bit 16 = no CRC + out.header_length = (buffer[1] & 0x1) ? 7 : 9; out.MPEG2 = (buffer[1] >> 3) & 0x1; // bit 17 to 18 out.profile = (buffer[2] >> 6) + 1; diff --git a/src/audio_core/hle/audiotoolbox_decoder.cpp b/src/audio_core/hle/audiotoolbox_decoder.cpp new file mode 100644 index 000000000..353c1d85f --- /dev/null +++ b/src/audio_core/hle/audiotoolbox_decoder.cpp @@ -0,0 +1,255 @@ +// Copyright 2023 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include "audio_core/audio_types.h" +#include "audio_core/hle/adts.h" +#include "audio_core/hle/audiotoolbox_decoder.h" + +namespace AudioCore::HLE { + +static constexpr auto bytes_per_sample = sizeof(s16); +static constexpr auto aac_frames_per_packet = 1024; +static constexpr auto error_out_of_data = -1932; + +class AudioToolboxDecoder::Impl { +public: + explicit Impl(Memory::MemorySystem& memory); + ~Impl(); + std::optional ProcessRequest(const BinaryRequest& request); + +private: + std::optional Initalize(const BinaryRequest& request); + std::optional Decode(const BinaryRequest& request); + + void Clear(); + bool InitializeDecoder(ADTSData& adts_header); + + static OSStatus DataFunc(AudioConverterRef in_audio_converter, u32* io_number_data_packets, + AudioBufferList* io_data, + AudioStreamPacketDescription** out_data_packet_description, + void* in_user_data); + + Memory::MemorySystem& memory; + + ADTSData adts_config; + AudioStreamBasicDescription output_format = {}; + AudioConverterRef converter = nullptr; + + u8* curr_data = nullptr; + u32 curr_data_len = 0; + + AudioStreamPacketDescription packet_description; +}; + +AudioToolboxDecoder::Impl::Impl(Memory::MemorySystem& memory) : memory(memory) {} + +std::optional AudioToolboxDecoder::Impl::Initalize(const BinaryRequest& request) { + BinaryResponse response; + std::memcpy(&response, &request, sizeof(response)); + response.unknown1 = 0x0; + + Clear(); + return response; +} + +AudioToolboxDecoder::Impl::~Impl() { + Clear(); +} + +void AudioToolboxDecoder::Impl::Clear() { + curr_data = nullptr; + curr_data_len = 0; + + adts_config = {}; + output_format = {}; + + if (converter) { + AudioConverterDispose(converter); + converter = nullptr; + } +} + +std::optional AudioToolboxDecoder::Impl::ProcessRequest( + const BinaryRequest& request) { + if (request.codec != DecoderCodec::AAC) { + LOG_ERROR(Audio_DSP, "AudioToolbox AAC Decoder cannot handle such codec: {}", + static_cast(request.codec)); + return {}; + } + + switch (request.cmd) { + case DecoderCommand::Init: { + return Initalize(request); + } + case DecoderCommand::Decode: { + return Decode(request); + } + case DecoderCommand::Unknown: { + BinaryResponse response; + std::memcpy(&response, &request, sizeof(response)); + response.unknown1 = 0x0; + return response; + } + default: + LOG_ERROR(Audio_DSP, "Got unknown binary request: {}", static_cast(request.cmd)); + return {}; + } +} + +bool AudioToolboxDecoder::Impl::InitializeDecoder(ADTSData& adts_header) { + if (converter) { + if (adts_config.channels == adts_header.channels && + adts_config.samplerate == adts_header.samplerate) { + return true; + } else { + Clear(); + } + } + + AudioStreamBasicDescription input_format = { + .mSampleRate = static_cast(adts_header.samplerate), + .mFormatID = kAudioFormatMPEG4AAC, + .mFramesPerPacket = aac_frames_per_packet, + .mChannelsPerFrame = adts_header.channels, + }; + + u32 bytes_per_frame = input_format.mChannelsPerFrame * bytes_per_sample; + output_format = { + .mSampleRate = input_format.mSampleRate, + .mFormatID = kAudioFormatLinearPCM, + .mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, + .mBytesPerPacket = bytes_per_frame, + .mFramesPerPacket = 1, + .mBytesPerFrame = bytes_per_frame, + .mChannelsPerFrame = input_format.mChannelsPerFrame, + .mBitsPerChannel = bytes_per_sample * 8, + }; + + auto status = AudioConverterNew(&input_format, &output_format, &converter); + if (status != noErr) { + LOG_ERROR(Audio_DSP, "Could not create AAC audio converter: {}", status); + Clear(); + return false; + } + + adts_config = adts_header; + return true; +} + +OSStatus AudioToolboxDecoder::Impl::DataFunc( + AudioConverterRef in_audio_converter, u32* io_number_data_packets, AudioBufferList* io_data, + AudioStreamPacketDescription** out_data_packet_description, void* in_user_data) { + auto impl = reinterpret_cast(in_user_data); + if (!impl || !impl->curr_data || impl->curr_data_len == 0) { + *io_number_data_packets = 0; + return error_out_of_data; + } + + io_data->mNumberBuffers = 1; + io_data->mBuffers[0].mNumberChannels = 0; + io_data->mBuffers[0].mDataByteSize = impl->curr_data_len; + io_data->mBuffers[0].mData = impl->curr_data; + *io_number_data_packets = 1; + + if (out_data_packet_description != nullptr) { + impl->packet_description.mStartOffset = 0; + impl->packet_description.mVariableFramesInPacket = 0; + impl->packet_description.mDataByteSize = impl->curr_data_len; + *out_data_packet_description = &impl->packet_description; + } + + impl->curr_data = nullptr; + impl->curr_data_len = 0; + + return noErr; +} + +std::optional AudioToolboxDecoder::Impl::Decode(const BinaryRequest& request) { + BinaryResponse response; + response.codec = request.codec; + response.cmd = request.cmd; + response.size = request.size; + + if (request.src_addr < Memory::FCRAM_PADDR || + request.src_addr + request.size > Memory::FCRAM_PADDR + Memory::FCRAM_SIZE) { + LOG_ERROR(Audio_DSP, "Got out of bounds src_addr {:08x}", request.src_addr); + return {}; + } + + auto data = memory.GetFCRAMPointer(request.src_addr - Memory::FCRAM_PADDR); + auto adts_header = ParseADTS(reinterpret_cast(data)); + curr_data = data + adts_header.header_length; + curr_data_len = request.size - adts_header.header_length; + + if (!InitializeDecoder(adts_header)) { + return std::nullopt; + } + + // 1024 samples, up to 2 channels each + s16 decoder_output[2048]; + AudioBufferList out_buffer{1, + {{ + output_format.mChannelsPerFrame, + sizeof(decoder_output), + decoder_output, + }}}; + + u32 num_packets = sizeof(decoder_output) / output_format.mBytesPerPacket; + auto status = AudioConverterFillComplexBuffer(converter, DataFunc, this, &num_packets, + &out_buffer, nullptr); + if (status != noErr && status != error_out_of_data) { + LOG_ERROR(Audio_DSP, "Could not decode AAC data: {}", status); + Clear(); + return std::nullopt; + } + + // De-interleave samples. + std::array, 2> out_streams; + auto num_frames = num_packets * output_format.mFramesPerPacket; + for (auto frame = 0; frame < num_frames; frame++) { + for (auto ch = 0; ch < output_format.mChannelsPerFrame; ch++) { + out_streams[ch].push_back( + decoder_output[(frame * output_format.mChannelsPerFrame) + ch]); + } + } + + curr_data = nullptr; + curr_data_len = 0; + + response.sample_rate = GetSampleRateEnum(static_cast(output_format.mSampleRate)); + response.num_channels = output_format.mChannelsPerFrame; + response.num_samples = num_frames; + + // transfer the decoded buffer from vector to the FCRAM + for (auto ch = 0; ch < out_streams.size(); ch++) { + if (!out_streams[ch].empty()) { + auto dst = ch == 0 ? request.dst_addr_ch0 : request.dst_addr_ch1; + if (dst < Memory::FCRAM_PADDR || + dst + out_streams[ch].size() > Memory::FCRAM_PADDR + Memory::FCRAM_SIZE) { + LOG_ERROR(Audio_DSP, "Got out of bounds dst_addr_ch{} {:08x}", ch, dst); + return {}; + } + std::memcpy(memory.GetFCRAMPointer(dst - Memory::FCRAM_PADDR), out_streams[ch].data(), + out_streams[ch].size() * bytes_per_sample); + } + } + + return response; +} + +AudioToolboxDecoder::AudioToolboxDecoder(Memory::MemorySystem& memory) + : impl(std::make_unique(memory)) {} + +AudioToolboxDecoder::~AudioToolboxDecoder() = default; + +std::optional AudioToolboxDecoder::ProcessRequest(const BinaryRequest& request) { + return impl->ProcessRequest(request); +} + +bool AudioToolboxDecoder::IsValid() const { + return true; +} + +} // namespace AudioCore::HLE diff --git a/src/audio_core/hle/audiotoolbox_decoder.h b/src/audio_core/hle/audiotoolbox_decoder.h new file mode 100644 index 000000000..10337691e --- /dev/null +++ b/src/audio_core/hle/audiotoolbox_decoder.h @@ -0,0 +1,23 @@ +// Copyright 2023 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include "audio_core/hle/decoder.h" + +namespace AudioCore::HLE { + +class AudioToolboxDecoder final : public DecoderBase { +public: + explicit AudioToolboxDecoder(Memory::MemorySystem& memory); + ~AudioToolboxDecoder() override; + std::optional ProcessRequest(const BinaryRequest& request) override; + bool IsValid() const override; + +private: + class Impl; + std::unique_ptr impl; +}; + +} // namespace AudioCore::HLE diff --git a/src/audio_core/hle/hle.cpp b/src/audio_core/hle/hle.cpp index 652e06bf3..18297ecee 100644 --- a/src/audio_core/hle/hle.cpp +++ b/src/audio_core/hle/hle.cpp @@ -10,6 +10,8 @@ #include "audio_core/audio_types.h" #ifdef HAVE_MF #include "audio_core/hle/wmf_decoder.h" +#elif HAVE_AUDIOTOOLBOX +#include "audio_core/hle/audiotoolbox_decoder.h" #elif HAVE_FFMPEG #include "audio_core/hle/ffmpeg_decoder.h" #elif ANDROID @@ -131,6 +133,8 @@ DspHle::Impl::Impl(DspHle& parent_, Memory::MemorySystem& memory) : parent(paren } #elif defined(HAVE_MF) decoder = std::make_unique(memory); +#elif defined(HAVE_AUDIOTOOLBOX) + decoder = std::make_unique(memory); #elif defined(HAVE_FFMPEG) decoder = std::make_unique(memory); #elif ANDROID diff --git a/src/audio_core/hle/mediandk_decoder.cpp b/src/audio_core/hle/mediandk_decoder.cpp index 16fd0c81e..f78db8e89 100644 --- a/src/audio_core/hle/mediandk_decoder.cpp +++ b/src/audio_core/hle/mediandk_decoder.cpp @@ -36,9 +36,10 @@ private: Memory::MemorySystem& mMemory; std::unique_ptr mDecoder; // default: 2 channles, 48000 samplerate - ADTSData mADTSData{/* MPEG2 */ false, /*profile*/ 2, /*channels*/ 2, - /*channel_idx*/ 2, /*framecount*/ 0, /*samplerate_idx*/ 3, - /*length*/ 0, /*samplerate*/ 48000}; + ADTSData mADTSData{ + /*header_length*/ 7, /*MPEG2*/ false, /*profile*/ 2, + /*channels*/ 2, /*channel_idx*/ 2, /*framecount*/ 0, + /*samplerate_idx*/ 3, /*length*/ 0, /*samplerate*/ 48000}; }; MediaNDKDecoder::Impl::Impl(Memory::MemorySystem& memory) : mMemory(memory) {