diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt index 4b8c516bc..21a32d642 100644 --- a/externals/CMakeLists.txt +++ b/externals/CMakeLists.txt @@ -214,8 +214,29 @@ else() set(ZSTD_BUILD_PROGRAMS OFF) set(ZSTD_BUILD_SHARED OFF) add_subdirectory(zstd/build/cmake EXCLUDE_FROM_ALL) - target_include_directories(libzstd_static INTERFACE $) + + target_include_directories(libzstd_static INTERFACE + $ + $ + ) + + add_library(zstd_seekable STATIC + $ + $ + ) + target_include_directories(zstd_seekable PUBLIC + $ + $ + ) + target_link_libraries(zstd_seekable PUBLIC libzstd_static) + + target_link_libraries(libzstd_static INTERFACE zstd_seekable) + add_library(zstd ALIAS libzstd_static) + + install(TARGETS zstd_seekable + EXPORT zstdExports + ) endif() # ENet diff --git a/src/citra_qt/citra_qt.cpp b/src/citra_qt/citra_qt.cpp index 784d19988..8fd54ba54 100644 --- a/src/citra_qt/citra_qt.cpp +++ b/src/citra_qt/citra_qt.cpp @@ -92,6 +92,7 @@ #endif #include "common/settings.h" #include "common/string_util.h" +#include "common/zstd_compression.h" #include "core/core.h" #include "core/dumping/backend.h" #include "core/file_sys/archive_extsavedata.h" @@ -991,6 +992,7 @@ void GMainWindow::ConnectWidgetEvents() { connect(this, &GMainWindow::UpdateProgress, this, &GMainWindow::OnUpdateProgress); connect(this, &GMainWindow::CIAInstallReport, this, &GMainWindow::OnCIAInstallReport); connect(this, &GMainWindow::CIAInstallFinished, this, &GMainWindow::OnCIAInstallFinished); + connect(this, &GMainWindow::CompressFinished, this, &GMainWindow::OnCompressFinished); connect(this, &GMainWindow::UpdateThemedIcons, multiplayer_state, &MultiplayerState::UpdateThemedIcons); } @@ -1082,6 +1084,10 @@ void GMainWindow::ConnectMenuEvents() { connect_menu(ui->action_Capture_Screenshot, &GMainWindow::OnCaptureScreenshot); connect_menu(ui->action_Dump_Video, &GMainWindow::OnDumpVideo); + // Tools + connect_menu(ui->action_Compress_ROM_File, &GMainWindow::OnCompressFile); + connect_menu(ui->action_Decompress_ROM_File, &GMainWindow::OnDecompressFile); + // Help connect_menu(ui->action_Open_Citra_Folder, &GMainWindow::OnOpenCitraFolder); connect_menu(ui->action_Open_Log_Folder, []() { @@ -2236,7 +2242,7 @@ void GMainWindow::OnMenuSetUpSystemFiles() { void GMainWindow::OnMenuInstallCIA() { QStringList filepaths = QFileDialog::getOpenFileNames( this, tr("Load Files"), UISettings::values.roms_path, - tr("3DS Installation File (*.CIA*)") + QStringLiteral(";;") + tr("All Files (*.*)")); + tr("3DS Installation File (*.cia *.zcia)") + QStringLiteral(";;") + tr("All Files (*.*)")); if (filepaths.isEmpty()) { return; @@ -2318,6 +2324,21 @@ void GMainWindow::OnCIAInstallReport(Service::AM::InstallStatus status, QString } } +void GMainWindow::OnCompressFinished(bool is_compress, bool success) { + progress_bar->hide(); + progress_bar->setValue(0); + + if (!success) { + if (is_compress) { + QMessageBox::critical(this, tr("Error compressing file"), + tr("File compress operation failed, check log for details.")); + } else { + QMessageBox::critical(this, tr("Error decompressing file"), + tr("File decompress operation failed, check log for details.")); + } + } +} + void GMainWindow::OnCIAInstallFinished() { progress_bar->hide(); progress_bar->setValue(0); @@ -3025,6 +3046,163 @@ void GMainWindow::OnDumpVideo() { } } +void GMainWindow::OnCompressFile() { + // NOTE: Encrypted files SHOULD NEVER be compressed, otherwise the resulting + // compressed file will have very poor compression ratios, due to the high + // entropy caused by encryption. This may cause confusion to the user as they + // will see the files do not compress well and blame the emulator. + // + // This is enforced using the loaders as they already return an error on encryption. + + QString filepath = QFileDialog::getOpenFileName( + this, tr("Load 3DS ROM File"), UISettings::values.roms_path, + tr("3DS ROM Files (*.cia *cci *3dsx *cxi)") + QStringLiteral(";;") + tr("All Files (*.*)")); + + if (filepath.isEmpty()) { + return; + } + std::string in_path = filepath.toStdString(); + + // Identify file type + Loader::AppLoader::CompressFileInfo compress_info{}; + compress_info.is_supported = false; + size_t frame_size{}; + { + auto loader = Loader::GetLoader(in_path); + if (loader) { + compress_info = loader->GetCompressFileInfo(); + frame_size = FileUtil::Z3DSWriteIOFile::DEFAULT_FRAME_SIZE; + } else { + bool is_compressed = false; + if (Service::AM::CheckCIAToInstall(in_path, is_compressed, true) == + Service::AM::InstallStatus::Success) { + compress_info.is_supported = true; + compress_info.is_compressed = is_compressed; + compress_info.recommended_compressed_extension = "zcia"; + compress_info.recommended_uncompressed_extension = "cia"; + compress_info.underlying_magic = std::array({'C', 'I', 'A', '\0'}); + frame_size = FileUtil::Z3DSWriteIOFile::MAX_FRAME_SIZE; + } + } + } + if (!compress_info.is_supported) { + QMessageBox::critical( + this, tr("Error compressing file"), + tr("The selected file is not a compatible 3DS ROM format. Make sure you have " + "chosen the right file, and that it is not encrypted.")); + return; + } + if (compress_info.is_compressed) { + QMessageBox::warning(this, tr("Error compressing file"), + tr("The selected file is already compressed.")); + return; + } + + QString out_filter = + tr("3DS Compressed ROM File (*.%1)") + .arg(QString::fromStdString(compress_info.recommended_compressed_extension)); + + QFileInfo fileinfo(filepath); + QString final_path = fileinfo.path() + QStringLiteral(DIR_SEP) + fileinfo.completeBaseName() + + QStringLiteral(".") + + QString::fromStdString(compress_info.recommended_compressed_extension); + + filepath = QFileDialog::getSaveFileName(this, tr("Save 3DS Compressed ROM File"), final_path, + out_filter); + if (filepath.isEmpty()) { + return; + } + std::string out_path = filepath.toStdString(); + + progress_bar->show(); + progress_bar->setMaximum(INT_MAX); + + (void)QtConcurrent::run([&, in_path, out_path, compress_info, frame_size] { + const auto progress = [&](std::size_t written, std::size_t total) { + emit UpdateProgress(written, total); + }; + bool success = FileUtil::CompressZ3DSFile(in_path, out_path, compress_info.underlying_magic, + frame_size, progress); + if (!success) { + FileUtil::Delete(out_path); + } + emit OnCompressFinished(true, success); + }); +} +void GMainWindow::OnDecompressFile() { + QString filepath = QFileDialog::getOpenFileName( + this, tr("Load 3DS Compressed ROM File"), UISettings::values.roms_path, + tr("3DS Compressed ROM Files (*.zcia *zcci *z3dsx *zcxi)") + QStringLiteral(";;") + + tr("All Files (*.*)")); + + if (filepath.isEmpty()) { + return; + } + std::string in_path = filepath.toStdString(); + + // Identify file type + Loader::AppLoader::CompressFileInfo compress_info{}; + compress_info.is_supported = false; + { + auto loader = Loader::GetLoader(in_path); + if (loader) { + compress_info = loader->GetCompressFileInfo(); + } else { + bool is_compressed = false; + if (Service::AM::CheckCIAToInstall(in_path, is_compressed, false) == + Service::AM::InstallStatus::Success) { + compress_info.is_supported = true; + compress_info.is_compressed = is_compressed; + compress_info.recommended_compressed_extension = "zcia"; + compress_info.recommended_uncompressed_extension = "cia"; + compress_info.underlying_magic = std::array({'C', 'I', 'A', '\0'}); + } + } + } + if (!compress_info.is_supported) { + QMessageBox::critical(this, tr("Error decompressing file"), + tr("The selected file is not a compatible compressed 3DS ROM format. " + "Make sure you have " + "chosen the right file.")); + return; + } + if (!compress_info.is_compressed) { + QMessageBox::warning(this, tr("Error decompressing file"), + tr("The selected file is already decompressed.")); + return; + } + + QString out_filter = + tr("3DS ROM File (*.%1)") + .arg(QString::fromStdString(compress_info.recommended_uncompressed_extension)); + + QFileInfo fileinfo(filepath); + QString final_path = fileinfo.path() + QStringLiteral(DIR_SEP) + fileinfo.completeBaseName() + + QStringLiteral(".") + + QString::fromStdString(compress_info.recommended_uncompressed_extension); + + filepath = QFileDialog::getSaveFileName(this, tr("Save 3DS ROM File"), final_path, out_filter); + if (filepath.isEmpty()) { + return; + } + std::string out_path = filepath.toStdString(); + + progress_bar->show(); + progress_bar->setMaximum(INT_MAX); + + (void)QtConcurrent::run([&, in_path, out_path, compress_info] { + const auto progress = [&](std::size_t written, std::size_t total) { + emit UpdateProgress(written, total); + }; + // TODO(PabloMK7): What should we do with the metadata? + bool success = FileUtil::DeCompressZ3DSFile(in_path, out_path, progress); + if (!success) { + FileUtil::Delete(out_path); + } + emit OnCompressFinished(false, success); + }); +} + #ifdef _WIN32 void GMainWindow::OnOpenFFmpeg() { auto filename = @@ -3514,8 +3692,8 @@ static bool IsSingleFileDropEvent(const QMimeData* mime) { return mime->hasUrls() && mime->urls().length() == 1; } -static const std::array AcceptedExtensions = {"cci", "cxi", "bin", "3dsx", - "app", "elf", "axf"}; +static const std::array AcceptedExtensions = { + "cci", "cxi", "bin", "3dsx", "app", "elf", "axf", "zcci", "zcxi", "z3dsx"}; static bool IsCorrectFileExtension(const QMimeData* mime) { const QString& filename = mime->urls().at(0).toLocalFile(); diff --git a/src/citra_qt/citra_qt.h b/src/citra_qt/citra_qt.h index c8d74a131..70e0493f3 100644 --- a/src/citra_qt/citra_qt.h +++ b/src/citra_qt/citra_qt.h @@ -141,6 +141,7 @@ signals: void UpdateProgress(std::size_t written, std::size_t total); void CIAInstallReport(Service::AM::InstallStatus status, QString filepath); + void CompressFinished(bool is_compress, bool success); void CIAInstallFinished(); // Signal that tells widgets to update icons to use the current theme void UpdateThemedIcons(); @@ -248,6 +249,7 @@ private slots: void OnMenuBootHomeMenu(u32 region); void OnUpdateProgress(std::size_t written, std::size_t total); void OnCIAInstallReport(Service::AM::InstallStatus status, QString filepath); + void OnCompressFinished(bool is_compress, bool success); void OnCIAInstallFinished(); void OnMenuRecentFile(); void OnConfigure(); @@ -281,6 +283,8 @@ private slots: void OnSaveMovie(); void OnCaptureScreenshot(); void OnDumpVideo(); + void OnCompressFile(); + void OnDecompressFile(); #ifdef _WIN32 void OnOpenFFmpeg(); #endif diff --git a/src/citra_qt/configuration/config.cpp b/src/citra_qt/configuration/config.cpp index 1ea045821..041085eec 100644 --- a/src/citra_qt/configuration/config.cpp +++ b/src/citra_qt/configuration/config.cpp @@ -475,6 +475,7 @@ void QtConfig::ReadDataStorageValues() { ReadBasicSetting(Settings::values.use_virtual_sd); ReadBasicSetting(Settings::values.use_custom_storage); + ReadBasicSetting(Settings::values.compress_cia_installs); const std::string nand_dir = ReadSetting(QStringLiteral("nand_directory"), QStringLiteral("")).toString().toStdString(); @@ -1045,6 +1046,7 @@ void QtConfig::SaveDataStorageValues() { WriteBasicSetting(Settings::values.use_virtual_sd); WriteBasicSetting(Settings::values.use_custom_storage); + WriteBasicSetting(Settings::values.compress_cia_installs); WriteSetting(QStringLiteral("nand_directory"), QString::fromStdString(FileUtil::GetUserPath(FileUtil::UserPath::NANDDir)), QStringLiteral("")); diff --git a/src/citra_qt/configuration/configure_storage.cpp b/src/citra_qt/configuration/configure_storage.cpp index 2346f06c2..a8cabd987 100644 --- a/src/citra_qt/configuration/configure_storage.cpp +++ b/src/citra_qt/configuration/configure_storage.cpp @@ -77,6 +77,7 @@ void ConfigureStorage::SetConfiguration() { ui->toggle_virtual_sd->setChecked(Settings::values.use_virtual_sd.GetValue()); ui->toggle_custom_storage->setChecked(Settings::values.use_custom_storage.GetValue()); + ui->toggle_compress_cia->setChecked(Settings::values.compress_cia_installs.GetValue()); ui->storage_group->setEnabled(!is_powered_on); } @@ -84,6 +85,7 @@ void ConfigureStorage::SetConfiguration() { void ConfigureStorage::ApplyConfiguration() { Settings::values.use_virtual_sd = ui->toggle_virtual_sd->isChecked(); Settings::values.use_custom_storage = ui->toggle_custom_storage->isChecked(); + Settings::values.compress_cia_installs = ui->toggle_compress_cia->isChecked(); if (!Settings::values.use_custom_storage) { FileUtil::UpdateUserPath(FileUtil::UserPath::NANDDir, diff --git a/src/citra_qt/configuration/configure_storage.ui b/src/citra_qt/configuration/configure_storage.ui index 75c5a8a3f..83d7ba123 100644 --- a/src/citra_qt/configuration/configure_storage.ui +++ b/src/citra_qt/configuration/configure_storage.ui @@ -179,6 +179,20 @@ + + + + + + Compress installed CIA contents + + + Enables compressing the contents of CIA files when they are installed to the emulated SD. + + + + + diff --git a/src/citra_qt/game_list.cpp b/src/citra_qt/game_list.cpp index cac036aaa..049f7579c 100644 --- a/src/citra_qt/game_list.cpp +++ b/src/citra_qt/game_list.cpp @@ -1039,8 +1039,10 @@ void GameList::LoadInterfaceLayout() { } const QStringList GameList::supported_file_extensions = { - QStringLiteral("3dsx"), QStringLiteral("elf"), QStringLiteral("axf"), - QStringLiteral("cci"), QStringLiteral("cxi"), QStringLiteral("app")}; + QStringLiteral("3dsx"), QStringLiteral("elf"), QStringLiteral("axf"), + QStringLiteral("cci"), QStringLiteral("cxi"), QStringLiteral("app"), + QStringLiteral("z3dsx"), QStringLiteral("zcci"), QStringLiteral("zcxi"), +}; void GameList::RefreshGameDirectory() { if (!UISettings::values.game_dirs.isEmpty() && current_worker != nullptr) { diff --git a/src/citra_qt/main.ui b/src/citra_qt/main.ui index 70f1915b4..acca5361d 100644 --- a/src/citra_qt/main.ui +++ b/src/citra_qt/main.ui @@ -208,6 +208,9 @@ + + + @@ -458,6 +461,16 @@ Dump Video + + + Compress ROM File... + + + + + Decompress ROM File... + + true diff --git a/src/citra_sdl/config.cpp b/src/citra_sdl/config.cpp index a84f746e5..da800cc30 100644 --- a/src/citra_sdl/config.cpp +++ b/src/citra_sdl/config.cpp @@ -213,6 +213,7 @@ void SdlConfig::ReadValues() { // Data Storage ReadSetting("Data Storage", Settings::values.use_virtual_sd); ReadSetting("Data Storage", Settings::values.use_custom_storage); + ReadSetting("Data Storage", Settings::values.compress_cia_installs); if (Settings::values.use_custom_storage) { FileUtil::UpdateUserPath(FileUtil::UserPath::NANDDir, diff --git a/src/common/file_util.cpp b/src/common/file_util.cpp index 847cdad55..c740e8416 100644 --- a/src/common/file_util.cpp +++ b/src/common/file_util.cpp @@ -1187,7 +1187,7 @@ bool IOFile::SeekImpl(s64 off, int origin) { return m_good; } -u64 IOFile::Tell() const { +u64 IOFile::TellImpl() const { if (IsOpen()) return ftello(m_file); @@ -1224,11 +1224,18 @@ static std::size_t pread(int fd, void* buf, std::size_t count, uint64_t offset) overlapped.OffsetHigh = static_cast(offset >> 32); overlapped.Offset = static_cast(offset & 0xFFFF'FFFFLL); + LARGE_INTEGER orig, dummy; + // TODO(PabloMK7): This is not fully async, windows being messy again... + // The file pos pointer will be undefined if ReadAt is used in multiple + // threads. Normally not problematic, but worth remembering. + SetFilePointerEx(file, {}, &orig, FILE_CURRENT); SetLastError(0); bool ret = ReadFile(file, buf, static_cast(count), &read_bytes, &overlapped); + DWORD last_error = GetLastError(); + SetFilePointerEx(file, orig, &dummy, FILE_BEGIN); - if (!ret && GetLastError() != ERROR_HANDLE_EOF) { - errno = GetLastError(); + if (!ret && last_error != ERROR_HANDLE_EOF) { + errno = last_error; return std::numeric_limits::max(); } return read_bytes; diff --git a/src/common/file_util.h b/src/common/file_util.h index ef36ce02f..57a1d67e1 100644 --- a/src/common/file_util.h +++ b/src/common/file_util.h @@ -301,7 +301,7 @@ public: void Swap(IOFile& other) noexcept; - bool Close(); + virtual bool Close(); template std::size_t ReadArray(T* data, std::size_t length) { @@ -412,15 +412,15 @@ public: return WriteImpl(data.data(), data.size(), sizeof(T)); } - [[nodiscard]] bool IsOpen() const { + [[nodiscard]] virtual bool IsOpen() const { return nullptr != m_file; } // m_good is set to false when a read, write or other function fails - [[nodiscard]] bool IsGood() const { + [[nodiscard]] virtual bool IsGood() const { return m_good; } - [[nodiscard]] int GetFd() const { + [[nodiscard]] virtual int GetFd() const { #ifdef ANDROID return m_fd; #else @@ -436,13 +436,15 @@ public: bool Seek(s64 off, int origin) { return SeekImpl(off, origin); } - [[nodiscard]] u64 Tell() const; - [[nodiscard]] u64 GetSize() const; - bool Resize(u64 size); - bool Flush(); + u64 Tell() const { + return TellImpl(); + } + virtual u64 GetSize() const; + virtual bool Resize(u64 size); + virtual bool Flush(); // clear error state - void Clear() { + virtual void Clear() { m_good = true; std::clearerr(m_file); } @@ -451,29 +453,35 @@ public: return false; } - const std::string& Filename() const { + virtual bool IsCompressed() { + return false; + } + + virtual const std::string& Filename() const { return filename; } protected: friend struct CryptoIOFileImpl; + + virtual bool Open(); + virtual std::size_t ReadImpl(void* data, std::size_t length, std::size_t data_size); virtual std::size_t ReadAtImpl(void* data, std::size_t length, std::size_t data_size, std::size_t offset); virtual std::size_t WriteImpl(const void* data, std::size_t length, std::size_t data_size); virtual bool SeekImpl(s64 off, int origin); + virtual u64 TellImpl() const; private: - bool Open(); - std::FILE* m_file = nullptr; int m_fd = -1; bool m_good = true; std::string filename; std::string openmode; - u32 flags; + u32 flags = 0; template void serialize(Archive& ar, const unsigned int) { diff --git a/src/common/settings.h b/src/common/settings.h index c8e76179c..552f89dbd 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -468,6 +468,7 @@ struct Values { // Data Storage Setting use_virtual_sd{true, "use_virtual_sd"}; Setting use_custom_storage{false, "use_custom_storage"}; + Setting compress_cia_installs{false, "compress_cia_installs"}; // System SwitchableSetting region_value{REGION_VALUE_AUTO_SELECT, "region_value"}; diff --git a/src/common/zstd_compression.cpp b/src/common/zstd_compression.cpp index 792a7f029..e3d5dac5f 100644 --- a/src/common/zstd_compression.cpp +++ b/src/common/zstd_compression.cpp @@ -1,15 +1,30 @@ +// Copyright Citra Emulator Project / Azahar Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + // Copyright 2019 yuzu Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include "common/alignment.h" +#include "common/archives.h" +#include "common/assert.h" #include "common/logging/log.h" +#include "common/scm_rev.h" #include "common/zstd_compression.h" namespace Common::Compression { - std::vector CompressDataZSTD(std::span source, s32 compression_level) { compression_level = std::clamp(compression_level, ZSTD_minCLevel(), ZSTD_maxCLevel()); const std::size_t max_compressed_size = ZSTD_compressBound(source.size()); @@ -71,3 +86,676 @@ std::vector DecompressDataZSTD(std::span compressed) { } } // namespace Common::Compression + +namespace FileUtil { + +template +void ReadFromIStream(std::istringstream& s, T* out, size_t out_size) { + s.read(reinterpret_cast(out), out_size); +} + +template +void WriteToOStream(std::ostringstream& s, const T* out, size_t out_size) { + s.write(reinterpret_cast(out), out_size); +} + +Z3DSMetadata::Z3DSMetadata(const std::span& source_data) { + if (source_data.empty()) + return; + std::string buf(reinterpret_cast(source_data.data()), source_data.size()); + std::istringstream in(buf, std::ios::binary); + + u8 version; + ReadFromIStream(in, &version, sizeof(version)); + + if (version != METADATA_VERSION) { + return; + } + + while (!in.eof()) { + Item item; + ReadFromIStream(in, &item, sizeof(Item)); + // Only binary type supported for now + if (item.type != Item::TYPE_BINARY) { + break; + } + std::string name(item.name_len, '\0'); + std::vector data(item.data_len); + ReadFromIStream(in, name.data(), name.size()); + ReadFromIStream(in, data.data(), data.size()); + items.insert({std::move(name), std::move(data)}); + } +} + +std::vector Z3DSMetadata::AsBinary() { + if (items.empty()) + return {}; + std::ostringstream out; + u8 version = METADATA_VERSION; + WriteToOStream(out, &version, sizeof(u8)); + + for (const auto& it : items) { + Item item{ + .type = Item::TYPE_BINARY, + .name_len = static_cast(std::min(0xFF, it.first.size())), + .data_len = static_cast(std::min(0xFFFF, it.second.size())), + }; + WriteToOStream(out, &item, sizeof(item)); + WriteToOStream(out, it.first.data(), item.name_len); + WriteToOStream(out, it.second.data(), item.data_len); + } + + std::string out_str = out.str(); + return std::vector(out_str.begin(), out_str.end()); +} + +struct Z3DSWriteIOFile::Z3DSWriteIOFileImpl { + Z3DSWriteIOFileImpl() {} + Z3DSWriteIOFileImpl(size_t frame_size) { + zstd_frame_size = frame_size; + cstream = ZSTD_seekable_createCStream(); + size_t init_result = ZSTD_seekable_initCStream(cstream, ZSTD_CLEVEL_DEFAULT, 0, + static_cast(frame_size)); + if (ZSTD_isError(init_result)) { + LOG_ERROR(Common_Filesystem, "ZSTD_seekable_initCStream() error : {}", + ZSTD_getErrorName(init_result)); + } + + write_header.magic = Z3DSFileHeader::EXPECTED_MAGIC; + write_header.version = Z3DSFileHeader::EXPECTED_VERSION; + write_header.header_size = sizeof(Z3DSFileHeader); + next_input_size_hint = ZSTD_CStreamInSize(); + } + + bool WriteHeader(IOFile* file) { + file->Seek(0, SEEK_SET); + return file->WriteBytes(&write_header, sizeof(write_header)) == sizeof(write_header); + } + + bool WriteMetadata(IOFile* file, const std::span& data) { + std::array tmp_data{}; + size_t total_size = Common::AlignUp(data.size(), 0x10); + write_header.metadata_size = static_cast(total_size); + size_t res_written = file->WriteBytes(data.data(), data.size()); + res_written += file->WriteBytes(tmp_data.data(), total_size - data.size()); + return res_written == total_size; + } + + size_t Write(IOFile* file, const void* data, std::size_t length) { + size_t ret = length; + + const size_t out_size = ZSTD_CStreamOutSize(); + const size_t in_size = ZSTD_CStreamInSize(); + + if (write_buffer.size() < out_size) { + write_buffer.resize(out_size); + } + + ZSTD_inBuffer input = {data, length, 0}; + while (input.pos < input.size) { + ZSTD_outBuffer output = {write_buffer.data(), write_buffer.size(), 0}; + next_input_size_hint = ZSTD_seekable_compressStream(cstream, &output, &input); + if (ZSTD_isError(next_input_size_hint)) { + LOG_ERROR(Common_Filesystem, "ZSTD_seekable_compressStream() error : {}", + ZSTD_getErrorName(next_input_size_hint)); + ret = 0; + next_input_size_hint = ZSTD_CStreamInSize(); + break; + } + if (next_input_size_hint > in_size) { + next_input_size_hint = in_size; + } + if (file->WriteBytes(static_cast(output.dst), output.pos) != output.pos) { + ret = 0; + break; + } + written_compressed += output.pos; + } + return ret; + } + + bool Close(IOFile* file, size_t written_uncompressed) { + const size_t out_size = ZSTD_CStreamOutSize(); + + if (write_buffer.size() < out_size) { + write_buffer.resize(out_size); + } + + size_t remaining; + do { + ZSTD_outBuffer output = {write_buffer.data(), write_buffer.size(), 0}; + remaining = ZSTD_seekable_endStream(cstream, &output); /* close stream */ + if (ZSTD_isError(remaining)) { + LOG_ERROR(Common_Filesystem, "ZSTD_seekable_endStream() error : {}", + ZSTD_getErrorName(remaining)); + return false; + } + + if (file->WriteBytes(static_cast(output.dst), output.pos) != output.pos) { + return false; + } + written_compressed += output.pos; + } while (remaining); + + write_header.compressed_size = written_compressed; + write_header.uncompressed_size = written_uncompressed; + + ZSTD_seekable_freeCStream(cstream); + + return WriteHeader(file); + } + + std::vector write_buffer; + size_t next_input_size_hint = 0; + size_t zstd_frame_size = 0; + u64 written_compressed = 0; + + ZSTD_seekable_CStream* cstream{}; + Z3DSFileHeader write_header{}; +}; + +Z3DSWriteIOFile::Z3DSWriteIOFile() + : IOFile(), file{std::make_unique()}, impl{std::make_unique()} {} + +Z3DSWriteIOFile::Z3DSWriteIOFile(std::unique_ptr&& underlying_file, + const std::array& underlying_magic, size_t frame_size) + : IOFile(), file{std::move(underlying_file)}, + impl{std::make_unique(frame_size)} { + ASSERT_MSG(!file->IsCompressed(), "Underlying file is already compressed!"); + impl->write_header.underlying_magic = underlying_magic; + impl->WriteHeader(file.get()); + + Metadata().Add("compressor", std::string("Azahar ") + Common::g_build_fullname); + + std::time_t tt = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + std::tm tm{}; +#if defined(_WIN32) + gmtime_s(&tm, &tt); +#else + gmtime_r(&tt, &tm); +#endif + char buf[0x20]; + std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &tm); + Metadata().Add("date", buf); + + Metadata().Add( + "maxframesize", + std::to_string(frame_size ? frame_size : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE)); +} + +Z3DSWriteIOFile::~Z3DSWriteIOFile() { + this->Close(); +} + +bool Z3DSWriteIOFile::Close() { + impl->Close(file.get(), written_uncompressed); + return file->Close(); +} + +u64 Z3DSWriteIOFile::GetSize() const { + return written_uncompressed; +} + +bool Z3DSWriteIOFile::Resize(u64 size) { + // Stubbed + UNIMPLEMENTED(); + return false; +} + +bool Z3DSWriteIOFile::Flush() { + return file->Flush(); +} + +void Z3DSWriteIOFile::Clear() { + return file->Clear(); +} + +bool Z3DSWriteIOFile::IsCrypto() { + return file->IsCrypto(); +} + +const std::string& Z3DSWriteIOFile::Filename() const { + return file->Filename(); +} + +bool Z3DSWriteIOFile::IsOpen() const { + return file->IsOpen(); +} + +bool Z3DSWriteIOFile::IsGood() const { + return file->IsGood(); +} + +int Z3DSWriteIOFile::GetFd() const { + return file->GetFd(); +} + +bool Z3DSWriteIOFile::Open() { + if (is_serializing) { + return true; + } + // Stubbed + UNIMPLEMENTED(); + return false; +} + +std::size_t Z3DSWriteIOFile::ReadImpl(void* data, std::size_t length, std::size_t data_size) { + // Stubbed + UNIMPLEMENTED(); + return 0; +} + +std::size_t Z3DSWriteIOFile::ReadAtImpl(void* data, std::size_t length, std::size_t data_size, + std::size_t offset) { + // Stubbed + UNIMPLEMENTED(); + return 0; +} + +std::size_t Z3DSWriteIOFile::WriteImpl(const void* data, std::size_t length, + std::size_t data_size) { + if (!metadata_written) { + metadata_written = true; + auto metadata_binary = metadata.AsBinary(); + if (!metadata_binary.empty()) { + impl->WriteMetadata(file.get(), metadata_binary); + } + } + + size_t ret = impl->Write(file.get(), data, length * data_size); + written_uncompressed += ret; + return ret; +} + +bool Z3DSWriteIOFile::SeekImpl(s64 off, int origin) { + if (is_serializing) { + return true; + } + // Stubbed + UNIMPLEMENTED(); + return false; +} + +u64 Z3DSWriteIOFile::TellImpl() const { + return written_uncompressed; +} + +size_t Z3DSWriteIOFile::GetNextWriteHint() { + return impl->next_input_size_hint; +} + +template +void Z3DSWriteIOFile::serialize(Archive& ar, const unsigned int) { + is_serializing = true; + ar& boost::serialization::base_object(*this); + + ar & file; + ar & written_uncompressed; + ar & metadata_written; + ar & metadata; + + Z3DSFileHeader hd; + size_t frame_size; + u64 written_compressed; + if (Archive::is_loading::value) { + ar & hd; + ar & frame_size; + ar & written_compressed; + impl = std::make_unique(frame_size); + impl->write_header = hd; + impl->written_compressed = written_compressed; + } else { + ar & impl->write_header; + ar & impl->zstd_frame_size; + ar & impl->written_compressed; + } + is_serializing = false; +} + +struct Z3DSReadIOFile::Z3DSReadIOFileImpl { + Z3DSReadIOFileImpl() {} + Z3DSReadIOFileImpl(IOFile* file, bool load_metadata = true) { + curr_file = file; + m_good = file->ReadAtBytes(&header, sizeof(header), 0) == sizeof(header); + m_good &= header.magic == Z3DSFileHeader::EXPECTED_MAGIC && + header.version == Z3DSFileHeader::EXPECTED_VERSION; + + if (!m_good) { + return; + } + + if (header.metadata_size && load_metadata) { + std::vector buff(header.metadata_size); + file->ReadAtBytes(buff.data(), buff.size(), header.header_size); + metadata = Z3DSMetadata(buff); + } + + seekable = ZSTD_seekable_create(); + + ZSTD_seekable_customFile custom_file{ + .opaque = this, + .read = [](void* opaque, void* buffer, size_t n) -> int { + return reinterpret_cast(opaque)->OnZSTDRead(buffer, n); + }, + .seek = [](void* opaque, long long offset, int origin) -> int { + return reinterpret_cast(opaque)->OnZSTDSeek(offset, origin); + }, + }; + size_t init_result = ZSTD_seekable_initAdvanced(seekable, custom_file); + if (ZSTD_isError(init_result)) { + LOG_ERROR(Common_Filesystem, "ZSTD_seekable_initCStream() error : {}", + ZSTD_getErrorName(init_result)); + m_good = false; + } + } + + int OnZSTDRead(void* buffer, size_t n) { + const size_t read = curr_file->ReadBytes(reinterpret_cast(buffer), n); + if (read != n) { + return -1; + } + return 0; + } + + int OnZSTDSeek(long long offset, int origin) { + if (origin == SEEK_SET) { + offset += static_cast(header.metadata_size) + header.header_size; + } + const bool res = curr_file->Seek(offset, origin); + return res ? 0 : -1; + } + + size_t Read(void* data, std::size_t length) { + if (!m_good) + return 0; + size_t result = ZSTD_seekable_decompress(seekable, data, length, uncompressed_pos); + if (ZSTD_isError(result)) { + LOG_ERROR(Common_Filesystem, "ZSTD_seekable_decompress() error : {}", + ZSTD_getErrorName(result)); + return 0; + } + uncompressed_pos += result; + return result; + } + + size_t ReadAt(void* data, std::size_t length, size_t pos) { + if (!m_good) + return 0; + // ReadAt should be thread safe, but seekable compression is not, + // so we are forced to use a lock. + std::scoped_lock lock(read_mutex); + + size_t result = ZSTD_seekable_decompress(seekable, data, length, pos); + if (ZSTD_isError(result)) { + LOG_ERROR(Common_Filesystem, "ZSTD_seekable_decompress() error : {}", + ZSTD_getErrorName(result)); + return 0; + } + return result; + } + + bool Seek(s64 off, int origin) { + s64 start = 0; + switch (origin) { + case SEEK_SET: + start = 0; + break; + case SEEK_CUR: + start = static_cast(uncompressed_pos); + break; + case SEEK_END: + start = static_cast(header.uncompressed_size); + break; + default: + return false; + } + s64 new_pos = start + off; + if (new_pos < 0) + return false; + uncompressed_pos = static_cast(new_pos); + return true; + } + + void Close() { + ZSTD_seekable_free(seekable); + } + + Z3DSFileHeader header{}; + ZSTD_seekable* seekable = nullptr; + bool m_good = true; + IOFile* curr_file = nullptr; + std::mutex read_mutex; + u64 uncompressed_pos = 0; + Z3DSMetadata metadata; +}; + +std::optional Z3DSReadIOFile::GetUnderlyingFileMagic(IOFile* underlying_file) { + Z3DSFileHeader header{}; + underlying_file->ReadAtBytes(&header, sizeof(header), 0); + if (header.magic != Z3DSFileHeader::EXPECTED_MAGIC || + header.version != Z3DSFileHeader::EXPECTED_VERSION) { + return std::nullopt; + } + + return MakeMagic(header.underlying_magic[0], header.underlying_magic[1], + header.underlying_magic[2], header.underlying_magic[3]); +} + +Z3DSReadIOFile::Z3DSReadIOFile() + : IOFile(), file{std::make_unique()}, impl{std::make_unique()} {} + +Z3DSReadIOFile::Z3DSReadIOFile(std::unique_ptr&& underlying_file) + : IOFile(), file{std::move(underlying_file)}, + impl{std::make_unique(file.get())} { + ASSERT_MSG(!file->IsCompressed(), "Underlying file is already compressed!"); +} + +Z3DSReadIOFile::~Z3DSReadIOFile() { + this->Close(); +} + +bool Z3DSReadIOFile::Close() { + impl->Close(); + return file->Close(); +} + +u64 Z3DSReadIOFile::GetSize() const { + return impl->header.uncompressed_size; +} + +bool Z3DSReadIOFile::Resize(u64 size) { + // Stubbed + UNIMPLEMENTED(); + return false; +} + +bool Z3DSReadIOFile::Flush() { + return file->Flush(); +} + +void Z3DSReadIOFile::Clear() { + return file->Clear(); +} + +bool Z3DSReadIOFile::IsCrypto() { + return file->IsCrypto(); +} + +const std::string& Z3DSReadIOFile::Filename() const { + return file->Filename(); +} + +bool Z3DSReadIOFile::IsOpen() const { + return file->IsOpen(); +} + +bool Z3DSReadIOFile::IsGood() const { + return file->IsGood() && impl->m_good; +} + +int Z3DSReadIOFile::GetFd() const { + return file->GetFd(); +} + +bool Z3DSReadIOFile::Open() { + if (is_serializing) { + return true; + } + // Stubbed + UNIMPLEMENTED(); + return false; +} + +std::size_t Z3DSReadIOFile::ReadImpl(void* data, std::size_t length, std::size_t data_size) { + return impl->Read(data, length * data_size); +} + +std::size_t Z3DSReadIOFile::ReadAtImpl(void* data, std::size_t length, std::size_t data_size, + std::size_t offset) { + return impl->ReadAt(data, length * data_size, offset); +} + +std::size_t Z3DSReadIOFile::WriteImpl(const void* data, std::size_t length, std::size_t data_size) { + // Stubbed + UNIMPLEMENTED(); + return 0; +} + +bool Z3DSReadIOFile::SeekImpl(s64 off, int origin) { + if (is_serializing) { + return true; + } + return impl->Seek(off, origin); +} + +u64 Z3DSReadIOFile::TellImpl() const { + return impl->uncompressed_pos; +} + +std::array Z3DSReadIOFile::GetFileMagic() { + return impl->header.underlying_magic; +} + +const Z3DSMetadata& Z3DSReadIOFile::Metadata() { + return impl->metadata; +} + +template +void Z3DSReadIOFile::serialize(Archive& ar, const unsigned int) { + is_serializing = true; + ar& boost::serialization::base_object(*this); + + ar & file; + + if (Archive::is_loading::value) { + impl = std::make_unique(file.get(), false); + } + ar & impl->uncompressed_pos; + ar & impl->metadata; + is_serializing = false; +} + +bool CompressZ3DSFile(const std::string& src_file_name, const std::string& dst_file_name, + const std::array& underlying_magic, size_t frame_size, + std::function&& update_callback) { + + IOFile in_file(src_file_name, "rb"); + if (!in_file.IsOpen()) { + LOG_ERROR(Common_Filesystem, "Failed to open source file: {}", src_file_name); + return false; + } + + std::unique_ptr out_file = std::make_unique(dst_file_name, "wb"); + if (!out_file->IsOpen()) { + LOG_ERROR(Common_Filesystem, "Failed to open destination file: {}", dst_file_name); + return false; + } + + if (Z3DSReadIOFile::GetUnderlyingFileMagic(&in_file) != std::nullopt) { + LOG_ERROR(Common_Filesystem, "Source file is already compressed, nothing to do: {}", + src_file_name); + return false; + } + + Z3DSWriteIOFile out_compress_file(std::move(out_file), underlying_magic, frame_size); + + size_t next_chunk = out_compress_file.GetNextWriteHint(); + std::vector buffer(next_chunk); + size_t in_size = in_file.GetSize(); + size_t written = 0; + + while (written != in_size) { + size_t to_read = ((in_size - written) > next_chunk) ? next_chunk : (in_size - written); + if (buffer.size() < to_read) { + buffer.resize(to_read); + } + if (in_file.ReadBytes(buffer.data(), to_read) != to_read) { + LOG_ERROR(Common_Filesystem, "Failed to read from source file"); + return false; + } + if (out_compress_file.WriteBytes(buffer.data(), to_read) != to_read) { + LOG_ERROR(Common_Filesystem, "Failed to write to destination file"); + } + written += to_read; + next_chunk = out_compress_file.GetNextWriteHint(); + if (update_callback) { + update_callback(written, in_size); + } + } + LOG_INFO(Common_Filesystem, "File {} compressed successfully to {}", src_file_name, + dst_file_name); + return true; +} + +bool DeCompressZ3DSFile(const std::string& src_file_name, const std::string& dst_file_name, + std::function&& update_callback) { + + std::unique_ptr in_file = std::make_unique(src_file_name, "rb"); + if (!in_file->IsOpen()) { + LOG_ERROR(Common_Filesystem, "Failed to open source file: {}", src_file_name); + return false; + } + + IOFile out_file(dst_file_name, "wb"); + if (!out_file.IsOpen()) { + LOG_ERROR(Common_Filesystem, "Failed to open destination file: {}", dst_file_name); + return false; + } + + if (Z3DSReadIOFile::GetUnderlyingFileMagic(in_file.get()) == std::nullopt) { + LOG_ERROR(Common_Filesystem, + "Source file is not compressed or is invalid, nothing to do: {}", src_file_name); + return false; + } + + Z3DSReadIOFile in_compress_file(std::move(in_file)); + size_t next_chunk = 64 * 1024 * 1024; + std::vector buffer(next_chunk); + size_t in_size = in_compress_file.GetSize(); + size_t written = 0; + + while (written != in_size) { + size_t to_read = (in_size - written) > next_chunk ? next_chunk : (in_size - written); + if (buffer.size() < to_read) { + buffer.resize(to_read); + } + if (in_compress_file.ReadBytes(buffer.data(), to_read) != to_read) { + LOG_ERROR(Common_Filesystem, "Failed to read from source file"); + return false; + } + if (out_file.WriteBytes(buffer.data(), to_read) != to_read) { + LOG_ERROR(Common_Filesystem, "Failed to write to destination file"); + } + written += to_read; + if (update_callback) { + update_callback(written, in_size); + } + } + LOG_INFO(Common_Filesystem, "File {} decompressed successfully to {}", src_file_name, + dst_file_name); + return true; +} +} // namespace FileUtil + +SERIALIZE_EXPORT_IMPL(FileUtil::Z3DSReadIOFile); +SERIALIZE_EXPORT_IMPL(FileUtil::Z3DSWriteIOFile); diff --git a/src/common/zstd_compression.h b/src/common/zstd_compression.h index 467b5d771..03b9eb61e 100644 --- a/src/common/zstd_compression.h +++ b/src/common/zstd_compression.h @@ -1,3 +1,7 @@ +// Copyright Citra Emulator Project / Azahar Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + // Copyright 2019 yuzu Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. @@ -5,9 +9,14 @@ #pragma once #include +#include #include +#include +#include +#include "common/archives.h" #include "common/common_types.h" +#include "common/file_util.h" namespace Common::Compression { @@ -41,3 +50,220 @@ namespace Common::Compression { [[nodiscard]] std::vector DecompressDataZSTD(std::span compressed); } // namespace Common::Compression + +namespace FileUtil { + +struct Z3DSFileHeader { + static constexpr std::array EXPECTED_MAGIC = {'Z', '3', 'D', 'S'}; + static constexpr u16 EXPECTED_VERSION = 1; + + std::array magic = EXPECTED_MAGIC; + std::array underlying_magic{}; + u16 version = EXPECTED_VERSION; + u16 header_size = 0; + u32 metadata_size = 0; + u64 compressed_size = 0; + u64 uncompressed_size = 0; + + template + void serialize(Archive& ar, const unsigned int) { + ar & magic; + ar & underlying_magic; + ar & version; + ar & header_size; + ar & metadata_size; + ar & compressed_size; + ar & uncompressed_size; + } +}; +static_assert(sizeof(Z3DSFileHeader) == 0x20, "Invalid Z3DSFileHeader size"); + +class Z3DSMetadata { +public: + static constexpr u8 METADATA_VERSION = 1; + Z3DSMetadata() {} + + Z3DSMetadata(const std::span& source_data); + + void Add(const std::string& name, const std::span& data) { + items.insert({name, std::vector(data.begin(), data.end())}); + } + + void Add(const std::string& name, const std::string& data) { + items.insert({name, std::vector(data.begin(), data.end())}); + } + + std::optional> Get(const std::string& name) const { + auto it = items.find(name); + if (it == items.end()) { + return std::nullopt; + } + return it->second; + } + + std::vector AsBinary(); + +private: + struct Item { + enum Type : u8 { + TYPE_END = 0, + TYPE_BINARY = 1, + }; + Type type{}; + u8 name_len{}; + u16 data_len{}; + }; + static_assert(sizeof(Item) == 4); + + std::unordered_map> items; + + template + void serialize(Archive& ar, const unsigned int) { + ar & items; + } + friend class boost::serialization::access; +}; + +class Z3DSWriteIOFile : public IOFile { +public: + static constexpr size_t DEFAULT_FRAME_SIZE = 256 * 1024; // 256KiB + static constexpr size_t MAX_FRAME_SIZE = 0; // Let the lib decide, usually 1GiB + + Z3DSWriteIOFile(); + + Z3DSWriteIOFile(std::unique_ptr&& underlying_file, + const std::array& underlying_magic, size_t frame_size); + + ~Z3DSWriteIOFile(); + + bool Close() override; + + u64 GetSize() const override; + + bool Resize(u64 size) override; + + bool Flush() override; + + void Clear() override; + + bool IsCrypto() override; + + bool IsCompressed() override { + return true; + } + + const std::string& Filename() const override; + + bool IsOpen() const override; + + bool IsGood() const override; + + int GetFd() const override; + + Z3DSMetadata& Metadata() { + return metadata; + } + + size_t GetNextWriteHint(); + +private: + struct Z3DSWriteIOFileImpl; + bool Open() override; + + std::size_t ReadImpl(void* data, std::size_t length, std::size_t data_size) override; + std::size_t ReadAtImpl(void* data, std::size_t length, std::size_t data_size, + std::size_t offset) override; + std::size_t WriteImpl(const void* data, std::size_t length, std::size_t data_size) override; + + bool SeekImpl(s64 off, int origin) override; + u64 TellImpl() const override; + + std::unique_ptr file; + std::unique_ptr impl; + u64 written_uncompressed = 0; + bool metadata_written = false; + Z3DSMetadata metadata; + + template + void serialize(Archive& ar, const unsigned int); + friend class boost::serialization::access; + bool is_serializing = false; +}; + +class Z3DSReadIOFile : public IOFile { +public: + static std::optional GetUnderlyingFileMagic(IOFile* underlying_file); + + Z3DSReadIOFile(); + + Z3DSReadIOFile(std::unique_ptr&& underlying_file); + + ~Z3DSReadIOFile(); + + bool Close() override; + + u64 GetSize() const override; + + bool Resize(u64 size) override; + + bool Flush() override; + + void Clear() override; + + bool IsCrypto() override; + + bool IsCompressed() override { + return true; + } + + const std::string& Filename() const override; + + bool IsOpen() const override; + + bool IsGood() const override; + + int GetFd() const override; + + std::array GetFileMagic(); + + const Z3DSMetadata& Metadata(); + +private: + struct Z3DSReadIOFileImpl; + + static constexpr u32 MakeMagic(char a, char b, char c, char d) { + return a | b << 8 | c << 16 | d << 24; + } + + bool Open() override; + + std::size_t ReadImpl(void* data, std::size_t length, std::size_t data_size) override; + std::size_t ReadAtImpl(void* data, std::size_t length, std::size_t data_size, + std::size_t offset) override; + std::size_t WriteImpl(const void* data, std::size_t length, std::size_t data_size) override; + + bool SeekImpl(s64 off, int origin) override; + u64 TellImpl() const override; + + std::unique_ptr file; + std::unique_ptr impl; + + template + void serialize(Archive& ar, const unsigned int); + friend class boost::serialization::access; + bool is_serializing = false; +}; + +using ProgressCallback = void(std::size_t, std::size_t); + +bool CompressZ3DSFile(const std::string& src_file, const std::string& dst_file, + const std::array& underlying_magic, size_t frame_size, + std::function&& update_callback = nullptr); + +bool DeCompressZ3DSFile(const std::string& src_file, const std::string& dst_file, + std::function&& update_callback = nullptr); + +} // namespace FileUtil + +BOOST_CLASS_EXPORT_KEY(FileUtil::Z3DSWriteIOFile) +BOOST_CLASS_EXPORT_KEY(FileUtil::Z3DSReadIOFile) \ No newline at end of file diff --git a/src/core/file_sys/cia_container.cpp b/src/core/file_sys/cia_container.cpp index 957c541b3..2c9f39853 100644 --- a/src/core/file_sys/cia_container.cpp +++ b/src/core/file_sys/cia_container.cpp @@ -59,14 +59,13 @@ Loader::ResultStatus CIAContainer::Load(const FileBackend& backend) { return Loader::ResultStatus::Success; } -Loader::ResultStatus CIAContainer::Load(const std::string& filepath) { - FileUtil::IOFile file(filepath, "rb"); - if (!file.IsOpen()) +Loader::ResultStatus CIAContainer::Load(FileUtil::IOFile* file) { + if (!file->IsOpen()) return Loader::ResultStatus::Error; // Load CIA Header std::vector header_data(sizeof(CIAHeader)); - if (file.ReadBytes(header_data.data(), sizeof(CIAHeader)) != sizeof(CIAHeader)) + if (file->ReadBytes(header_data.data(), sizeof(CIAHeader)) != sizeof(CIAHeader)) return Loader::ResultStatus::Error; Loader::ResultStatus result = LoadHeader(header_data); @@ -75,8 +74,8 @@ Loader::ResultStatus CIAContainer::Load(const std::string& filepath) { // Load Ticket std::vector ticket_data(cia_header.tik_size); - file.Seek(GetTicketOffset(), SEEK_SET); - if (file.ReadBytes(ticket_data.data(), cia_header.tik_size) != cia_header.tik_size) + file->Seek(GetTicketOffset(), SEEK_SET); + if (file->ReadBytes(ticket_data.data(), cia_header.tik_size) != cia_header.tik_size) return Loader::ResultStatus::Error; result = LoadTicket(ticket_data); @@ -85,8 +84,8 @@ Loader::ResultStatus CIAContainer::Load(const std::string& filepath) { // Load Title Metadata std::vector tmd_data(cia_header.tmd_size); - file.Seek(GetTitleMetadataOffset(), SEEK_SET); - if (file.ReadBytes(tmd_data.data(), cia_header.tmd_size) != cia_header.tmd_size) + file->Seek(GetTitleMetadataOffset(), SEEK_SET); + if (file->ReadBytes(tmd_data.data(), cia_header.tmd_size) != cia_header.tmd_size) return Loader::ResultStatus::Error; result = LoadTitleMetadata(tmd_data); @@ -96,8 +95,8 @@ Loader::ResultStatus CIAContainer::Load(const std::string& filepath) { // Load CIA Metadata if (cia_header.meta_size) { std::vector meta_data(sizeof(Metadata)); - file.Seek(GetMetadataOffset(), SEEK_SET); - if (file.ReadBytes(meta_data.data(), sizeof(Metadata)) != sizeof(Metadata)) + file->Seek(GetMetadataOffset(), SEEK_SET); + if (file->ReadBytes(meta_data.data(), sizeof(Metadata)) != sizeof(Metadata)) return Loader::ResultStatus::Error; result = LoadMetadata(meta_data); diff --git a/src/core/file_sys/cia_container.h b/src/core/file_sys/cia_container.h index e562dc9be..a2d210a77 100644 --- a/src/core/file_sys/cia_container.h +++ b/src/core/file_sys/cia_container.h @@ -9,6 +9,7 @@ #include #include #include "common/common_types.h" +#include "common/file_util.h" #include "common/swap.h" #include "core/file_sys/ticket.h" #include "core/file_sys/title_metadata.h" @@ -62,7 +63,7 @@ class CIAContainer { public: // Load whole CIAs outright Loader::ResultStatus Load(const FileBackend& backend); - Loader::ResultStatus Load(const std::string& filepath); + Loader::ResultStatus Load(FileUtil::IOFile* file); Loader::ResultStatus Load(std::span header_data); // Load parts of CIAs (for CIAs streamed in) diff --git a/src/core/file_sys/ncch_container.cpp b/src/core/file_sys/ncch_container.cpp index a3f97a4f1..29232d4be 100644 --- a/src/core/file_sys/ncch_container.cpp +++ b/src/core/file_sys/ncch_container.cpp @@ -10,6 +10,7 @@ #include #include "common/common_types.h" #include "common/logging/log.h" +#include "common/zstd_compression.h" #include "core/core.h" #include "core/file_sys/layered_fs.h" #include "core/file_sys/ncch_container.h" @@ -137,6 +138,15 @@ Loader::ResultStatus NCCHContainer::LoadHeader() { return Loader::ResultStatus::Success; } + if (!file->IsOpen()) { + return Loader::ResultStatus::Error; + } + + if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt) { + // The file is compressed + file = std::make_unique(std::move(file)); + } + for (int i = 0; i < 2; i++) { if (!file->IsOpen()) { return Loader::ResultStatus::Error; @@ -151,6 +161,7 @@ Loader::ResultStatus NCCHContainer::LoadHeader() { // Skip NCSD header and load first NCCH (NCSD is just a container of NCCH files)... if (Loader::MakeMagic('N', 'C', 'S', 'D') == ncch_header.magic) { + is_ncsd = true; NCSD_Header ncsd_header; file->Seek(ncch_offset, SEEK_SET); file->ReadBytes(&ncsd_header, sizeof(NCSD_Header)); @@ -166,9 +177,12 @@ Loader::ResultStatus NCCHContainer::LoadHeader() { if (Loader::MakeMagic('N', 'C', 'C', 'H') != ncch_header.magic) { // We may be loading a crypto file, try again if (i == 0) { - file.reset(); file = HW::UniqueData::OpenUniqueCryptoFile( filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH); + if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt) { + // The file is compressed + file = std::make_unique(std::move(file)); + } } else { return Loader::ResultStatus::ErrorInvalidFormat; } @@ -179,6 +193,11 @@ Loader::ResultStatus NCCHContainer::LoadHeader() { LOG_DEBUG(Service_FS, "NCCH file has console unique crypto"); } + if (!ncch_header.no_crypto) { + // Encrypted NCCH are not supported + return Loader::ResultStatus::ErrorEncrypted; + } + has_header = true; return Loader::ResultStatus::Success; } @@ -190,9 +209,18 @@ Loader::ResultStatus NCCHContainer::Load() { int block_size = kBlockSize; if (file->IsOpen()) { - size_t file_size; + if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt) { + // The file is compressed + file = std::make_unique(std::move(file)); + } + + size_t file_size; for (int i = 0; i < 2; i++) { + if (!file->IsOpen()) { + return Loader::ResultStatus::Error; + } + file_size = file->GetSize(); // Reset read pointer in case this file has been read before. @@ -203,6 +231,7 @@ Loader::ResultStatus NCCHContainer::Load() { // Skip NCSD header and load first NCCH (NCSD is just a container of NCCH files)... if (Loader::MakeMagic('N', 'C', 'S', 'D') == ncch_header.magic) { + is_ncsd = true; NCSD_Header ncsd_header; file->Seek(ncch_offset, SEEK_SET); file->ReadBytes(&ncsd_header, sizeof(NCSD_Header)); @@ -219,15 +248,25 @@ Loader::ResultStatus NCCHContainer::Load() { if (i == 0) { file = HW::UniqueData::OpenUniqueCryptoFile( filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH); + if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != + std::nullopt) { + // The file is compressed + file = std::make_unique(std::move(file)); + } } else { return Loader::ResultStatus::ErrorInvalidFormat; } + } else { + break; } } if (file->IsCrypto()) { LOG_DEBUG(Service_FS, "NCCH file has console unique crypto"); } + if (file->IsCompressed()) { + LOG_DEBUG(Service_FS, "NCCH file is compressed"); + } has_header = true; @@ -323,12 +362,7 @@ Loader::ResultStatus NCCHContainer::Load() { if (file->ReadBytes(&exefs_header, sizeof(ExeFs_Header)) != sizeof(ExeFs_Header)) return Loader::ResultStatus::Error; - if (file->IsCrypto()) { - exefs_file = HW::UniqueData::OpenUniqueCryptoFile( - filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH); - } else { - exefs_file = std::make_unique(filepath, "rb"); - } + exefs_file = Reopen(file, filepath); has_exefs = true; } @@ -366,12 +400,7 @@ Loader::ResultStatus NCCHContainer::LoadOverrides() { is_tainted = true; has_exefs = true; } else { - if (file->IsCrypto()) { - exefs_file = HW::UniqueData::OpenUniqueCryptoFile( - filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH); - } else { - exefs_file = std::make_unique(filepath, "rb"); - } + exefs_file = Reopen(file, filepath); } } else if (FileUtil::Exists(exefsdir_override) && FileUtil::IsDirectory(exefsdir_override)) { is_tainted = true; @@ -607,12 +636,7 @@ Loader::ResultStatus NCCHContainer::ReadRomFS(std::shared_ptr& romf // We reopen the file, to allow its position to be independent from file's std::unique_ptr romfs_file_inner; - if (file->IsCrypto()) { - romfs_file_inner = HW::UniqueData::OpenUniqueCryptoFile( - filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH); - } else { - romfs_file_inner = std::make_unique(filepath, "rb"); - } + romfs_file_inner = Reopen(file, filepath); if (!romfs_file_inner->IsOpen()) return Loader::ResultStatus::Error; @@ -742,4 +766,24 @@ bool NCCHContainer::HasExHeader() { return has_exheader; } +std::unique_ptr NCCHContainer::Reopen( + const std::unique_ptr& orig_file, const std::string& new_filename) { + const bool is_compressed = orig_file->IsCompressed(); + const bool is_crypto = orig_file->IsCrypto(); + const std::string filename = new_filename.empty() ? orig_file->Filename() : new_filename; + + std::unique_ptr out_file; + if (is_crypto) { + out_file = HW::UniqueData::OpenUniqueCryptoFile(filename, "rb", + HW::UniqueData::UniqueCryptoFileID::NCCH); + } else { + out_file = std::make_unique(filename, "rb"); + } + if (is_compressed) { + out_file = std::make_unique(std::move(out_file)); + } + + return out_file; +} + } // namespace FileSys diff --git a/src/core/file_sys/ncch_container.h b/src/core/file_sys/ncch_container.h index 214f91998..675228499 100644 --- a/src/core/file_sys/ncch_container.h +++ b/src/core/file_sys/ncch_container.h @@ -333,16 +333,28 @@ public: */ bool HasExHeader(); + bool IsNCSD() { + return is_ncsd; + } + + bool IsFileCompressed() { + return file->IsCompressed(); + } + NCCH_Header ncch_header; ExeFs_Header exefs_header; ExHeader_Header exheader_header; private: + std::unique_ptr Reopen(const std::unique_ptr& orig_file, + const std::string& new_filename = ""); + bool has_header = false; bool has_exheader = false; bool has_exefs = false; bool has_romfs = false; + bool is_ncsd = false; bool is_proto = false; bool is_tainted = false; // Are there parts of this container being overridden? bool is_loaded = false; diff --git a/src/core/hle/service/am/am.cpp b/src/core/hle/service/am/am.cpp index 8d9ec4bc5..211855899 100644 --- a/src/core/hle/service/am/am.cpp +++ b/src/core/hle/service/am/am.cpp @@ -16,6 +16,7 @@ #include "common/hacks/hack_manager.h" #include "common/logging/log.h" #include "common/string_util.h" +#include "common/zstd_compression.h" #include "core/core.h" #include "core/file_sys/certificate.h" #include "core/file_sys/errors.h" @@ -105,6 +106,12 @@ NCCHCryptoFile::NCCHCryptoFile(const std::string& out_file, bool encrypted_conte file = std::make_unique(out_file, "wb"); } + if (Settings::values.compress_cia_installs) { + std::array magic = {'N', 'C', 'C', 'H'}; + file = std::make_unique( + std::move(file), magic, FileUtil::Z3DSWriteIOFile::DEFAULT_FRAME_SIZE); + } + if (!file->IsOpen()) { is_error = true; } @@ -116,6 +123,7 @@ void NCCHCryptoFile::Write(const u8* buffer, std::size_t length) { if (is_not_ncch) { file->WriteBytes(buffer, length); + return; } const int kBlockSize = 0x200; ///< Size of ExeFS blocks (in bytes) @@ -1061,8 +1069,16 @@ InstallStatus InstallCIA(const std::string& path, return InstallStatus::ErrorFileNotFound; } + std::unique_ptr in_file = std::make_unique(path, "rb"); + bool is_compressed = + FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(in_file.get()) != std::nullopt; + if (is_compressed) { + in_file = std::make_unique(std::move(in_file)); + } + FileSys::CIAContainer container; - if (container.Load(path) == Loader::ResultStatus::Success) { + if (container.Load(in_file.get()) == Loader::ResultStatus::Success) { + in_file->Seek(0, SEEK_SET); Service::AM::CIAFile installFile( Core::System::GetInstance(), Service::AM::GetTitleMediaType(container.GetTitleMetadata().GetTitleID())); @@ -1072,18 +1088,12 @@ InstallStatus InstallCIA(const std::string& path, return InstallStatus::ErrorEncrypted; } - FileUtil::IOFile file(path, "rb"); - if (!file.IsOpen()) { - LOG_ERROR(Service_AM, "Could not open CIA file '{}'.", path); - return InstallStatus::ErrorFailedToOpenFile; - } - std::vector buffer; buffer.resize(0x10000); - auto file_size = file.GetSize(); + auto file_size = in_file->GetSize(); std::size_t total_bytes_read = 0; while (total_bytes_read != file_size) { - std::size_t bytes_read = file.ReadBytes(buffer.data(), buffer.size()); + std::size_t bytes_read = in_file->ReadBytes(buffer.data(), buffer.size()); auto result = installFile.Write(static_cast(total_bytes_read), bytes_read, true, false, static_cast(buffer.data())); @@ -1128,6 +1138,51 @@ InstallStatus InstallCIA(const std::string& path, return InstallStatus::ErrorInvalid; } +InstallStatus CheckCIAToInstall(const std::string& path, bool& is_compressed, + bool check_encryption) { + if (!FileUtil::Exists(path)) { + LOG_ERROR(Service_AM, "File {} does not exist!", path); + return InstallStatus::ErrorFileNotFound; + } + + std::unique_ptr in_file = std::make_unique(path, "rb"); + is_compressed = FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(in_file.get()) != std::nullopt; + if (is_compressed) { + in_file = std::make_unique(std::move(in_file)); + } + + FileSys::CIAContainer container; + if (container.Load(in_file.get()) == Loader::ResultStatus::Success) { + in_file->Seek(0, SEEK_SET); + const FileSys::TitleMetadata& tmd = container.GetTitleMetadata(); + + if (check_encryption) { + if (tmd.HasEncryptedContent(container.GetHeader())) { + return InstallStatus::ErrorEncrypted; + } + + for (size_t i = 0; i < tmd.GetContentCount(); i++) { + u64 offset = container.GetContentOffset(i); + NCCH_Header ncch; + const auto read = in_file->ReadAtBytes(&ncch, sizeof(ncch), offset); + if (read != sizeof(ncch)) { + return InstallStatus::ErrorInvalid; + } + if (ncch.magic != Loader::MakeMagic('N', 'C', 'C', 'H')) { + return InstallStatus::ErrorInvalid; + } + if (!ncch.no_crypto) { + return InstallStatus::ErrorEncrypted; + } + } + } + + return InstallStatus::Success; + } + + return InstallStatus::ErrorInvalid; +} + u64 GetTitleUpdateId(u64 title_id) { // Real services seem to just discard and replace the whole high word. return (title_id & 0xFFFFFFFF) | (static_cast(TID_HIGH_UPDATE) << 32); diff --git a/src/core/hle/service/am/am.h b/src/core/hle/service/am/am.h index 1798e596b..5fd52811a 100644 --- a/src/core/hle/service/am/am.h +++ b/src/core/hle/service/am/am.h @@ -359,6 +359,14 @@ private: InstallStatus InstallCIA(const std::string& path, std::function&& update_callback = nullptr); +/** + * Checks if the provided path is a valid CIA file + * that can be installed. + * @param path file path of the CIA file to check to install + */ +InstallStatus CheckCIAToInstall(const std::string& path, bool& is_compressed, + bool check_encryption); + /** * Get the update title ID for a title * @param titleId the title ID diff --git a/src/core/loader/3dsx.cpp b/src/core/loader/3dsx.cpp index cbfd0e1f8..0a3a7b662 100644 --- a/src/core/loader/3dsx.cpp +++ b/src/core/loader/3dsx.cpp @@ -5,6 +5,7 @@ #include #include #include "common/logging/log.h" +#include "common/zstd_compression.h" #include "core/core.h" #include "core/hle/kernel/process.h" #include "core/hle/kernel/resource_limit.h" @@ -94,16 +95,16 @@ static u32 TranslateAddr(u32 addr, const THREEloadinfo* loadinfo, u32* offsets) using Kernel::CodeSet; -static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file, u32 base_addr, +static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile* file, u32 base_addr, std::shared_ptr* out_codeset) { - if (!file.IsOpen()) + if (!file->IsOpen()) return ERROR_FILE; // Reset read pointer in case this file has been read before. - file.Seek(0, SEEK_SET); + file->Seek(0, SEEK_SET); THREEDSX_Header hdr; - if (file.ReadBytes(&hdr, sizeof(hdr)) != sizeof(hdr)) + if (file->ReadBytes(&hdr, sizeof(hdr)) != sizeof(hdr)) return ERROR_READ; THREEloadinfo loadinfo; @@ -129,22 +130,22 @@ static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file, loadinfo.seg_ptrs[2] = loadinfo.seg_ptrs[1] + loadinfo.seg_sizes[1]; // Skip header for future compatibility - file.Seek(hdr.header_size, SEEK_SET); + file->Seek(hdr.header_size, SEEK_SET); // Read the relocation headers std::vector relocs(n_reloc_tables * NUM_SEGMENTS); for (unsigned int current_segment = 0; current_segment < NUM_SEGMENTS; ++current_segment) { std::size_t size = n_reloc_tables * sizeof(u32); - if (file.ReadBytes(&relocs[current_segment * n_reloc_tables], size) != size) + if (file->ReadBytes(&relocs[current_segment * n_reloc_tables], size) != size) return ERROR_READ; } // Read the segments - if (file.ReadBytes(loadinfo.seg_ptrs[0], hdr.code_seg_size) != hdr.code_seg_size) + if (file->ReadBytes(loadinfo.seg_ptrs[0], hdr.code_seg_size) != hdr.code_seg_size) return ERROR_READ; - if (file.ReadBytes(loadinfo.seg_ptrs[1], hdr.rodata_seg_size) != hdr.rodata_seg_size) + if (file->ReadBytes(loadinfo.seg_ptrs[1], hdr.rodata_seg_size) != hdr.rodata_seg_size) return ERROR_READ; - if (file.ReadBytes(loadinfo.seg_ptrs[2], hdr.data_seg_size - hdr.bss_size) != + if (file->ReadBytes(loadinfo.seg_ptrs[2], hdr.data_seg_size - hdr.bss_size) != hdr.data_seg_size - hdr.bss_size) return ERROR_READ; @@ -158,7 +159,7 @@ static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file, u32 n_relocs = relocs[current_segment * n_reloc_tables + current_segment_reloc_table]; if (current_segment_reloc_table >= 2) { // We are not using this table - ignore it because we don't know what it dose - file.Seek(n_relocs * sizeof(THREEDSX_Reloc), SEEK_CUR); + file->Seek(n_relocs * sizeof(THREEDSX_Reloc), SEEK_CUR); continue; } THREEDSX_Reloc reloc_table[RELOCBUFSIZE]; @@ -170,7 +171,7 @@ static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file, u32 remaining = std::min(RELOCBUFSIZE, n_relocs); n_relocs -= remaining; - if (file.ReadBytes(reloc_table, remaining * sizeof(THREEDSX_Reloc)) != + if (file->ReadBytes(reloc_table, remaining * sizeof(THREEDSX_Reloc)) != remaining * sizeof(THREEDSX_Reloc)) return ERROR_READ; @@ -248,13 +249,15 @@ static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file, return ERROR_NONE; } -FileType AppLoader_THREEDSX::IdentifyType(FileUtil::IOFile& file) { +FileType AppLoader_THREEDSX::IdentifyType(FileUtil::IOFile* file) { u32 magic; - file.Seek(0, SEEK_SET); - if (1 != file.ReadArray(&magic, 1)) + file->Seek(0, SEEK_SET); + if (1 != file->ReadArray(&magic, 1)) return FileType::Error; - if (MakeMagic('3', 'D', 'S', 'X') == magic) + if (MakeMagic('3', 'D', 'S', 'X') == magic || + (MakeMagic('Z', '3', 'D', 'S') == magic && + FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file) == MakeMagic('3', 'D', 'S', 'X'))) return FileType::THREEDSX; return FileType::Error; @@ -264,11 +267,15 @@ ResultStatus AppLoader_THREEDSX::Load(std::shared_ptr& process) if (is_loaded) return ResultStatus::ErrorAlreadyLoaded; - if (!file.IsOpen()) + if (!file->IsOpen()) return ResultStatus::Error; + if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt) { + file = std::make_unique(std::move(file)); + } + std::shared_ptr codeset; - if (Load3DSXFile(system, file, Memory::PROCESS_IMAGE_VADDR, &codeset) != ERROR_NONE) + if (Load3DSXFile(system, file.get(), Memory::PROCESS_IMAGE_VADDR, &codeset) != ERROR_NONE) return ResultStatus::Error; codeset->name = filename; @@ -292,14 +299,14 @@ ResultStatus AppLoader_THREEDSX::Load(std::shared_ptr& process) } ResultStatus AppLoader_THREEDSX::ReadRomFS(std::shared_ptr& romfs_file) { - if (!file.IsOpen()) + if (!file->IsOpen()) return ResultStatus::Error; // Reset read pointer in case this file has been read before. - file.Seek(0, SEEK_SET); + file->Seek(0, SEEK_SET); THREEDSX_Header hdr; - if (file.ReadBytes(&hdr, sizeof(THREEDSX_Header)) != sizeof(THREEDSX_Header)) + if (file->ReadBytes(&hdr, sizeof(THREEDSX_Header)) != sizeof(THREEDSX_Header)) return ResultStatus::Error; if (hdr.header_size != sizeof(THREEDSX_Header)) @@ -308,7 +315,7 @@ ResultStatus AppLoader_THREEDSX::ReadRomFS(std::shared_ptr // Check if the 3DSX has a RomFS... if (hdr.fs_offset != 0) { u32 romfs_offset = hdr.fs_offset; - u32 romfs_size = static_cast(file.GetSize()) - hdr.fs_offset; + u32 romfs_size = static_cast(file->GetSize()) - hdr.fs_offset; LOG_DEBUG(Loader, "RomFS offset: {:#010X}", romfs_offset); LOG_DEBUG(Loader, "RomFS size: {:#010X}", romfs_size); @@ -328,15 +335,26 @@ ResultStatus AppLoader_THREEDSX::ReadRomFS(std::shared_ptr return ResultStatus::ErrorNotUsed; } +AppLoader::CompressFileInfo AppLoader_THREEDSX::GetCompressFileInfo() { + CompressFileInfo info; + info.is_supported = true; + info.recommended_compressed_extension = "z3dsx"; + info.recommended_uncompressed_extension = "3dsx"; + info.underlying_magic = std::array({'3', 'D', 'S', 'X'}); + info.is_compressed = + FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt; + return info; +} + ResultStatus AppLoader_THREEDSX::ReadIcon(std::vector& buffer) { - if (!file.IsOpen()) + if (!file->IsOpen()) return ResultStatus::Error; // Reset read pointer in case this file has been read before. - file.Seek(0, SEEK_SET); + file->Seek(0, SEEK_SET); THREEDSX_Header hdr; - if (file.ReadBytes(&hdr, sizeof(THREEDSX_Header)) != sizeof(THREEDSX_Header)) + if (file->ReadBytes(&hdr, sizeof(THREEDSX_Header)) != sizeof(THREEDSX_Header)) return ResultStatus::Error; if (hdr.header_size != sizeof(THREEDSX_Header)) @@ -344,10 +362,10 @@ ResultStatus AppLoader_THREEDSX::ReadIcon(std::vector& buffer) { // Check if the 3DSX has a SMDH... if (hdr.smdh_offset != 0) { - file.Seek(hdr.smdh_offset, SEEK_SET); + file->Seek(hdr.smdh_offset, SEEK_SET); buffer.resize(hdr.smdh_size); - if (file.ReadBytes(buffer.data(), hdr.smdh_size) != hdr.smdh_size) + if (file->ReadBytes(buffer.data(), hdr.smdh_size) != hdr.smdh_size) return ResultStatus::Error; return ResultStatus::Success; diff --git a/src/core/loader/3dsx.h b/src/core/loader/3dsx.h index d7162f4d7..4403a6763 100644 --- a/src/core/loader/3dsx.h +++ b/src/core/loader/3dsx.h @@ -1,3 +1,7 @@ +// Copyright Citra Emulator Project / Azahar Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + // Copyright 2014 Dolphin Emulator Project / Citra Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. @@ -23,10 +27,10 @@ public: * @param file FileUtil::IOFile open file * @return FileType found, or FileType::Error if this loader doesn't know it */ - static FileType IdentifyType(FileUtil::IOFile& file); + static FileType IdentifyType(FileUtil::IOFile* file); FileType GetFileType() override { - return IdentifyType(file); + return IdentifyType(file.get()); } ResultStatus Load(std::shared_ptr& process) override; @@ -35,6 +39,8 @@ public: ResultStatus ReadRomFS(std::shared_ptr& romfs_file) override; + CompressFileInfo GetCompressFileInfo() override; + private: std::string filename; std::string filepath; diff --git a/src/core/loader/artic.cpp b/src/core/loader/artic.cpp index 9d8aa240c..30f62dc57 100644 --- a/src/core/loader/artic.cpp +++ b/src/core/loader/artic.cpp @@ -80,7 +80,7 @@ Apploader_Artic::~Apploader_Artic() { client->Stop(); } -FileType Apploader_Artic::IdentifyType(FileUtil::IOFile& file) { +FileType Apploader_Artic::IdentifyType(FileUtil::IOFile* file) { return FileType::ARTIC; } diff --git a/src/core/loader/artic.h b/src/core/loader/artic.h index 18ba31d87..13b92c189 100644 --- a/src/core/loader/artic.h +++ b/src/core/loader/artic.h @@ -33,10 +33,10 @@ public: * @param file FileUtil::IOFile open file * @return FileType found, or FileType::Error if this loader doesn't know it */ - static FileType IdentifyType(FileUtil::IOFile& file); + static FileType IdentifyType(FileUtil::IOFile* file); FileType GetFileType() override { - return IdentifyType(file); + return IdentifyType(file.get()); } [[nodiscard]] std::span GetPreferredRegions() const override { diff --git a/src/core/loader/elf.cpp b/src/core/loader/elf.cpp index 30b98e483..c03ef3267 100644 --- a/src/core/loader/elf.cpp +++ b/src/core/loader/elf.cpp @@ -1,3 +1,7 @@ +// Copyright Citra Emulator Project / Azahar Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + // Copyright 2013 Dolphin Emulator Project / 2014 Citra Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. @@ -352,10 +356,10 @@ SectionID ElfReader::GetSectionByName(const char* name, int firstSection) const namespace Loader { -FileType AppLoader_ELF::IdentifyType(FileUtil::IOFile& file) { +FileType AppLoader_ELF::IdentifyType(FileUtil::IOFile* file) { u32 magic; - file.Seek(0, SEEK_SET); - if (1 != file.ReadArray(&magic, 1)) + file->Seek(0, SEEK_SET); + if (1 != file->ReadArray(&magic, 1)) return FileType::Error; if (MakeMagic('\x7f', 'E', 'L', 'F') == magic) @@ -368,15 +372,15 @@ ResultStatus AppLoader_ELF::Load(std::shared_ptr& process) { if (is_loaded) return ResultStatus::ErrorAlreadyLoaded; - if (!file.IsOpen()) + if (!file->IsOpen()) return ResultStatus::Error; // Reset read pointer in case this file has been read before. - file.Seek(0, SEEK_SET); + file->Seek(0, SEEK_SET); - std::size_t size = file.GetSize(); + std::size_t size = file->GetSize(); std::unique_ptr buffer(new u8[size]); - if (file.ReadBytes(&buffer[0], size) != size) + if (file->ReadBytes(&buffer[0], size) != size) return ResultStatus::Error; ElfReader elf_reader(&buffer[0]); diff --git a/src/core/loader/elf.h b/src/core/loader/elf.h index ad5799d14..6e36c5292 100644 --- a/src/core/loader/elf.h +++ b/src/core/loader/elf.h @@ -1,3 +1,7 @@ +// Copyright Citra Emulator Project / Azahar Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + // Copyright 2013 Dolphin Emulator Project / 2014 Citra Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. @@ -22,10 +26,10 @@ public: * @param file FileUtil::IOFile open file * @return FileType found, or FileType::Error if this loader doesn't know it */ - static FileType IdentifyType(FileUtil::IOFile& file); + static FileType IdentifyType(FileUtil::IOFile* file); FileType GetFileType() override { - return IdentifyType(file); + return IdentifyType(file.get()); } ResultStatus Load(std::shared_ptr& process) override; diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp index c09760e36..205f81a00 100644 --- a/src/core/loader/loader.cpp +++ b/src/core/loader/loader.cpp @@ -19,7 +19,7 @@ FileType IdentifyFile(FileUtil::IOFile& file) { FileType type; #define CHECK_TYPE(loader) \ - type = AppLoader_##loader::IdentifyType(file); \ + type = AppLoader_##loader::IdentifyType(&file); \ if (FileType::Error != type) \ return type; @@ -48,16 +48,16 @@ FileType GuessFromExtension(const std::string& extension_) { if (extension == ".elf" || extension == ".axf") return FileType::ELF; - if (extension == ".cci") + if (extension == ".cci" || extension == ".zcci") return FileType::CCI; - if (extension == ".cxi" || extension == ".app") + if (extension == ".cxi" || extension == ".app" || extension == ".zcxi") return FileType::CXI; - if (extension == ".3dsx") + if (extension == ".3dsx" || extension == ".z3dsx") return FileType::THREEDSX; - if (extension == ".cia") + if (extension == ".cia" || extension == ".zcia") return FileType::CIA; return FileType::Unknown; diff --git a/src/core/loader/loader.h b/src/core/loader/loader.h index 76a7ea2df..2f2d6f3c3 100644 --- a/src/core/loader/loader.h +++ b/src/core/loader/loader.h @@ -85,8 +85,16 @@ constexpr u32 MakeMagic(char a, char b, char c, char d) { /// Interface for loading an application class AppLoader : NonCopyable { public: + struct CompressFileInfo { + bool is_supported{}; + bool is_compressed{}; + std::array underlying_magic{}; + std::string recommended_compressed_extension; + std::string recommended_uncompressed_extension; + }; + explicit AppLoader(Core::System& system_, FileUtil::IOFile&& file) - : system(system_), file(std::move(file)) {} + : system(system_), file(std::make_unique(std::move(file))) {} virtual ~AppLoader() {} /** @@ -279,9 +287,15 @@ public: return false; } + virtual CompressFileInfo GetCompressFileInfo() { + CompressFileInfo info{}; + info.is_supported = false; + return info; + } + protected: Core::System& system; - FileUtil::IOFile file; + std::unique_ptr file; bool is_loaded = false; std::optional memory_mode_override = std::nullopt; }; diff --git a/src/core/loader/ncch.cpp b/src/core/loader/ncch.cpp index 9ecf64685..9e4defaf6 100644 --- a/src/core/loader/ncch.cpp +++ b/src/core/loader/ncch.cpp @@ -12,6 +12,7 @@ #include "common/settings.h" #include "common/string_util.h" #include "common/swap.h" +#include "common/zstd_compression.h" #include "core/core.h" #include "core/file_sys/ncch_container.h" #include "core/file_sys/title_metadata.h" @@ -34,10 +35,10 @@ namespace Loader { using namespace Common::Literals; static constexpr u64 UPDATE_TID_HIGH = 0x0004000e00000000; -FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile& file) { +FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile* file) { u32 magic; - file.Seek(0x100, SEEK_SET); - if (1 != file.ReadArray(&magic, 1)) + file->Seek(0x100, SEEK_SET); + if (1 != file->ReadArray(&magic, 1)) return FileType::Error; if (MakeMagic('N', 'C', 'S', 'D') == magic) @@ -47,7 +48,7 @@ FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile& file) { return FileType::CXI; std::unique_ptr file_crypto = HW::UniqueData::OpenUniqueCryptoFile( - file.Filename(), "rb", HW::UniqueData::UniqueCryptoFileID::NCCH); + file->Filename(), "rb", HW::UniqueData::UniqueCryptoFileID::NCCH); file_crypto->Seek(0x100, SEEK_SET); if (1 != file_crypto->ReadArray(&magic, 1)) @@ -59,6 +60,16 @@ FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile& file) { if (MakeMagic('N', 'C', 'C', 'H') == magic) return FileType::CXI; + std::optional magic_zstd; + if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file) != std::nullopt || + FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file_crypto.get()) != std::nullopt) { + if (MakeMagic('N', 'C', 'S', 'D') == magic_zstd) + return FileType::CCI; + + if (MakeMagic('N', 'C', 'C', 'H') == magic_zstd) + return FileType::CXI; + } + return FileType::Error; } @@ -396,4 +407,24 @@ ResultStatus AppLoader_NCCH::ReadTitle(std::string& title) { return ResultStatus::Success; } +AppLoader::CompressFileInfo AppLoader_NCCH::GetCompressFileInfo() { + CompressFileInfo info{}; + if (base_ncch.LoadHeader() != ResultStatus::Success) { + info.is_supported = false; + return info; + } + info.is_supported = true; + info.is_compressed = base_ncch.IsFileCompressed(); + if (base_ncch.IsNCSD()) { + info.underlying_magic = std::array({'N', 'C', 'S', 'D'}); + info.recommended_compressed_extension = "zcci"; + info.recommended_uncompressed_extension = "cci"; + } else { + info.underlying_magic = std::array({'N', 'C', 'C', 'H'}); + info.recommended_compressed_extension = "zcxi"; + info.recommended_uncompressed_extension = "cxi"; + } + return info; +} + } // namespace Loader diff --git a/src/core/loader/ncch.h b/src/core/loader/ncch.h index 099644b75..2610baeca 100644 --- a/src/core/loader/ncch.h +++ b/src/core/loader/ncch.h @@ -1,4 +1,4 @@ -// Copyright 2014 Citra Emulator Project +// Copyright Citra Emulator Project / Azahar Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. @@ -17,17 +17,20 @@ class AppLoader_NCCH final : public AppLoader { public: AppLoader_NCCH(Core::System& system_, FileUtil::IOFile&& file, const std::string& filepath) : AppLoader(system_, std::move(file)), base_ncch(filepath), overlay_ncch(&base_ncch), - filepath(filepath) {} + filepath(filepath) { + filetype = IdentifyType(this->file.get()); + this->file.reset(); + } /** * Returns the type of the file * @param file FileUtil::IOFile open file * @return FileType found, or FileType::Error if this loader doesn't know it */ - static FileType IdentifyType(FileUtil::IOFile& file); + static FileType IdentifyType(FileUtil::IOFile* file); FileType GetFileType() override { - return IdentifyType(file); + return filetype; } [[nodiscard]] std::span GetPreferredRegions() const override { @@ -71,6 +74,8 @@ public: ResultStatus ReadTitle(std::string& title) override; + CompressFileInfo GetCompressFileInfo() override; + private: /** * Loads .code section into memory for booting @@ -94,6 +99,7 @@ private: std::vector preferred_regions; std::string filepath; + FileType filetype; }; } // namespace Loader