Azahar-Mirror/azahar
1
0
Fork 0
mirror of https://github.com/azahar-emu/azahar synced 2025-11-06 23:19:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

loader: Add compressed ROM (Z3DS) support

Browse Source
This commit is contained in:
PabloMK7 2025-07-02 20:24:42 +02:00
parent fcdfe03674
commit 8dafecd528
30 changed files with 1487 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
externals/CMakeLists.txt vendored
View File

@ -214,8 +214,29 @@ else()
set(ZSTD_BUILD_PROGRAMS OFF)
set(ZSTD_BUILD_SHARED OFF)
add_subdirectory(zstd/build/cmake EXCLUDE_FROM_ALL)
target_include_directories(libzstd_static INTERFACE $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/externals/zstd/lib>)
target_include_directories(libzstd_static INTERFACE
$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/externals/zstd/lib>
$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/externals/zstd/lib/common>
)
add_library(zstd_seekable STATIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/zstd/contrib/seekable_format/zstdseek_compress.c>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/zstd/contrib/seekable_format/zstdseek_decompress.c>
)
target_include_directories(zstd_seekable PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/zstd/contrib/seekable_format>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/zstd/lib/common>
)
target_link_libraries(zstd_seekable PUBLIC libzstd_static)
target_link_libraries(libzstd_static INTERFACE zstd_seekable)
add_library(zstd ALIAS libzstd_static)
install(TARGETS zstd_seekable
EXPORT zstdExports
)
endif()
# ENet

184
src/citra_qt/citra_qt.cpp
View File

@ -92,6 +92,7 @@
#endif
#include "common/settings.h"
#include "common/string_util.h"
#include "common/zstd_compression.h"
#include "core/core.h"
#include "core/dumping/backend.h"
#include "core/file_sys/archive_extsavedata.h"
@ -991,6 +992,7 @@ void GMainWindow::ConnectWidgetEvents() {
connect(this, &GMainWindow::UpdateProgress, this, &GMainWindow::OnUpdateProgress);
connect(this, &GMainWindow::CIAInstallReport, this, &GMainWindow::OnCIAInstallReport);
connect(this, &GMainWindow::CIAInstallFinished, this, &GMainWindow::OnCIAInstallFinished);
connect(this, &GMainWindow::CompressFinished, this, &GMainWindow::OnCompressFinished);
connect(this, &GMainWindow::UpdateThemedIcons, multiplayer_state,
&MultiplayerState::UpdateThemedIcons);
}
@ -1082,6 +1084,10 @@ void GMainWindow::ConnectMenuEvents() {
connect_menu(ui->action_Capture_Screenshot, &GMainWindow::OnCaptureScreenshot);
connect_menu(ui->action_Dump_Video, &GMainWindow::OnDumpVideo);
// Tools
connect_menu(ui->action_Compress_ROM_File, &GMainWindow::OnCompressFile);
connect_menu(ui->action_Decompress_ROM_File, &GMainWindow::OnDecompressFile);
// Help
connect_menu(ui->action_Open_Citra_Folder, &GMainWindow::OnOpenCitraFolder);
connect_menu(ui->action_Open_Log_Folder, []() {
@ -2236,7 +2242,7 @@ void GMainWindow::OnMenuSetUpSystemFiles() {
void GMainWindow::OnMenuInstallCIA() {
QStringList filepaths = QFileDialog::getOpenFileNames(
this, tr("Load Files"), UISettings::values.roms_path,
tr("3DS Installation File (*.CIA*)") + QStringLiteral(";;") + tr("All Files (*.*)"));
tr("3DS Installation File (*.cia *.zcia)") + QStringLiteral(";;") + tr("All Files (*.*)"));
if (filepaths.isEmpty()) {
return;
@ -2318,6 +2324,21 @@ void GMainWindow::OnCIAInstallReport(Service::AM::InstallStatus status, QString
}
}
void GMainWindow::OnCompressFinished(bool is_compress, bool success) {
progress_bar->hide();
progress_bar->setValue(0);
if (!success) {
if (is_compress) {
QMessageBox::critical(this, tr("Error compressing file"),
tr("File compress operation failed, check log for details."));
} else {
QMessageBox::critical(this, tr("Error decompressing file"),
tr("File decompress operation failed, check log for details."));
}
}
}
void GMainWindow::OnCIAInstallFinished() {
progress_bar->hide();
progress_bar->setValue(0);
@ -3025,6 +3046,163 @@ void GMainWindow::OnDumpVideo() {
}
}
void GMainWindow::OnCompressFile() {
// NOTE: Encrypted files SHOULD NEVER be compressed, otherwise the resulting
// compressed file will have very poor compression ratios, due to the high
// entropy caused by encryption. This may cause confusion to the user as they
// will see the files do not compress well and blame the emulator.
//
// This is enforced using the loaders as they already return an error on encryption.
QString filepath = QFileDialog::getOpenFileName(
this, tr("Load 3DS ROM File"), UISettings::values.roms_path,
tr("3DS ROM Files (*.cia *cci *3dsx *cxi)") + QStringLiteral(";;") + tr("All Files (*.*)"));
if (filepath.isEmpty()) {
return;
}
std::string in_path = filepath.toStdString();
// Identify file type
Loader::AppLoader::CompressFileInfo compress_info{};
compress_info.is_supported = false;
size_t frame_size{};
{
auto loader = Loader::GetLoader(in_path);
if (loader) {
compress_info = loader->GetCompressFileInfo();
frame_size = FileUtil::Z3DSWriteIOFile::DEFAULT_FRAME_SIZE;
} else {
bool is_compressed = false;
if (Service::AM::CheckCIAToInstall(in_path, is_compressed, true) ==
Service::AM::InstallStatus::Success) {
compress_info.is_supported = true;
compress_info.is_compressed = is_compressed;
compress_info.recommended_compressed_extension = "zcia";
compress_info.recommended_uncompressed_extension = "cia";
compress_info.underlying_magic = std::array<u8, 4>({'C', 'I', 'A', '\0'});
frame_size = FileUtil::Z3DSWriteIOFile::MAX_FRAME_SIZE;
}
}
}
if (!compress_info.is_supported) {
QMessageBox::critical(
this, tr("Error compressing file"),
tr("The selected file is not a compatible 3DS ROM format. Make sure you have "
"chosen the right file, and that it is not encrypted."));
return;
}
if (compress_info.is_compressed) {
QMessageBox::warning(this, tr("Error compressing file"),
tr("The selected file is already compressed."));
return;
}
QString out_filter =
tr("3DS Compressed ROM File (*.%1)")
.arg(QString::fromStdString(compress_info.recommended_compressed_extension));
QFileInfo fileinfo(filepath);
QString final_path = fileinfo.path() + QStringLiteral(DIR_SEP) + fileinfo.completeBaseName() +
QStringLiteral(".") +
QString::fromStdString(compress_info.recommended_compressed_extension);
filepath = QFileDialog::getSaveFileName(this, tr("Save 3DS Compressed ROM File"), final_path,
out_filter);
if (filepath.isEmpty()) {
return;
}
std::string out_path = filepath.toStdString();
progress_bar->show();
progress_bar->setMaximum(INT_MAX);
(void)QtConcurrent::run([&, in_path, out_path, compress_info, frame_size] {
const auto progress = [&](std::size_t written, std::size_t total) {
emit UpdateProgress(written, total);
};
bool success = FileUtil::CompressZ3DSFile(in_path, out_path, compress_info.underlying_magic,
frame_size, progress);
if (!success) {
FileUtil::Delete(out_path);
}
emit OnCompressFinished(true, success);
});
}
void GMainWindow::OnDecompressFile() {
QString filepath = QFileDialog::getOpenFileName(
this, tr("Load 3DS Compressed ROM File"), UISettings::values.roms_path,
tr("3DS Compressed ROM Files (*.zcia *zcci *z3dsx *zcxi)") + QStringLiteral(";;") +
tr("All Files (*.*)"));
if (filepath.isEmpty()) {
return;
}
std::string in_path = filepath.toStdString();
// Identify file type
Loader::AppLoader::CompressFileInfo compress_info{};
compress_info.is_supported = false;
{
auto loader = Loader::GetLoader(in_path);
if (loader) {
compress_info = loader->GetCompressFileInfo();
} else {
bool is_compressed = false;
if (Service::AM::CheckCIAToInstall(in_path, is_compressed, false) ==
Service::AM::InstallStatus::Success) {
compress_info.is_supported = true;
compress_info.is_compressed = is_compressed;
compress_info.recommended_compressed_extension = "zcia";
compress_info.recommended_uncompressed_extension = "cia";
compress_info.underlying_magic = std::array<u8, 4>({'C', 'I', 'A', '\0'});
}
}
}
if (!compress_info.is_supported) {
QMessageBox::critical(this, tr("Error decompressing file"),
tr("The selected file is not a compatible compressed 3DS ROM format. "
"Make sure you have "
"chosen the right file."));
return;
}
if (!compress_info.is_compressed) {
QMessageBox::warning(this, tr("Error decompressing file"),
tr("The selected file is already decompressed."));
return;
}
QString out_filter =
tr("3DS ROM File (*.%1)")
.arg(QString::fromStdString(compress_info.recommended_uncompressed_extension));
QFileInfo fileinfo(filepath);
QString final_path = fileinfo.path() + QStringLiteral(DIR_SEP) + fileinfo.completeBaseName() +
QStringLiteral(".") +
QString::fromStdString(compress_info.recommended_uncompressed_extension);
filepath = QFileDialog::getSaveFileName(this, tr("Save 3DS ROM File"), final_path, out_filter);
if (filepath.isEmpty()) {
return;
}
std::string out_path = filepath.toStdString();
progress_bar->show();
progress_bar->setMaximum(INT_MAX);
(void)QtConcurrent::run([&, in_path, out_path, compress_info] {
const auto progress = [&](std::size_t written, std::size_t total) {
emit UpdateProgress(written, total);
};
// TODO(PabloMK7): What should we do with the metadata?
bool success = FileUtil::DeCompressZ3DSFile(in_path, out_path, progress);
if (!success) {
FileUtil::Delete(out_path);
}
emit OnCompressFinished(false, success);
});
}
#ifdef _WIN32
void GMainWindow::OnOpenFFmpeg() {
auto filename =
@ -3514,8 +3692,8 @@ static bool IsSingleFileDropEvent(const QMimeData* mime) {
return mime->hasUrls() && mime->urls().length() == 1;
}
static const std::array<std::string, 8> AcceptedExtensions = {"cci", "cxi", "bin", "3dsx",
"app", "elf", "axf"};
static const std::array<std::string, 10> AcceptedExtensions = {
"cci", "cxi", "bin", "3dsx", "app", "elf", "axf", "zcci", "zcxi", "z3dsx"};
static bool IsCorrectFileExtension(const QMimeData* mime) {
const QString& filename = mime->urls().at(0).toLocalFile();

4
src/citra_qt/citra_qt.h
View File

@ -141,6 +141,7 @@ signals:
void UpdateProgress(std::size_t written, std::size_t total);
void CIAInstallReport(Service::AM::InstallStatus status, QString filepath);
void CompressFinished(bool is_compress, bool success);
void CIAInstallFinished();
// Signal that tells widgets to update icons to use the current theme
void UpdateThemedIcons();
@ -248,6 +249,7 @@ private slots:
void OnMenuBootHomeMenu(u32 region);
void OnUpdateProgress(std::size_t written, std::size_t total);
void OnCIAInstallReport(Service::AM::InstallStatus status, QString filepath);
void OnCompressFinished(bool is_compress, bool success);
void OnCIAInstallFinished();
void OnMenuRecentFile();
void OnConfigure();
@ -281,6 +283,8 @@ private slots:
void OnSaveMovie();
void OnCaptureScreenshot();
void OnDumpVideo();
void OnCompressFile();
void OnDecompressFile();
#ifdef _WIN32
void OnOpenFFmpeg();
#endif

2
src/citra_qt/configuration/config.cpp
View File

@ -475,6 +475,7 @@ void QtConfig::ReadDataStorageValues() {
ReadBasicSetting(Settings::values.use_virtual_sd);
ReadBasicSetting(Settings::values.use_custom_storage);
ReadBasicSetting(Settings::values.compress_cia_installs);
const std::string nand_dir =
ReadSetting(QStringLiteral("nand_directory"), QStringLiteral("")).toString().toStdString();
@ -1045,6 +1046,7 @@ void QtConfig::SaveDataStorageValues() {
WriteBasicSetting(Settings::values.use_virtual_sd);
WriteBasicSetting(Settings::values.use_custom_storage);
WriteBasicSetting(Settings::values.compress_cia_installs);
WriteSetting(QStringLiteral("nand_directory"),
QString::fromStdString(FileUtil::GetUserPath(FileUtil::UserPath::NANDDir)),
QStringLiteral(""));

2
src/citra_qt/configuration/configure_storage.cpp
View File

@ -77,6 +77,7 @@ void ConfigureStorage::SetConfiguration() {
ui->toggle_virtual_sd->setChecked(Settings::values.use_virtual_sd.GetValue());
ui->toggle_custom_storage->setChecked(Settings::values.use_custom_storage.GetValue());
ui->toggle_compress_cia->setChecked(Settings::values.compress_cia_installs.GetValue());
ui->storage_group->setEnabled(!is_powered_on);
}
@ -84,6 +85,7 @@ void ConfigureStorage::SetConfiguration() {
void ConfigureStorage::ApplyConfiguration() {
Settings::values.use_virtual_sd = ui->toggle_virtual_sd->isChecked();
Settings::values.use_custom_storage = ui->toggle_custom_storage->isChecked();
Settings::values.compress_cia_installs = ui->toggle_compress_cia->isChecked();
if (!Settings::values.use_custom_storage) {
FileUtil::UpdateUserPath(FileUtil::UserPath::NANDDir,

14
src/citra_qt/configuration/configure_storage.ui
View File

@ -179,6 +179,20 @@
</layout>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_5">
<item>
<widget class="QCheckBox" name="toggle_compress_cia">
<property name="text">
<string>Compress installed CIA contents</string>
</property>
<property name="toolTip">
<string>Enables compressing the contents of CIA files when they are installed to the emulated SD.</string>
</property>
</widget>
</item>
</layout>
</item>
</layout>
</widget>
</item>

6
src/citra_qt/game_list.cpp
View File

@ -1039,8 +1039,10 @@ void GameList::LoadInterfaceLayout() {
}
const QStringList GameList::supported_file_extensions = {
QStringLiteral("3dsx"), QStringLiteral("elf"), QStringLiteral("axf"),
QStringLiteral("cci"), QStringLiteral("cxi"), QStringLiteral("app")};
QStringLiteral("3dsx"), QStringLiteral("elf"), QStringLiteral("axf"),
QStringLiteral("cci"), QStringLiteral("cxi"), QStringLiteral("app"),
QStringLiteral("z3dsx"), QStringLiteral("zcci"), QStringLiteral("zcxi"),
};
void GameList::RefreshGameDirectory() {
if (!UISettings::values.game_dirs.isEmpty() && current_worker != nullptr) {

13
src/citra_qt/main.ui
View File

@ -208,6 +208,9 @@
<addaction name="separator"/>
<addaction name="action_Capture_Screenshot"/>
<addaction name="action_Dump_Video"/>
<addaction name="separator"/>
<addaction name="action_Compress_ROM_File"/>
<addaction name="action_Decompress_ROM_File"/>
</widget>
<widget class="QMenu" name="menu_Help">
<property name="title">
@ -458,6 +461,16 @@
<string>Dump Video</string>
</property>
</action>
<action name="action_Compress_ROM_File">
<property name="text">
<string>Compress ROM File...</string>
</property>
</action>
<action name="action_Decompress_ROM_File">
<property name="text">
<string>Decompress ROM File...</string>
</property>
</action>
<action name="action_View_Lobby">
<property name="enabled">
<bool>true</bool>

1
src/citra_sdl/config.cpp
View File

@ -213,6 +213,7 @@ void SdlConfig::ReadValues() {
// Data Storage
ReadSetting("Data Storage", Settings::values.use_virtual_sd);
ReadSetting("Data Storage", Settings::values.use_custom_storage);
ReadSetting("Data Storage", Settings::values.compress_cia_installs);
if (Settings::values.use_custom_storage) {
FileUtil::UpdateUserPath(FileUtil::UserPath::NANDDir,

13
src/common/file_util.cpp
View File

@ -1187,7 +1187,7 @@ bool IOFile::SeekImpl(s64 off, int origin) {
return m_good;
}
u64 IOFile::Tell() const {
u64 IOFile::TellImpl() const {
if (IsOpen())
return ftello(m_file);
@ -1224,11 +1224,18 @@ static std::size_t pread(int fd, void* buf, std::size_t count, uint64_t offset)
overlapped.OffsetHigh = static_cast<uint32_t>(offset >> 32);
overlapped.Offset = static_cast<uint32_t>(offset & 0xFFFF'FFFFLL);
LARGE_INTEGER orig, dummy;
// TODO(PabloMK7): This is not fully async, windows being messy again...
// The file pos pointer will be undefined if ReadAt is used in multiple
// threads. Normally not problematic, but worth remembering.
SetFilePointerEx(file, {}, &orig, FILE_CURRENT);
SetLastError(0);
bool ret = ReadFile(file, buf, static_cast<uint32_t>(count), &read_bytes, &overlapped);
DWORD last_error = GetLastError();
SetFilePointerEx(file, orig, &dummy, FILE_BEGIN);
if (!ret && GetLastError() != ERROR_HANDLE_EOF) {
errno = GetLastError();
if (!ret && last_error != ERROR_HANDLE_EOF) {
errno = last_error;
return std::numeric_limits<std::size_t>::max();
}
return read_bytes;

34
src/common/file_util.h
View File

@ -301,7 +301,7 @@ public:
void Swap(IOFile& other) noexcept;
bool Close();
virtual bool Close();
template <typename T>
std::size_t ReadArray(T* data, std::size_t length) {
@ -412,15 +412,15 @@ public:
return WriteImpl(data.data(), data.size(), sizeof(T));
}
[[nodiscard]] bool IsOpen() const {
[[nodiscard]] virtual bool IsOpen() const {
return nullptr != m_file;
}
// m_good is set to false when a read, write or other function fails
[[nodiscard]] bool IsGood() const {
[[nodiscard]] virtual bool IsGood() const {
return m_good;
}
[[nodiscard]] int GetFd() const {
[[nodiscard]] virtual int GetFd() const {
#ifdef ANDROID
return m_fd;
#else
@ -436,13 +436,15 @@ public:
bool Seek(s64 off, int origin) {
return SeekImpl(off, origin);
}
[[nodiscard]] u64 Tell() const;
[[nodiscard]] u64 GetSize() const;
bool Resize(u64 size);
bool Flush();
u64 Tell() const {
return TellImpl();
}
virtual u64 GetSize() const;
virtual bool Resize(u64 size);
virtual bool Flush();
// clear error state
void Clear() {
virtual void Clear() {
m_good = true;
std::clearerr(m_file);
}
@ -451,29 +453,35 @@ public:
return false;
}
const std::string& Filename() const {
virtual bool IsCompressed() {
return false;
}
virtual const std::string& Filename() const {
return filename;
}
protected:
friend struct CryptoIOFileImpl;
virtual bool Open();
virtual std::size_t ReadImpl(void* data, std::size_t length, std::size_t data_size);
virtual std::size_t ReadAtImpl(void* data, std::size_t length, std::size_t data_size,
std::size_t offset);
virtual std::size_t WriteImpl(const void* data, std::size_t length, std::size_t data_size);
virtual bool SeekImpl(s64 off, int origin);
virtual u64 TellImpl() const;
private:
bool Open();
std::FILE* m_file = nullptr;
int m_fd = -1;
bool m_good = true;
std::string filename;
std::string openmode;
u32 flags;
u32 flags = 0;
template <class Archive>
void serialize(Archive& ar, const unsigned int) {

1
src/common/settings.h
View File

@ -468,6 +468,7 @@ struct Values {
// Data Storage
Setting<bool> use_virtual_sd{true, "use_virtual_sd"};
Setting<bool> use_custom_storage{false, "use_custom_storage"};
Setting<bool> compress_cia_installs{false, "compress_cia_installs"};
// System
SwitchableSetting<s32> region_value{REGION_VALUE_AUTO_SELECT, "region_value"};

690
src/common/zstd_compression.cpp
View File

@ -1,15 +1,30 @@
// Copyright Citra Emulator Project / Azahar Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
// Copyright 2019 yuzu Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include <algorithm>
#include <chrono>
#include <ctime>
#include <format>
#include <mutex>
#include <sstream>
#include <zstd.h>
#include <zstd/contrib/seekable_format/zstd_seekable.h>
#include <boost/serialization/base_object.hpp>
#include <boost/serialization/unique_ptr.hpp>
#include "common/alignment.h"
#include "common/archives.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "common/scm_rev.h"
#include "common/zstd_compression.h"
namespace Common::Compression {
std::vector<u8> CompressDataZSTD(std::span<const u8> source, s32 compression_level) {
compression_level = std::clamp(compression_level, ZSTD_minCLevel(), ZSTD_maxCLevel());
const std::size_t max_compressed_size = ZSTD_compressBound(source.size());
@ -71,3 +86,676 @@ std::vector<u8> DecompressDataZSTD(std::span<const u8> compressed) {
}
} // namespace Common::Compression
namespace FileUtil {
template <typename T>
void ReadFromIStream(std::istringstream& s, T* out, size_t out_size) {
s.read(reinterpret_cast<char*>(out), out_size);
}
template <typename T>
void WriteToOStream(std::ostringstream& s, const T* out, size_t out_size) {
s.write(reinterpret_cast<const char*>(out), out_size);
}
Z3DSMetadata::Z3DSMetadata(const std::span<u8>& source_data) {
if (source_data.empty())
return;
std::string buf(reinterpret_cast<const char*>(source_data.data()), source_data.size());
std::istringstream in(buf, std::ios::binary);
u8 version;
ReadFromIStream(in, &version, sizeof(version));
if (version != METADATA_VERSION) {
return;
}
while (!in.eof()) {
Item item;
ReadFromIStream(in, &item, sizeof(Item));
// Only binary type supported for now
if (item.type != Item::TYPE_BINARY) {
break;
}
std::string name(item.name_len, '\0');
std::vector<u8> data(item.data_len);
ReadFromIStream(in, name.data(), name.size());
ReadFromIStream(in, data.data(), data.size());
items.insert({std::move(name), std::move(data)});
}
}
std::vector<u8> Z3DSMetadata::AsBinary() {
if (items.empty())
return {};
std::ostringstream out;
u8 version = METADATA_VERSION;
WriteToOStream(out, &version, sizeof(u8));
for (const auto& it : items) {
Item item{
.type = Item::TYPE_BINARY,
.name_len = static_cast<u8>(std::min<size_t>(0xFF, it.first.size())),
.data_len = static_cast<u16>(std::min<size_t>(0xFFFF, it.second.size())),
};
WriteToOStream(out, &item, sizeof(item));
WriteToOStream(out, it.first.data(), item.name_len);
WriteToOStream(out, it.second.data(), item.data_len);
}
std::string out_str = out.str();
return std::vector<u8>(out_str.begin(), out_str.end());
}
struct Z3DSWriteIOFile::Z3DSWriteIOFileImpl {
Z3DSWriteIOFileImpl() {}
Z3DSWriteIOFileImpl(size_t frame_size) {
zstd_frame_size = frame_size;
cstream = ZSTD_seekable_createCStream();
size_t init_result = ZSTD_seekable_initCStream(cstream, ZSTD_CLEVEL_DEFAULT, 0,
static_cast<unsigned int>(frame_size));
if (ZSTD_isError(init_result)) {
LOG_ERROR(Common_Filesystem, "ZSTD_seekable_initCStream() error : {}",
ZSTD_getErrorName(init_result));
}
write_header.magic = Z3DSFileHeader::EXPECTED_MAGIC;
write_header.version = Z3DSFileHeader::EXPECTED_VERSION;
write_header.header_size = sizeof(Z3DSFileHeader);
next_input_size_hint = ZSTD_CStreamInSize();
}
bool WriteHeader(IOFile* file) {
file->Seek(0, SEEK_SET);
return file->WriteBytes(&write_header, sizeof(write_header)) == sizeof(write_header);
}
bool WriteMetadata(IOFile* file, const std::span<u8>& data) {
std::array<u8, 0x10> tmp_data{};
size_t total_size = Common::AlignUp(data.size(), 0x10);
write_header.metadata_size = static_cast<u32>(total_size);
size_t res_written = file->WriteBytes(data.data(), data.size());
res_written += file->WriteBytes(tmp_data.data(), total_size - data.size());
return res_written == total_size;
}
size_t Write(IOFile* file, const void* data, std::size_t length) {
size_t ret = length;
const size_t out_size = ZSTD_CStreamOutSize();
const size_t in_size = ZSTD_CStreamInSize();
if (write_buffer.size() < out_size) {
write_buffer.resize(out_size);
}
ZSTD_inBuffer input = {data, length, 0};
while (input.pos < input.size) {
ZSTD_outBuffer output = {write_buffer.data(), write_buffer.size(), 0};
next_input_size_hint = ZSTD_seekable_compressStream(cstream, &output, &input);
if (ZSTD_isError(next_input_size_hint)) {
LOG_ERROR(Common_Filesystem, "ZSTD_seekable_compressStream() error : {}",
ZSTD_getErrorName(next_input_size_hint));
ret = 0;
next_input_size_hint = ZSTD_CStreamInSize();
break;
}
if (next_input_size_hint > in_size) {
next_input_size_hint = in_size;
}
if (file->WriteBytes(static_cast<u8*>(output.dst), output.pos) != output.pos) {
ret = 0;
break;
}
written_compressed += output.pos;
}
return ret;
}
bool Close(IOFile* file, size_t written_uncompressed) {
const size_t out_size = ZSTD_CStreamOutSize();
if (write_buffer.size() < out_size) {
write_buffer.resize(out_size);
}
size_t remaining;
do {
ZSTD_outBuffer output = {write_buffer.data(), write_buffer.size(), 0};
remaining = ZSTD_seekable_endStream(cstream, &output); /* close stream */
if (ZSTD_isError(remaining)) {
LOG_ERROR(Common_Filesystem, "ZSTD_seekable_endStream() error : {}",
ZSTD_getErrorName(remaining));
return false;
}
if (file->WriteBytes(static_cast<u8*>(output.dst), output.pos) != output.pos) {
return false;
}
written_compressed += output.pos;
} while (remaining);
write_header.compressed_size = written_compressed;
write_header.uncompressed_size = written_uncompressed;
ZSTD_seekable_freeCStream(cstream);
return WriteHeader(file);
}
std::vector<u8> write_buffer;
size_t next_input_size_hint = 0;
size_t zstd_frame_size = 0;
u64 written_compressed = 0;
ZSTD_seekable_CStream* cstream{};
Z3DSFileHeader write_header{};
};
Z3DSWriteIOFile::Z3DSWriteIOFile()
: IOFile(), file{std::make_unique<IOFile>()}, impl{std::make_unique<Z3DSWriteIOFileImpl>()} {}
Z3DSWriteIOFile::Z3DSWriteIOFile(std::unique_ptr<IOFile>&& underlying_file,
const std::array<u8, 4>& underlying_magic, size_t frame_size)
: IOFile(), file{std::move(underlying_file)},
impl{std::make_unique<Z3DSWriteIOFileImpl>(frame_size)} {
ASSERT_MSG(!file->IsCompressed(), "Underlying file is already compressed!");
impl->write_header.underlying_magic = underlying_magic;
impl->WriteHeader(file.get());
Metadata().Add("compressor", std::string("Azahar ") + Common::g_build_fullname);
std::time_t tt = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
std::tm tm{};
#if defined(_WIN32)
gmtime_s(&tm, &tt);
#else
gmtime_r(&tt, &tm);
#endif
char buf[0x20];
std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &tm);
Metadata().Add("date", buf);
Metadata().Add(
"maxframesize",
std::to_string(frame_size ? frame_size : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE));
}
Z3DSWriteIOFile::~Z3DSWriteIOFile() {
this->Close();
}
bool Z3DSWriteIOFile::Close() {
impl->Close(file.get(), written_uncompressed);
return file->Close();
}
u64 Z3DSWriteIOFile::GetSize() const {
return written_uncompressed;
}
bool Z3DSWriteIOFile::Resize(u64 size) {
// Stubbed
UNIMPLEMENTED();
return false;
}
bool Z3DSWriteIOFile::Flush() {
return file->Flush();
}
void Z3DSWriteIOFile::Clear() {
return file->Clear();
}
bool Z3DSWriteIOFile::IsCrypto() {
return file->IsCrypto();
}
const std::string& Z3DSWriteIOFile::Filename() const {
return file->Filename();
}
bool Z3DSWriteIOFile::IsOpen() const {
return file->IsOpen();
}
bool Z3DSWriteIOFile::IsGood() const {
return file->IsGood();
}
int Z3DSWriteIOFile::GetFd() const {
return file->GetFd();
}
bool Z3DSWriteIOFile::Open() {
if (is_serializing) {
return true;
}
// Stubbed
UNIMPLEMENTED();
return false;
}
std::size_t Z3DSWriteIOFile::ReadImpl(void* data, std::size_t length, std::size_t data_size) {
// Stubbed
UNIMPLEMENTED();
return 0;
}
std::size_t Z3DSWriteIOFile::ReadAtImpl(void* data, std::size_t length, std::size_t data_size,
std::size_t offset) {
// Stubbed
UNIMPLEMENTED();
return 0;
}
std::size_t Z3DSWriteIOFile::WriteImpl(const void* data, std::size_t length,
std::size_t data_size) {
if (!metadata_written) {
metadata_written = true;
auto metadata_binary = metadata.AsBinary();
if (!metadata_binary.empty()) {
impl->WriteMetadata(file.get(), metadata_binary);
}
}
size_t ret = impl->Write(file.get(), data, length * data_size);
written_uncompressed += ret;
return ret;
}
bool Z3DSWriteIOFile::SeekImpl(s64 off, int origin) {
if (is_serializing) {
return true;
}
// Stubbed
UNIMPLEMENTED();
return false;
}
u64 Z3DSWriteIOFile::TellImpl() const {
return written_uncompressed;
}
size_t Z3DSWriteIOFile::GetNextWriteHint() {
return impl->next_input_size_hint;
}
template <class Archive>
void Z3DSWriteIOFile::serialize(Archive& ar, const unsigned int) {
is_serializing = true;
ar& boost::serialization::base_object<IOFile>(*this);
ar & file;
ar & written_uncompressed;
ar & metadata_written;
ar & metadata;
Z3DSFileHeader hd;
size_t frame_size;
u64 written_compressed;
if (Archive::is_loading::value) {
ar & hd;
ar & frame_size;
ar & written_compressed;
impl = std::make_unique<Z3DSWriteIOFileImpl>(frame_size);
impl->write_header = hd;
impl->written_compressed = written_compressed;
} else {
ar & impl->write_header;
ar & impl->zstd_frame_size;
ar & impl->written_compressed;
}
is_serializing = false;
}
struct Z3DSReadIOFile::Z3DSReadIOFileImpl {
Z3DSReadIOFileImpl() {}
Z3DSReadIOFileImpl(IOFile* file, bool load_metadata = true) {
curr_file = file;
m_good = file->ReadAtBytes(&header, sizeof(header), 0) == sizeof(header);
m_good &= header.magic == Z3DSFileHeader::EXPECTED_MAGIC &&
header.version == Z3DSFileHeader::EXPECTED_VERSION;
if (!m_good) {
return;
}
if (header.metadata_size && load_metadata) {
std::vector<u8> buff(header.metadata_size);
file->ReadAtBytes(buff.data(), buff.size(), header.header_size);
metadata = Z3DSMetadata(buff);
}
seekable = ZSTD_seekable_create();
ZSTD_seekable_customFile custom_file{
.opaque = this,
.read = [](void* opaque, void* buffer, size_t n) -> int {
return reinterpret_cast<Z3DSReadIOFileImpl*>(opaque)->OnZSTDRead(buffer, n);
},
.seek = [](void* opaque, long long offset, int origin) -> int {
return reinterpret_cast<Z3DSReadIOFileImpl*>(opaque)->OnZSTDSeek(offset, origin);
},
};
size_t init_result = ZSTD_seekable_initAdvanced(seekable, custom_file);
if (ZSTD_isError(init_result)) {
LOG_ERROR(Common_Filesystem, "ZSTD_seekable_initCStream() error : {}",
ZSTD_getErrorName(init_result));
m_good = false;
}
}
int OnZSTDRead(void* buffer, size_t n) {
const size_t read = curr_file->ReadBytes(reinterpret_cast<uint8_t*>(buffer), n);
if (read != n) {
return -1;
}
return 0;
}
int OnZSTDSeek(long long offset, int origin) {
if (origin == SEEK_SET) {
offset += static_cast<long long>(header.metadata_size) + header.header_size;
}
const bool res = curr_file->Seek(offset, origin);
return res ? 0 : -1;
}
size_t Read(void* data, std::size_t length) {
if (!m_good)
return 0;
size_t result = ZSTD_seekable_decompress(seekable, data, length, uncompressed_pos);
if (ZSTD_isError(result)) {
LOG_ERROR(Common_Filesystem, "ZSTD_seekable_decompress() error : {}",
ZSTD_getErrorName(result));
return 0;
}
uncompressed_pos += result;
return result;
}
size_t ReadAt(void* data, std::size_t length, size_t pos) {
if (!m_good)
return 0;
// ReadAt should be thread safe, but seekable compression is not,
// so we are forced to use a lock.
std::scoped_lock lock(read_mutex);
size_t result = ZSTD_seekable_decompress(seekable, data, length, pos);
if (ZSTD_isError(result)) {
LOG_ERROR(Common_Filesystem, "ZSTD_seekable_decompress() error : {}",
ZSTD_getErrorName(result));
return 0;
}
return result;
}
bool Seek(s64 off, int origin) {
s64 start = 0;
switch (origin) {
case SEEK_SET:
start = 0;
break;
case SEEK_CUR:
start = static_cast<s64>(uncompressed_pos);
break;
case SEEK_END:
start = static_cast<s64>(header.uncompressed_size);
break;
default:
return false;
}
s64 new_pos = start + off;
if (new_pos < 0)
return false;
uncompressed_pos = static_cast<u64>(new_pos);
return true;
}
void Close() {
ZSTD_seekable_free(seekable);
}
Z3DSFileHeader header{};
ZSTD_seekable* seekable = nullptr;
bool m_good = true;
IOFile* curr_file = nullptr;
std::mutex read_mutex;
u64 uncompressed_pos = 0;
Z3DSMetadata metadata;
};
std::optional<u32> Z3DSReadIOFile::GetUnderlyingFileMagic(IOFile* underlying_file) {
Z3DSFileHeader header{};
underlying_file->ReadAtBytes(&header, sizeof(header), 0);
if (header.magic != Z3DSFileHeader::EXPECTED_MAGIC ||
header.version != Z3DSFileHeader::EXPECTED_VERSION) {
return std::nullopt;
}
return MakeMagic(header.underlying_magic[0], header.underlying_magic[1],
header.underlying_magic[2], header.underlying_magic[3]);
}
Z3DSReadIOFile::Z3DSReadIOFile()
: IOFile(), file{std::make_unique<IOFile>()}, impl{std::make_unique<Z3DSReadIOFileImpl>()} {}
Z3DSReadIOFile::Z3DSReadIOFile(std::unique_ptr<IOFile>&& underlying_file)
: IOFile(), file{std::move(underlying_file)},
impl{std::make_unique<Z3DSReadIOFileImpl>(file.get())} {
ASSERT_MSG(!file->IsCompressed(), "Underlying file is already compressed!");
}
Z3DSReadIOFile::~Z3DSReadIOFile() {
this->Close();
}
bool Z3DSReadIOFile::Close() {
impl->Close();
return file->Close();
}
u64 Z3DSReadIOFile::GetSize() const {
return impl->header.uncompressed_size;
}
bool Z3DSReadIOFile::Resize(u64 size) {
// Stubbed
UNIMPLEMENTED();
return false;
}
bool Z3DSReadIOFile::Flush() {
return file->Flush();
}
void Z3DSReadIOFile::Clear() {
return file->Clear();
}
bool Z3DSReadIOFile::IsCrypto() {
return file->IsCrypto();
}
const std::string& Z3DSReadIOFile::Filename() const {
return file->Filename();
}
bool Z3DSReadIOFile::IsOpen() const {
return file->IsOpen();
}
bool Z3DSReadIOFile::IsGood() const {
return file->IsGood() && impl->m_good;
}
int Z3DSReadIOFile::GetFd() const {
return file->GetFd();
}
bool Z3DSReadIOFile::Open() {
if (is_serializing) {
return true;
}
// Stubbed
UNIMPLEMENTED();
return false;
}
std::size_t Z3DSReadIOFile::ReadImpl(void* data, std::size_t length, std::size_t data_size) {
return impl->Read(data, length * data_size);
}
std::size_t Z3DSReadIOFile::ReadAtImpl(void* data, std::size_t length, std::size_t data_size,
std::size_t offset) {
return impl->ReadAt(data, length * data_size, offset);
}
std::size_t Z3DSReadIOFile::WriteImpl(const void* data, std::size_t length, std::size_t data_size) {
// Stubbed
UNIMPLEMENTED();
return 0;
}
bool Z3DSReadIOFile::SeekImpl(s64 off, int origin) {
if (is_serializing) {
return true;
}
return impl->Seek(off, origin);
}
u64 Z3DSReadIOFile::TellImpl() const {
return impl->uncompressed_pos;
}
std::array<u8, 4> Z3DSReadIOFile::GetFileMagic() {
return impl->header.underlying_magic;
}
const Z3DSMetadata& Z3DSReadIOFile::Metadata() {
return impl->metadata;
}
template <class Archive>
void Z3DSReadIOFile::serialize(Archive& ar, const unsigned int) {
is_serializing = true;
ar& boost::serialization::base_object<IOFile>(*this);
ar & file;
if (Archive::is_loading::value) {
impl = std::make_unique<Z3DSReadIOFileImpl>(file.get(), false);
}
ar & impl->uncompressed_pos;
ar & impl->metadata;
is_serializing = false;
}
bool CompressZ3DSFile(const std::string& src_file_name, const std::string& dst_file_name,
const std::array<u8, 4>& underlying_magic, size_t frame_size,
std::function<ProgressCallback>&& update_callback) {
IOFile in_file(src_file_name, "rb");
if (!in_file.IsOpen()) {
LOG_ERROR(Common_Filesystem, "Failed to open source file: {}", src_file_name);
return false;
}
std::unique_ptr<IOFile> out_file = std::make_unique<IOFile>(dst_file_name, "wb");
if (!out_file->IsOpen()) {
LOG_ERROR(Common_Filesystem, "Failed to open destination file: {}", dst_file_name);
return false;
}
if (Z3DSReadIOFile::GetUnderlyingFileMagic(&in_file) != std::nullopt) {
LOG_ERROR(Common_Filesystem, "Source file is already compressed, nothing to do: {}",
src_file_name);
return false;
}
Z3DSWriteIOFile out_compress_file(std::move(out_file), underlying_magic, frame_size);
size_t next_chunk = out_compress_file.GetNextWriteHint();
std::vector<u8> buffer(next_chunk);
size_t in_size = in_file.GetSize();
size_t written = 0;
while (written != in_size) {
size_t to_read = ((in_size - written) > next_chunk) ? next_chunk : (in_size - written);
if (buffer.size() < to_read) {
buffer.resize(to_read);
}
if (in_file.ReadBytes(buffer.data(), to_read) != to_read) {
LOG_ERROR(Common_Filesystem, "Failed to read from source file");
return false;
}
if (out_compress_file.WriteBytes(buffer.data(), to_read) != to_read) {
LOG_ERROR(Common_Filesystem, "Failed to write to destination file");
}
written += to_read;
next_chunk = out_compress_file.GetNextWriteHint();
if (update_callback) {
update_callback(written, in_size);
}
}
LOG_INFO(Common_Filesystem, "File {} compressed successfully to {}", src_file_name,
dst_file_name);
return true;
}
bool DeCompressZ3DSFile(const std::string& src_file_name, const std::string& dst_file_name,
std::function<ProgressCallback>&& update_callback) {
std::unique_ptr<IOFile> in_file = std::make_unique<IOFile>(src_file_name, "rb");
if (!in_file->IsOpen()) {
LOG_ERROR(Common_Filesystem, "Failed to open source file: {}", src_file_name);
return false;
}
IOFile out_file(dst_file_name, "wb");
if (!out_file.IsOpen()) {
LOG_ERROR(Common_Filesystem, "Failed to open destination file: {}", dst_file_name);
return false;
}
if (Z3DSReadIOFile::GetUnderlyingFileMagic(in_file.get()) == std::nullopt) {
LOG_ERROR(Common_Filesystem,
"Source file is not compressed or is invalid, nothing to do: {}", src_file_name);
return false;
}
Z3DSReadIOFile in_compress_file(std::move(in_file));
size_t next_chunk = 64 * 1024 * 1024;
std::vector<u8> buffer(next_chunk);
size_t in_size = in_compress_file.GetSize();
size_t written = 0;
while (written != in_size) {
size_t to_read = (in_size - written) > next_chunk ? next_chunk : (in_size - written);
if (buffer.size() < to_read) {
buffer.resize(to_read);
}
if (in_compress_file.ReadBytes(buffer.data(), to_read) != to_read) {
LOG_ERROR(Common_Filesystem, "Failed to read from source file");
return false;
}
if (out_file.WriteBytes(buffer.data(), to_read) != to_read) {
LOG_ERROR(Common_Filesystem, "Failed to write to destination file");
}
written += to_read;
if (update_callback) {
update_callback(written, in_size);
}
}
LOG_INFO(Common_Filesystem, "File {} decompressed successfully to {}", src_file_name,
dst_file_name);
return true;
}
} // namespace FileUtil
SERIALIZE_EXPORT_IMPL(FileUtil::Z3DSReadIOFile);
SERIALIZE_EXPORT_IMPL(FileUtil::Z3DSWriteIOFile);

226
src/common/zstd_compression.h
View File

@ -1,3 +1,7 @@
// Copyright Citra Emulator Project / Azahar Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
// Copyright 2019 yuzu Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
@ -5,9 +9,14 @@
#pragma once
#include <span>
#include <unordered_map>
#include <vector>
#include <boost/serialization/array.hpp>
#include <boost/serialization/unordered_map.hpp>
#include "common/archives.h"
#include "common/common_types.h"
#include "common/file_util.h"
namespace Common::Compression {
@ -41,3 +50,220 @@ namespace Common::Compression {
[[nodiscard]] std::vector<u8> DecompressDataZSTD(std::span<const u8> compressed);
} // namespace Common::Compression
namespace FileUtil {
struct Z3DSFileHeader {
static constexpr std::array<u8, 4> EXPECTED_MAGIC = {'Z', '3', 'D', 'S'};
static constexpr u16 EXPECTED_VERSION = 1;
std::array<u8, 4> magic = EXPECTED_MAGIC;
std::array<u8, 4> underlying_magic{};
u16 version = EXPECTED_VERSION;
u16 header_size = 0;
u32 metadata_size = 0;
u64 compressed_size = 0;
u64 uncompressed_size = 0;
template <class Archive>
void serialize(Archive& ar, const unsigned int) {
ar & magic;
ar & underlying_magic;
ar & version;
ar & header_size;
ar & metadata_size;
ar & compressed_size;
ar & uncompressed_size;
}
};
static_assert(sizeof(Z3DSFileHeader) == 0x20, "Invalid Z3DSFileHeader size");
class Z3DSMetadata {
public:
static constexpr u8 METADATA_VERSION = 1;
Z3DSMetadata() {}
Z3DSMetadata(const std::span<u8>& source_data);
void Add(const std::string& name, const std::span<u8>& data) {
items.insert({name, std::vector<u8>(data.begin(), data.end())});
}
void Add(const std::string& name, const std::string& data) {
items.insert({name, std::vector<u8>(data.begin(), data.end())});
}
std::optional<std::vector<u8>> Get(const std::string& name) const {
auto it = items.find(name);
if (it == items.end()) {
return std::nullopt;
}
return it->second;
}
std::vector<u8> AsBinary();
private:
struct Item {
enum Type : u8 {
TYPE_END = 0,
TYPE_BINARY = 1,
};
Type type{};
u8 name_len{};
u16 data_len{};
};
static_assert(sizeof(Item) == 4);
std::unordered_map<std::string, std::vector<u8>> items;
template <class Archive>
void serialize(Archive& ar, const unsigned int) {
ar & items;
}
friend class boost::serialization::access;
};
class Z3DSWriteIOFile : public IOFile {
public:
static constexpr size_t DEFAULT_FRAME_SIZE = 256 * 1024; // 256KiB
static constexpr size_t MAX_FRAME_SIZE = 0; // Let the lib decide, usually 1GiB
Z3DSWriteIOFile();
Z3DSWriteIOFile(std::unique_ptr<IOFile>&& underlying_file,
const std::array<u8, 4>& underlying_magic, size_t frame_size);
~Z3DSWriteIOFile();
bool Close() override;
u64 GetSize() const override;
bool Resize(u64 size) override;
bool Flush() override;
void Clear() override;
bool IsCrypto() override;
bool IsCompressed() override {
return true;
}
const std::string& Filename() const override;
bool IsOpen() const override;
bool IsGood() const override;
int GetFd() const override;
Z3DSMetadata& Metadata() {
return metadata;
}
size_t GetNextWriteHint();
private:
struct Z3DSWriteIOFileImpl;
bool Open() override;
std::size_t ReadImpl(void* data, std::size_t length, std::size_t data_size) override;
std::size_t ReadAtImpl(void* data, std::size_t length, std::size_t data_size,
std::size_t offset) override;
std::size_t WriteImpl(const void* data, std::size_t length, std::size_t data_size) override;
bool SeekImpl(s64 off, int origin) override;
u64 TellImpl() const override;
std::unique_ptr<IOFile> file;
std::unique_ptr<Z3DSWriteIOFileImpl> impl;
u64 written_uncompressed = 0;
bool metadata_written = false;
Z3DSMetadata metadata;
template <class Archive>
void serialize(Archive& ar, const unsigned int);
friend class boost::serialization::access;
bool is_serializing = false;
};
class Z3DSReadIOFile : public IOFile {
public:
static std::optional<u32> GetUnderlyingFileMagic(IOFile* underlying_file);
Z3DSReadIOFile();
Z3DSReadIOFile(std::unique_ptr<IOFile>&& underlying_file);
~Z3DSReadIOFile();
bool Close() override;
u64 GetSize() const override;
bool Resize(u64 size) override;
bool Flush() override;
void Clear() override;
bool IsCrypto() override;
bool IsCompressed() override {
return true;
}
const std::string& Filename() const override;
bool IsOpen() const override;
bool IsGood() const override;
int GetFd() const override;
std::array<u8, 4> GetFileMagic();
const Z3DSMetadata& Metadata();
private:
struct Z3DSReadIOFileImpl;
static constexpr u32 MakeMagic(char a, char b, char c, char d) {
return a | b << 8 | c << 16 | d << 24;
}
bool Open() override;
std::size_t ReadImpl(void* data, std::size_t length, std::size_t data_size) override;
std::size_t ReadAtImpl(void* data, std::size_t length, std::size_t data_size,
std::size_t offset) override;
std::size_t WriteImpl(const void* data, std::size_t length, std::size_t data_size) override;
bool SeekImpl(s64 off, int origin) override;
u64 TellImpl() const override;
std::unique_ptr<IOFile> file;
std::unique_ptr<Z3DSReadIOFileImpl> impl;
template <class Archive>
void serialize(Archive& ar, const unsigned int);
friend class boost::serialization::access;
bool is_serializing = false;
};
using ProgressCallback = void(std::size_t, std::size_t);
bool CompressZ3DSFile(const std::string& src_file, const std::string& dst_file,
const std::array<u8, 4>& underlying_magic, size_t frame_size,
std::function<ProgressCallback>&& update_callback = nullptr);
bool DeCompressZ3DSFile(const std::string& src_file, const std::string& dst_file,
std::function<ProgressCallback>&& update_callback = nullptr);
} // namespace FileUtil
BOOST_CLASS_EXPORT_KEY(FileUtil::Z3DSWriteIOFile)
BOOST_CLASS_EXPORT_KEY(FileUtil::Z3DSReadIOFile)

19
src/core/file_sys/cia_container.cpp
View File

@ -59,14 +59,13 @@ Loader::ResultStatus CIAContainer::Load(const FileBackend& backend) {
return Loader::ResultStatus::Success;
}
Loader::ResultStatus CIAContainer::Load(const std::string& filepath) {
FileUtil::IOFile file(filepath, "rb");
if (!file.IsOpen())
Loader::ResultStatus CIAContainer::Load(FileUtil::IOFile* file) {
if (!file->IsOpen())
return Loader::ResultStatus::Error;
// Load CIA Header
std::vector<u8> header_data(sizeof(CIAHeader));
if (file.ReadBytes(header_data.data(), sizeof(CIAHeader)) != sizeof(CIAHeader))
if (file->ReadBytes(header_data.data(), sizeof(CIAHeader)) != sizeof(CIAHeader))
return Loader::ResultStatus::Error;
Loader::ResultStatus result = LoadHeader(header_data);
@ -75,8 +74,8 @@ Loader::ResultStatus CIAContainer::Load(const std::string& filepath) {
// Load Ticket
std::vector<u8> ticket_data(cia_header.tik_size);
file.Seek(GetTicketOffset(), SEEK_SET);
if (file.ReadBytes(ticket_data.data(), cia_header.tik_size) != cia_header.tik_size)
file->Seek(GetTicketOffset(), SEEK_SET);
if (file->ReadBytes(ticket_data.data(), cia_header.tik_size) != cia_header.tik_size)
return Loader::ResultStatus::Error;
result = LoadTicket(ticket_data);
@ -85,8 +84,8 @@ Loader::ResultStatus CIAContainer::Load(const std::string& filepath) {
// Load Title Metadata
std::vector<u8> tmd_data(cia_header.tmd_size);
file.Seek(GetTitleMetadataOffset(), SEEK_SET);
if (file.ReadBytes(tmd_data.data(), cia_header.tmd_size) != cia_header.tmd_size)
file->Seek(GetTitleMetadataOffset(), SEEK_SET);
if (file->ReadBytes(tmd_data.data(), cia_header.tmd_size) != cia_header.tmd_size)
return Loader::ResultStatus::Error;
result = LoadTitleMetadata(tmd_data);
@ -96,8 +95,8 @@ Loader::ResultStatus CIAContainer::Load(const std::string& filepath) {
// Load CIA Metadata
if (cia_header.meta_size) {
std::vector<u8> meta_data(sizeof(Metadata));
file.Seek(GetMetadataOffset(), SEEK_SET);
if (file.ReadBytes(meta_data.data(), sizeof(Metadata)) != sizeof(Metadata))
file->Seek(GetMetadataOffset(), SEEK_SET);
if (file->ReadBytes(meta_data.data(), sizeof(Metadata)) != sizeof(Metadata))
return Loader::ResultStatus::Error;
result = LoadMetadata(meta_data);

3
src/core/file_sys/cia_container.h
View File

@ -9,6 +9,7 @@
#include <span>
#include <string>
#include "common/common_types.h"
#include "common/file_util.h"
#include "common/swap.h"
#include "core/file_sys/ticket.h"
#include "core/file_sys/title_metadata.h"
@ -62,7 +63,7 @@ class CIAContainer {
public:
// Load whole CIAs outright
Loader::ResultStatus Load(const FileBackend& backend);
Loader::ResultStatus Load(const std::string& filepath);
Loader::ResultStatus Load(FileUtil::IOFile* file);
Loader::ResultStatus Load(std::span<const u8> header_data);
// Load parts of CIAs (for CIAs streamed in)

84
src/core/file_sys/ncch_container.cpp
View File

@ -10,6 +10,7 @@
#include <cryptopp/sha.h>
#include "common/common_types.h"
#include "common/logging/log.h"
#include "common/zstd_compression.h"
#include "core/core.h"
#include "core/file_sys/layered_fs.h"
#include "core/file_sys/ncch_container.h"
@ -137,6 +138,15 @@ Loader::ResultStatus NCCHContainer::LoadHeader() {
return Loader::ResultStatus::Success;
}
if (!file->IsOpen()) {
return Loader::ResultStatus::Error;
}
if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt) {
// The file is compressed
file = std::make_unique<FileUtil::Z3DSReadIOFile>(std::move(file));
}
for (int i = 0; i < 2; i++) {
if (!file->IsOpen()) {
return Loader::ResultStatus::Error;
@ -151,6 +161,7 @@ Loader::ResultStatus NCCHContainer::LoadHeader() {
// Skip NCSD header and load first NCCH (NCSD is just a container of NCCH files)...
if (Loader::MakeMagic('N', 'C', 'S', 'D') == ncch_header.magic) {
is_ncsd = true;
NCSD_Header ncsd_header;
file->Seek(ncch_offset, SEEK_SET);
file->ReadBytes(&ncsd_header, sizeof(NCSD_Header));
@ -166,9 +177,12 @@ Loader::ResultStatus NCCHContainer::LoadHeader() {
if (Loader::MakeMagic('N', 'C', 'C', 'H') != ncch_header.magic) {
// We may be loading a crypto file, try again
if (i == 0) {
file.reset();
file = HW::UniqueData::OpenUniqueCryptoFile(
filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH);
if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt) {
// The file is compressed
file = std::make_unique<FileUtil::Z3DSReadIOFile>(std::move(file));
}
} else {
return Loader::ResultStatus::ErrorInvalidFormat;
}
@ -179,6 +193,11 @@ Loader::ResultStatus NCCHContainer::LoadHeader() {
LOG_DEBUG(Service_FS, "NCCH file has console unique crypto");
}
if (!ncch_header.no_crypto) {
// Encrypted NCCH are not supported
return Loader::ResultStatus::ErrorEncrypted;
}
has_header = true;
return Loader::ResultStatus::Success;
}
@ -190,9 +209,18 @@ Loader::ResultStatus NCCHContainer::Load() {
int block_size = kBlockSize;
if (file->IsOpen()) {
size_t file_size;
if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt) {
// The file is compressed
file = std::make_unique<FileUtil::Z3DSReadIOFile>(std::move(file));
}
size_t file_size;
for (int i = 0; i < 2; i++) {
if (!file->IsOpen()) {
return Loader::ResultStatus::Error;
}
file_size = file->GetSize();
// Reset read pointer in case this file has been read before.
@ -203,6 +231,7 @@ Loader::ResultStatus NCCHContainer::Load() {
// Skip NCSD header and load first NCCH (NCSD is just a container of NCCH files)...
if (Loader::MakeMagic('N', 'C', 'S', 'D') == ncch_header.magic) {
is_ncsd = true;
NCSD_Header ncsd_header;
file->Seek(ncch_offset, SEEK_SET);
file->ReadBytes(&ncsd_header, sizeof(NCSD_Header));
@ -219,15 +248,25 @@ Loader::ResultStatus NCCHContainer::Load() {
if (i == 0) {
file = HW::UniqueData::OpenUniqueCryptoFile(
filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH);
if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) !=
std::nullopt) {
// The file is compressed
file = std::make_unique<FileUtil::Z3DSReadIOFile>(std::move(file));
}
} else {
return Loader::ResultStatus::ErrorInvalidFormat;
}
} else {
break;
}
}
if (file->IsCrypto()) {
LOG_DEBUG(Service_FS, "NCCH file has console unique crypto");
}
if (file->IsCompressed()) {
LOG_DEBUG(Service_FS, "NCCH file is compressed");
}
has_header = true;
@ -323,12 +362,7 @@ Loader::ResultStatus NCCHContainer::Load() {
if (file->ReadBytes(&exefs_header, sizeof(ExeFs_Header)) != sizeof(ExeFs_Header))
return Loader::ResultStatus::Error;
if (file->IsCrypto()) {
exefs_file = HW::UniqueData::OpenUniqueCryptoFile(
filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH);
} else {
exefs_file = std::make_unique<FileUtil::IOFile>(filepath, "rb");
}
exefs_file = Reopen(file, filepath);
has_exefs = true;
}
@ -366,12 +400,7 @@ Loader::ResultStatus NCCHContainer::LoadOverrides() {
is_tainted = true;
has_exefs = true;
} else {
if (file->IsCrypto()) {
exefs_file = HW::UniqueData::OpenUniqueCryptoFile(
filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH);
} else {
exefs_file = std::make_unique<FileUtil::IOFile>(filepath, "rb");
}
exefs_file = Reopen(file, filepath);
}
} else if (FileUtil::Exists(exefsdir_override) && FileUtil::IsDirectory(exefsdir_override)) {
is_tainted = true;
@ -607,12 +636,7 @@ Loader::ResultStatus NCCHContainer::ReadRomFS(std::shared_ptr<RomFSReader>& romf
// We reopen the file, to allow its position to be independent from file's
std::unique_ptr<FileUtil::IOFile> romfs_file_inner;
if (file->IsCrypto()) {
romfs_file_inner = HW::UniqueData::OpenUniqueCryptoFile(
filepath, "rb", HW::UniqueData::UniqueCryptoFileID::NCCH);
} else {
romfs_file_inner = std::make_unique<FileUtil::IOFile>(filepath, "rb");
}
romfs_file_inner = Reopen(file, filepath);
if (!romfs_file_inner->IsOpen())
return Loader::ResultStatus::Error;
@ -742,4 +766,24 @@ bool NCCHContainer::HasExHeader() {
return has_exheader;
}
std::unique_ptr<FileUtil::IOFile> NCCHContainer::Reopen(
const std::unique_ptr<FileUtil::IOFile>& orig_file, const std::string& new_filename) {
const bool is_compressed = orig_file->IsCompressed();
const bool is_crypto = orig_file->IsCrypto();
const std::string filename = new_filename.empty() ? orig_file->Filename() : new_filename;
std::unique_ptr<FileUtil::IOFile> out_file;
if (is_crypto) {
out_file = HW::UniqueData::OpenUniqueCryptoFile(filename, "rb",
HW::UniqueData::UniqueCryptoFileID::NCCH);
} else {
out_file = std::make_unique<FileUtil::IOFile>(filename, "rb");
}
if (is_compressed) {
out_file = std::make_unique<FileUtil::Z3DSReadIOFile>(std::move(out_file));
}
return out_file;
}
} // namespace FileSys

12
src/core/file_sys/ncch_container.h
View File

@ -333,16 +333,28 @@ public:
*/
bool HasExHeader();
bool IsNCSD() {
return is_ncsd;
}
bool IsFileCompressed() {
return file->IsCompressed();
}
NCCH_Header ncch_header;
ExeFs_Header exefs_header;
ExHeader_Header exheader_header;
private:
std::unique_ptr<FileUtil::IOFile> Reopen(const std::unique_ptr<FileUtil::IOFile>& orig_file,
const std::string& new_filename = "");
bool has_header = false;
bool has_exheader = false;
bool has_exefs = false;
bool has_romfs = false;
bool is_ncsd = false;
bool is_proto = false;
bool is_tainted = false; // Are there parts of this container being overridden?
bool is_loaded = false;

73
src/core/hle/service/am/am.cpp
View File

@ -16,6 +16,7 @@
#include "common/hacks/hack_manager.h"
#include "common/logging/log.h"
#include "common/string_util.h"
#include "common/zstd_compression.h"
#include "core/core.h"
#include "core/file_sys/certificate.h"
#include "core/file_sys/errors.h"
@ -105,6 +106,12 @@ NCCHCryptoFile::NCCHCryptoFile(const std::string& out_file, bool encrypted_conte
file = std::make_unique<FileUtil::IOFile>(out_file, "wb");
}
if (Settings::values.compress_cia_installs) {
std::array<u8, 4> magic = {'N', 'C', 'C', 'H'};
file = std::make_unique<FileUtil::Z3DSWriteIOFile>(
std::move(file), magic, FileUtil::Z3DSWriteIOFile::DEFAULT_FRAME_SIZE);
}
if (!file->IsOpen()) {
is_error = true;
}
@ -116,6 +123,7 @@ void NCCHCryptoFile::Write(const u8* buffer, std::size_t length) {
if (is_not_ncch) {
file->WriteBytes(buffer, length);
return;
}
const int kBlockSize = 0x200; ///< Size of ExeFS blocks (in bytes)
@ -1061,8 +1069,16 @@ InstallStatus InstallCIA(const std::string& path,
return InstallStatus::ErrorFileNotFound;
}
std::unique_ptr<FileUtil::IOFile> in_file = std::make_unique<FileUtil::IOFile>(path, "rb");
bool is_compressed =
FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(in_file.get()) != std::nullopt;
if (is_compressed) {
in_file = std::make_unique<FileUtil::Z3DSReadIOFile>(std::move(in_file));
}
FileSys::CIAContainer container;
if (container.Load(path) == Loader::ResultStatus::Success) {
if (container.Load(in_file.get()) == Loader::ResultStatus::Success) {
in_file->Seek(0, SEEK_SET);
Service::AM::CIAFile installFile(
Core::System::GetInstance(),
Service::AM::GetTitleMediaType(container.GetTitleMetadata().GetTitleID()));
@ -1072,18 +1088,12 @@ InstallStatus InstallCIA(const std::string& path,
return InstallStatus::ErrorEncrypted;
}
FileUtil::IOFile file(path, "rb");
if (!file.IsOpen()) {
LOG_ERROR(Service_AM, "Could not open CIA file '{}'.", path);
return InstallStatus::ErrorFailedToOpenFile;
}
std::vector<u8> buffer;
buffer.resize(0x10000);
auto file_size = file.GetSize();
auto file_size = in_file->GetSize();
std::size_t total_bytes_read = 0;
while (total_bytes_read != file_size) {
std::size_t bytes_read = file.ReadBytes(buffer.data(), buffer.size());
std::size_t bytes_read = in_file->ReadBytes(buffer.data(), buffer.size());
auto result = installFile.Write(static_cast<u64>(total_bytes_read), bytes_read, true,
false, static_cast<u8*>(buffer.data()));
@ -1128,6 +1138,51 @@ InstallStatus InstallCIA(const std::string& path,
return InstallStatus::ErrorInvalid;
}
InstallStatus CheckCIAToInstall(const std::string& path, bool& is_compressed,
bool check_encryption) {
if (!FileUtil::Exists(path)) {
LOG_ERROR(Service_AM, "File {} does not exist!", path);
return InstallStatus::ErrorFileNotFound;
}
std::unique_ptr<FileUtil::IOFile> in_file = std::make_unique<FileUtil::IOFile>(path, "rb");
is_compressed = FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(in_file.get()) != std::nullopt;
if (is_compressed) {
in_file = std::make_unique<FileUtil::Z3DSReadIOFile>(std::move(in_file));
}
FileSys::CIAContainer container;
if (container.Load(in_file.get()) == Loader::ResultStatus::Success) {
in_file->Seek(0, SEEK_SET);
const FileSys::TitleMetadata& tmd = container.GetTitleMetadata();
if (check_encryption) {
if (tmd.HasEncryptedContent(container.GetHeader())) {
return InstallStatus::ErrorEncrypted;
}
for (size_t i = 0; i < tmd.GetContentCount(); i++) {
u64 offset = container.GetContentOffset(i);
NCCH_Header ncch;
const auto read = in_file->ReadAtBytes(&ncch, sizeof(ncch), offset);
if (read != sizeof(ncch)) {
return InstallStatus::ErrorInvalid;
}
if (ncch.magic != Loader::MakeMagic('N', 'C', 'C', 'H')) {
return InstallStatus::ErrorInvalid;
}
if (!ncch.no_crypto) {
return InstallStatus::ErrorEncrypted;
}
}
}
return InstallStatus::Success;
}
return InstallStatus::ErrorInvalid;
}
u64 GetTitleUpdateId(u64 title_id) {
// Real services seem to just discard and replace the whole high word.
return (title_id & 0xFFFFFFFF) | (static_cast<u64>(TID_HIGH_UPDATE) << 32);

8
src/core/hle/service/am/am.h
View File

@ -359,6 +359,14 @@ private:
InstallStatus InstallCIA(const std::string& path,
std::function<ProgressCallback>&& update_callback = nullptr);
/**
* Checks if the provided path is a valid CIA file
* that can be installed.
* @param path file path of the CIA file to check to install
*/
InstallStatus CheckCIAToInstall(const std::string& path, bool& is_compressed,
bool check_encryption);
/**
* Get the update title ID for a title
* @param titleId the title ID

70
src/core/loader/3dsx.cpp
View File

@ -5,6 +5,7 @@
#include <algorithm>
#include <vector>
#include "common/logging/log.h"
#include "common/zstd_compression.h"
#include "core/core.h"
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/resource_limit.h"
@ -94,16 +95,16 @@ static u32 TranslateAddr(u32 addr, const THREEloadinfo* loadinfo, u32* offsets)
using Kernel::CodeSet;
static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file, u32 base_addr,
static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile* file, u32 base_addr,
std::shared_ptr<CodeSet>* out_codeset) {
if (!file.IsOpen())
if (!file->IsOpen())
return ERROR_FILE;
// Reset read pointer in case this file has been read before.
file.Seek(0, SEEK_SET);
file->Seek(0, SEEK_SET);
THREEDSX_Header hdr;
if (file.ReadBytes(&hdr, sizeof(hdr)) != sizeof(hdr))
if (file->ReadBytes(&hdr, sizeof(hdr)) != sizeof(hdr))
return ERROR_READ;
THREEloadinfo loadinfo;
@ -129,22 +130,22 @@ static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file,
loadinfo.seg_ptrs[2] = loadinfo.seg_ptrs[1] + loadinfo.seg_sizes[1];
// Skip header for future compatibility
file.Seek(hdr.header_size, SEEK_SET);
file->Seek(hdr.header_size, SEEK_SET);
// Read the relocation headers
std::vector<u32> relocs(n_reloc_tables * NUM_SEGMENTS);
for (unsigned int current_segment = 0; current_segment < NUM_SEGMENTS; ++current_segment) {
std::size_t size = n_reloc_tables * sizeof(u32);
if (file.ReadBytes(&relocs[current_segment * n_reloc_tables], size) != size)
if (file->ReadBytes(&relocs[current_segment * n_reloc_tables], size) != size)
return ERROR_READ;
}
// Read the segments
if (file.ReadBytes(loadinfo.seg_ptrs[0], hdr.code_seg_size) != hdr.code_seg_size)
if (file->ReadBytes(loadinfo.seg_ptrs[0], hdr.code_seg_size) != hdr.code_seg_size)
return ERROR_READ;
if (file.ReadBytes(loadinfo.seg_ptrs[1], hdr.rodata_seg_size) != hdr.rodata_seg_size)
if (file->ReadBytes(loadinfo.seg_ptrs[1], hdr.rodata_seg_size) != hdr.rodata_seg_size)
return ERROR_READ;
if (file.ReadBytes(loadinfo.seg_ptrs[2], hdr.data_seg_size - hdr.bss_size) !=
if (file->ReadBytes(loadinfo.seg_ptrs[2], hdr.data_seg_size - hdr.bss_size) !=
hdr.data_seg_size - hdr.bss_size)
return ERROR_READ;
@ -158,7 +159,7 @@ static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file,
u32 n_relocs = relocs[current_segment * n_reloc_tables + current_segment_reloc_table];
if (current_segment_reloc_table >= 2) {
// We are not using this table - ignore it because we don't know what it dose
file.Seek(n_relocs * sizeof(THREEDSX_Reloc), SEEK_CUR);
file->Seek(n_relocs * sizeof(THREEDSX_Reloc), SEEK_CUR);
continue;
}
THREEDSX_Reloc reloc_table[RELOCBUFSIZE];
@ -170,7 +171,7 @@ static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file,
u32 remaining = std::min(RELOCBUFSIZE, n_relocs);
n_relocs -= remaining;
if (file.ReadBytes(reloc_table, remaining * sizeof(THREEDSX_Reloc)) !=
if (file->ReadBytes(reloc_table, remaining * sizeof(THREEDSX_Reloc)) !=
remaining * sizeof(THREEDSX_Reloc))
return ERROR_READ;
@ -248,13 +249,15 @@ static THREEDSX_Error Load3DSXFile(Core::System& system, FileUtil::IOFile& file,
return ERROR_NONE;
}
FileType AppLoader_THREEDSX::IdentifyType(FileUtil::IOFile& file) {
FileType AppLoader_THREEDSX::IdentifyType(FileUtil::IOFile* file) {
u32 magic;
file.Seek(0, SEEK_SET);
if (1 != file.ReadArray<u32>(&magic, 1))
file->Seek(0, SEEK_SET);
if (1 != file->ReadArray<u32>(&magic, 1))
return FileType::Error;
if (MakeMagic('3', 'D', 'S', 'X') == magic)
if (MakeMagic('3', 'D', 'S', 'X') == magic ||
(MakeMagic('Z', '3', 'D', 'S') == magic &&
FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file) == MakeMagic('3', 'D', 'S', 'X')))
return FileType::THREEDSX;
return FileType::Error;
@ -264,11 +267,15 @@ ResultStatus AppLoader_THREEDSX::Load(std::shared_ptr<Kernel::Process>& process)
if (is_loaded)
return ResultStatus::ErrorAlreadyLoaded;
if (!file.IsOpen())
if (!file->IsOpen())
return ResultStatus::Error;
if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt) {
file = std::make_unique<FileUtil::Z3DSReadIOFile>(std::move(file));
}
std::shared_ptr<CodeSet> codeset;
if (Load3DSXFile(system, file, Memory::PROCESS_IMAGE_VADDR, &codeset) != ERROR_NONE)
if (Load3DSXFile(system, file.get(), Memory::PROCESS_IMAGE_VADDR, &codeset) != ERROR_NONE)
return ResultStatus::Error;
codeset->name = filename;
@ -292,14 +299,14 @@ ResultStatus AppLoader_THREEDSX::Load(std::shared_ptr<Kernel::Process>& process)
}
ResultStatus AppLoader_THREEDSX::ReadRomFS(std::shared_ptr<FileSys::RomFSReader>& romfs_file) {
if (!file.IsOpen())
if (!file->IsOpen())
return ResultStatus::Error;
// Reset read pointer in case this file has been read before.
file.Seek(0, SEEK_SET);
file->Seek(0, SEEK_SET);
THREEDSX_Header hdr;
if (file.ReadBytes(&hdr, sizeof(THREEDSX_Header)) != sizeof(THREEDSX_Header))
if (file->ReadBytes(&hdr, sizeof(THREEDSX_Header)) != sizeof(THREEDSX_Header))
return ResultStatus::Error;
if (hdr.header_size != sizeof(THREEDSX_Header))
@ -308,7 +315,7 @@ ResultStatus AppLoader_THREEDSX::ReadRomFS(std::shared_ptr<FileSys::RomFSReader>
// Check if the 3DSX has a RomFS...
if (hdr.fs_offset != 0) {
u32 romfs_offset = hdr.fs_offset;
u32 romfs_size = static_cast<u32>(file.GetSize()) - hdr.fs_offset;
u32 romfs_size = static_cast<u32>(file->GetSize()) - hdr.fs_offset;
LOG_DEBUG(Loader, "RomFS offset: {:#010X}", romfs_offset);
LOG_DEBUG(Loader, "RomFS size: {:#010X}", romfs_size);
@ -328,15 +335,26 @@ ResultStatus AppLoader_THREEDSX::ReadRomFS(std::shared_ptr<FileSys::RomFSReader>
return ResultStatus::ErrorNotUsed;
}
AppLoader::CompressFileInfo AppLoader_THREEDSX::GetCompressFileInfo() {
CompressFileInfo info;
info.is_supported = true;
info.recommended_compressed_extension = "z3dsx";
info.recommended_uncompressed_extension = "3dsx";
info.underlying_magic = std::array<u8, 4>({'3', 'D', 'S', 'X'});
info.is_compressed =
FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file.get()) != std::nullopt;
return info;
}
ResultStatus AppLoader_THREEDSX::ReadIcon(std::vector<u8>& buffer) {
if (!file.IsOpen())
if (!file->IsOpen())
return ResultStatus::Error;
// Reset read pointer in case this file has been read before.
file.Seek(0, SEEK_SET);
file->Seek(0, SEEK_SET);
THREEDSX_Header hdr;
if (file.ReadBytes(&hdr, sizeof(THREEDSX_Header)) != sizeof(THREEDSX_Header))
if (file->ReadBytes(&hdr, sizeof(THREEDSX_Header)) != sizeof(THREEDSX_Header))
return ResultStatus::Error;
if (hdr.header_size != sizeof(THREEDSX_Header))
@ -344,10 +362,10 @@ ResultStatus AppLoader_THREEDSX::ReadIcon(std::vector<u8>& buffer) {
// Check if the 3DSX has a SMDH...
if (hdr.smdh_offset != 0) {
file.Seek(hdr.smdh_offset, SEEK_SET);
file->Seek(hdr.smdh_offset, SEEK_SET);
buffer.resize(hdr.smdh_size);
if (file.ReadBytes(buffer.data(), hdr.smdh_size) != hdr.smdh_size)
if (file->ReadBytes(buffer.data(), hdr.smdh_size) != hdr.smdh_size)
return ResultStatus::Error;
return ResultStatus::Success;

10
src/core/loader/3dsx.h
View File

@ -1,3 +1,7 @@
// Copyright Citra Emulator Project / Azahar Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
// Copyright 2014 Dolphin Emulator Project / Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
@ -23,10 +27,10 @@ public:
* @param file FileUtil::IOFile open file
* @return FileType found, or FileType::Error if this loader doesn't know it
*/
static FileType IdentifyType(FileUtil::IOFile& file);
static FileType IdentifyType(FileUtil::IOFile* file);
FileType GetFileType() override {
return IdentifyType(file);
return IdentifyType(file.get());
}
ResultStatus Load(std::shared_ptr<Kernel::Process>& process) override;
@ -35,6 +39,8 @@ public:
ResultStatus ReadRomFS(std::shared_ptr<FileSys::RomFSReader>& romfs_file) override;
CompressFileInfo GetCompressFileInfo() override;
private:
std::string filename;
std::string filepath;

2
src/core/loader/artic.cpp
View File

@ -80,7 +80,7 @@ Apploader_Artic::~Apploader_Artic() {
client->Stop();
}
FileType Apploader_Artic::IdentifyType(FileUtil::IOFile& file) {
FileType Apploader_Artic::IdentifyType(FileUtil::IOFile* file) {
return FileType::ARTIC;
}

4
src/core/loader/artic.h
View File

@ -33,10 +33,10 @@ public:
* @param file FileUtil::IOFile open file
* @return FileType found, or FileType::Error if this loader doesn't know it
*/
static FileType IdentifyType(FileUtil::IOFile& file);
static FileType IdentifyType(FileUtil::IOFile* file);
FileType GetFileType() override {
return IdentifyType(file);
return IdentifyType(file.get());
}
[[nodiscard]] std::span<const u32> GetPreferredRegions() const override {

18
src/core/loader/elf.cpp
View File

@ -1,3 +1,7 @@
// Copyright Citra Emulator Project / Azahar Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
// Copyright 2013 Dolphin Emulator Project / 2014 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
@ -352,10 +356,10 @@ SectionID ElfReader::GetSectionByName(const char* name, int firstSection) const
namespace Loader {
FileType AppLoader_ELF::IdentifyType(FileUtil::IOFile& file) {
FileType AppLoader_ELF::IdentifyType(FileUtil::IOFile* file) {
u32 magic;
file.Seek(0, SEEK_SET);
if (1 != file.ReadArray<u32>(&magic, 1))
file->Seek(0, SEEK_SET);
if (1 != file->ReadArray<u32>(&magic, 1))
return FileType::Error;
if (MakeMagic('\x7f', 'E', 'L', 'F') == magic)
@ -368,15 +372,15 @@ ResultStatus AppLoader_ELF::Load(std::shared_ptr<Kernel::Process>& process) {
if (is_loaded)
return ResultStatus::ErrorAlreadyLoaded;
if (!file.IsOpen())
if (!file->IsOpen())
return ResultStatus::Error;
// Reset read pointer in case this file has been read before.
file.Seek(0, SEEK_SET);
file->Seek(0, SEEK_SET);
std::size_t size = file.GetSize();
std::size_t size = file->GetSize();
std::unique_ptr<u8[]> buffer(new u8[size]);
if (file.ReadBytes(&buffer[0], size) != size)
if (file->ReadBytes(&buffer[0], size) != size)
return ResultStatus::Error;
ElfReader elf_reader(&buffer[0]);

8
src/core/loader/elf.h
View File

@ -1,3 +1,7 @@
// Copyright Citra Emulator Project / Azahar Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
// Copyright 2013 Dolphin Emulator Project / 2014 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
@ -22,10 +26,10 @@ public:
* @param file FileUtil::IOFile open file
* @return FileType found, or FileType::Error if this loader doesn't know it
*/
static FileType IdentifyType(FileUtil::IOFile& file);
static FileType IdentifyType(FileUtil::IOFile* file);
FileType GetFileType() override {
return IdentifyType(file);
return IdentifyType(file.get());
}
ResultStatus Load(std::shared_ptr<Kernel::Process>& process) override;

10
src/core/loader/loader.cpp
View File

@ -19,7 +19,7 @@ FileType IdentifyFile(FileUtil::IOFile& file) {
FileType type;
#define CHECK_TYPE(loader) \
type = AppLoader_##loader::IdentifyType(file); \
type = AppLoader_##loader::IdentifyType(&file); \
if (FileType::Error != type) \
return type;
@ -48,16 +48,16 @@ FileType GuessFromExtension(const std::string& extension_) {
if (extension == ".elf" || extension == ".axf")
return FileType::ELF;
if (extension == ".cci")
if (extension == ".cci" || extension == ".zcci")
return FileType::CCI;
if (extension == ".cxi" || extension == ".app")
if (extension == ".cxi" || extension == ".app" || extension == ".zcxi")
return FileType::CXI;
if (extension == ".3dsx")
if (extension == ".3dsx" || extension == ".z3dsx")
return FileType::THREEDSX;
if (extension == ".cia")
if (extension == ".cia" || extension == ".zcia")
return FileType::CIA;
return FileType::Unknown;

18
src/core/loader/loader.h
View File

@ -85,8 +85,16 @@ constexpr u32 MakeMagic(char a, char b, char c, char d) {
/// Interface for loading an application
class AppLoader : NonCopyable {
public:
struct CompressFileInfo {
bool is_supported{};
bool is_compressed{};
std::array<u8, 4> underlying_magic{};
std::string recommended_compressed_extension;
std::string recommended_uncompressed_extension;
};
explicit AppLoader(Core::System& system_, FileUtil::IOFile&& file)
: system(system_), file(std::move(file)) {}
: system(system_), file(std::make_unique<FileUtil::IOFile>(std::move(file))) {}
virtual ~AppLoader() {}
/**
@ -279,9 +287,15 @@ public:
return false;
}
virtual CompressFileInfo GetCompressFileInfo() {
CompressFileInfo info{};
info.is_supported = false;
return info;
}
protected:
Core::System& system;
FileUtil::IOFile file;
std::unique_ptr<FileUtil::IOFile> file;
bool is_loaded = false;
std::optional<Kernel::MemoryMode> memory_mode_override = std::nullopt;
};

39
src/core/loader/ncch.cpp
View File

@ -12,6 +12,7 @@
#include "common/settings.h"
#include "common/string_util.h"
#include "common/swap.h"
#include "common/zstd_compression.h"
#include "core/core.h"
#include "core/file_sys/ncch_container.h"
#include "core/file_sys/title_metadata.h"
@ -34,10 +35,10 @@ namespace Loader {
using namespace Common::Literals;
static constexpr u64 UPDATE_TID_HIGH = 0x0004000e00000000;
FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile& file) {
FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile* file) {
u32 magic;
file.Seek(0x100, SEEK_SET);
if (1 != file.ReadArray<u32>(&magic, 1))
file->Seek(0x100, SEEK_SET);
if (1 != file->ReadArray<u32>(&magic, 1))
return FileType::Error;
if (MakeMagic('N', 'C', 'S', 'D') == magic)
@ -47,7 +48,7 @@ FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile& file) {
return FileType::CXI;
std::unique_ptr<FileUtil::IOFile> file_crypto = HW::UniqueData::OpenUniqueCryptoFile(
file.Filename(), "rb", HW::UniqueData::UniqueCryptoFileID::NCCH);
file->Filename(), "rb", HW::UniqueData::UniqueCryptoFileID::NCCH);
file_crypto->Seek(0x100, SEEK_SET);
if (1 != file_crypto->ReadArray<u32>(&magic, 1))
@ -59,6 +60,16 @@ FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile& file) {
if (MakeMagic('N', 'C', 'C', 'H') == magic)
return FileType::CXI;
std::optional<u32> magic_zstd;
if (FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file) != std::nullopt ||
FileUtil::Z3DSReadIOFile::GetUnderlyingFileMagic(file_crypto.get()) != std::nullopt) {
if (MakeMagic('N', 'C', 'S', 'D') == magic_zstd)
return FileType::CCI;
if (MakeMagic('N', 'C', 'C', 'H') == magic_zstd)
return FileType::CXI;
}
return FileType::Error;
}
@ -396,4 +407,24 @@ ResultStatus AppLoader_NCCH::ReadTitle(std::string& title) {
return ResultStatus::Success;
}
AppLoader::CompressFileInfo AppLoader_NCCH::GetCompressFileInfo() {
CompressFileInfo info{};
if (base_ncch.LoadHeader() != ResultStatus::Success) {
info.is_supported = false;
return info;
}
info.is_supported = true;
info.is_compressed = base_ncch.IsFileCompressed();
if (base_ncch.IsNCSD()) {
info.underlying_magic = std::array<u8, 4>({'N', 'C', 'S', 'D'});
info.recommended_compressed_extension = "zcci";
info.recommended_uncompressed_extension = "cci";
} else {
info.underlying_magic = std::array<u8, 4>({'N', 'C', 'C', 'H'});
info.recommended_compressed_extension = "zcxi";
info.recommended_uncompressed_extension = "cxi";
}
return info;
}
} // namespace Loader

14
src/core/loader/ncch.h
View File

@ -1,4 +1,4 @@
// Copyright 2014 Citra Emulator Project
// Copyright Citra Emulator Project / Azahar Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
@ -17,17 +17,20 @@ class AppLoader_NCCH final : public AppLoader {
public:
AppLoader_NCCH(Core::System& system_, FileUtil::IOFile&& file, const std::string& filepath)
: AppLoader(system_, std::move(file)), base_ncch(filepath), overlay_ncch(&base_ncch),
filepath(filepath) {}
filepath(filepath) {
filetype = IdentifyType(this->file.get());
this->file.reset();
}
/**
* Returns the type of the file
* @param file FileUtil::IOFile open file
* @return FileType found, or FileType::Error if this loader doesn't know it
*/
static FileType IdentifyType(FileUtil::IOFile& file);
static FileType IdentifyType(FileUtil::IOFile* file);
FileType GetFileType() override {
return IdentifyType(file);
return filetype;
}
[[nodiscard]] std::span<const u32> GetPreferredRegions() const override {
@ -71,6 +74,8 @@ public:
ResultStatus ReadTitle(std::string& title) override;
CompressFileInfo GetCompressFileInfo() override;
private:
/**
* Loads .code section into memory for booting
@ -94,6 +99,7 @@ private:
std::vector<u32> preferred_regions;
std::string filepath;
FileType filetype;
};
} // namespace Loader