Various savestate bugfixes

src/android/app/src/main/java/org/citra/citra_emu/features/settings/model/BooleanSetting.kt

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -15,7 +15,9 @@ enum class BooleanSetting(
     ALLOW_PLUGIN_LOADER("allow_plugin_loader", Settings.SECTION_SYSTEM, true),
     SWAP_SCREEN("swap_screen", Settings.SECTION_LAYOUT, false),
     INSTANT_DEBUG_LOG("instant_debug_log", Settings.SECTION_DEBUG, false),
-    CUSTOM_LAYOUT("custom_layout",Settings.SECTION_LAYOUT,false);
+    CUSTOM_LAYOUT("custom_layout",Settings.SECTION_LAYOUT,false),
+    DELAY_START_LLE_MODULES("delay_start_for_lle_modules", Settings.SECTION_DEBUG, true),
+    DETERMINISTIC_ASYNC_OPERATIONS("deterministic_async_operations", Settings.SECTION_DEBUG, false);
 
     override var boolean: Boolean = defaultValue
 
@@ -36,7 +38,9 @@ enum class BooleanSetting(
         private val NOT_RUNTIME_EDITABLE = listOf(
             PLUGIN_LOADER,
             ALLOW_PLUGIN_LOADER, 
-            ASYNC_SHADERS
+            ASYNC_SHADERS,
+            DELAY_START_LLE_MODULES,
+            DETERMINISTIC_ASYNC_OPERATIONS,
         )
 
         fun from(key: String): BooleanSetting? =

src/android/app/src/main/java/org/citra/citra_emu/features/settings/ui/SettingsFragmentPresenter.kt

@@ -1360,6 +1360,25 @@ class SettingsFragmentPresenter(private val fragmentView: SettingsFragmentView)
                     BooleanSetting.INSTANT_DEBUG_LOG.defaultValue
                 )
             )
+            add(
+                SwitchSetting(
+                    BooleanSetting.DELAY_START_LLE_MODULES,
+                    R.string.delay_start_lle_modules,
+                    R.string.delay_start_lle_modules_desc,
+                    BooleanSetting.DELAY_START_LLE_MODULES.key,
+                    BooleanSetting.DELAY_START_LLE_MODULES.defaultValue
+                )
+            )
+            add(
+                SwitchSetting(
+                    BooleanSetting.DETERMINISTIC_ASYNC_OPERATIONS,
+                    R.string.deterministic_async_operations,
+                    R.string.deterministic_async_operations_desc,
+                    BooleanSetting.DETERMINISTIC_ASYNC_OPERATIONS.key,
+                    BooleanSetting.DETERMINISTIC_ASYNC_OPERATIONS.defaultValue
+                )
+            )
+
         }
     }
 

src/android/app/src/main/jni/default_ini.h

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -413,6 +413,15 @@ gdbstub_port=24689
 # Immediately commits the debug log to file. Use this if Azahar crashes and the log output is being cut.
 instant_debug_log =
 
+# Delay the start of apps when LLE modules are enabled
+# 0: Off, 1 (default): On
+delay_start_for_lle_modules =
+
+# Force deterministic async operations
+# Only needed for debugging, makes performance worse if enabled
+# 0: Off (default), 1: On
+deterministic_async_operations =
+
 # To LLE a service module add "LLE\<module name>=true"
 
 [WebService]

src/android/app/src/main/res/values/strings.xml

@@ -781,5 +781,9 @@
     <string name="instant_debug_log_desc">Immediately commits the debug log to file. Use this if Azahar crashes and the log output is being cut.</string>
     <string name="disable_right_eye_render">Disable Right Eye Render</string>
     <string name="disable_right_eye_render_description">Greatly improves performance in some applications, but can cause flickering in others.</string>
+    <string name="delay_start_lle_modules">Delay start with LLE modules</string>
+    <string name="delay_start_lle_modules_desc">Delays the start of the app when LLE modules are enabled</string>
+    <string name="deterministic_async_operations">Deterministic Async Operations</string>
+    <string name="deterministic_async_operations_desc">Makes async operations deterministic for debugging. Enabling this may cause freezes.</string>
 
 </resources>

src/citra_qt/configuration/configure_debug.cpp

@@ -102,6 +102,8 @@ void ConfigureDebug::SetConfiguration() {
     ui->toggle_cpu_jit->setChecked(Settings::values.use_cpu_jit.GetValue());
     ui->delay_start_for_lle_modules->setChecked(
         Settings::values.delay_start_for_lle_modules.GetValue());
+    ui->deterministic_async_operations->setChecked(
+        Settings::values.deterministic_async_operations.GetValue());
     ui->toggle_renderer_debug->setChecked(Settings::values.renderer_debug.GetValue());
     ui->toggle_dump_command_buffers->setChecked(Settings::values.dump_command_buffers.GetValue());
 
@@ -137,6 +139,8 @@ void ConfigureDebug::ApplyConfiguration() {
     Common::Log::SetRegexFilter(Settings::values.log_regex_filter.GetValue());
     Settings::values.use_cpu_jit = ui->toggle_cpu_jit->isChecked();
     Settings::values.delay_start_for_lle_modules = ui->delay_start_for_lle_modules->isChecked();
+    Settings::values.deterministic_async_operations =
+        ui->deterministic_async_operations->isChecked();
     Settings::values.renderer_debug = ui->toggle_renderer_debug->isChecked();
     Settings::values.dump_command_buffers = ui->toggle_dump_command_buffers->isChecked();
     Settings::values.instant_debug_log = ui->instant_debug_log->isChecked();

src/citra_qt/configuration/configure_debug.ui

@@ -259,6 +259,16 @@
               </property>
             </widget>
           </item>
+          <item row="2" column="0">
+            <widget class="QCheckBox" name="deterministic_async_operations">
+              <property name="text">
+                <string>Force deterministic async operations</string>
+              </property>
+              <property name="toolTip">
+                <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Forces all async operations to run on the main thread, making them deterministic. Do not enable if you don't know what you are doing.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+              </property>
+            </widget>
+          </item>
         </layout>
       </widget>
     </item>

src/citra_sdl/config.cpp

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -328,6 +328,8 @@ void SdlConfig::ReadValues() {
     // Miscellaneous
     ReadSetting("Miscellaneous", Settings::values.log_filter);
     ReadSetting("Miscellaneous", Settings::values.log_regex_filter);
+    ReadSetting("Miscellaneous", Settings::values.delay_start_for_lle_modules);
+    ReadSetting("Miscellaneous", Settings::values.deterministic_async_operations);
 
     // Apply the log_filter setting as the logger has already been initialized
     // and doesn't pick up the filter on its own.

src/common/archives.h

@@ -1,4 +1,4 @@
-// Copyright 2020 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -19,3 +19,8 @@ using oarchive = boost::archive::binary_oarchive;
     BOOST_CLASS_EXPORT_IMPLEMENT(A)                                                                \
     BOOST_SERIALIZATION_REGISTER_ARCHIVE(iarchive)                                                 \
     BOOST_SERIALIZATION_REGISTER_ARCHIVE(oarchive)
+
+#define DEBUG_SERIALIZATION_POINT                                                                  \
+    do {                                                                                           \
+        LOG_DEBUG(Savestate, "");                                                                  \
+    } while (0)

src/common/logging/filter.cpp

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -74,6 +74,7 @@ bool ParseFilterRule(Filter& instance, Iterator begin, Iterator end) {
     SUB(Core, Timing)                                                                              \
     SUB(Core, Cheats)                                                                              \
     CLS(Config)                                                                                    \
+    CLS(Savestate)                                                                                 \
     CLS(Debug)                                                                                     \
     SUB(Debug, Emulated)                                                                           \
     SUB(Debug, GPU)                                                                                \

src/common/logging/types.h

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -39,6 +39,7 @@ enum class Class : u8 {
     Core_Timing,       ///< CoreTiming functions
     Core_Cheats,       ///< Cheat functions
     Config,            ///< Emulator configuration
+    Savestate,         ///< Savestates
     Debug,             ///< Debugging tools
     Debug_Emulated,    ///< Debug messages from the emulated programs
     Debug_GPU,         ///< GPU debugging tools

src/common/settings.h

@@ -451,6 +451,7 @@ struct Values {
     SwitchableSetting<s32, true> cpu_clock_percentage{100, 5, 400, "cpu_clock_percentage"};
     SwitchableSetting<bool> is_new_3ds{true, "is_new_3ds"};
     SwitchableSetting<bool> lle_applets{false, "lle_applets"};
+    SwitchableSetting<bool> deterministic_async_operations{false, "deterministic_async_operations"};
 
     // Data Storage
     Setting<bool> use_virtual_sd{true, "use_virtual_sd"};

src/core/core.cpp

@@ -121,7 +121,35 @@ System::ResultStatus System::RunLoop(bool tight_loop) {
     case Signal::Shutdown:
         return ResultStatus::ShutdownRequested;
     case Signal::Load: {
-        const u32 slot = param;
+        if (save_state_request_status != SaveStateStatus::NONE) {
+            LOG_ERROR(Core, "A pending save state operation has not finished yet");
+            status_details = "A pending save state operation has not finished yet";
+            return ResultStatus::ErrorSavestate;
+        }
+        save_state_slot = param;
+        save_state_request_time = std::chrono::steady_clock::now();
+        save_state_request_status = SaveStateStatus::LOADING;
+        break;
+    }
+    case Signal::Save: {
+        if (save_state_request_status != SaveStateStatus::NONE) {
+            LOG_ERROR(Core, "A pending save state operation has not finished yet");
+            status_details = "A pending save state operation has not finished yet";
+            return ResultStatus::ErrorSavestate;
+        }
+        save_state_slot = param;
+        save_state_request_time = std::chrono::steady_clock::now();
+        save_state_request_status = SaveStateStatus::SAVING;
+        break;
+    }
+    default:
+        break;
+    }
+
+    if (save_state_request_status == SaveStateStatus::LOADING && kernel.get() &&
+        !kernel->AreAsyncOperationsPending()) {
+        const u32 slot = save_state_slot;
+        save_state_request_status = SaveStateStatus::NONE;
         LOG_INFO(Core, "Begin load of slot {}", slot);
         try {
             System::LoadState(slot);
@@ -133,9 +161,10 @@ System::ResultStatus System::RunLoop(bool tight_loop) {
         }
         frame_limiter.WaitOnce();
         return ResultStatus::Success;
-    }
+    } else if (save_state_request_status == SaveStateStatus::SAVING && kernel.get() &&
-    case Signal::Save: {
+               !kernel->AreAsyncOperationsPending()) {
-        const u32 slot = param;
+        save_state_request_status = SaveStateStatus::NONE;
+        const u32 slot = save_state_slot;
         LOG_INFO(Core, "Begin save to slot {}", slot);
         try {
             System::SaveState(slot);
@@ -147,9 +176,13 @@ System::ResultStatus System::RunLoop(bool tight_loop) {
         }
         frame_limiter.WaitOnce();
         return ResultStatus::Success;
-    }
+    } else if (save_state_request_status != SaveStateStatus::NONE &&
-    default:
+               (std::chrono::steady_clock::now() - save_state_request_time) >
-        break;
+                   std::chrono::seconds(5)) {
+        save_state_request_status = SaveStateStatus::NONE;
+        LOG_ERROR(Core, "Cannot perform save state operation due to pending async operations");
+        status_details = "Cannot perform save state operation due to pending async operations";
+        return ResultStatus::ErrorSavestate;
     }
 
     // All cores should have executed the same amount of ticks. If this is not the case an event was
@@ -476,7 +509,7 @@ System::ResultStatus System::Init(Frontend::EmuWindow& emu_window,
     archive_manager = std::make_unique<Service::FS::ArchiveManager>(*this);
 
     HW::AES::InitKeys();
-    Service::Init(*this, !app_loader->DoingInitialSetup());
+    Service::Init(*this, lle_modules, !app_loader->DoingInitialSetup());
     GDBStub::DeferStart();
 
     if (!registered_image_interface) {
@@ -598,6 +631,7 @@ void System::Shutdown(bool is_deserializing) {
 
     gpu.reset();
     if (!is_deserializing) {
+        lle_modules.clear();
         GDBStub::Shutdown();
         perf_stats.reset();
         app_loader.reset();
@@ -720,12 +754,25 @@ bool System::IsInitialSetup() {
 template <class Archive>
 void System::serialize(Archive& ar, const unsigned int file_version) {
 
+    if (Archive::is_loading::value) {
+        save_state_status = SaveStateStatus::LOADING;
+    } else {
+        save_state_status = SaveStateStatus::SAVING;
+    }
+
     u32 num_cores;
     if (Archive::is_saving::value) {
         num_cores = this->GetNumCores();
     }
     ar & num_cores;
 
+    // TODO(PabloMK7): Figure out why this is the case
+    if (!lle_modules.empty()) {
+        throw std::runtime_error("Savestates are not supported with LLE modules enabled");
+    }
+
+    ar & lle_modules;
+
     if (Archive::is_loading::value) {
         // When loading, we want to make sure any lingering state gets cleared out before we begin.
         // Shutdown, but persist a few things between loads...
@@ -775,6 +822,8 @@ void System::serialize(Archive& ar, const unsigned int file_version) {
         gpu->SetInterruptHandler(
             [gsp](Service::GSP::InterruptId interrupt_id) { gsp->SignalInterrupt(interrupt_id); });
     }
+
+    save_state_status = SaveStateStatus::NONE;
 }
 
 SERIALIZE_IMPL(System)

src/core/core.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <atomic>
+#include <chrono>
 #include <memory>
 #include <mutex>
 #include <string>
@@ -345,6 +346,16 @@ public:
                (mic_permission_granted = mic_permission_func());
     }
 
+    enum class SaveStateStatus {
+        NONE,
+        LOADING,
+        SAVING,
+    };
+
+    SaveStateStatus GetSaveStateStatus() {
+        return save_state_status;
+    }
+
     void SaveState(u32 slot) const;
 
     void LoadState(u32 slot);
@@ -440,6 +451,11 @@ private:
 
     std::atomic_bool is_powered_on{};
 
+    SaveStateStatus save_state_status = SaveStateStatus::NONE;
+    SaveStateStatus save_state_request_status = SaveStateStatus::NONE;
+    u32 save_state_slot = 0;
+    std::chrono::steady_clock::time_point save_state_request_time{};
+
     ResultStatus status = ResultStatus::Success;
     std::string status_details = "";
     /// Saved variables for reset
@@ -460,6 +476,8 @@ private:
     boost::optional<Service::APT::DeliverArg> restore_deliver_arg;
     boost::optional<Service::PLGLDR::PLG_LDR::PluginLoaderContext> restore_plugin_context;
 
+    std::vector<u64> lle_modules;
+
     friend class boost::serialization::access;
     template <typename Archive>
     void serialize(Archive& ar, const unsigned int file_version);

src/core/file_sys/archive_extsavedata.cpp

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -54,7 +54,14 @@ public:
     }
 
 private:
+    FixSizeDiskFile() : DiskFile() {};
     u64 size{};
+    template <class Archive>
+    void serialize(Archive& ar, const unsigned int) {
+        ar& boost::serialization::base_object<DiskFile>(*this);
+        ar & size;
+    }
+    friend class boost::serialization::access;
 };
 
 class ExtSaveDataDelayGenerator : public DelayGenerator {
@@ -424,5 +431,6 @@ ResultVal<ArchiveFormatInfo> ArchiveFactory_ExtSaveData::GetFormatInfo(const Pat
 }
 } // namespace FileSys
 
+BOOST_CLASS_EXPORT(FileSys::FixSizeDiskFile)
 SERIALIZE_EXPORT_IMPL(FileSys::ExtSaveDataDelayGenerator)
 SERIALIZE_EXPORT_IMPL(FileSys::ExtSaveDataArchive)

src/core/file_sys/disk_archive.h

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -44,9 +44,9 @@ protected:
     Mode mode;
     std::unique_ptr<FileUtil::IOFile> file;
 
-private:
     DiskFile() = default;
 
+private:
     template <class Archive>
     void serialize(Archive& ar, const unsigned int) {
         ar& boost::serialization::base_object<FileBackend>(*this);

src/core/hle/kernel/hle_ipc.cpp

@@ -1,4 +1,4 @@
-// Copyright 2017 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -56,6 +56,10 @@ public:
                           cmd_buff.size() * sizeof(u32));
     }
 
+    bool SupportsSerialization() {
+        return !callback.get() || callback->SupportsSerialization();
+    }
+
 private:
     ThreadCallback() = default;
     std::shared_ptr<HLERequestContext::WakeupCallback> callback{};

src/core/hle/kernel/hle_ipc.h

@@ -1,4 +1,4 @@
-// Copyright 2017 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -15,6 +15,7 @@
 #include <boost/serialization/export.hpp>
 #include "common/common_types.h"
 #include "common/serialization/boost_small_vector.hpp"
+#include "common/settings.h"
 #include "common/swap.h"
 #include "core/hle/ipc.h"
 #include "core/hle/kernel/object.h"
@@ -231,6 +232,9 @@ public:
         virtual ~WakeupCallback() = default;
         virtual void WakeUp(std::shared_ptr<Thread> thread, HLERequestContext& context,
                             ThreadWakeupReason reason) = 0;
+        virtual bool SupportsSerialization() {
+            return true;
+        }
 
     private:
         template <class Archive>
@@ -257,28 +261,26 @@ private:
     template <typename ResultFunctor>
     class AsyncWakeUpCallback : public WakeupCallback {
     public:
-        explicit AsyncWakeUpCallback(ResultFunctor res_functor, std::future<void> fut)
+        explicit AsyncWakeUpCallback(KernelSystem& kernel, ResultFunctor res_functor,
-            : functor(res_functor) {
+                                     std::future<void> fut)
+            : kernel(kernel), functor(res_functor) {
             future = std::move(fut);
         }
 
         void WakeUp(std::shared_ptr<Kernel::Thread> thread, Kernel::HLERequestContext& ctx,
-                    Kernel::ThreadWakeupReason reason) {
+                    Kernel::ThreadWakeupReason reason) override {
             functor(ctx);
+            kernel.ReportAsyncState(false);
+        }
+
+        bool SupportsSerialization() override {
+            return false;
         }
 
     private:
+        KernelSystem& kernel;
         ResultFunctor functor;
         std::future<void> future;
-
-        template <class Archive>
-        void serialize(Archive& ar, const unsigned int) {
-            if (!Archive::is_loading::value && future.valid()) {
-                future.wait();
-            }
-            ar & functor;
-        }
-        friend class boost::serialization::access;
     };
 
 public:
@@ -300,11 +302,12 @@ public:
     void RunAsync(AsyncFunctor async_section, ResultFunctor result_function,
                   bool really_async = true) {
 
-        if (really_async) {
+        if (!Settings::values.deterministic_async_operations && really_async) {
+            kernel.ReportAsyncState(true);
             this->SleepClientThread(
                 "RunAsync", std::chrono::nanoseconds(-1),
                 std::make_shared<AsyncWakeUpCallback<ResultFunctor>>(
-                    result_function,
+                    kernel, result_function,
                     std::move(std::async(std::launch::async, [this, async_section] {
                         s64 sleep_for = async_section(*this);
                         this->thread->WakeAfterDelay(sleep_for, true);
@@ -313,8 +316,9 @@ public:
         } else {
             s64 sleep_for = async_section(*this);
             if (sleep_for > 0) {
+                kernel.ReportAsyncState(true);
                 auto parallel_wakeup = std::make_shared<AsyncWakeUpCallback<ResultFunctor>>(
-                    result_function, std::move(std::future<void>()));
+                    kernel, result_function, std::move(std::future<void>()));
                 this->SleepClientThread("RunAsync", std::chrono::nanoseconds(sleep_for),
                                         parallel_wakeup);
             } else {

src/core/hle/kernel/kernel.h

@@ -1,4 +1,8 @@
-// Copyright 2014 Citra Emulator Project / PPSSPP Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+// PPSSPP Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -346,6 +350,18 @@ public:
         return main_thread_extended_sleep;
     }
 
+    void ReportAsyncState(bool state) {
+        if (state) {
+            pending_async_operations++;
+        } else {
+            pending_async_operations--;
+        }
+    }
+
+    bool AreAsyncOperationsPending() {
+        return pending_async_operations != 0;
+    }
+
 private:
     void MemoryInit(MemoryMode memory_mode, New3dsMemoryMode n3ds_mode, u64 override_init_time);
 
@@ -354,6 +370,8 @@ private:
     std::unique_ptr<ResourceLimitList> resource_limits;
     std::atomic<u32> next_object_id{0};
 
+    std::atomic<int> pending_async_operations{};
+
     // Note: keep the member order below in order to perform correct destruction.
     // Thread manager is destructed before process list in order to Stop threads and clear thread
     // info from their parent processes first. Timer manager is destructed after process list

src/core/hle/kernel/thread.cpp

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project / PPSSPP Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -59,6 +59,19 @@ void Thread::serialize(Archive& ar, const unsigned int file_version) {
     ar & wait_objects;
     ar & wait_address;
     ar & name;
+    if (Archive::is_loading::value) {
+        bool serialize_blocked;
+        ar & serialize_blocked;
+        if (!serialize_blocked) {
+            ar & wakeup_callback;
+        }
+    } else {
+        bool serialize_blocked = wakeup_callback.get() && !wakeup_callback->SupportsSerialization();
+        ar & serialize_blocked;
+        if (!serialize_blocked) {
+            ar & wakeup_callback;
+        }
+    }
     ar & wakeup_callback;
 }
 SERIALIZE_IMPL(Thread)

src/core/hle/kernel/thread.h

@@ -1,4 +1,8 @@
-// Copyright 2014 Citra Emulator Project / PPSSPP Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+// PPSSPP Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -67,6 +71,10 @@ public:
     virtual void WakeUp(ThreadWakeupReason reason, std::shared_ptr<Thread> thread,
                         std::shared_ptr<WaitObject> object) = 0;
 
+    virtual bool SupportsSerialization() {
+        return true;
+    }
+
 private:
     template <class Archive>
     void serialize(Archive& ar, const unsigned int);

src/core/hle/service/ac/ac.cpp

@@ -1,4 +1,4 @@
-// Copyright 2016 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -206,6 +206,7 @@ Module::Module(Core::System& system_) : system(system_) {}
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar & ac_connected;
     ar & close_event;
     ar & connect_event;

src/core/hle/service/act/act.cpp

@@ -1,7 +1,8 @@
-// Copyright 2016 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include "common/archives.h"
 #include "core/core.h"
 #include "core/hle/ipc_helpers.h"
 #include "core/hle/kernel/shared_memory.h"
@@ -10,8 +11,13 @@
 #include "core/hle/service/act/act_errors.h"
 #include "core/hle/service/act/act_u.h"
 
+SERIALIZE_EXPORT_IMPL(Service::ACT::Module)
+SERVICE_CONSTRUCT_IMPL(Service::ACT::Module)
+
 namespace Service::ACT {
 
+Module::Module(Core::System& system_) : system(system_) {}
+
 Module::Interface::Interface(std::shared_ptr<Module> act, const char* name)
     : ServiceFramework(name, 3), act(std::move(act)) {}
 
@@ -59,9 +65,15 @@ void Module::Interface::GetAccountInfo(Kernel::HLERequestContext& ctx) {
     rb.Push(ResultSuccess);
 }
 
+template <class Archive>
+void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
+}
+SERIALIZE_IMPL(Module)
+
 void InstallInterfaces(Core::System& system) {
     auto& service_manager = system.ServiceManager();
-    auto act = std::make_shared<Module>();
+    auto act = std::make_shared<Module>(system);
     std::make_shared<ACT_A>(act)->InstallAsService(service_manager);
     std::make_shared<ACT_U>(act)->InstallAsService(service_manager);
 }

src/core/hle/service/act/act.h

@@ -1,4 +1,4 @@
-// Copyright 2016 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -15,6 +15,9 @@ namespace Service::ACT {
 /// Initializes all ACT services
 class Module final {
 public:
+    explicit Module(Core::System& system_);
+    ~Module() = default;
+
     class Interface : public ServiceFramework<Interface> {
     public:
         Interface(std::shared_ptr<Module> act, const char* name);
@@ -62,11 +65,17 @@ public:
     };
 
 private:
+    [[maybe_unused]]
+    Core::System& system;
+
     template <class Archive>
-    void serialize(Archive& ar, const unsigned int file_version) {}
+    void serialize(Archive& ar, const unsigned int);
     friend class boost::serialization::access;
 };
 
 void InstallInterfaces(Core::System& system);
 
 } // namespace Service::ACT
+
+BOOST_CLASS_EXPORT_KEY(Service::ACT::Module)
+SERVICE_CONSTRUCT(Service::ACT::Module)

src/core/hle/service/am/am.cpp

@@ -3764,6 +3764,7 @@ void Module::Interface::Sign(Kernel::HLERequestContext& ctx) {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar & cia_installing;
     ar & force_old_device_id;
     ar & force_new_device_id;

src/core/hle/service/apt/apt.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -41,6 +41,7 @@ namespace Service::APT {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int file_version) {
+    DEBUG_SERIALIZATION_POINT;
     ar & shared_font_mem;
     ar & shared_font_loaded;
     ar & shared_font_relocated;

src/core/hle/service/boss/boss.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -19,6 +19,7 @@ namespace Service::BOSS {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar & task_finish_event;
     ar & new_arrival_flag;
     ar & ns_data_new_flag;

src/core/hle/service/cam/cam.cpp

@@ -1,4 +1,4 @@
-// Copyright Citra Emulator Project / Lime3DS Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -26,6 +26,7 @@ namespace Service::CAM {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int file_version) {
+    DEBUG_SERIALIZATION_POINT;
     ar & cameras;
     ar & ports;
     ar & is_camera_reload_pending;

src/core/hle/service/cecd/cecd.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -35,6 +35,7 @@ namespace Service::CECD {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar & cecd_system_save_data_archive;
     ar & cecinfo_event;
     ar & cecinfosys_event;

src/core/hle/service/cfg/cfg.cpp

@@ -38,6 +38,7 @@ namespace Service::CFG {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar & cfg_config_file_buffer;
     ar & cfg_system_save_data_archive;
     ar & mac_address;

src/core/hle/service/csnd/csnd_snd.h

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -7,6 +7,7 @@
 #include <memory>
 #include <boost/serialization/array.hpp>
 #include <boost/serialization/shared_ptr.hpp>
+#include "common/archives.h"
 #include "core/hle/kernel/mutex.h"
 #include "core/hle/kernel/shared_memory.h"
 #include "core/hle/service/service.h"
@@ -257,6 +258,7 @@ private:
 
     template <class Archive>
     void serialize(Archive& ar, const unsigned int) {
+        DEBUG_SERIALIZATION_POINT;
         ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
         ar & mutex;
         ar & shared_memory;

src/core/hle/service/dsp/dsp_dsp.h

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -269,6 +269,7 @@ private:
 
     template <class Archive>
     void serialize(Archive& ar, const unsigned int) {
+        DEBUG_SERIALIZATION_POINT;
         ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
         ar & semaphore_event;
         ar & preset_semaphore;

src/core/hle/service/frd/frd.h

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -271,6 +271,7 @@ private:
 
     template <class Archive>
     void serialize(Archive& ar, const unsigned int) {
+        DEBUG_SERIALIZATION_POINT;
         ar & my_friend_key;
         ar & my_presence;
         ar & logged_in;

src/core/hle/service/fs/fs_user.cpp

@@ -1885,6 +1885,7 @@ FS_USER::FS_USER(Core::System& system)
 }
 template <class Archive>
 void Service::FS::FS_USER::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     ar & priority;
     ar & secure_value_backend;

src/core/hle/service/gsp/gsp_gpu.cpp

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -716,6 +716,7 @@ SessionData* GSP_GPU::FindRegisteredThreadData(u32 thread_id) {
 
 template <class Archive>
 void GSP_GPU::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     ar & shared_memory;
     ar & active_thread_id;

src/core/hle/service/hid/hid.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -32,6 +32,7 @@ namespace Service::HID {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int file_version) {
+    DEBUG_SERIALIZATION_POINT;
     ar & shared_mem;
     ar & event_pad_or_touch_1;
     ar & event_pad_or_touch_2;

src/core/hle/service/http/http_c.h

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -904,6 +904,7 @@ private:
         // NOTE: Serialization of the HTTP service is on a 'best effort' basis.
         // There is a very good chance that saving/loading during a network connection will break,
         // regardless!
+        DEBUG_SERIALIZATION_POINT;
         ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
         ar & ClCertA.certificate;
         ar & ClCertA.private_key;

src/core/hle/service/ir/ir_rst.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -22,6 +22,7 @@ namespace Service::IR {
 
 template <class Archive>
 void IR_RST::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     ar & update_event;
     ar & shared_memory;

src/core/hle/service/ir/ir_user.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -25,6 +25,7 @@ namespace Service::IR {
 
 template <class Archive>
 void IR_USER::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     ar & conn_status_event;
     ar & send_event;

src/core/hle/service/ldr_ro/ldr_ro.h

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -164,6 +164,7 @@ private:
 private:
     template <class Archive>
     void serialize(Archive& ar, const unsigned int) {
+        DEBUG_SERIALIZATION_POINT;
         ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     }
     friend class boost::serialization::access;

src/core/hle/service/mic/mic_u.cpp

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -25,6 +25,7 @@ namespace Service::MIC {
 
 template <class Archive>
 void MIC_U::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     ar&* impl.get();
 }

src/core/hle/service/ndm/ndm_u.h

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -274,6 +274,7 @@ private:
 
     template <class Archive>
     void serialize(Archive& ar, const unsigned int) {
+        DEBUG_SERIALIZATION_POINT;
         ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
         ar & daemon_bit_mask;
         ar & default_daemon_bit_mask;

src/core/hle/service/news/news.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -45,6 +45,7 @@ constexpr std::array<u8, 8> news_system_savedata_id{
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar & db;
     ar & notification_ids;
     ar & automatic_sync_flag;

src/core/hle/service/nfc/nfc.cpp

@@ -1,4 +1,4 @@
-// Copyright 2016 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -16,6 +16,7 @@ namespace Service::NFC {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar & nfc_mode;
     ar & device;
 }

src/core/hle/service/nim/nim_u.h

@@ -1,4 +1,4 @@
-// Copyright 2015 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -532,6 +532,7 @@ private:
 
     template <class Archive>
     void serialize(Archive& ar, const unsigned int) {
+        DEBUG_SERIALIZATION_POINT;
         ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
         ar & nim_system_update_event_for_menu;
         ar & nim_system_update_event_for_news;

src/core/hle/service/nwm/nwm_uds.cpp

@@ -33,6 +33,7 @@ namespace Service::NWM {
 
 template <class Archive>
 void NWM_UDS::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     ar & node_map;
     ar & connection_event;

src/core/hle/service/plgldr/plgldr.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -85,6 +85,7 @@ SERIALIZE_IMPL(PLG_LDR::PluginLoaderContext)
 
 template <class Archive>
 void PLG_LDR::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     ar & plgldr_context;
 }

src/core/hle/service/ptm/ptm.cpp

@@ -225,6 +225,7 @@ Module::Module(Core::System& system_) : system(system_) {
 
 template <class Archive>
 void Module::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar & shell_open;
     ar & battery_is_charging;
     ar & pedometer_is_counting;

src/core/hle/service/service.cpp

@@ -220,13 +220,27 @@ static bool AttemptLLE(const ServiceModuleInfo& service_module) {
 }
 
 /// Initialize ServiceManager
-void Init(Core::System& core, bool allow_lle) {
+void Init(Core::System& core, std::vector<u64>& lle_modules, bool allow_lle) {
     SM::ServiceManager::InstallInterfaces(core);
     core.Kernel().SetAppMainThreadExtendedSleep(false);
     bool lle_module_present = false;
 
     for (const auto& service_module : service_module_map) {
-        const bool has_lle = allow_lle && AttemptLLE(service_module);
+        if (core.GetSaveStateStatus() == Core::System::SaveStateStatus::LOADING &&
+            std::find(lle_modules.begin(), lle_modules.end(), service_module.title_id) !=
+                lle_modules.end()) {
+            // The system module has already been loaded before, do not attempt to load again as the
+            // process, threads, etc are already serialized in the kernel structures.
+            lle_module_present |= true;
+            continue;
+        }
+
+        const bool has_lle = allow_lle &&
+                             core.GetSaveStateStatus() != Core::System::SaveStateStatus::LOADING &&
+                             AttemptLLE(service_module);
+        if (has_lle) {
+            lle_modules.push_back(service_module.title_id);
+        }
         if (!has_lle && service_module.init_function != nullptr) {
             service_module.init_function(core);
         }

src/core/hle/service/service.h

@@ -12,6 +12,7 @@
 #include <boost/container/flat_map.hpp>
 #include <boost/serialization/base_object.hpp>
 #include <boost/serialization/shared_ptr.hpp>
+#include "common/archives.h"
 #include "common/common_types.h"
 #include "common/construct.h"
 #include "core/hle/kernel/hle_ipc.h"
@@ -183,7 +184,7 @@ private:
 };
 
 /// Initialize ServiceManager
-void Init(Core::System& system, bool allow_lle);
+void Init(Core::System& system, std::vector<u64>& lle_modules, bool allow_lle);
 
 struct ServiceModuleInfo {
     std::string name;
@@ -218,6 +219,7 @@ extern const std::array<ServiceModuleInfo, 41> service_module_map;
 #define SERVICE_SERIALIZATION_SIMPLE                                                               \
     template <class Archive>                                                                       \
     void serialize(Archive& ar, const unsigned int) {                                              \
+        DEBUG_SERIALIZATION_POINT;                                                                 \
         ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);               \
     }                                                                                              \
     friend class boost::serialization::access;

src/core/hle/service/sm/sm.h

@@ -1,4 +1,4 @@
-// Copyright 2017 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -89,11 +89,13 @@ private:
 
     template <class Archive>
     void save(Archive& ar, const unsigned int file_version) const {
+        DEBUG_SERIALIZATION_POINT;
         ar << registered_services;
     }
 
     template <class Archive>
     void load(Archive& ar, const unsigned int file_version) {
+        DEBUG_SERIALIZATION_POINT;
         ar >> registered_services;
         registered_services_inverse.clear();
         for (const auto& pair : registered_services) {

src/core/hle/service/sm/srv.cpp

@@ -1,4 +1,4 @@
-// Copyright 2016 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -31,6 +31,7 @@ namespace Service::SM {
 
 template <class Archive>
 void SRV::serialize(Archive& ar, const unsigned int) {
+    DEBUG_SERIALIZATION_POINT;
     ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
     ar & notification_semaphore;
     ar & get_service_handle_delayed_map;

src/core/hle/service/soc/soc_u.h

@@ -1,4 +1,4 @@
-// Copyright 2014 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -169,6 +169,7 @@ private:
 
     template <class Archive>
     void serialize(Archive& ar, const unsigned int) {
+        DEBUG_SERIALIZATION_POINT;
         ar& boost::serialization::base_object<Kernel::SessionRequestHandler>(*this);
         ar & created_sockets;
         ar & initialized_processes;