From e00a49e1e574f1861cd97ad1a8e578192f97adc3 Mon Sep 17 00:00:00 2001
From: GPUCode <geoster3d@gmail.com>
Date: Tue, 2 Apr 2024 22:39:06 +0300
Subject: [PATCH] core: Remove usage of MemoryRef

---
 externals/teakra                              |   2 +-
 src/audio_core/dsp_interface.h                |   2 +-
 src/audio_core/hle/hle.cpp                    |  24 ++-
 src/audio_core/hle/hle.h                      |   5 +-
 src/audio_core/lle/lle.cpp                    |  19 +--
 src/audio_core/lle/lle.h                      |   2 +-
 src/core/arm/dynarmic/arm_dynarmic.cpp        |   2 +-
 src/core/core.cpp                             |   2 -
 src/core/file_sys/plugin_3gx.cpp              |  14 +-
 src/core/hle/kernel/ipc.cpp                   |  16 +-
 src/core/hle/kernel/ipc.h                     |   3 +-
 src/core/hle/kernel/memory.cpp                |  20 +--
 src/core/hle/kernel/process.cpp               |  12 +-
 src/core/hle/kernel/shared_memory.cpp         |   4 +-
 src/core/hle/kernel/shared_memory.h           |   3 +-
 src/core/hle/kernel/svc.cpp                   |   7 +-
 src/core/hle/kernel/vm_manager.cpp            |  28 ++--
 src/core/hle/kernel/vm_manager.h              |  16 +-
 src/core/memory.cpp                           | 145 +++++-------------
 src/core/memory.h                             |  76 +++------
 src/tests/core/memory/vm_manager.cpp          |   4 +-
 .../rasterizer_cache/rasterizer_cache.h       |  30 ++--
 .../renderer_vulkan/vk_rasterizer.cpp         |   8 +-
 23 files changed, 151 insertions(+), 293 deletions(-)

diff --git a/externals/teakra b/externals/teakra
index 01db7cdd0..ad825418e 160000
--- a/externals/teakra
+++ b/externals/teakra
@@ -1 +1 @@
-Subproject commit 01db7cdd00aabcce559a8dddce8798dabb71949b
+Subproject commit ad825418ef8b91e31bea678a9f470988bac6b568
diff --git a/src/audio_core/dsp_interface.h b/src/audio_core/dsp_interface.h
index e49a0cd8c..abc64dc60 100644
--- a/src/audio_core/dsp_interface.h
+++ b/src/audio_core/dsp_interface.h
@@ -87,7 +87,7 @@ public:
     virtual void PipeWrite(DspPipe pipe_number, std::span<const u8> buffer) = 0;
 
     /// Returns a reference to the array backing DSP memory
-    virtual std::array<u8, Memory::DSP_RAM_SIZE>& GetDspMemory() = 0;
+    virtual std::span<u8, Memory::DSP_RAM_SIZE> GetDspMemory() = 0;
 
     /// Sets the handler for the interrupts we trigger
     virtual void SetInterruptHandler(
diff --git a/src/audio_core/hle/hle.cpp b/src/audio_core/hle/hle.cpp
index d21de3b23..17947c7fc 100644
--- a/src/audio_core/hle/hle.cpp
+++ b/src/audio_core/hle/hle.cpp
@@ -44,12 +44,9 @@ public:
     std::size_t GetPipeReadableSize(DspPipe pipe_number) const;
     void PipeWrite(DspPipe pipe_number, std::span<const u8> buffer);
 
-    std::array<u8, Memory::DSP_RAM_SIZE>& GetDspMemory();
-
     void SetInterruptHandler(
         std::function<void(Service::DSP::InterruptType type, DspPipe pipe)> handler);
 
-private:
     void ResetPipes();
     void WriteU16(DspPipe pipe_number, u16 value);
     void AudioPipeWriteStructAddresses();
@@ -65,7 +62,7 @@ private:
     DspState dsp_state = DspState::Off;
     std::array<std::vector<u8>, num_dsp_pipe> pipe_data{};
 
-    HLE::DspMemory dsp_memory;
+    HLE::DspMemory* dsp_memory;
     std::array<HLE::Source, HLE::num_sources> sources{{
         HLE::Source(0),  HLE::Source(1),  HLE::Source(2),  HLE::Source(3),  HLE::Source(4),
         HLE::Source(5),  HLE::Source(6),  HLE::Source(7),  HLE::Source(8),  HLE::Source(9),
@@ -86,7 +83,8 @@ private:
 
 DspHle::Impl::Impl(DspHle& parent_, Memory::MemorySystem& memory, Core::Timing& timing)
     : parent(parent_), core_timing(timing) {
-    dsp_memory.raw_memory.fill(0);
+    dsp_memory = reinterpret_cast<HLE::DspMemory*>(memory.GetDspMemory().data());
+    dsp_memory->raw_memory.fill(0);
 
     for (auto& source : sources) {
         source.SetMemory(memory);
@@ -257,10 +255,6 @@ void DspHle::Impl::PipeWrite(DspPipe pipe_number, std::span<const u8> buffer) {
     }
 }
 
-std::array<u8, Memory::DSP_RAM_SIZE>& DspHle::Impl::GetDspMemory() {
-    return dsp_memory.raw_memory;
-}
-
 void DspHle::Impl::SetInterruptHandler(
     std::function<void(Service::DSP::InterruptType type, DspPipe pipe)> handler) {
     interrupt_handler = handler;
@@ -316,8 +310,8 @@ void DspHle::Impl::AudioPipeWriteStructAddresses() {
 size_t DspHle::Impl::CurrentRegionIndex() const {
     // The region with the higher frame counter is chosen unless there is wraparound.
     // This function only returns a 0 or 1.
-    const u16 frame_counter_0 = dsp_memory.region_0.frame_counter;
-    const u16 frame_counter_1 = dsp_memory.region_1.frame_counter;
+    const u16 frame_counter_0 = dsp_memory->region_0.frame_counter;
+    const u16 frame_counter_1 = dsp_memory->region_1.frame_counter;
 
     if (frame_counter_0 == 0xFFFFu && frame_counter_1 != 0xFFFEu) {
         // Wraparound has occurred.
@@ -333,11 +327,11 @@ size_t DspHle::Impl::CurrentRegionIndex() const {
 }
 
 HLE::SharedMemory& DspHle::Impl::ReadRegion() {
-    return CurrentRegionIndex() == 0 ? dsp_memory.region_0 : dsp_memory.region_1;
+    return CurrentRegionIndex() == 0 ? dsp_memory->region_0 : dsp_memory->region_1;
 }
 
 HLE::SharedMemory& DspHle::Impl::WriteRegion() {
-    return CurrentRegionIndex() != 0 ? dsp_memory.region_0 : dsp_memory.region_1;
+    return CurrentRegionIndex() != 0 ? dsp_memory->region_0 : dsp_memory->region_1;
 }
 
 StereoFrame16 DspHle::Impl::GenerateCurrentFrame() {
@@ -421,8 +415,8 @@ void DspHle::PipeWrite(DspPipe pipe_number, std::span<const u8> buffer) {
     impl->PipeWrite(pipe_number, buffer);
 }
 
-std::array<u8, Memory::DSP_RAM_SIZE>& DspHle::GetDspMemory() {
-    return impl->GetDspMemory();
+std::span<u8, Memory::DSP_RAM_SIZE> DspHle::GetDspMemory() {
+    return impl->dsp_memory->raw_memory;
 }
 
 void DspHle::SetInterruptHandler(
diff --git a/src/audio_core/hle/hle.h b/src/audio_core/hle/hle.h
index f66f1a590..b01faa1da 100644
--- a/src/audio_core/hle/hle.h
+++ b/src/audio_core/hle/hle.h
@@ -4,14 +4,11 @@
 
 #pragma once
 
-#include <array>
 #include <memory>
 #include <vector>
 #include "audio_core/audio_types.h"
 #include "audio_core/dsp_interface.h"
-#include "common/common_types.h"
 #include "core/hle/service/dsp/dsp_dsp.h"
-#include "core/memory.h"
 
 namespace Core {
 class Timing;
@@ -36,7 +33,7 @@ public:
     std::size_t GetPipeReadableSize(DspPipe pipe_number) const override;
     void PipeWrite(DspPipe pipe_number, std::span<const u8> buffer) override;
 
-    std::array<u8, Memory::DSP_RAM_SIZE>& GetDspMemory() override;
+    std::span<u8, Memory::DSP_RAM_SIZE> GetDspMemory() override;
 
     void SetInterruptHandler(
         std::function<void(Service::DSP::InterruptType type, DspPipe pipe)> handler) override;
diff --git a/src/audio_core/lle/lle.cpp b/src/audio_core/lle/lle.cpp
index 91ac5dc73..869068130 100644
--- a/src/audio_core/lle/lle.cpp
+++ b/src/audio_core/lle/lle.cpp
@@ -121,7 +121,9 @@ static u8 PipeIndexToSlotIndex(u8 pipe_index, PipeDirection direction) {
 }
 
 struct DspLle::Impl final {
-    Impl(Core::Timing& timing, bool multithread) : core_timing(timing), multithread(multithread) {
+    Impl(Memory::MemorySystem& memory, Core::Timing& timing, bool multithread_)
+        : dsp_memory{memory.GetDspMemory()}, config{dsp_memory.data()}, teakra{config},
+          core_timing{timing}, multithread{multithread_} {
         teakra_slice_event = core_timing.RegisterEvent(
             "DSP slice", [this](u64, int late) { TeakraSliceEvent(static_cast<u64>(late)); });
     }
@@ -130,6 +132,8 @@ struct DspLle::Impl final {
         StopTeakraThread();
     }
 
+    std::span<u8, Memory::DSP_RAM_SIZE> dsp_memory;
+    Teakra::UserConfig config;
     Teakra::Teakra teakra;
     u16 pipe_base_waddr = 0;
 
@@ -189,13 +193,11 @@ struct DspLle::Impl final {
     }
 
     u8* GetDspDataPointer(u32 baddr) {
-        auto& memory = teakra.GetDspMemory();
-        return &memory[DspDataOffset + baddr];
+        return &dsp_memory[DspDataOffset + baddr];
     }
 
     const u8* GetDspDataPointer(u32 baddr) const {
-        auto& memory = teakra.GetDspMemory();
-        return &memory[DspDataOffset + baddr];
+        return &dsp_memory[DspDataOffset + baddr];
     }
 
     PipeStatus GetPipeStatus(u8 pipe_index, PipeDirection direction) const {
@@ -312,7 +314,6 @@ struct DspLle::Impl final {
         teakra.Reset();
 
         Dsp1 dsp(buffer);
-        auto& dsp_memory = teakra.GetDspMemory();
         u8* program = dsp_memory.data();
         u8* data = dsp_memory.data() + DspDataOffset;
         for (const auto& segment : dsp.segments) {
@@ -403,8 +404,8 @@ void DspLle::PipeWrite(DspPipe pipe_number, std::span<const u8> buffer) {
     impl->WritePipe(static_cast<u8>(pipe_number), buffer);
 }
 
-std::array<u8, Memory::DSP_RAM_SIZE>& DspLle::GetDspMemory() {
-    return impl->teakra.GetDspMemory();
+std::span<u8, Memory::DSP_RAM_SIZE> DspLle::GetDspMemory() {
+    return impl->dsp_memory;
 }
 
 void DspLle::SetInterruptHandler(
@@ -469,7 +470,7 @@ DspLle::DspLle(Core::System& system, bool multithread)
 
 DspLle::DspLle(Core::System& system, Memory::MemorySystem& memory, Core::Timing& timing,
                bool multithread)
-    : DspInterface(system), impl(std::make_unique<Impl>(timing, multithread)) {
+    : DspInterface(system), impl(std::make_unique<Impl>(memory, timing, multithread)) {
     Teakra::AHBMCallback ahbm;
     ahbm.read8 = [&memory](u32 address) -> u8 {
         return *memory.GetFCRAMPointer(address - Memory::FCRAM_PADDR);
diff --git a/src/audio_core/lle/lle.h b/src/audio_core/lle/lle.h
index 7ae64575d..d2f9c37c2 100644
--- a/src/audio_core/lle/lle.h
+++ b/src/audio_core/lle/lle.h
@@ -31,7 +31,7 @@ public:
     std::size_t GetPipeReadableSize(DspPipe pipe_number) const override;
     void PipeWrite(DspPipe pipe_number, std::span<const u8> buffer) override;
 
-    std::array<u8, Memory::DSP_RAM_SIZE>& GetDspMemory() override;
+    std::span<u8, Memory::DSP_RAM_SIZE> GetDspMemory() override;
 
     void SetInterruptHandler(
         std::function<void(Service::DSP::InterruptType type, DspPipe pipe)> handler) override;
diff --git a/src/core/arm/dynarmic/arm_dynarmic.cpp b/src/core/arm/dynarmic/arm_dynarmic.cpp
index 8dd81a4c9..d39dd1187 100644
--- a/src/core/arm/dynarmic/arm_dynarmic.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic.cpp
@@ -298,7 +298,7 @@ std::unique_ptr<Dynarmic::A32::Jit> ARM_Dynarmic::MakeJit() {
     Dynarmic::A32::UserConfig config;
     config.callbacks = cb.get();
     if (current_page_table) {
-        config.page_table = &current_page_table->GetPointerArray();
+        config.page_table = &current_page_table->pointers;
     }
     config.coprocessors[15] = std::make_shared<DynarmicCP15>(cp15_state);
     config.define_unpredictable_behaviour = true;
diff --git a/src/core/core.cpp b/src/core/core.cpp
index c36db49cd..9b19a1607 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -429,8 +429,6 @@ System::ResultStatus System::Init(Frontend::EmuWindow& emu_window,
         dsp_core = std::make_unique<AudioCore::DspLle>(*this, multithread);
     }
 
-    memory->SetDSP(*dsp_core);
-
     dsp_core->SetSink(Settings::values.output_type.GetValue(),
                       Settings::values.output_device.GetValue());
     dsp_core->EnableStretching(Settings::values.enable_audio_stretching.GetValue());
diff --git a/src/core/file_sys/plugin_3gx.cpp b/src/core/file_sys/plugin_3gx.cpp
index e325d7fca..fbda94f2a 100644
--- a/src/core/file_sys/plugin_3gx.cpp
+++ b/src/core/file_sys/plugin_3gx.cpp
@@ -194,9 +194,9 @@ Loader::ResultStatus FileSys::Plugin3GXLoader::Map(
                   plg_context.plugin_path);
         return Loader::ResultStatus::ErrorMemoryAllocationFailed;
     }
-    auto backing_memory_fb = kernel.memory.GetFCRAMRef(*offset_fb);
+    auto backing_memory_fb = kernel.memory.GetFCRAMPointer(*offset_fb);
     plg_ldr.SetPluginFBAddr(Memory::FCRAM_PADDR + *offset_fb);
-    std::fill(backing_memory_fb.GetPtr(), backing_memory_fb.GetPtr() + _3GX_fb_size, 0);
+    std::memset(backing_memory_fb, 0, _3GX_fb_size);
 
     auto vma_heap_fb = process.vm_manager.MapBackingMemory(
         _3GX_heap_load_addr, backing_memory_fb, _3GX_fb_size, Kernel::MemoryState::Continuous);
@@ -212,8 +212,8 @@ Loader::ResultStatus FileSys::Plugin3GXLoader::Map(
                   plg_context.plugin_path);
         return Loader::ResultStatus::ErrorMemoryAllocationFailed;
     }
-    auto backing_memory = kernel.memory.GetFCRAMRef(*offset);
-    std::fill(backing_memory.GetPtr(), backing_memory.GetPtr() + block_size - _3GX_fb_size, 0);
+    auto backing_memory = kernel.memory.GetFCRAMPointer(*offset);
+    std::memset(backing_memory, 0, block_size - _3GX_fb_size);
 
     // Then we map part of the memory, which contains the executable
     auto vma = process.vm_manager.MapBackingMemory(_3GX_exe_load_addr, backing_memory, exe_size,
@@ -251,7 +251,7 @@ Loader::ResultStatus FileSys::Plugin3GXLoader::Map(
     kernel.memory.WriteBlock(process, _3GX_exe_load_addr, &plugin_header, sizeof(PluginHeader));
 
     // Map plugin heap
-    auto backing_memory_heap = kernel.memory.GetFCRAMRef(*offset + exe_size);
+    auto backing_memory_heap = kernel.memory.GetFCRAMPointer(*offset + exe_size);
 
     // Map the rest of the memory at the heap location
     auto vma_heap = process.vm_manager.MapBackingMemory(
@@ -346,8 +346,8 @@ void FileSys::Plugin3GXLoader::MapBootloader(Kernel::Process& process, Kernel::K
     }
 
     // Map bootloader to the offset provided
-    auto backing_memory = kernel.memory.GetFCRAMRef(memory_offset);
-    std::fill(backing_memory.GetPtr(), backing_memory.GetPtr() + bootloader_memory_size, 0);
+    auto backing_memory = kernel.memory.GetFCRAMPointer(memory_offset);
+    std::memset(backing_memory, 0, bootloader_memory_size);
     auto vma = process.vm_manager.MapBackingMemory(_3GX_exe_load_addr - bootloader_memory_size,
                                                    backing_memory, bootloader_memory_size,
                                                    Kernel::MemoryState::Continuous);
diff --git a/src/core/hle/kernel/ipc.cpp b/src/core/hle/kernel/ipc.cpp
index 1d26df7a2..a1bf033cc 100644
--- a/src/core/hle/kernel/ipc.cpp
+++ b/src/core/hle/kernel/ipc.cpp
@@ -5,7 +5,6 @@
 #include <algorithm>
 #include "common/alignment.h"
 
-#include "common/memory_ref.h"
 #include "core/core.h"
 #include "core/hle/ipc.h"
 #include "core/hle/kernel/handle_table.h"
@@ -196,23 +195,22 @@ Result TranslateCommandBuffer(Kernel::KernelSystem& kernel, Memory::MemorySystem
             // TODO(Subv): Perform permission checks.
 
             // Create a buffer which contains the mapped buffer and two additional guard pages.
-            std::shared_ptr<BackingMem> buffer =
-                std::make_shared<BufferMem>((num_pages + 2) * Memory::CITRA_PAGE_SIZE);
+            const u32 buffer_size = (num_pages + 2) * Memory::CITRA_PAGE_SIZE;
+            auto buffer = std::make_unique<u8[]>(buffer_size);
             memory.ReadBlock(*src_process, source_address,
-                             buffer->GetPtr() + Memory::CITRA_PAGE_SIZE + page_offset, size);
+                             buffer.get() + Memory::CITRA_PAGE_SIZE + page_offset, size);
 
             // Map the guard pages and mapped pages at once.
             target_address =
                 dst_process->vm_manager
                     .MapBackingMemoryToBase(Memory::IPC_MAPPING_VADDR, Memory::IPC_MAPPING_SIZE,
-                                            buffer, static_cast<u32>(buffer->GetSize()),
-                                            Kernel::MemoryState::Shared)
+                                            buffer.get(), buffer_size, Kernel::MemoryState::Shared)
                     .Unwrap();
 
             // Change the permissions and state of the guard pages.
             const VAddr low_guard_address = target_address;
             const VAddr high_guard_address =
-                low_guard_address + static_cast<VAddr>(buffer->GetSize()) - Memory::CITRA_PAGE_SIZE;
+                low_guard_address + buffer_size - Memory::CITRA_PAGE_SIZE;
             ASSERT(dst_process->vm_manager.ChangeMemoryState(
                        low_guard_address, Memory::CITRA_PAGE_SIZE, Kernel::MemoryState::Shared,
                        Kernel::VMAPermission::ReadWrite, Kernel::MemoryState::Reserved,
@@ -226,8 +224,8 @@ Result TranslateCommandBuffer(Kernel::KernelSystem& kernel, Memory::MemorySystem
             target_address += Memory::CITRA_PAGE_SIZE;
             cmd_buf[i++] = target_address + page_offset;
 
-            mapped_buffer_context.push_back({permissions, size, source_address,
-                                             target_address + page_offset, std::move(buffer)});
+            mapped_buffer_context.emplace_back(permissions, size, source_address,
+                                               target_address + page_offset, std::move(buffer));
 
             break;
         }
diff --git a/src/core/hle/kernel/ipc.h b/src/core/hle/kernel/ipc.h
index ec2a8e947..7a4dee09f 100644
--- a/src/core/hle/kernel/ipc.h
+++ b/src/core/hle/kernel/ipc.h
@@ -23,8 +23,7 @@ struct MappedBufferContext {
     u32 size;
     VAddr source_address;
     VAddr target_address;
-
-    std::shared_ptr<BackingMem> buffer;
+    std::unique_ptr<u8[]> buffer;
 };
 
 /// Performs IPC command buffer translation from one process to another.
diff --git a/src/core/hle/kernel/memory.cpp b/src/core/hle/kernel/memory.cpp
index 29aa454b7..3fcb38e23 100644
--- a/src/core/hle/kernel/memory.cpp
+++ b/src/core/hle/kernel/memory.cpp
@@ -135,7 +135,7 @@ void KernelSystem::HandleSpecialMapping(VMManager& address_space, const AddressM
         return;
     }
 
-    auto target_pointer = memory.GetPhysicalRef(area->paddr_base + offset_into_region);
+    u8* target_pointer = memory.GetPhysicalPointer(area->paddr_base + offset_into_region);
 
     // TODO(yuriks): This flag seems to have some other effect, but it's unknown what
     MemoryState memory_state = mapping.unk_flag ? MemoryState::Static : MemoryState::IO;
@@ -148,16 +148,18 @@ void KernelSystem::HandleSpecialMapping(VMManager& address_space, const AddressM
 }
 
 void KernelSystem::MapSharedPages(VMManager& address_space) {
-    auto cfg_mem_vma = address_space
-                           .MapBackingMemory(Memory::CONFIG_MEMORY_VADDR, {config_mem_handler},
-                                             Memory::CONFIG_MEMORY_SIZE, MemoryState::Shared)
-                           .Unwrap();
+    auto cfg_mem_vma =
+        address_space
+            .MapBackingMemory(Memory::CONFIG_MEMORY_VADDR, config_mem_handler->GetPtr(),
+                              Memory::CONFIG_MEMORY_SIZE, MemoryState::Shared)
+            .Unwrap();
     address_space.Reprotect(cfg_mem_vma, VMAPermission::Read);
 
-    auto shared_page_vma = address_space
-                               .MapBackingMemory(Memory::SHARED_PAGE_VADDR, {shared_page_handler},
-                                                 Memory::SHARED_PAGE_SIZE, MemoryState::Shared)
-                               .Unwrap();
+    auto shared_page_vma =
+        address_space
+            .MapBackingMemory(Memory::SHARED_PAGE_VADDR, shared_page_handler->GetPtr(),
+                              Memory::SHARED_PAGE_SIZE, MemoryState::Shared)
+            .Unwrap();
     address_space.Reprotect(shared_page_vma, VMAPermission::Read);
 }
 
diff --git a/src/core/hle/kernel/process.cpp b/src/core/hle/kernel/process.cpp
index 5f489b888..985fc30dc 100644
--- a/src/core/hle/kernel/process.cpp
+++ b/src/core/hle/kernel/process.cpp
@@ -242,13 +242,13 @@ Result Process::HeapAllocate(VAddr* out_addr, VAddr target, u32 size, VMAPermiss
     // Maps heap block by block
     VAddr interval_target = target;
     for (const auto& interval : allocated_fcram) {
-        u32 interval_size = interval.upper() - interval.lower();
+        const u32 interval_size = interval.upper() - interval.lower();
         LOG_DEBUG(Kernel, "Allocated FCRAM region lower={:08X}, upper={:08X}", interval.lower(),
                   interval.upper());
         std::fill(kernel.memory.GetFCRAMPointer(interval.lower()),
                   kernel.memory.GetFCRAMPointer(interval.upper()), 0);
         auto vma = vm_manager.MapBackingMemory(interval_target,
-                                               kernel.memory.GetFCRAMRef(interval.lower()),
+                                               kernel.memory.GetFCRAMPointer(interval.lower()),
                                                interval_size, memory_state);
         ASSERT(vma.Succeeded());
         vm_manager.Reprotect(vma.Unwrap(), perms);
@@ -276,7 +276,7 @@ Result Process::HeapFree(VAddr target, u32 size) {
     // Free heaps block by block
     CASCADE_RESULT(auto backing_blocks, vm_manager.GetBackingBlocksForRange(target, size));
     for (const auto& [backing_memory, block_size] : backing_blocks) {
-        const auto backing_offset = kernel.memory.GetFCRAMOffset(backing_memory.GetPtr());
+        const auto backing_offset = kernel.memory.GetFCRAMOffset(backing_memory);
         memory_region->Free(backing_offset, block_size);
         holding_memory -= MemoryRegionInfo::Interval(backing_offset, backing_offset + block_size);
     }
@@ -322,9 +322,9 @@ Result Process::LinearAllocate(VAddr* out_addr, VAddr target, u32 size, VMAPermi
         }
     }
 
-    auto backing_memory = kernel.memory.GetFCRAMRef(physical_offset);
+    auto backing_memory = kernel.memory.GetFCRAMPointer(physical_offset);
 
-    std::fill(backing_memory.GetPtr(), backing_memory.GetPtr() + size, 0);
+    std::fill(backing_memory, backing_memory + size, 0);
     auto vma = vm_manager.MapBackingMemory(target, backing_memory, size, MemoryState::Continuous);
     ASSERT(vma.Succeeded());
     vm_manager.Reprotect(vma.Unwrap(), perms);
@@ -410,7 +410,7 @@ ResultVal<VAddr> Process::AllocateThreadLocalStorage() {
         // Map the page to the current process' address space.
         auto tls_page_addr =
             Memory::TLS_AREA_VADDR + static_cast<VAddr>(tls_page) * Memory::CITRA_PAGE_SIZE;
-        vm_manager.MapBackingMemory(tls_page_addr, kernel.memory.GetFCRAMRef(*offset),
+        vm_manager.MapBackingMemory(tls_page_addr, kernel.memory.GetFCRAMPointer(*offset),
                                     Memory::CITRA_PAGE_SIZE, MemoryState::Locked);
 
         LOG_DEBUG(Kernel, "Allocated TLS page at addr={:08X}", tls_page_addr);
diff --git a/src/core/hle/kernel/shared_memory.cpp b/src/core/hle/kernel/shared_memory.cpp
index 3ac901463..b18cfab39 100644
--- a/src/core/hle/kernel/shared_memory.cpp
+++ b/src/core/hle/kernel/shared_memory.cpp
@@ -52,7 +52,7 @@ ResultVal<std::shared_ptr<SharedMemory>> KernelSystem::CreateSharedMemory(
         ASSERT_MSG(offset, "Not enough space in region to allocate shared memory!");
 
         std::fill(memory.GetFCRAMPointer(*offset), memory.GetFCRAMPointer(*offset + size), 0);
-        shared_memory->backing_blocks = {{memory.GetFCRAMRef(*offset), size}};
+        shared_memory->backing_blocks = {{memory.GetFCRAMPointer(*offset), size}};
         shared_memory->holding_memory += MemoryRegionInfo::Interval(*offset, *offset + size);
         shared_memory->linear_heap_phys_offset = *offset;
 
@@ -94,7 +94,7 @@ std::shared_ptr<SharedMemory> KernelSystem::CreateSharedMemoryForApplet(
     shared_memory->permissions = permissions;
     shared_memory->other_permissions = other_permissions;
     for (const auto& interval : backing_blocks) {
-        shared_memory->backing_blocks.emplace_back(memory.GetFCRAMRef(interval.lower()),
+        shared_memory->backing_blocks.emplace_back(memory.GetFCRAMPointer(interval.lower()),
                                                    interval.upper() - interval.lower());
         std::fill(memory.GetFCRAMPointer(interval.lower()),
                   memory.GetFCRAMPointer(interval.upper()), 0);
diff --git a/src/core/hle/kernel/shared_memory.h b/src/core/hle/kernel/shared_memory.h
index edbc93869..53751974b 100644
--- a/src/core/hle/kernel/shared_memory.h
+++ b/src/core/hle/kernel/shared_memory.h
@@ -7,7 +7,6 @@
 #include <string>
 #include <utility>
 #include "common/common_types.h"
-#include "common/memory_ref.h"
 #include "core/hle/kernel/object.h"
 #include "core/hle/kernel/process.h"
 #include "core/hle/result.h"
@@ -87,7 +86,7 @@ private:
     /// during creation.
     PAddr linear_heap_phys_offset = 0;
     /// Backing memory for this shared memory block.
-    std::vector<std::pair<MemoryRef, u32>> backing_blocks;
+    std::vector<std::pair<u8*, u32>> backing_blocks;
     /// Size of the memory block. Page-aligned.
     u32 size = 0;
     /// Region of memory this block exists in.
diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp
index 0429b9741..f69a22f19 100644
--- a/src/core/hle/kernel/svc.cpp
+++ b/src/core/hle/kernel/svc.cpp
@@ -1937,8 +1937,7 @@ u32 SVC::ConvertVaToPa(u32 addr) {
         vma->second.type != VMAType::BackingMemory) {
         return 0;
     }
-    return kernel.memory.GetFCRAMOffset(vma->second.backing_memory.GetPtr() + addr -
-                                        vma->second.base) +
+    return kernel.memory.GetFCRAMOffset(vma->second.backing_memory + addr - vma->second.base) +
            Memory::FCRAM_PADDR;
 }
 
@@ -1967,8 +1966,8 @@ Result SVC::MapProcessMemoryEx(Handle dst_process_handle, u32 dst_address,
 
     auto vma_res = dst_process->vm_manager.MapBackingMemory(
         dst_address,
-        memory.GetFCRAMRef(vma->second.backing_memory.GetPtr() + offset -
-                           kernel.memory.GetFCRAMPointer(0)),
+        memory.GetFCRAMPointer(vma->second.backing_memory + offset -
+                               kernel.memory.GetFCRAMPointer(0)),
         size, Kernel::MemoryState::Continuous);
 
     if (!vma_res.Succeeded()) {
diff --git a/src/core/hle/kernel/vm_manager.cpp b/src/core/hle/kernel/vm_manager.cpp
index 89abc813d..014386821 100644
--- a/src/core/hle/kernel/vm_manager.cpp
+++ b/src/core/hle/kernel/vm_manager.cpp
@@ -29,8 +29,7 @@ bool VirtualMemoryArea::CanBeMergedWith(const VirtualMemoryArea& next) const {
         type != next.type) {
         return false;
     }
-    if (type == VMAType::BackingMemory &&
-        backing_memory.GetPtr() + size != next.backing_memory.GetPtr()) {
+    if (type == VMAType::BackingMemory && backing_memory + size != next.backing_memory) {
         return false;
     }
     return true;
@@ -38,24 +37,16 @@ bool VirtualMemoryArea::CanBeMergedWith(const VirtualMemoryArea& next) const {
 
 VMManager::VMManager(Memory::MemorySystem& memory, Kernel::Process& proc)
     : page_table(std::make_shared<Memory::PageTable>()), memory(memory), process(proc) {
-    Reset();
-}
-
-VMManager::~VMManager() = default;
-
-void VMManager::Reset() {
-    vma_map.clear();
-
     // Initialize the map with a single free region covering the entire managed space.
     VirtualMemoryArea initial_vma;
     initial_vma.size = MAX_ADDRESS;
     vma_map.emplace(initial_vma.base, initial_vma);
 
-    page_table->Clear();
-
     UpdatePageTableForVMA(initial_vma);
 }
 
+VMManager::~VMManager() = default;
+
 VMManager::VMAHandle VMManager::FindVMA(VAddr target) const {
     if (target >= MAX_ADDRESS) {
         return vma_map.end();
@@ -64,7 +55,7 @@ VMManager::VMAHandle VMManager::FindVMA(VAddr target) const {
     }
 }
 
-ResultVal<VAddr> VMManager::MapBackingMemoryToBase(VAddr base, u32 region_size, MemoryRef memory,
+ResultVal<VAddr> VMManager::MapBackingMemoryToBase(VAddr base, u32 region_size, u8* memory,
                                                    u32 size, MemoryState state) {
     // Find the first Free VMA.
     VMAHandle vma_handle = std::find_if(vma_map.begin(), vma_map.end(), [&](const auto& vma) {
@@ -92,9 +83,9 @@ ResultVal<VAddr> VMManager::MapBackingMemoryToBase(VAddr base, u32 region_size,
     return target;
 }
 
-ResultVal<VMManager::VMAHandle> VMManager::MapBackingMemory(VAddr target, MemoryRef memory,
-                                                            u32 size, MemoryState state) {
-    ASSERT(memory.GetPtr() != nullptr);
+ResultVal<VMManager::VMAHandle> VMManager::MapBackingMemory(VAddr target, u8* memory, u32 size,
+                                                            MemoryState state) {
+    ASSERT(memory != nullptr);
 
     // This is the appropriately sized VMA that will turn into our allocation.
     CASCADE_RESULT(VMAIter vma_handle, CarveVMA(target, size));
@@ -339,9 +330,8 @@ void VMManager::UpdatePageTableForVMA(const VirtualMemoryArea& vma) {
         plgldr->OnMemoryChanged(process, Core::System::GetInstance().Kernel());
 }
 
-ResultVal<std::vector<std::pair<MemoryRef, u32>>> VMManager::GetBackingBlocksForRange(VAddr address,
-                                                                                      u32 size) {
-    std::vector<std::pair<MemoryRef, u32>> backing_blocks;
+ResultVal<VMManager::BackingBlocks> VMManager::GetBackingBlocksForRange(VAddr address, u32 size) {
+    BackingBlocks backing_blocks;
     VAddr interval_target = address;
     while (interval_target != address + size) {
         auto vma = FindVMA(interval_target);
diff --git a/src/core/hle/kernel/vm_manager.h b/src/core/hle/kernel/vm_manager.h
index 8ce61c384..c5ea84649 100644
--- a/src/core/hle/kernel/vm_manager.h
+++ b/src/core/hle/kernel/vm_manager.h
@@ -68,7 +68,7 @@ struct VirtualMemoryArea {
 
     /// Settings for type = BackingMemory
     /// Pointer backing this VMA. It will not be destroyed or freed when the VMA is removed.
-    MemoryRef backing_memory{};
+    u8* backing_memory{};
 
     /// Tests if this area can be merged to the right with `next`.
     bool CanBeMergedWith(const VirtualMemoryArea& next) const;
@@ -108,14 +108,9 @@ public:
     explicit VMManager(Memory::MemorySystem& memory, Kernel::Process& proc);
     ~VMManager();
 
-    /// Clears the address space map, re-initializing with a single free area.
-    void Reset();
-
     /// Finds the VMA in which the given address is included in, or `vma_map.end()`.
     VMAHandle FindVMA(VAddr target) const;
 
-    // TODO(yuriks): Should these functions actually return the handle?
-
     /**
      * Maps part of a ref-counted block of memory at the first free address after the given base.
      *
@@ -126,7 +121,7 @@ public:
      * @param state MemoryState tag to attach to the VMA.
      * @returns The address at which the memory was mapped.
      */
-    ResultVal<VAddr> MapBackingMemoryToBase(VAddr base, u32 region_size, MemoryRef memory, u32 size,
+    ResultVal<VAddr> MapBackingMemoryToBase(VAddr base, u32 region_size, u8* memory, u32 size,
                                             MemoryState state);
     /**
      * Maps an unmanaged host memory pointer at a given address.
@@ -136,8 +131,7 @@ public:
      * @param size Size of the mapping.
      * @param state MemoryState tag to attach to the VMA.
      */
-    ResultVal<VMAHandle> MapBackingMemory(VAddr target, MemoryRef memory, u32 size,
-                                          MemoryState state);
+    ResultVal<VMAHandle> MapBackingMemory(VAddr target, u8* memory, u32 size, MemoryState state);
 
     /**
      * Updates the memory state and permissions of the specified range. The range's original memory
@@ -167,8 +161,8 @@ public:
     void LogLayout(Common::Log::Level log_level) const;
 
     /// Gets a list of backing memory blocks for the specified range
-    ResultVal<std::vector<std::pair<MemoryRef, u32>>> GetBackingBlocksForRange(VAddr address,
-                                                                               u32 size);
+    using BackingBlocks = std::vector<std::pair<u8*, u32>>;
+    ResultVal<BackingBlocks> GetBackingBlocksForRange(VAddr address, u32 size);
 
     /// Each VMManager has its own page table, which is set as the main one when the owning process
     /// is scheduled.
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 2e023d6ad..c7f4fc3a5 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -4,7 +4,6 @@
 
 #include <array>
 #include <cstring>
-#include "audio_core/dsp_interface.h"
 
 #include "common/assert.h"
 #include "common/atomic_ops.h"
@@ -22,12 +21,6 @@
 
 namespace Memory {
 
-void PageTable::Clear() {
-    pointers.raw.fill(nullptr);
-    pointers.refs.fill(MemoryRef());
-    attributes.fill(PageType::Unmapped);
-}
-
 class RasterizerCacheMarker {
 public:
     void Mark(VAddr addr, bool cached) {
@@ -68,53 +61,17 @@ private:
 
 class MemorySystem::Impl {
 public:
+    Core::System& system;
     std::unique_ptr<u8[]> fcram = std::make_unique<u8[]>(Memory::FCRAM_N3DS_SIZE);
     std::unique_ptr<u8[]> vram = std::make_unique<u8[]>(Memory::VRAM_SIZE);
     std::unique_ptr<u8[]> n3ds_extra_ram = std::make_unique<u8[]>(Memory::N3DS_EXTRA_RAM_SIZE);
+    std::unique_ptr<u8[]> dsp_mem = std::make_unique<u8[]>(Memory::DSP_RAM_SIZE);
 
-    Core::System& system;
     std::shared_ptr<PageTable> current_page_table = nullptr;
     RasterizerCacheMarker cache_marker;
     std::vector<std::shared_ptr<PageTable>> page_table_list;
 
-    AudioCore::DspInterface* dsp = nullptr;
-
-    std::shared_ptr<BackingMem> fcram_mem;
-    std::shared_ptr<BackingMem> vram_mem;
-    std::shared_ptr<BackingMem> n3ds_extra_ram_mem;
-    std::shared_ptr<BackingMem> dsp_mem;
-
-    Impl(Core::System& system_);
-
-    const u8* GetPtr(Region r) const {
-        switch (r) {
-        case Region::VRAM:
-            return vram.get();
-        case Region::DSP:
-            return dsp->GetDspMemory().data();
-        case Region::FCRAM:
-            return fcram.get();
-        case Region::N3DS:
-            return n3ds_extra_ram.get();
-        default:
-            UNREACHABLE();
-        }
-    }
-
-    u8* GetPtr(Region r) {
-        switch (r) {
-        case Region::VRAM:
-            return vram.get();
-        case Region::DSP:
-            return dsp->GetDspMemory().data();
-        case Region::FCRAM:
-            return fcram.get();
-        case Region::N3DS:
-            return n3ds_extra_ram.get();
-        default:
-            UNREACHABLE();
-        }
-    }
+    Impl(Core::System& system_) : system{system_} {}
 
     u32 GetSize(Region r) const {
         switch (r) {
@@ -233,26 +190,26 @@ public:
         }
     }
 
-    MemoryRef GetPointerForRasterizerCache(VAddr addr) const {
+    u8* GetPointerForRasterizerCache(VAddr addr) const {
         if (addr >= LINEAR_HEAP_VADDR && addr < LINEAR_HEAP_VADDR_END) {
-            return {fcram_mem, addr - LINEAR_HEAP_VADDR};
+            return fcram.get() + addr - LINEAR_HEAP_VADDR;
         }
         if (addr >= NEW_LINEAR_HEAP_VADDR && addr < NEW_LINEAR_HEAP_VADDR_END) {
-            return {fcram_mem, addr - NEW_LINEAR_HEAP_VADDR};
+            return fcram.get() + addr - NEW_LINEAR_HEAP_VADDR;
         }
         if (addr >= VRAM_VADDR && addr < VRAM_VADDR_END) {
-            return {vram_mem, addr - VRAM_VADDR};
+            return vram.get() + addr - VRAM_VADDR;
         }
         if (addr >= PLUGIN_3GX_FB_VADDR && addr < PLUGIN_3GX_FB_VADDR_END) {
             auto plg_ldr = Service::PLGLDR::GetService(system);
             if (plg_ldr) {
-                return {fcram_mem,
-                        addr - PLUGIN_3GX_FB_VADDR + plg_ldr->GetPluginFBAddr() - FCRAM_PADDR};
+                return fcram.get() + addr - PLUGIN_3GX_FB_VADDR + plg_ldr->GetPluginFBAddr() -
+                       FCRAM_PADDR;
             }
         }
 
         UNREACHABLE();
-        return MemoryRef{};
+        return nullptr;
     }
 
     void RasterizerFlushVirtualRegion(VAddr start, u32 size, FlushMode mode) {
@@ -294,33 +251,8 @@ public:
     }
 };
 
-// We use this rather than BufferMem because we don't want new objects to be allocated when
-// deserializing. This avoids unnecessary memory thrashing.
-template <Region R>
-class MemorySystem::BackingMemImpl : public BackingMem {
-public:
-    explicit BackingMemImpl(MemorySystem::Impl& impl_) : impl(impl_) {}
-    u8* GetPtr() override {
-        return impl.GetPtr(R);
-    }
-    const u8* GetPtr() const override {
-        return impl.GetPtr(R);
-    }
-    std::size_t GetSize() const override {
-        return impl.GetSize(R);
-    }
-
-private:
-    MemorySystem::Impl& impl;
-};
-
-MemorySystem::Impl::Impl(Core::System& system_)
-    : system{system_}, fcram_mem(std::make_shared<BackingMemImpl<Region::FCRAM>>(*this)),
-      vram_mem(std::make_shared<BackingMemImpl<Region::VRAM>>(*this)),
-      n3ds_extra_ram_mem(std::make_shared<BackingMemImpl<Region::N3DS>>(*this)),
-      dsp_mem(std::make_shared<BackingMemImpl<Region::DSP>>(*this)) {}
-
 MemorySystem::MemorySystem(Core::System& system) : impl(std::make_unique<Impl>(system)) {}
+
 MemorySystem::~MemorySystem() = default;
 
 void MemorySystem::SetCurrentPageTable(std::shared_ptr<PageTable> page_table) {
@@ -335,10 +267,9 @@ void MemorySystem::RasterizerFlushVirtualRegion(VAddr start, u32 size, FlushMode
     impl->RasterizerFlushVirtualRegion(start, size, mode);
 }
 
-void MemorySystem::MapPages(PageTable& page_table, u32 base, u32 size, MemoryRef memory,
-                            PageType type) {
-    LOG_DEBUG(HW_Memory, "Mapping {} onto {:08X}-{:08X}", (void*)memory.GetPtr(),
-              base * CITRA_PAGE_SIZE, (base + size) * CITRA_PAGE_SIZE);
+void MemorySystem::MapPages(PageTable& page_table, u32 base, u32 size, u8* memory, PageType type) {
+    LOG_DEBUG(HW_Memory, "Mapping {} onto {:08X}-{:08X}", fmt::ptr(memory), base * CITRA_PAGE_SIZE,
+              (base + size) * CITRA_PAGE_SIZE);
 
     if (impl->system.IsPoweredOn()) {
         RasterizerFlushVirtualRegion(base << CITRA_PAGE_BITS, size * CITRA_PAGE_SIZE,
@@ -347,7 +278,7 @@ void MemorySystem::MapPages(PageTable& page_table, u32 base, u32 size, MemoryRef
 
     u32 end = base + size;
     while (base != end) {
-        ASSERT_MSG(base < PAGE_TABLE_NUM_ENTRIES, "out of range mapping at {:08X}", base);
+        ASSERT_MSG(base < PageTable::NUM_ENTRIES, "out of range mapping at {:08X}", base);
 
         page_table.attributes[base] = type;
         page_table.pointers[base] = memory;
@@ -359,12 +290,13 @@ void MemorySystem::MapPages(PageTable& page_table, u32 base, u32 size, MemoryRef
         }
 
         base += 1;
-        if (memory != nullptr && memory.GetSize() > CITRA_PAGE_SIZE)
+        if (memory != nullptr /*&& memory.GetSize() > CITRA_PAGE_SIZE*/) {
             memory += CITRA_PAGE_SIZE;
+        }
     }
 }
 
-void MemorySystem::MapMemoryRegion(PageTable& page_table, VAddr base, u32 size, MemoryRef target) {
+void MemorySystem::MapMemoryRegion(PageTable& page_table, VAddr base, u32 size, u8* target) {
     ASSERT_MSG((size & CITRA_PAGE_MASK) == 0, "non-page aligned size: {:08X}", size);
     ASSERT_MSG((base & CITRA_PAGE_MASK) == 0, "non-page aligned base: {:08X}", base);
     MapPages(page_table, base / CITRA_PAGE_SIZE, size / CITRA_PAGE_SIZE, target, PageType::Memory);
@@ -377,7 +309,7 @@ void MemorySystem::UnmapRegion(PageTable& page_table, VAddr base, u32 size) {
              PageType::Unmapped);
 }
 
-MemoryRef MemorySystem::GetPointerForRasterizerCache(VAddr addr) const {
+u8* MemorySystem::GetPointerForRasterizerCache(VAddr addr) const {
     return impl->GetPointerForRasterizerCache(addr);
 }
 
@@ -507,7 +439,7 @@ bool MemorySystem::WriteExclusive(const VAddr vaddr, const T data, const T expec
     case PageType::RasterizerCachedMemory: {
         RasterizerFlushVirtualRegion(vaddr, sizeof(T), FlushMode::Invalidate);
         const auto volatile_pointer =
-            reinterpret_cast<volatile T*>(GetPointerForRasterizerCache(vaddr).GetPtr());
+            reinterpret_cast<volatile T*>(GetPointerForRasterizerCache(vaddr));
         return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
     }
     default:
@@ -532,7 +464,7 @@ bool MemorySystem::IsValidVirtualAddress(const Kernel::Process& process, const V
 }
 
 bool MemorySystem::IsValidPhysicalAddress(const PAddr paddr) const {
-    return GetPhysicalRef(paddr);
+    return !GetPhysicalSpan(paddr).empty();
 }
 
 u8* MemorySystem::GetPointer(const VAddr vaddr) {
@@ -583,10 +515,10 @@ std::string MemorySystem::ReadCString(VAddr vaddr, std::size_t max_length) {
 }
 
 u8* MemorySystem::GetPhysicalPointer(PAddr address) const {
-    return GetPhysicalRef(address);
+    return GetPhysicalSpan(address).data();
 }
 
-MemoryRef MemorySystem::GetPhysicalRef(PAddr address) const {
+std::span<u8> MemorySystem::GetPhysicalSpan(PAddr address) const {
     constexpr std::array memory_areas = {
         std::make_pair(VRAM_PADDR, VRAM_SIZE),
         std::make_pair(DSP_RAM_PADDR, DSP_RAM_SIZE),
@@ -603,33 +535,33 @@ MemoryRef MemorySystem::GetPhysicalRef(PAddr address) const {
     if (area == memory_areas.end()) {
         LOG_ERROR(HW_Memory, "Unknown GetPhysicalPointer @ {:#08X} at PC {:#08X}", address,
                   impl->GetPC());
-        return nullptr;
+        return {};
     }
 
-    u32 offset_into_region = address - area->first;
+    const u32 offset_into_region = address - area->first;
+    if (offset_into_region > area->second) {
+        return {};
+    }
 
-    std::shared_ptr<BackingMem> target_mem = nullptr;
+    u8* target_mem = nullptr;
     switch (area->first) {
     case VRAM_PADDR:
-        target_mem = impl->vram_mem;
+        target_mem = impl->vram.get();
         break;
     case DSP_RAM_PADDR:
-        target_mem = impl->dsp_mem;
+        target_mem = impl->dsp_mem.get();
         break;
     case FCRAM_PADDR:
-        target_mem = impl->fcram_mem;
+        target_mem = impl->fcram.get();
         break;
     case N3DS_EXTRA_RAM_PADDR:
-        target_mem = impl->n3ds_extra_ram_mem;
+        target_mem = impl->n3ds_extra_ram.get();
         break;
     default:
         UNREACHABLE();
     }
-    if (offset_into_region > target_mem->GetSize()) {
-        return {nullptr};
-    }
 
-    return {target_mem, offset_into_region};
+    return std::span{target_mem + offset_into_region, area->second - offset_into_region};
 }
 
 std::vector<VAddr> MemorySystem::PhysicalToVirtualAddressForRasterizer(PAddr addr) {
@@ -889,13 +821,8 @@ const u8* MemorySystem::GetFCRAMPointer(std::size_t offset) const {
     return impl->fcram.get() + offset;
 }
 
-MemoryRef MemorySystem::GetFCRAMRef(std::size_t offset) const {
-    ASSERT(offset <= Memory::FCRAM_N3DS_SIZE);
-    return MemoryRef(impl->fcram_mem, offset);
-}
-
-void MemorySystem::SetDSP(AudioCore::DspInterface& dsp) {
-    impl->dsp = &dsp;
+std::span<u8, DSP_RAM_SIZE> MemorySystem::GetDspMemory() const {
+    return std::span<u8, DSP_RAM_SIZE>{impl->dsp_mem.get(), DSP_RAM_SIZE};
 }
 
 } // namespace Memory
diff --git a/src/core/memory.h b/src/core/memory.h
index d482f0bc4..b6d2027cc 100644
--- a/src/core/memory.h
+++ b/src/core/memory.h
@@ -7,7 +7,6 @@
 #include <cstddef>
 #include <string>
 #include "common/common_types.h"
-#include "common/memory_ref.h"
 
 namespace Kernel {
 class Process;
@@ -30,7 +29,6 @@ namespace Memory {
 constexpr u32 CITRA_PAGE_SIZE = 0x1000;
 constexpr u32 CITRA_PAGE_MASK = CITRA_PAGE_SIZE - 1;
 constexpr int CITRA_PAGE_BITS = 12;
-constexpr std::size_t PAGE_TABLE_NUM_ENTRIES = 1 << (32 - CITRA_PAGE_BITS);
 
 enum class PageType {
     /// Page is unmapped and should cause an access error.
@@ -49,54 +47,23 @@ enum class PageType {
  * requires an indexed fetch and a check for NULL.
  */
 struct PageTable {
+    PageTable() = default;
+    ~PageTable() noexcept = default;
+
+    PageTable(const PageTable&) = delete;
+    PageTable& operator=(const PageTable&) = delete;
+
+    PageTable(PageTable&&) noexcept = default;
+    PageTable& operator=(PageTable&&) noexcept = default;
+
+    static constexpr std::size_t NUM_ENTRIES = 1 << (32 - CITRA_PAGE_BITS);
+
     /**
      * Array of memory pointers backing each page. An entry can only be non-null if the
      * corresponding entry in the `attributes` array is of type `Memory`.
      */
-
-    // The reason for this rigmarole is to keep the 'raw' and 'refs' arrays in sync.
-    // We need 'raw' for dynarmic and 'refs' for serialization
-    struct Pointers {
-
-        struct Entry {
-            Entry(Pointers& pointers_, VAddr idx_) : pointers(pointers_), idx(idx_) {}
-
-            Entry& operator=(MemoryRef value) {
-                pointers.raw[idx] = value.GetPtr();
-                pointers.refs[idx] = std::move(value);
-                return *this;
-            }
-
-            operator u8*() {
-                return pointers.raw[idx];
-            }
-
-        private:
-            Pointers& pointers;
-            VAddr idx;
-        };
-
-        Entry operator[](std::size_t idx) {
-            return Entry(*this, static_cast<VAddr>(idx));
-        }
-
-    private:
-        std::array<u8*, PAGE_TABLE_NUM_ENTRIES> raw;
-        std::array<MemoryRef, PAGE_TABLE_NUM_ENTRIES> refs;
-        friend struct PageTable;
-    };
-
-    Pointers pointers;
-
-    /**
-     * Array of fine grained page attributes. If it is set to any value other than `Memory`, then
-     * the corresponding entry in `pointers` MUST be set to null.
-     */
-    std::array<PageType, PAGE_TABLE_NUM_ENTRIES> attributes;
-
-    std::array<u8*, PAGE_TABLE_NUM_ENTRIES>& GetPointerArray() {
-        return pointers.raw;
-    }
+    std::array<u8*, NUM_ENTRIES> pointers{};
+    std::array<PageType, NUM_ENTRIES> attributes{};
 
     void Clear();
 };
@@ -235,7 +202,7 @@ public:
      * @param size The amount of bytes to map. Must be page-aligned.
      * @param target Buffer with the memory backing the mapping. Must be of length at least `size`.
      */
-    void MapMemoryRegion(PageTable& page_table, VAddr base, u32 size, MemoryRef target);
+    void MapMemoryRegion(PageTable& page_table, VAddr base, u32 size, u8* target);
 
     void UnmapRegion(PageTable& page_table, VAddr base, u32 size);
 
@@ -510,7 +477,7 @@ public:
     u8* GetPhysicalPointer(PAddr address) const;
 
     /// Returns a reference to the memory region beginning at the specified physical address
-    MemoryRef GetPhysicalRef(PAddr address) const;
+    std::span<u8> GetPhysicalSpan(PAddr address) const;
 
     /// Determines if the given VAddr is valid for the specified process.
     bool IsValidVirtualAddress(const Kernel::Process& process, VAddr vaddr);
@@ -527,16 +494,13 @@ public:
     /// Gets pointer in FCRAM with given offset
     const u8* GetFCRAMPointer(std::size_t offset) const;
 
-    /// Gets a serializable ref to FCRAM with the given offset
-    MemoryRef GetFCRAMRef(std::size_t offset) const;
-
     /// Registers page table for rasterizer cache marking
     void RegisterPageTable(std::shared_ptr<PageTable> page_table);
 
     /// Unregisters page table for rasterizer cache marking
     void UnregisterPageTable(std::shared_ptr<PageTable> page_table);
 
-    void SetDSP(AudioCore::DspInterface& dsp);
+    std::span<u8, DSP_RAM_SIZE> GetDspMemory() const;
 
     void RasterizerFlushVirtualRegion(VAddr start, u32 size, FlushMode mode);
 
@@ -556,17 +520,13 @@ private:
      * Since the cache only happens on linear heap or VRAM, we know the exact physical address and
      * pointer of such virtual address
      */
-    MemoryRef GetPointerForRasterizerCache(VAddr addr) const;
+    u8* GetPointerForRasterizerCache(VAddr addr) const;
 
-    void MapPages(PageTable& page_table, u32 base, u32 size, MemoryRef memory, PageType type);
+    void MapPages(PageTable& page_table, u32 base, u32 size, u8* memory, PageType type);
 
 private:
     class Impl;
     std::unique_ptr<Impl> impl;
-
-public:
-    template <Region R>
-    class BackingMemImpl;
 };
 
 } // namespace Memory
diff --git a/src/tests/core/memory/vm_manager.cpp b/src/tests/core/memory/vm_manager.cpp
index f58ad2e5b..6245a8c85 100644
--- a/src/tests/core/memory/vm_manager.cpp
+++ b/src/tests/core/memory/vm_manager.cpp
@@ -32,7 +32,7 @@ TEST_CASE("Memory Basics", "[kernel][memory]") {
         CHECK(vma != manager->vma_map.end());
         CHECK(vma->second.size == static_cast<u32>(block.GetSize()));
         CHECK(vma->second.type == Kernel::VMAType::BackingMemory);
-        CHECK(vma->second.backing_memory.GetPtr() == block.GetPtr());
+        CHECK(vma->second.backing_memory == block.GetPtr());
         CHECK(vma->second.meminfo_state == Kernel::MemoryState::Private);
     }
 
@@ -50,7 +50,7 @@ TEST_CASE("Memory Basics", "[kernel][memory]") {
         auto vma = manager->FindVMA(Memory::HEAP_VADDR);
         CHECK(vma != manager->vma_map.end());
         CHECK(vma->second.type == Kernel::VMAType::Free);
-        CHECK(vma->second.backing_memory.GetPtr() == nullptr);
+        CHECK(vma->second.backing_memory == nullptr);
     }
 
     SECTION("changing memory permissions") {
diff --git a/src/video_core/rasterizer_cache/rasterizer_cache.h b/src/video_core/rasterizer_cache/rasterizer_cache.h
index 16027fd93..3760621eb 100644
--- a/src/video_core/rasterizer_cache/rasterizer_cache.h
+++ b/src/video_core/rasterizer_cache/rasterizer_cache.h
@@ -1000,12 +1000,12 @@ void RasterizerCache<T>::UploadSurface(Surface& surface, SurfaceInterval interva
     const auto staging = runtime.FindStaging(
         load_info.width * load_info.height * surface.GetInternalBytesPerPixel(), true);
 
-    MemoryRef source_ptr = memory.GetPhysicalRef(load_info.addr);
-    if (!source_ptr) [[unlikely]] {
+    auto source_span = memory.GetPhysicalSpan(load_info.addr);
+    if (source_span.empty()) [[unlikely]] {
         return;
     }
 
-    const auto upload_data = source_ptr.GetWriteBytes(load_info.end - load_info.addr);
+    const auto upload_data = source_span.subspan(0, load_info.end - load_info.addr);
     DecodeTexture(load_info, load_info.addr, load_info.end, upload_data, staging.mapped,
                   runtime.NeedsConversion(surface.pixel_format));
 
@@ -1048,12 +1048,12 @@ bool RasterizerCache<T>::UploadCustomSurface(SurfaceId surface_id, SurfaceInterv
     const SurfaceParams load_info = surface.FromInterval(interval);
     ASSERT(load_info.addr >= surface.addr && load_info.end <= surface.end);
 
-    MemoryRef source_ptr = memory.GetPhysicalRef(load_info.addr);
-    if (!source_ptr) [[unlikely]] {
+    auto source_span = memory.GetPhysicalSpan(load_info.addr);
+    if (source_span.empty()) [[unlikely]] {
         return false;
     }
 
-    const auto upload_data = source_ptr.GetWriteBytes(load_info.end - load_info.addr);
+    const auto upload_data = source_span.subspan(0, load_info.end - load_info.addr);
     const u64 hash = ComputeHash(load_info, upload_data);
 
     const u32 level = surface.LevelOf(load_info.addr);
@@ -1108,12 +1108,12 @@ void RasterizerCache<T>::DownloadSurface(Surface& surface, SurfaceInterval inter
     };
     surface.Download(download, staging);
 
-    MemoryRef dest_ptr = memory.GetPhysicalRef(flush_start);
-    if (!dest_ptr) [[unlikely]] {
+    auto dest_span = memory.GetPhysicalSpan(flush_start);
+    if (dest_span.empty()) [[unlikely]] {
         return;
     }
 
-    const auto download_dest = dest_ptr.GetWriteBytes(flush_end - flush_start);
+    const auto download_dest = dest_span.subspan(0, flush_end - flush_start);
     EncodeTexture(flush_info, flush_start, flush_end, staging.mapped, download_dest,
                   runtime.NeedsConversion(surface.pixel_format));
 }
@@ -1124,29 +1124,29 @@ void RasterizerCache<T>::DownloadFillSurface(Surface& surface, SurfaceInterval i
     const u32 flush_end = boost::icl::last_next(interval);
     ASSERT(flush_start >= surface.addr && flush_end <= surface.end);
 
-    MemoryRef dest_ptr = memory.GetPhysicalRef(flush_start);
-    if (!dest_ptr) [[unlikely]] {
+    auto dest_span = memory.GetPhysicalSpan(flush_start);
+    if (dest_span.empty()) [[unlikely]] {
         return;
     }
 
     const u32 start_offset = flush_start - surface.addr;
     const u32 download_size =
-        std::clamp(flush_end - flush_start, 0u, static_cast<u32>(dest_ptr.GetSize()));
+        std::clamp(flush_end - flush_start, 0u, static_cast<u32>(dest_span.size()));
     const u32 coarse_start_offset = start_offset - (start_offset % surface.fill_size);
     const u32 backup_bytes = start_offset % surface.fill_size;
 
     std::array<u8, 4> backup_data;
     if (backup_bytes) {
-        std::memcpy(backup_data.data(), &dest_ptr[coarse_start_offset], backup_bytes);
+        std::memcpy(backup_data.data(), &dest_span[coarse_start_offset], backup_bytes);
     }
 
     for (u32 offset = coarse_start_offset; offset < download_size; offset += surface.fill_size) {
-        std::memcpy(&dest_ptr[offset], &surface.fill_data[0],
+        std::memcpy(&dest_span[offset], &surface.fill_data[0],
                     std::min(surface.fill_size, download_size - offset));
     }
 
     if (backup_bytes) {
-        std::memcpy(&dest_ptr[coarse_start_offset], &backup_data[0], backup_bytes);
+        std::memcpy(&dest_span[coarse_start_offset], &backup_data[0], backup_bytes);
     }
 }
 
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 0208c8cde..f2a51f1ac 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -219,14 +219,14 @@ void RasterizerVulkan::SetupVertexArray() {
         u32 data_size = loader.byte_count * vertex_num;
         res_cache.FlushRegion(data_addr, data_size);
 
-        const MemoryRef src_ref = memory.GetPhysicalRef(data_addr);
-        if (src_ref.GetSize() < data_size) {
+        const auto src_span = memory.GetPhysicalSpan(data_addr);
+        if (src_span.size() < data_size) {
             LOG_ERROR(Render_Vulkan,
                       "Vertex buffer size {} exceeds available space {} at address {:#016X}",
-                      data_size, src_ref.GetSize(), data_addr);
+                      data_size, src_span.size(), data_addr);
         }
 
-        const u8* src_ptr = src_ref.GetPtr();
+        const u8* src_ptr = src_span.data();
         u8* dst_ptr = array_ptr + buffer_offset;
 
         // Align stride up if required by Vulkan implementation.