diff --git a/src/core/core.cpp b/src/core/core.cpp
index 109f937f9..286364018 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -189,7 +189,6 @@ System::ResultStatus System::Init(Frontend::EmuWindow& emu_window, u32 system_mo
     }
 
     kernel->GetThreadManager().SetCPU(*cpu_core);
-    memory->SetCPU(*cpu_core);
 
     if (Settings::values.enable_dsp_lle) {
         dsp_core = std::make_unique<AudioCore::DspLle>(*memory,
diff --git a/src/core/hle/kernel/thread.cpp b/src/core/hle/kernel/thread.cpp
index e3ee705c3..e4b14417f 100644
--- a/src/core/hle/kernel/thread.cpp
+++ b/src/core/hle/kernel/thread.cpp
@@ -114,6 +114,7 @@ void ThreadManager::SwitchContext(Thread* new_thread) {
             kernel.SetCurrentProcess(SharedFrom(current_thread->owner_process));
             kernel.memory.SetCurrentPageTable(
                 &current_thread->owner_process->vm_manager.page_table);
+            cpu->PageTableChanged(); // notify the CPU the page table in memory has changed
         }
 
         cpu->LoadContext(new_thread->context);
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 77cb7198c..096f4c697 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -66,22 +66,14 @@ public:
     RasterizerCacheMarker cache_marker;
     std::vector<PageTable*> page_table_list;
 
-    ARM_Interface* cpu = nullptr;
     AudioCore::DspInterface* dsp = nullptr;
 };
 
 MemorySystem::MemorySystem() : impl(std::make_unique<Impl>()) {}
 MemorySystem::~MemorySystem() = default;
 
-void MemorySystem::SetCPU(ARM_Interface& cpu) {
-    impl->cpu = &cpu;
-}
-
 void MemorySystem::SetCurrentPageTable(PageTable* page_table) {
     impl->current_page_table = page_table;
-    if (impl->cpu != nullptr) {
-        impl->cpu->PageTableChanged();
-    }
 }
 
 PageTable* MemorySystem::GetCurrentPageTable() const {
diff --git a/src/core/memory.h b/src/core/memory.h
index 019a9f7e7..6caca5a2b 100644
--- a/src/core/memory.h
+++ b/src/core/memory.h
@@ -220,9 +220,6 @@ public:
     MemorySystem();
     ~MemorySystem();
 
-    /// Sets CPU to notify page table change
-    void SetCPU(ARM_Interface& cpu);
-
     /**
      * Maps an allocated buffer onto a region of the emulated process address space.
      *