diff --git a/src/sfizz/FilePool.cpp b/src/sfizz/FilePool.cpp
index db1a1dfb..ff7492d6 100644
--- a/src/sfizz/FilePool.cpp
+++ b/src/sfizz/FilePool.cpp
@@ -428,14 +428,17 @@ void sfz::FilePool::loadingJob(QueuedFileData data) noexcept
 
     data.data->status = FileData::Status::Done;
 
-    std::lock_guard<SpinMutex> guard { lastUsedMutex };
+    std::lock_guard<SpinMutex> guard { garbageAndLastUsedMutex };
     if (absl::c_find(lastUsedFiles, *id) == lastUsedFiles.end())
         lastUsedFiles.push_back(*id);
 }
 
 void sfz::FilePool::clear()
 {
+    std::lock_guard<SpinMutex> guard { garbageAndLastUsedMutex };
     emptyFileLoadingQueues();
+    garbageToCollect.clear();
+    lastUsedFiles.clear();
     preloadedFiles.clear();
 }
 
@@ -514,10 +517,7 @@ void sfz::FilePool::dispatchingJob() noexcept
 void sfz::FilePool::garbageJob() noexcept
 {
     while (semGarbageBarrier.wait(), garbageFlag) {
-        std::lock_guard<SpinMutex> guard { garbageMutex };
-        for (auto& g: garbageToCollect)
-            g.reset();
-
+        std::lock_guard<SpinMutex> guard { garbageAndLastUsedMutex };
         garbageToCollect.clear();
     }
 }
@@ -587,9 +587,8 @@ void sfz::FilePool::setRamLoading(bool loadInRam) noexcept
 
 void sfz::FilePool::triggerGarbageCollection() noexcept
 {
-    const std::unique_lock<SpinMutex> lastUsedLock { lastUsedMutex, std::try_to_lock };
-    const std::unique_lock<SpinMutex> garbageLock { garbageMutex, std::try_to_lock };
-    if (!lastUsedLock.owns_lock() || !garbageLock.owns_lock())
+    const std::unique_lock<SpinMutex> guard { garbageAndLastUsedMutex, std::try_to_lock };
+    if (!guard.owns_lock())
         return;
 
     const auto now = std::chrono::high_resolution_clock::now();
@@ -597,7 +596,15 @@ void sfz::FilePool::triggerGarbageCollection() noexcept
         if (garbageToCollect.size() == garbageToCollect.capacity())
            return false;
 
-        auto& data = preloadedFiles[id];
+        auto it = preloadedFiles.find(id);
+        if (it == preloadedFiles.end()) {
+            // Getting here means that the preloadedFiles got changed (probably cleared)
+            // while the lastUsedFiles were untouched.
+            ASSERTFALSE;
+            return true;
+        }
+
+        sfz::FileData& data = it->second;
         if (data.status == FileData::Status::Preloaded)
             return true;
 
diff --git a/src/sfizz/FilePool.h b/src/sfizz/FilePool.h
index 8c14f1ce..41068598 100644
--- a/src/sfizz/FilePool.h
+++ b/src/sfizz/FilePool.h
@@ -362,9 +362,8 @@ private:
     std::thread dispatchThread { &FilePool::dispatchingJob, this };
     std::thread garbageThread { &FilePool::garbageJob, this };
 
-    SpinMutex lastUsedMutex;
+    SpinMutex garbageAndLastUsedMutex;
     std::vector<FileId> lastUsedFiles;
-    SpinMutex garbageMutex;
     std::vector<FileAudioBuffer> garbageToCollect;
 
     std::shared_ptr<ThreadPool> threadPool;
diff --git a/src/sfizz/Synth.cpp b/src/sfizz/Synth.cpp
index 3365f791..4103f388 100644
--- a/src/sfizz/Synth.cpp
+++ b/src/sfizz/Synth.cpp
@@ -194,6 +194,9 @@ void sfz::Synth::clear()
 {
     const std::lock_guard<SpinMutex> disableCallback { callbackGuard };
 
+    // Clear the background queues before removing everyone
+    resources.filePool.waitForBackgroundLoading();
+
     for (auto& voice : voices)
         voice->reset();
     for (auto& list : noteActivationLists)