diff --git a/src/sfizz/Voice.cpp b/src/sfizz/Voice.cpp
index 54c5ebc2..caaa718f 100644
--- a/src/sfizz/Voice.cpp
+++ b/src/sfizz/Voice.cpp
@@ -237,6 +237,14 @@ void sfz::Voice::registerTempo(int delay, float secondsPerQuarter) noexcept
     UNUSED(secondsPerQuarter);
 }
 
+void sfz::Voice::updateTrackingFactor() noexcept
+{
+    // Protect the envelope follower against blowups
+    const auto maxTrackingFactor = sampleRate / samplesPerBlock;
+    attackTrackingFactor =  min(config::powerFollowerAttackFactor, maxTrackingFactor) / sampleRate;
+    releaseTrackingFactor =  min(config::powerFollowerReleaseFactor, maxTrackingFactor) / sampleRate;
+}
+
 void sfz::Voice::setSampleRate(float sampleRate) noexcept
 {
     this->sampleRate = sampleRate;
@@ -249,13 +257,13 @@ void sfz::Voice::setSampleRate(float sampleRate) noexcept
     for (auto& lfo : lfos)
         lfo->setSampleRate(sampleRate);
 
-    attackTrackingFactor =  config::powerFollowerAttackFactor / sampleRate;
-    releaseTrackingFactor =  config::powerFollowerReleaseFactor / sampleRate;
+    updateTrackingFactor();
 }
 
 void sfz::Voice::setSamplesPerBlock(int samplesPerBlock) noexcept
 {
     this->samplesPerBlock = samplesPerBlock;
+    updateTrackingFactor();
 }
 
 void sfz::Voice::renderBlock(AudioSpan<float> buffer) noexcept
diff --git a/src/sfizz/Voice.h b/src/sfizz/Voice.h
index c2b3d1c4..5881e79c 100644
--- a/src/sfizz/Voice.h
+++ b/src/sfizz/Voice.h
@@ -485,6 +485,12 @@ private:
     Smoother xfadeSmoother;
     void resetSmoothers() noexcept;
 
+    /**
+     * @brief Update and clamp the tracking factors to ensure the power
+     *          follower does not blow up.
+     *
+     */
+    void updateTrackingFactor() noexcept;
     float attackTrackingFactor { config::powerFollowerAttackFactor / config::defaultSampleRate };
     float releaseTrackingFactor { config::powerFollowerReleaseFactor / config::defaultSampleRate };
     float meanChannelPower;