diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java
index 002b0e2dcf039..3779d8720a9d9 100644
--- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java
+++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java
@@ -482,34 +482,6 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
         return status;
     }
 
-    /**
-     * Stops all recognitions active currently and clears the internal state.
-     */
-    void stopAllRecognitions() {
-        synchronized (mLock) {
-            MetricsLogger.count(mContext, "sth_stop_all_recognitions", 1);
-            if (mModuleProperties == null || mModule == null) {
-                return;
-            }
-
-            // Stop all recognition models.
-            for (ModelData model : mModelDataMap.values()) {
-                if (model.isModelStarted()) {
-                    int status = stopRecognitionLocked(model,
-                            false /* do not notify for synchronous calls */);
-                    if (status != STATUS_OK) {
-                        Slog.w(TAG, "Error stopping model: " + model.getHandle());
-                    }
-                    model.setStopped();
-                    model.setRequested(false);
-                    model.clearCallback();
-                    model.setRecognitionConfig(null);
-                }
-            }
-            internalClearGlobalStateLocked();
-        }
-    }
-
     public ModuleProperties getModuleProperties() {
         return mModuleProperties;
     }
diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerInternal.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerInternal.java
index 113431f2de7e8..d05e044499ab1 100644
--- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerInternal.java
+++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerInternal.java
@@ -68,13 +68,11 @@ public abstract class SoundTriggerInternal {
      */
     public abstract int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener);
 
-    /**
-     * Stops all recognitions active currently and clears the internal state.
-     */
-    public abstract void stopAllRecognitions();
-
     public abstract ModuleProperties getModuleProperties();
 
+    /**
+     * Unloads (and stops if running) the given keyphraseId
+     */
     public abstract int unloadKeyphraseModel(int keyphaseId);
 
     public abstract void dump(FileDescriptor fd, PrintWriter pw, String[] args);
diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java
index a4c1210ba1888..9bca0128cc438 100644
--- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java
+++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java
@@ -205,12 +205,6 @@ public class SoundTriggerService extends SystemService {
             return mSoundTriggerHelper.stopKeyphraseRecognition(keyphraseId, listener);
         }
 
-        @Override
-        public void stopAllRecognitions() {
-            if (!isInitialized()) return;
-            mSoundTriggerHelper.stopAllRecognitions();
-        }
-
         @Override
         public ModuleProperties getModuleProperties() {
             if (!isInitialized()) return null;
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
index 386c3aa1b2a40..57d68fd9e37eb 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
@@ -69,6 +69,7 @@ import com.android.server.UiThread;
 import java.io.FileDescriptor;
 import java.io.PrintWriter;
 import java.util.List;
+import java.util.TreeSet;
 
 /**
  * SystemService that publishes an IVoiceInteractionManagerService.
@@ -81,6 +82,7 @@ public class VoiceInteractionManagerService extends SystemService {
     final ContentResolver mResolver;
     final DatabaseHelper mDbHelper;
     final ActivityManagerInternal mAmInternal;
+    final TreeSet<Integer> mLoadedKeyphraseIds;
     SoundTriggerInternal mSoundTriggerInternal;
 
     public VoiceInteractionManagerService(Context context) {
@@ -90,6 +92,7 @@ public class VoiceInteractionManagerService extends SystemService {
         mDbHelper = new DatabaseHelper(context);
         mServiceStub = new VoiceInteractionManagerServiceStub();
         mAmInternal = LocalServices.getService(ActivityManagerInternal.class);
+        mLoadedKeyphraseIds = new TreeSet<Integer>();
 
         PackageManagerInternal packageManagerInternal = LocalServices.getService(
                 PackageManagerInternal.class);
@@ -394,7 +397,7 @@ public class VoiceInteractionManagerService extends SystemService {
 
                 if (force || mImpl == null || mImpl.mUser != mCurUser
                         || !mImpl.mComponent.equals(serviceComponent)) {
-                    mSoundTriggerInternal.stopAllRecognitions();
+                    unloadAllKeyphraseModels();
                     if (mImpl != null) {
                         mImpl.shutdownLocked();
                     }
@@ -785,6 +788,7 @@ public class VoiceInteractionManagerService extends SystemService {
                         if (mImpl != null && mImpl.mService != null) {
                             mImpl.notifySoundModelsChangedLocked();
                         }
+                        mLoadedKeyphraseIds.remove(keyphraseId);
                     }
                 }
                 Binder.restoreCallingIdentity(caller);
@@ -865,6 +869,11 @@ public class VoiceInteractionManagerService extends SystemService {
                     Slog.w(TAG, "No matching sound model found in startRecognition");
                     return SoundTriggerInternal.STATUS_ERROR;
                 } else {
+                    // Regardless of the status of the start recognition, we need to make sure
+                    // that we unload this model if needed later.
+                    synchronized (this) {
+                        mLoadedKeyphraseIds.add(keyphraseId);
+                    }
                     return mSoundTriggerInternal.startRecognition(
                             keyphraseId, soundModel, callback, recognitionConfig);
                 }
@@ -893,6 +902,21 @@ public class VoiceInteractionManagerService extends SystemService {
             }
         }
 
+        private synchronized void unloadAllKeyphraseModels() {
+            for (int keyphraseId : mLoadedKeyphraseIds) {
+                final long caller = Binder.clearCallingIdentity();
+                try {
+                    int status = mSoundTriggerInternal.unloadKeyphraseModel(keyphraseId);
+                    if (status != SoundTriggerInternal.STATUS_OK) {
+                        Slog.w(TAG, "Failed to unload keyphrase " + keyphraseId + ":" + status);
+                    }
+                } finally {
+                    Binder.restoreCallingIdentity(caller);
+                }
+            }
+            mLoadedKeyphraseIds.clear();
+        }
+
         @Override
         public ComponentName getActiveServiceComponentName() {
             enforceCallingPermission(Manifest.permission.ACCESS_VOICE_INTERACTION_SERVICE);
@@ -1078,7 +1102,7 @@ public class VoiceInteractionManagerService extends SystemService {
                     // The user is force stopping our current interactor/recognizer.
                     // Clear the current settings and restore default state.
                     synchronized (VoiceInteractionManagerService.this) {
-                        mSoundTriggerInternal.stopAllRecognitions();
+                        unloadAllKeyphraseModels();
                         if (mImpl != null) {
                             mImpl.shutdownLocked();
                             mImpl = null;