diff --git a/packages/SystemUI/src/com/android/systemui/statusbar/phone/PhoneStatusBar.java b/packages/SystemUI/src/com/android/systemui/statusbar/phone/PhoneStatusBar.java
index 12063a2f1f16e..6fc32de100e2d 100644
--- a/packages/SystemUI/src/com/android/systemui/statusbar/phone/PhoneStatusBar.java
+++ b/packages/SystemUI/src/com/android/systemui/statusbar/phone/PhoneStatusBar.java
@@ -1441,7 +1441,11 @@ public class PhoneStatusBar extends BaseStatusBar implements DemoMode,
     public void removeNotification(String key, RankingMap ranking) {
         boolean deferRemoval = false;
         if (mHeadsUpManager.isHeadsUp(key)) {
-            deferRemoval = !mHeadsUpManager.removeNotification(key);
+            // A cancel() in repsonse to a remote input shouldn't be delayed, as it makes the
+            // sending look longer than it takes.
+            boolean ignoreEarliestRemovalTime = mRemoteInputController.isSpinning(key)
+                    && !FORCE_REMOTE_INPUT_HISTORY;
+            deferRemoval = !mHeadsUpManager.removeNotification(key,  ignoreEarliestRemovalTime);
         }
         if (key.equals(mMediaNotificationKey)) {
             clearCurrentMediaNotification();
@@ -2368,7 +2372,7 @@ public class PhoneStatusBar extends BaseStatusBar implements DemoMode,
         if (wasHeadsUp) {
             if (!shouldPeek) {
                 // We don't want this to be interrupting anymore, lets remove it
-                mHeadsUpManager.removeNotification(key);
+                mHeadsUpManager.removeNotification(key, false /* ignoreEarliestRemovalTime */);
             } else {
                 mHeadsUpManager.updateNotification(entry, alertAgain);
             }
diff --git a/packages/SystemUI/src/com/android/systemui/statusbar/policy/HeadsUpManager.java b/packages/SystemUI/src/com/android/systemui/statusbar/policy/HeadsUpManager.java
index 52fb470222752..d3ae54984e994 100644
--- a/packages/SystemUI/src/com/android/systemui/statusbar/policy/HeadsUpManager.java
+++ b/packages/SystemUI/src/com/android/systemui/statusbar/policy/HeadsUpManager.java
@@ -264,9 +264,9 @@ public class HeadsUpManager implements ViewTreeObserver.OnComputeInternalInsetsL
      * @return true if the notification was removed and false if it still needs to be kept around
      * for a bit since it wasn't shown long enough
      */
-    public boolean removeNotification(String key) {
+    public boolean removeNotification(String key, boolean ignoreEarliestRemovalTime) {
         if (DEBUG) Log.v(TAG, "remove");
-        if (wasShownLongEnough(key)) {
+        if (wasShownLongEnough(key) || ignoreEarliestRemovalTime) {
             releaseImmediately(key);
             return true;
         } else {