diff --git a/packages/DocumentsUI/res/layout/fragment_directory.xml b/packages/DocumentsUI/res/layout/fragment_directory.xml
index 8eb46ddffe48a..1c086a614a5c3 100644
--- a/packages/DocumentsUI/res/layout/fragment_directory.xml
+++ b/packages/DocumentsUI/res/layout/fragment_directory.xml
@@ -39,65 +39,70 @@
         android:background="@color/material_grey_50"
         android:visibility="gone"/>
 
-    <!-- The empty container view -->
-    <FrameLayout
-        android:id="@android:id/empty"
-        android:gravity="center"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        android:orientation="vertical"
-        android:background="@color/directory_background"
-        android:focusable="true"
-        android:focusableInTouchMode="true"
-        android:visibility="gone">
-
-        <LinearLayout
-            android:id="@+id/content"
-            android:gravity="center"
-            android:layout_width="match_parent"
-            android:layout_height="match_parent"
-            android:orientation="vertical">
-
-            <ImageView
-                android:id="@+id/artwork"
-                android:src="@drawable/cabinet"
-                android:adjustViewBounds="true"
-                android:layout_height="250dp"
-                android:layout_width="fill_parent"
-                android:alpha="1"
-                android:layout_centerVertical="true"
-                android:layout_marginBottom="25dp"
-                android:scaleType="fitCenter"
-                android:contentDescription="@null" />
-
-            <TextView
-                android:id="@+id/message"
-                android:layout_width="wrap_content"
-                android:layout_height="wrap_content"
-                android:text="@string/empty"
-                style="@android:style/TextAppearance.Material.Subhead" />
-
-        </LinearLayout>
-    </FrameLayout>
-
-    <!-- This FrameLayout works around b/24189541 -->
-    <FrameLayout
+    <com.android.documentsui.dirlist.TouchSwipeRefreshLayout
+        android:id="@+id/refresh_layout"
         android:layout_width="match_parent"
         android:layout_height="match_parent">
 
-        <android.support.v7.widget.RecyclerView
-            android:id="@+id/dir_list"
-            android:scrollbars="vertical"
+        <FrameLayout
             android:layout_width="match_parent"
-            android:layout_height="match_parent"
-            android:paddingStart="0dp"
-            android:paddingEnd="0dp"
-            android:paddingTop="0dp"
-            android:paddingBottom="0dp"
-            android:clipToPadding="false"
-            android:scrollbarStyle="outsideOverlay"
-            android:drawSelectorOnTop="true" />
+            android:layout_height="match_parent">
+            <!-- The empty container view -->
+            <FrameLayout
+                android:id="@android:id/empty"
+                android:gravity="center"
+                android:layout_width="match_parent"
+                android:layout_height="match_parent"
+                android:orientation="vertical"
+                android:background="@color/directory_background"
+                android:focusable="true"
+                android:focusableInTouchMode="true"
+                android:visibility="gone"
+                android:clickable="true">
 
-    </FrameLayout>
+                <LinearLayout
+                    android:id="@+id/content"
+                    android:gravity="center"
+                    android:layout_width="match_parent"
+                    android:layout_height="match_parent"
+                    android:orientation="vertical">
+
+                    <ImageView
+                        android:id="@+id/artwork"
+                        android:src="@drawable/cabinet"
+                        android:adjustViewBounds="true"
+                        android:layout_height="250dp"
+                        android:layout_width="fill_parent"
+                        android:alpha="1"
+                        android:layout_centerVertical="true"
+                        android:layout_marginBottom="25dp"
+                        android:scaleType="fitCenter"
+                        android:contentDescription="@null"/>
+
+                    <TextView
+                        android:id="@+id/message"
+                        android:layout_width="wrap_content"
+                        android:layout_height="wrap_content"
+                        android:text="@string/empty"
+                        style="@android:style/TextAppearance.Material.Subhead"/>
+
+                </LinearLayout>
+            </FrameLayout>
+
+            <android.support.v7.widget.RecyclerView
+                android:id="@+id/dir_list"
+                android:scrollbars="vertical"
+                android:layout_width="match_parent"
+                android:layout_height="match_parent"
+                android:paddingStart="0dp"
+                android:paddingEnd="0dp"
+                android:paddingTop="0dp"
+                android:paddingBottom="0dp"
+                android:clipToPadding="false"
+                android:scrollbarStyle="outsideOverlay"
+                android:drawSelectorOnTop="true"/>
+        </FrameLayout>
+
+    </com.android.documentsui.dirlist.TouchSwipeRefreshLayout>
 
 </com.android.documentsui.dirlist.AnimationView>
diff --git a/packages/DocumentsUI/src/com/android/documentsui/dirlist/DirectoryFragment.java b/packages/DocumentsUI/src/com/android/documentsui/dirlist/DirectoryFragment.java
index 85c988e6fdaa1..ca7b2ca6a7ba4 100644
--- a/packages/DocumentsUI/src/com/android/documentsui/dirlist/DirectoryFragment.java
+++ b/packages/DocumentsUI/src/com/android/documentsui/dirlist/DirectoryFragment.java
@@ -46,6 +46,7 @@ import android.os.Parcelable;
 import android.provider.DocumentsContract;
 import android.provider.DocumentsContract.Document;
 import android.support.v13.view.DragStartHelper;
+import android.support.v4.widget.SwipeRefreshLayout;
 import android.support.v7.widget.GridLayoutManager;
 import android.support.v7.widget.GridLayoutManager.SpanSizeLookup;
 import android.support.v7.widget.RecyclerView;
@@ -118,7 +119,7 @@ import javax.annotation.Nullable;
  */
 public class DirectoryFragment extends Fragment
         implements DocumentsAdapter.Environment, LoaderCallbacks<DirectoryResult>,
-        ItemDragListener.DragHost {
+        ItemDragListener.DragHost, SwipeRefreshLayout.OnRefreshListener {
 
     @IntDef(flag = true, value = {
             TYPE_NORMAL,
@@ -148,6 +149,7 @@ public class DirectoryFragment extends Fragment
 
     private IconHelper mIconHelper;
 
+    private SwipeRefreshLayout mRefreshLayout;
     private View mEmptyView;
     private RecyclerView mRecView;
     private ListeningGestureDetector mGestureDetector;
@@ -192,6 +194,10 @@ public class DirectoryFragment extends Fragment
 
         mMessageBar = MessageBar.create(getChildFragmentManager());
         mProgressBar = view.findViewById(R.id.progressbar);
+
+        mRefreshLayout = (SwipeRefreshLayout) view.findViewById(R.id.refresh_layout);
+        mRefreshLayout.setOnRefreshListener(this);
+
         mEmptyView = view.findViewById(android.R.id.empty);
         mRecView = (RecyclerView) view.findViewById(R.id.dir_list);
         mRecView.setRecyclerListener(
@@ -1601,6 +1607,11 @@ public class DirectoryFragment extends Fragment
         return R.id.container_directory;
     }
 
+    @Override
+    public void onRefresh() {
+        getLoaderManager().restartLoader(LOADER_ID, null, this);
+    }
+
     @Override
     public Loader<DirectoryResult> onCreateLoader(int id, Bundle args) {
         Context context = getActivity();
@@ -1667,10 +1678,13 @@ public class DirectoryFragment extends Fragment
 
         mTuner.onModelLoaded(mModel, mType, mSearchMode);
 
+        mRefreshLayout.setRefreshing(false);
     }
 
     @Override
     public void onLoaderReset(Loader<DirectoryResult> loader) {
         mModel.update(null);
+
+        mRefreshLayout.setRefreshing(false);
     }
   }
diff --git a/packages/DocumentsUI/src/com/android/documentsui/dirlist/TouchSwipeRefreshLayout.java b/packages/DocumentsUI/src/com/android/documentsui/dirlist/TouchSwipeRefreshLayout.java
new file mode 100644
index 0000000000000..42634ba08854e
--- /dev/null
+++ b/packages/DocumentsUI/src/com/android/documentsui/dirlist/TouchSwipeRefreshLayout.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.documentsui.dirlist;
+
+import android.content.Context;
+import android.support.v4.widget.SwipeRefreshLayout;
+import android.util.AttributeSet;
+import android.view.MotionEvent;
+
+import com.android.documentsui.Events;
+
+/**
+ * A {@link SwipeRefreshLayout} that only refresh on touch events.
+ */
+public class TouchSwipeRefreshLayout extends SwipeRefreshLayout {
+
+    public TouchSwipeRefreshLayout(Context context) {
+        this(context, null);
+    }
+
+    public TouchSwipeRefreshLayout(Context context, AttributeSet attrs) {
+        super(context, attrs);
+    }
+
+    @Override
+    public boolean onInterceptTouchEvent(MotionEvent e) {
+        return Events.isMouseEvent(e) ? false : super.onInterceptTouchEvent(e);
+    }
+}