From ff6423b27c4bdc7f7c5109f212d4a3b360cbea0d Mon Sep 17 00:00:00 2001 From: Jeff Sharkey Date: Tue, 8 Oct 2019 16:10:05 -0600 Subject: [PATCH] Progress towards per-volume database separation. After lots of feedback from a diverse set of stakeholders, the following goals have become clear for MediaProvider: -- When an SD card is ejected, we shouldn't be leaving "stale" metadata around, such as artists, albums, or genres that are no longer relevant. -- We need to avoid heavy full re-indexing of content when an SD card is only temporarily ejected and reinserted within a week. -- We need to support "merged" queries against the virtual VOLUME_EXTERNAL view of all currently mounted volumes. For example, developers should be able to quickly list all available artists without needing to manually merge cursors. With these goals in mind, we spent a lot of time pondering various approaches, and landed upon defining "ID" columns using stable values that enable instant merging between databases. This is implemented by running the 64-bit version of FarmHash against the relevant KEY values, which have already been stripped to aid clustering of equal values. (These keys are how "The Beatles" and "Beatles, The" and "Beatles" are already merged together today.) The approach above now lets us define our artists, albums, and genres in terms of the underlying audio table/view, meaning they're always accurate. Note that the above approach means that developers no longer have dynamic control over genre membership, since it's now always derived from the underlying media files. This follows the design principle we've been using for the last year of saying the source of truth is the underlying media files, to ensure that we can accurately reconstruct the database after corruption or wiping. Developers that want to edit artist, album, or genre information should edit the metadata in the underlying file, which will trigger a rescan. This change also deprecates the various _KEY constants, since they're not reliable for sorting; developers should be using COLLATE LOCALIZED or other methods. (This places us in an awkward position where we're adding newly-deprecated GENRE_KEY columns, but they're needed for completeness.) Fix keyFor() generation to use a consistent Locale.ROOT. Bug: 136964095, 141520122, 140850497, 140127429, 138130722 Test: atest --test-mapping packages/providers/MediaProvider Change-Id: Id4945a04c6996c6ea4a909dda32aa1dd02759d08 --- api/current.txt | 17 +- core/java/android/provider/MediaStore.java | 212 ++++++++++++++------- 2 files changed, 151 insertions(+), 78 deletions(-) diff --git a/api/current.txt b/api/current.txt index 8733aab02b004..4305709a6798a 100644 --- a/api/current.txt +++ b/api/current.txt @@ -38457,16 +38457,17 @@ package android.provider { public static final class MediaStore.Audio { ctor public MediaStore.Audio(); - method public static String keyFor(String); + method @Deprecated @Nullable public static String keyFor(@Nullable String); } public static interface MediaStore.Audio.AlbumColumns { field public static final String ALBUM = "album"; field @Deprecated public static final String ALBUM_ART = "album_art"; field public static final String ALBUM_ID = "album_id"; - field public static final String ALBUM_KEY = "album_key"; + field @Deprecated public static final String ALBUM_KEY = "album_key"; field public static final String ARTIST = "artist"; field public static final String ARTIST_ID = "artist_id"; + field @Deprecated public static final String ARTIST_KEY = "artist_key"; field public static final String FIRST_YEAR = "minyear"; field public static final String LAST_YEAR = "maxyear"; field public static final String NUMBER_OF_SONGS = "numsongs"; @@ -38485,7 +38486,7 @@ package android.provider { public static interface MediaStore.Audio.ArtistColumns { field public static final String ARTIST = "artist"; - field public static final String ARTIST_KEY = "artist_key"; + field @Deprecated public static final String ARTIST_KEY = "artist_key"; field public static final String NUMBER_OF_ALBUMS = "number_of_albums"; field public static final String NUMBER_OF_TRACKS = "number_of_tracks"; } @@ -38508,19 +38509,23 @@ package android.provider { public static interface MediaStore.Audio.AudioColumns extends android.provider.MediaStore.MediaColumns { field public static final String ALBUM = "album"; field public static final String ALBUM_ID = "album_id"; - field public static final String ALBUM_KEY = "album_key"; + field @Deprecated public static final String ALBUM_KEY = "album_key"; field public static final String ARTIST = "artist"; field public static final String ARTIST_ID = "artist_id"; - field public static final String ARTIST_KEY = "artist_key"; + field @Deprecated public static final String ARTIST_KEY = "artist_key"; field public static final String BOOKMARK = "bookmark"; field public static final String COMPOSER = "composer"; + field public static final String GENRE = "genre"; + field public static final String GENRE_ID = "genre_id"; + field @Deprecated public static final String GENRE_KEY = "genre_key"; field public static final String IS_ALARM = "is_alarm"; field public static final String IS_AUDIOBOOK = "is_audiobook"; field public static final String IS_MUSIC = "is_music"; field public static final String IS_NOTIFICATION = "is_notification"; field public static final String IS_PODCAST = "is_podcast"; field public static final String IS_RINGTONE = "is_ringtone"; - field public static final String TITLE_KEY = "title_key"; + field @Deprecated public static final String TITLE_KEY = "title_key"; + field public static final String TITLE_RESOURCE_URI = "title_resource_uri"; field public static final String TRACK = "track"; field public static final String YEAR = "year"; } diff --git a/core/java/android/provider/MediaStore.java b/core/java/android/provider/MediaStore.java index 079a42ddaa6c9..7309555ad69c7 100644 --- a/core/java/android/provider/MediaStore.java +++ b/core/java/android/provider/MediaStore.java @@ -39,7 +39,6 @@ import android.content.Context; import android.content.Intent; import android.content.UriPermission; import android.database.Cursor; -import android.database.DatabaseUtils; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.ImageDecoder; @@ -69,15 +68,19 @@ import android.util.Log; import com.android.internal.annotations.GuardedBy; +import libcore.util.HexEncoding; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.text.Collator; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Set; import java.util.regex.Pattern; @@ -2058,7 +2061,17 @@ public final class MediaStore { /** * A non human readable key calculated from the TITLE, used for * searching, sorting and grouping + * + * @see Audio#keyFor(String) + * @deprecated These keys are generated using + * {@link java.util.Locale#ROOT}, which means they don't + * reflect locale-specific sorting preferences. To apply + * locale-specific sorting preferences, use + * {@link ContentResolver#QUERY_ARG_SQL_SORT_ORDER} with + * {@code COLLATE LOCALIZED}, or + * {@link ContentResolver#QUERY_ARG_SORT_LOCALE}. */ + @Deprecated @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) public static final String TITLE_KEY = "title_key"; @@ -2103,7 +2116,17 @@ public final class MediaStore { /** * A non human readable key calculated from the ARTIST, used for * searching, sorting and grouping + * + * @see Audio#keyFor(String) + * @deprecated These keys are generated using + * {@link java.util.Locale#ROOT}, which means they don't + * reflect locale-specific sorting preferences. To apply + * locale-specific sorting preferences, use + * {@link ContentResolver#QUERY_ARG_SQL_SORT_ORDER} with + * {@code COLLATE LOCALIZED}, or + * {@link ContentResolver#QUERY_ARG_SORT_LOCALE}. */ + @Deprecated @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) public static final String ARTIST_KEY = "artist_key"; @@ -2128,7 +2151,17 @@ public final class MediaStore { /** * A non human readable key calculated from the ALBUM, used for * searching, sorting and grouping + * + * @see Audio#keyFor(String) + * @deprecated These keys are generated using + * {@link java.util.Locale#ROOT}, which means they don't + * reflect locale-specific sorting preferences. To apply + * locale-specific sorting preferences, use + * {@link ContentResolver#QUERY_ARG_SQL_SORT_ORDER} with + * {@code COLLATE LOCALIZED}, or + * {@link ContentResolver#QUERY_ARG_SORT_LOCALE}. */ + @Deprecated @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) public static final String ALBUM_KEY = "album_key"; @@ -2185,91 +2218,89 @@ public final class MediaStore { public static final String IS_AUDIOBOOK = "is_audiobook"; /** - * The genre of the audio file, if any - * Does not exist in the database - only used by the media scanner for inserts. - * @hide + * The id of the genre the audio file is from, if any */ - @Deprecated - // @Column(Cursor.FIELD_TYPE_STRING) + @Column(value = Cursor.FIELD_TYPE_INTEGER, readOnly = true) + public static final String GENRE_ID = "genre_id"; + + /** + * The genre of the audio file, if any. + */ + @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) public static final String GENRE = "genre"; /** - * The resource URI of a localized title, if any + * A non human readable key calculated from the GENRE, used for + * searching, sorting and grouping + * + * @see Audio#keyFor(String) + * @deprecated These keys are generated using + * {@link java.util.Locale#ROOT}, which means they don't + * reflect locale-specific sorting preferences. To apply + * locale-specific sorting preferences, use + * {@link ContentResolver#QUERY_ARG_SQL_SORT_ORDER} with + * {@code COLLATE LOCALIZED}, or + * {@link ContentResolver#QUERY_ARG_SORT_LOCALE}. + */ + @Deprecated + @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) + public static final String GENRE_KEY = "genre_key"; + + /** + * The resource URI of a localized title, if any. + *

* Conforms to this pattern: - * Scheme: {@link ContentResolver.SCHEME_ANDROID_RESOURCE} - * Authority: Package Name of ringtone title provider - * First Path Segment: Type of resource (must be "string") - * Second Path Segment: Resource ID of title - * @hide + *

*/ @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) public static final String TITLE_RESOURCE_URI = "title_resource_uri"; } + private static final Pattern PATTERN_TRIM_BEFORE = Pattern.compile( + "(?i)(^(the|an|a) |,\\s*(the|an|a)$|[^\\w\\s]|^\\s+|\\s+$)"); + private static final Pattern PATTERN_TRIM_AFTER = Pattern.compile( + "(^(00)+|(00)+$)"); + /** - * Converts a name to a "key" that can be used for grouping, sorting - * and searching. - * The rules that govern this conversion are: - * - remove 'special' characters like ()[]'!?., - * - remove leading/trailing spaces - * - convert everything to lowercase - * - remove leading "the ", "an " and "a " - * - remove trailing ", the|an|a" - * - remove accents. This step leaves us with CollationKey data, - * which is not human readable + * Converts a user-visible string into a "key" that can be used for + * grouping, sorting, and searching. * - * @param name The artist or album name to convert - * @return The "key" for the given name. + * @return Opaque token that should not be parsed or displayed to users. + * @deprecated These keys are generated using + * {@link java.util.Locale#ROOT}, which means they don't + * reflect locale-specific sorting preferences. To apply + * locale-specific sorting preferences, use + * {@link ContentResolver#QUERY_ARG_SQL_SORT_ORDER} with + * {@code COLLATE LOCALIZED}, or + * {@link ContentResolver#QUERY_ARG_SORT_LOCALE}. */ - public static String keyFor(String name) { - if (name != null) { - boolean sortfirst = false; - if (name.equals(UNKNOWN_STRING)) { - return "\001"; - } - // Check if the first character is \001. We use this to - // force sorting of certain special files, like the silent ringtone. - if (name.startsWith("\001")) { - sortfirst = true; - } - name = name.trim().toLowerCase(); - if (name.startsWith("the ")) { - name = name.substring(4); - } - if (name.startsWith("an ")) { - name = name.substring(3); - } - if (name.startsWith("a ")) { - name = name.substring(2); - } - if (name.endsWith(", the") || name.endsWith(",the") || - name.endsWith(", an") || name.endsWith(",an") || - name.endsWith(", a") || name.endsWith(",a")) { - name = name.substring(0, name.lastIndexOf(',')); - } - name = name.replaceAll("[\\[\\]\\(\\)\"'.,?!]", "").trim(); - if (name.length() > 0) { - // Insert a separator between the characters to avoid - // matches on a partial character. If we ever change - // to start-of-word-only matches, this can be removed. - StringBuilder b = new StringBuilder(); - b.append('.'); - int nl = name.length(); - for (int i = 0; i < nl; i++) { - b.append(name.charAt(i)); - b.append('.'); - } - name = b.toString(); - String key = DatabaseUtils.getCollationKey(name); - if (sortfirst) { - key = "\001" + key; - } - return key; - } else { - return ""; - } + @Deprecated + public static @Nullable String keyFor(@Nullable String name) { + if (TextUtils.isEmpty(name)) return null; + + if (UNKNOWN_STRING.equals(name)) { + return "01"; } - return null; + + final boolean sortFirst = name.startsWith("\001"); + + name = PATTERN_TRIM_BEFORE.matcher(name).replaceAll(""); + if (TextUtils.isEmpty(name)) return null; + + final Collator c = Collator.getInstance(Locale.ROOT); + c.setStrength(Collator.PRIMARY); + name = HexEncoding.encodeToString(c.getCollationKey(name).toByteArray(), false); + + name = PATTERN_TRIM_AFTER.matcher(name).replaceAll(""); + if (sortFirst) { + name = "01" + name; + } + return name; } public static final class Media implements AudioColumns { @@ -2631,7 +2662,17 @@ public final class MediaStore { /** * A non human readable key calculated from the ARTIST, used for * searching, sorting and grouping + * + * @see Audio#keyFor(String) + * @deprecated These keys are generated using + * {@link java.util.Locale#ROOT}, which means they don't + * reflect locale-specific sorting preferences. To apply + * locale-specific sorting preferences, use + * {@link ContentResolver#QUERY_ARG_SQL_SORT_ORDER} with + * {@code COLLATE LOCALIZED}, or + * {@link ContentResolver#QUERY_ARG_SORT_LOCALE}. */ + @Deprecated @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) public static final String ARTIST_KEY = "artist_key"; @@ -2734,6 +2775,23 @@ public final class MediaStore { @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) public static final String ARTIST = "artist"; + /** + * A non human readable key calculated from the ARTIST, used for + * searching, sorting and grouping + * + * @see Audio#keyFor(String) + * @deprecated These keys are generated using + * {@link java.util.Locale#ROOT}, which means they don't + * reflect locale-specific sorting preferences. To apply + * locale-specific sorting preferences, use + * {@link ContentResolver#QUERY_ARG_SQL_SORT_ORDER} with + * {@code COLLATE LOCALIZED}, or + * {@link ContentResolver#QUERY_ARG_SORT_LOCALE}. + */ + @Deprecated + @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) + public static final String ARTIST_KEY = "artist_key"; + /** * The number of songs on this album */ @@ -2769,7 +2827,17 @@ public final class MediaStore { /** * A non human readable key calculated from the ALBUM, used for * searching, sorting and grouping + * + * @see Audio#keyFor(String) + * @deprecated These keys are generated using + * {@link java.util.Locale#ROOT}, which means they don't + * reflect locale-specific sorting preferences. To apply + * locale-specific sorting preferences, use + * {@link ContentResolver#QUERY_ARG_SQL_SORT_ORDER} with + * {@code COLLATE LOCALIZED}, or + * {@link ContentResolver#QUERY_ARG_SORT_LOCALE}. */ + @Deprecated @Column(value = Cursor.FIELD_TYPE_STRING, readOnly = true) public static final String ALBUM_KEY = "album_key";