From 721bc8e6a8b1350c62b4a570f523541d1ce1124a Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Tue, 8 May 2018 15:04:20 +0100 Subject: [PATCH 1/3] Refactor build rules for hidden API lists The build rules were getting messy and too difficult to extend. Extract assertions into own functions, create shared variables. Bug: 64382372 Test: make out/target/common/obj/PACKAGING/hiddenapi-blacklist.txt Change-Id: Ie34adb8b5349b30daa2e9e4976d285f6d9711f0e --- Android.mk | 119 ++++++++++++++++++----------- config/hiddenapi-dark-greylist.txt | 0 2 files changed, 75 insertions(+), 44 deletions(-) delete mode 100644 config/hiddenapi-dark-greylist.txt diff --git a/Android.mk b/Android.mk index bb37fa21c4637..ff2a62c30a62d 100644 --- a/Android.mk +++ b/Android.mk @@ -867,23 +867,65 @@ LOCAL_ERROR_PRONE_FLAGS := -XepDisableAllChecks include $(BUILD_STATIC_JAVA_LIBRARY) # ==== hiddenapi lists ======================================= +include $(CLEAR_VARS) -# Copy light and dark greylist over into the build folder. -# This is for ART buildbots which need to mock these lists and have alternative -# rules for building them. Other rules in the build system should depend on the -# files in the build folder. +# File names of final API lists +LOCAL_LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) +LOCAL_DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) +LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) + +# File names of source files we will use to generate the final API lists. +LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt +LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt +LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) +LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) + +LOCAL_SRC_ALL := \ + $(LOCAL_SRC_GREYLIST) \ + $(LOCAL_SRC_VENDOR_LIST) \ + $(LOCAL_SRC_PRIVATE_API) \ + $(LOCAL_SRC_REMOVED_API) + +define assert-has-no-overlap +if [ ! -z "`comm -12 <(sort $(1)) <(sort $(2))`" ]; then \ + echo "$(1) and $(2) should not overlap" 1>&2; \ + comm -12 <(sort $(1)) <(sort $(2)) 1>&2; \ + exit 1; \ +fi +endef + +define assert-is-subset +if [ ! -z "`comm -23 <(sort $(1)) <(sort $(2))`" ]; then \ + echo "$(1) must be a subset of $(2)" 1>&2; \ + comm -23 <(sort $(1)) <(sort $(2)) 1>&2; \ + exit 1; \ +fi +endef + +define assert-has-no-duplicates +if [ ! -z "`sort $(1) | uniq -D`" ]; then \ + echo "$(1) has duplicate entries" 1>&2; \ + sort $(1) | uniq -D 1>&2; \ + exit 1; \ +fi +endef + +# The following rules build API lists in the build folder. +# By not using files from the source tree, ART buildbots can mock these lists +# or have alternative rules for building them. Other rules in the build system +# should depend on the files in the build folder. # Merge light greylist from multiple files: -# (1) manual light greylist -# (2) list of usages from vendor apps -# (3) list of removed APIs +# (1) manual greylist LOCAL_SRC_GREYLIST +# (2) list of usages from vendor apps LOCAL_SRC_VENDOR_LIST +# (3) list of removed APIs in LOCAL_SRC_REMOVED_API # @removed does not imply private in Doclava. We must take the subset also -# in PRIVATE_API. +# in LOCAL_SRC_PRIVATE_API. # (4) list of serialization APIs # Automatically adds all methods which match the signatures in # REGEX_SERIALIZATION. These are greylisted in order to allow applications # to write their own serializers. -$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ +$(LOCAL_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ "readObject\(Ljava/io/ObjectInputStream;\)V" \ "readObjectNoData\(\)V" \ "readResolve\(\)Ljava/lang/Object;" \ @@ -891,43 +933,32 @@ $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ "serialPersistentFields:\[Ljava/io/ObjectStreamField;" \ "writeObject\(Ljava/io/ObjectOutputStream;\)V" \ "writeReplace\(\)Ljava/lang/Object;" -$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): PRIVATE_API := $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) -$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) -$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): frameworks/base/config/hiddenapi-light-greylist.txt \ - frameworks/base/config/hiddenapi-vendor-list.txt \ - $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) \ - $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) - sort frameworks/base/config/hiddenapi-light-greylist.txt \ - frameworks/base/config/hiddenapi-vendor-list.txt \ - <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" $(PRIVATE_API)) \ - <(comm -12 <(sort $(REMOVED_API)) <(sort $(PRIVATE_API))) \ - > $@ +$(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) + sort $(LOCAL_SRC_GREYLIST) $(LOCAL_SRC_VENDOR_LIST) \ + <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" \ + $(LOCAL_SRC_PRIVATE_API)) \ + <(comm -12 <(sort $(LOCAL_SRC_REMOVED_API)) <(sort $(LOCAL_SRC_PRIVATE_API))) \ + > $@ + $(call assert-has-no-duplicates,$@) + $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) -$(eval $(call copy-one-file,frameworks/base/config/hiddenapi-dark-greylist.txt,\ - $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST))) +# Generate an empty dark greylist. +$(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) + rm -f $@ + touch $@ + $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) + $(call assert-has-no-duplicates,$@) + $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) -# Generate dark greylist as private API minus (blacklist plus light greylist). - -$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST): PRIVATE_API := $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) -$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST): LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) -$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST): DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) -$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST): $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) \ - $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \ - $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) - if [ ! -z "`comm -12 <(sort $(LIGHT_GREYLIST)) <(sort $(DARK_GREYLIST))`" ]; then \ - echo "There should be no overlap between $(LIGHT_GREYLIST) and $(DARK_GREYLIST)" 1>&2; \ - comm -12 <(sort $(LIGHT_GREYLIST)) <(sort $(DARK_GREYLIST)) 1>&2; \ - exit 1; \ - elif [ ! -z "`comm -13 <(sort $(PRIVATE_API)) <(sort $(LIGHT_GREYLIST))`" ]; then \ - echo "$(LIGHT_GREYLIST) must be a subset of $(PRIVATE_API)" 1>&2; \ - comm -13 <(sort $(PRIVATE_API)) <(sort $(LIGHT_GREYLIST)) 1>&2; \ - exit 2; \ - elif [ ! -z "`comm -13 <(sort $(PRIVATE_API)) <(sort $(DARK_GREYLIST))`" ]; then \ - echo "$(DARK_GREYLIST) must be a subset of $(PRIVATE_API)" 1>&2; \ - comm -13 <(sort $(PRIVATE_API)) <(sort $(DARK_GREYLIST)) 1>&2; \ - exit 3; \ - fi - comm -23 <(sort $(PRIVATE_API)) <(sort $(LIGHT_GREYLIST) $(DARK_GREYLIST)) > $@ +# Generate blacklist as private API minus (light greylist plus dark greylist). +$(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST) + comm -13 <(sort $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)) \ + <(sort $(LOCAL_SRC_PRIVATE_API)) \ + > $@ + $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) + $(call assert-has-no-duplicates,$@) + $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) + $(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST)) # Include subdirectory makefiles # ============================================================ diff --git a/config/hiddenapi-dark-greylist.txt b/config/hiddenapi-dark-greylist.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 185f00e6677cea3fb135648f4a19d22dec54bbb5 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Tue, 8 May 2018 15:28:31 +0100 Subject: [PATCH 2/3] Populate hidden API dark greylist This patch will iterate over all classes in the light greylist and add the remaining (currently blacklisted) class members into the dark greylist. This is meant to mitigate the impact of the feature on legacy apps, in case their hidden API uses have not been detected by our analyses. The dark greylist entries will be available to apps targeting pre-P SDK versions, but blocked for apps targeting P or later. Bug: 64382372 Test: make out/target/common/obj/PACKAGING/hiddenapi-dark-greylist.txt Change-Id: Id1ef4ab523b3b4d1333b5fbf2b3e6622ef2be607 --- Android.mk | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Android.mk b/Android.mk index ff2a62c30a62d..218c5944f31da 100644 --- a/Android.mk +++ b/Android.mk @@ -942,10 +942,23 @@ $(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) $(call assert-has-no-duplicates,$@) $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) -# Generate an empty dark greylist. +# Generate dark greylist as remaining members of classes on the light greylist, +# as well as the members of their inner classes. +# The algorithm is as follows: +# (1) extract the class descriptor from each entry in LOCAL_LIGHT_GREYLIST +# (2) strip the final semicolon and anything after (and including) a dollar sign, +# e.g. 'Lpackage/class$inner;' turns into 'Lpackage/class' +# (3) insert all entries from LOCAL_SRC_PRIVATE_API which begin with the stripped +# descriptor followed by a semi-colon or a dollar sign, e.g. matching a regex +# '^Lpackage/class[;$]' +# (4) subtract entries shared with LOCAL_LIGHT_GREYLIST $(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) - rm -f $@ - touch $@ + comm -13 $(LOCAL_LIGHT_GREYLIST) \ + <(sed 's/;\->.*//' $(LOCAL_LIGHT_GREYLIST) | sed 's/$$.*//' | sort | uniq | \ + while read CLASS_DESC; do \ + grep -E "^$${CLASS_DESC}[;$$]" $(LOCAL_SRC_PRIVATE_API); \ + done | sort | uniq) \ + > $@ $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) From 4b34f79a35a6602962ed1df1accd9d6ea6e41e77 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Tue, 8 May 2018 16:49:02 +0100 Subject: [PATCH 3/3] Manualy blacklist certain hidden APIs Previous CL introduced the concept of populating the dark greylist with remaining members of classes on the light greylist. This breaks the assumption that some methods/fields will always remain blacklisted. Introduce a new file which forces their blacklisting and adjust the build rules accordingly. Bug: 64382372 Test: make out/target/common/obj/PACKAGING/hiddenapi-blacklist.txt Change-Id: I8b20e03c50f4027a24d2a6081bedb08bc8a34011 --- Android.mk | 7 ++++++- config/hiddenapi-force-blacklist.txt | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 config/hiddenapi-force-blacklist.txt diff --git a/Android.mk b/Android.mk index 218c5944f31da..ccf3894a3d020 100644 --- a/Android.mk +++ b/Android.mk @@ -877,12 +877,14 @@ LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) # File names of source files we will use to generate the final API lists. LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt +LOCAL_SRC_FORCE_BLACKLIST := frameworks/base/config/hiddenapi-force-blacklist.txt LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) LOCAL_SRC_ALL := \ $(LOCAL_SRC_GREYLIST) \ $(LOCAL_SRC_VENDOR_LIST) \ + $(LOCAL_SRC_FORCE_BLACKLIST) \ $(LOCAL_SRC_PRIVATE_API) \ $(LOCAL_SRC_REMOVED_API) @@ -941,6 +943,7 @@ $(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) > $@ $(call assert-has-no-duplicates,$@) $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) + $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) # Generate dark greylist as remaining members of classes on the light greylist, # as well as the members of their inner classes. @@ -953,7 +956,7 @@ $(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) # '^Lpackage/class[;$]' # (4) subtract entries shared with LOCAL_LIGHT_GREYLIST $(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) - comm -13 $(LOCAL_LIGHT_GREYLIST) \ + comm -13 <(sort $(LOCAL_LIGHT_GREYLIST) $(LOCAL_SRC_FORCE_BLACKLIST)) \ <(sed 's/;\->.*//' $(LOCAL_LIGHT_GREYLIST) | sed 's/$$.*//' | sort | uniq | \ while read CLASS_DESC; do \ grep -E "^$${CLASS_DESC}[;$$]" $(LOCAL_SRC_PRIVATE_API); \ @@ -962,6 +965,7 @@ $(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) + $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) # Generate blacklist as private API minus (light greylist plus dark greylist). $(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST) @@ -972,6 +976,7 @@ $(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLI $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) $(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST)) + $(call assert-is-subset,$(LOCAL_SRC_FORCE_BLACKLIST),$@) # Include subdirectory makefiles # ============================================================ diff --git a/config/hiddenapi-force-blacklist.txt b/config/hiddenapi-force-blacklist.txt new file mode 100644 index 0000000000000..2a10f77b862fa --- /dev/null +++ b/config/hiddenapi-force-blacklist.txt @@ -0,0 +1 @@ +Ldalvik/system/VMRuntime;->setHiddenApiExemptions([Ljava/lang/String;)V