From 74502c75003ff88c0dfd3d7f5e0b0c4c46058943 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Tue, 8 May 2018 15:04:20 +0100 Subject: [PATCH 1/3] Refactor build rules for hidden API lists The build rules were getting messy and too difficult to extend. Extract assertions into own functions, create shared variables. Bug: 64382372 Test: make out/target/common/obj/PACKAGING/hiddenapi-blacklist.txt Merged-In: Ie34adb8b5349b30daa2e9e4976d285f6d9711f0e Change-Id: Ie34adb8b5349b30daa2e9e4976d285f6d9711f0e (cherry picked from commit 721bc8e6a8b1350c62b4a570f523541d1ce1124a) --- Android.mk | 119 ++++++++++++++++++----------- config/hiddenapi-dark-greylist.txt | 0 2 files changed, 75 insertions(+), 44 deletions(-) delete mode 100644 config/hiddenapi-dark-greylist.txt diff --git a/Android.mk b/Android.mk index a7d639b35177d..6639d7d1cf12c 100644 --- a/Android.mk +++ b/Android.mk @@ -583,23 +583,65 @@ include $(BUILD_HOST_JAVA_LIBRARY) # ==== hiddenapi lists ======================================= +include $(CLEAR_VARS) -# Copy light and dark greylist over into the build folder. -# This is for ART buildbots which need to mock these lists and have alternative -# rules for building them. Other rules in the build system should depend on the -# files in the build folder. +# File names of final API lists +LOCAL_LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) +LOCAL_DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) +LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) + +# File names of source files we will use to generate the final API lists. +LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt +LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt +LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) +LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) + +LOCAL_SRC_ALL := \ + $(LOCAL_SRC_GREYLIST) \ + $(LOCAL_SRC_VENDOR_LIST) \ + $(LOCAL_SRC_PRIVATE_API) \ + $(LOCAL_SRC_REMOVED_API) + +define assert-has-no-overlap +if [ ! -z "`comm -12 <(sort $(1)) <(sort $(2))`" ]; then \ + echo "$(1) and $(2) should not overlap" 1>&2; \ + comm -12 <(sort $(1)) <(sort $(2)) 1>&2; \ + exit 1; \ +fi +endef + +define assert-is-subset +if [ ! -z "`comm -23 <(sort $(1)) <(sort $(2))`" ]; then \ + echo "$(1) must be a subset of $(2)" 1>&2; \ + comm -23 <(sort $(1)) <(sort $(2)) 1>&2; \ + exit 1; \ +fi +endef + +define assert-has-no-duplicates +if [ ! -z "`sort $(1) | uniq -D`" ]; then \ + echo "$(1) has duplicate entries" 1>&2; \ + sort $(1) | uniq -D 1>&2; \ + exit 1; \ +fi +endef + +# The following rules build API lists in the build folder. +# By not using files from the source tree, ART buildbots can mock these lists +# or have alternative rules for building them. Other rules in the build system +# should depend on the files in the build folder. # Merge light greylist from multiple files: -# (1) manual light greylist -# (2) list of usages from vendor apps -# (3) list of removed APIs +# (1) manual greylist LOCAL_SRC_GREYLIST +# (2) list of usages from vendor apps LOCAL_SRC_VENDOR_LIST +# (3) list of removed APIs in LOCAL_SRC_REMOVED_API # @removed does not imply private in Doclava. We must take the subset also -# in PRIVATE_API. +# in LOCAL_SRC_PRIVATE_API. # (4) list of serialization APIs # Automatically adds all methods which match the signatures in # REGEX_SERIALIZATION. These are greylisted in order to allow applications # to write their own serializers. -$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ +$(LOCAL_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ "readObject\(Ljava/io/ObjectInputStream;\)V" \ "readObjectNoData\(\)V" \ "readResolve\(\)Ljava/lang/Object;" \ @@ -607,43 +649,32 @@ $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ "serialPersistentFields:\[Ljava/io/ObjectStreamField;" \ "writeObject\(Ljava/io/ObjectOutputStream;\)V" \ "writeReplace\(\)Ljava/lang/Object;" -$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): PRIVATE_API := $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) -$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) -$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST): frameworks/base/config/hiddenapi-light-greylist.txt \ - frameworks/base/config/hiddenapi-vendor-list.txt \ - $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) \ - $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) - sort frameworks/base/config/hiddenapi-light-greylist.txt \ - frameworks/base/config/hiddenapi-vendor-list.txt \ - <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" $(PRIVATE_API)) \ - <(comm -12 <(sort $(REMOVED_API)) <(sort $(PRIVATE_API))) \ - > $@ +$(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) + sort $(LOCAL_SRC_GREYLIST) $(LOCAL_SRC_VENDOR_LIST) \ + <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" \ + $(LOCAL_SRC_PRIVATE_API)) \ + <(comm -12 <(sort $(LOCAL_SRC_REMOVED_API)) <(sort $(LOCAL_SRC_PRIVATE_API))) \ + > $@ + $(call assert-has-no-duplicates,$@) + $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) -$(eval $(call copy-one-file,frameworks/base/config/hiddenapi-dark-greylist.txt,\ - $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST))) +# Generate an empty dark greylist. +$(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) + rm -f $@ + touch $@ + $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) + $(call assert-has-no-duplicates,$@) + $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) -# Generate dark greylist as private API minus (blacklist plus light greylist). - -$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST): PRIVATE_API := $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) -$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST): LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) -$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST): DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) -$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST): $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) \ - $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \ - $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) - if [ ! -z "`comm -12 <(sort $(LIGHT_GREYLIST)) <(sort $(DARK_GREYLIST))`" ]; then \ - echo "There should be no overlap between $(LIGHT_GREYLIST) and $(DARK_GREYLIST)" 1>&2; \ - comm -12 <(sort $(LIGHT_GREYLIST)) <(sort $(DARK_GREYLIST)) 1>&2; \ - exit 1; \ - elif [ ! -z "`comm -13 <(sort $(PRIVATE_API)) <(sort $(LIGHT_GREYLIST))`" ]; then \ - echo "$(LIGHT_GREYLIST) must be a subset of $(PRIVATE_API)" 1>&2; \ - comm -13 <(sort $(PRIVATE_API)) <(sort $(LIGHT_GREYLIST)) 1>&2; \ - exit 2; \ - elif [ ! -z "`comm -13 <(sort $(PRIVATE_API)) <(sort $(DARK_GREYLIST))`" ]; then \ - echo "$(DARK_GREYLIST) must be a subset of $(PRIVATE_API)" 1>&2; \ - comm -13 <(sort $(PRIVATE_API)) <(sort $(DARK_GREYLIST)) 1>&2; \ - exit 3; \ - fi - comm -23 <(sort $(PRIVATE_API)) <(sort $(LIGHT_GREYLIST) $(DARK_GREYLIST)) > $@ +# Generate blacklist as private API minus (light greylist plus dark greylist). +$(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST) + comm -13 <(sort $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)) \ + <(sort $(LOCAL_SRC_PRIVATE_API)) \ + > $@ + $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) + $(call assert-has-no-duplicates,$@) + $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) + $(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST)) # Build AOSP blacklist # ============================================================ diff --git a/config/hiddenapi-dark-greylist.txt b/config/hiddenapi-dark-greylist.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 6afca53da14b764225b80d250b373e79fa214e68 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Tue, 8 May 2018 15:28:31 +0100 Subject: [PATCH 2/3] Populate hidden API dark greylist This patch will iterate over all classes in the light greylist and add the remaining (currently blacklisted) class members into the dark greylist. This is meant to mitigate the impact of the feature on legacy apps, in case their hidden API uses have not been detected by our analyses. The dark greylist entries will be available to apps targeting pre-P SDK versions, but blocked for apps targeting P or later. Bug: 64382372 Test: make out/target/common/obj/PACKAGING/hiddenapi-dark-greylist.txt Merged-In: Id1ef4ab523b3b4d1333b5fbf2b3e6622ef2be607 Change-Id: Id1ef4ab523b3b4d1333b5fbf2b3e6622ef2be607 (cherry picked from commit 185f00e6677cea3fb135648f4a19d22dec54bbb5) --- Android.mk | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Android.mk b/Android.mk index 6639d7d1cf12c..bf17fba69f4b9 100644 --- a/Android.mk +++ b/Android.mk @@ -658,10 +658,23 @@ $(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) $(call assert-has-no-duplicates,$@) $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) -# Generate an empty dark greylist. +# Generate dark greylist as remaining members of classes on the light greylist, +# as well as the members of their inner classes. +# The algorithm is as follows: +# (1) extract the class descriptor from each entry in LOCAL_LIGHT_GREYLIST +# (2) strip the final semicolon and anything after (and including) a dollar sign, +# e.g. 'Lpackage/class$inner;' turns into 'Lpackage/class' +# (3) insert all entries from LOCAL_SRC_PRIVATE_API which begin with the stripped +# descriptor followed by a semi-colon or a dollar sign, e.g. matching a regex +# '^Lpackage/class[;$]' +# (4) subtract entries shared with LOCAL_LIGHT_GREYLIST $(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) - rm -f $@ - touch $@ + comm -13 $(LOCAL_LIGHT_GREYLIST) \ + <(sed 's/;\->.*//' $(LOCAL_LIGHT_GREYLIST) | sed 's/$$.*//' | sort | uniq | \ + while read CLASS_DESC; do \ + grep -E "^$${CLASS_DESC}[;$$]" $(LOCAL_SRC_PRIVATE_API); \ + done | sort | uniq) \ + > $@ $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) From 259e1629af9cedeccab4d147b23bc74c50d17b97 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Tue, 8 May 2018 16:49:02 +0100 Subject: [PATCH 3/3] Manualy blacklist certain hidden APIs Previous CL introduced the concept of populating the dark greylist with remaining members of classes on the light greylist. This breaks the assumption that some methods/fields will always remain blacklisted. Introduce a new file which forces their blacklisting and adjust the build rules accordingly. Bug: 64382372 Test: make out/target/common/obj/PACKAGING/hiddenapi-blacklist.txt Merged-In: I8b20e03c50f4027a24d2a6081bedb08bc8a34011 Change-Id: I8b20e03c50f4027a24d2a6081bedb08bc8a34011 (cherry picked from commit 4b34f79a35a6602962ed1df1accd9d6ea6e41e77) --- Android.mk | 7 ++++++- config/hiddenapi-force-blacklist.txt | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 config/hiddenapi-force-blacklist.txt diff --git a/Android.mk b/Android.mk index bf17fba69f4b9..e8c5b183be6aa 100644 --- a/Android.mk +++ b/Android.mk @@ -593,12 +593,14 @@ LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) # File names of source files we will use to generate the final API lists. LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt +LOCAL_SRC_FORCE_BLACKLIST := frameworks/base/config/hiddenapi-force-blacklist.txt LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_PRIVATE_DEX_API_FILE) LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) LOCAL_SRC_ALL := \ $(LOCAL_SRC_GREYLIST) \ $(LOCAL_SRC_VENDOR_LIST) \ + $(LOCAL_SRC_FORCE_BLACKLIST) \ $(LOCAL_SRC_PRIVATE_API) \ $(LOCAL_SRC_REMOVED_API) @@ -657,6 +659,7 @@ $(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) > $@ $(call assert-has-no-duplicates,$@) $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) + $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) # Generate dark greylist as remaining members of classes on the light greylist, # as well as the members of their inner classes. @@ -669,7 +672,7 @@ $(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) # '^Lpackage/class[;$]' # (4) subtract entries shared with LOCAL_LIGHT_GREYLIST $(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) - comm -13 $(LOCAL_LIGHT_GREYLIST) \ + comm -13 <(sort $(LOCAL_LIGHT_GREYLIST) $(LOCAL_SRC_FORCE_BLACKLIST)) \ <(sed 's/;\->.*//' $(LOCAL_LIGHT_GREYLIST) | sed 's/$$.*//' | sort | uniq | \ while read CLASS_DESC; do \ grep -E "^$${CLASS_DESC}[;$$]" $(LOCAL_SRC_PRIVATE_API); \ @@ -678,6 +681,7 @@ $(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) + $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) # Generate blacklist as private API minus (light greylist plus dark greylist). $(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST) @@ -688,6 +692,7 @@ $(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLI $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) $(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST)) + $(call assert-is-subset,$(LOCAL_SRC_FORCE_BLACKLIST),$@) # Build AOSP blacklist # ============================================================ diff --git a/config/hiddenapi-force-blacklist.txt b/config/hiddenapi-force-blacklist.txt new file mode 100644 index 0000000000000..2a10f77b862fa --- /dev/null +++ b/config/hiddenapi-force-blacklist.txt @@ -0,0 +1 @@ +Ldalvik/system/VMRuntime;->setHiddenApiExemptions([Ljava/lang/String;)V