Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Rewrite hidden API list generation in Python"

Browse Source
This commit is contained in:
David Brazdil
2018-08-30 17:59:02 +00:00
committed by Gerrit Code Review
parent 9137ecafbd 8503b9045b
commit a264c2c83e
3 changed files with 358 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
Android.mk
View File

@@ -322,131 +322,34 @@ $(OUT_DOCS)/offline-sdk-timestamp: $(OUT_DOCS)/offline-sdk-docs-docs.zip
( unzip -qo $< -d $(OUT_DOCS)/offline-sdk && touch -f $@ ) || exit 1
# ==== hiddenapi lists =======================================
include $(CLEAR_VARS)
# File names of final API lists
LOCAL_WHITELIST := $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST)
LOCAL_LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST)
LOCAL_DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST)
LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)
# File names of source files we will use to generate the final API lists.
LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt
LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt
LOCAL_SRC_FORCE_BLACKLIST := frameworks/base/config/hiddenapi-force-blacklist.txt
LOCAL_SRC_PUBLIC_API := $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST)
LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)
LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)
LOCAL_SRC_ALL := \
$(LOCAL_SRC_GREYLIST) \
$(LOCAL_SRC_VENDOR_LIST) \
$(LOCAL_SRC_FORCE_BLACKLIST) \
$(LOCAL_SRC_PUBLIC_API) \
$(LOCAL_SRC_PRIVATE_API) \
$(LOCAL_SRC_REMOVED_API)
define assert-has-no-overlap
if [ ! -z "`comm -12 <(sort $(1)) <(sort $(2))`" ]; then \
echo "$(1) and $(2) should not overlap" 1>&2; \
comm -12 <(sort $(1)) <(sort $(2)) 1>&2; \
exit 1; \
fi
endef
define assert-is-subset
if [ ! -z "`comm -23 <(sort $(1)) <(sort $(2))`" ]; then \
echo "$(1) must be a subset of $(2)" 1>&2; \
comm -23 <(sort $(1)) <(sort $(2)) 1>&2; \
exit 1; \
fi
endef
define assert-has-no-duplicates
if [ ! -z "`sort $(1) | uniq -D`" ]; then \
echo "$(1) has duplicate entries" 1>&2; \
sort $(1) | uniq -D 1>&2; \
exit 1; \
fi
endef
# The following rules build API lists in the build folder.
# By not using files from the source tree, ART buildbots can mock these lists
# or have alternative rules for building them. Other rules in the build system
# should depend on the files in the build folder.
# Merge whitelist from:
# (1) public API stubs
# (2) whitelist entries generated by class2greylist (PRIVATE_WHITELIST_INPUTS)
$(LOCAL_WHITELIST): $(LOCAL_SRC_PUBLIC_API)
sort $(LOCAL_SRC_PUBLIC_API) $(PRIVATE_WHITELIST_INPUTS) > $@
$(call assert-has-no-duplicates,$@)
# Merge light greylist from multiple files:
# (1) manual greylist LOCAL_SRC_GREYLIST
# (2) list of usages from vendor apps LOCAL_SRC_VENDOR_LIST
# (3) list of removed APIs in LOCAL_SRC_REMOVED_API
# @removed does not imply private in Doclava. We must take the subset also
# in LOCAL_SRC_PRIVATE_API.
# (4) list of serialization APIs
# Automatically adds all methods which match the signatures in
# REGEX_SERIALIZATION. These are greylisted in order to allow applications
# to write their own serializers.
# (5) greylist entries generated by class2greylist (PRIVATE_GREYLIST_INPUTS)
$(LOCAL_LIGHT_GREYLIST): REGEX_SERIALIZATION := \
"readObject\(Ljava/io/ObjectInputStream;\)V" \
"readObjectNoData\(\)V" \
"readResolve\(\)Ljava/lang/Object;" \
"serialVersionUID:J" \
"serialPersistentFields:\[Ljava/io/ObjectStreamField;" \
"writeObject\(Ljava/io/ObjectOutputStream;\)V" \
"writeReplace\(\)Ljava/lang/Object;"
$(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST)
sort $(LOCAL_SRC_GREYLIST) $(LOCAL_SRC_VENDOR_LIST) $(PRIVATE_GREYLIST_INPUTS) \
<(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" \
$(LOCAL_SRC_PRIVATE_API)) \
<(comm -12 <(sort $(LOCAL_SRC_REMOVED_API)) <(sort $(LOCAL_SRC_PRIVATE_API))) \
> $@
$(call assert-has-no-duplicates,$@)
$(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))
$(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))
$(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST))
# Generate dark greylist as remaining classes and class members in the same
# package as classes listed in the light greylist.
# The algorithm is as follows:
# (1) extract the class descriptor from each entry in LOCAL_LIGHT_GREYLIST
# (2) strip everything after the last forward-slash,
# e.g. 'Lpackage/subpackage/class$inner;' turns into 'Lpackage/subpackage/'
# (3) insert all entries from LOCAL_SRC_PRIVATE_API which begin with the package
# name but do not contain another forward-slash in the class name, e.g.
# matching '^Lpackage/subpackage/[^/;]*;'
# (4) subtract entries shared with LOCAL_LIGHT_GREYLIST
$(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST)
comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_SRC_FORCE_BLACKLIST)) \
<(cat $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) | \
sed 's/\->.*//' | sed 's/\(.*\/\).*/\1/' | sort | uniq | \
while read PKG_NAME; do \
grep -E "^$${PKG_NAME}[^/;]*;" $(LOCAL_SRC_PRIVATE_API); \
done | sort | uniq) \
> $@
$(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))
$(call assert-has-no-duplicates,$@)
$(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))
$(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST))
$(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST))
# Generate blacklist as private API minus (light greylist plus dark greylist).
$(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)
comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)) \
<(sort $(LOCAL_SRC_PRIVATE_API)) \
> $@
$(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))
$(call assert-has-no-duplicates,$@)
$(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))
$(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST))
$(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST))
$(call assert-is-subset,$(LOCAL_SRC_FORCE_BLACKLIST),$@)
$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \
.KATI_IMPLICIT_OUTPUTS := \
$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \
$(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \
$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)
$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \
frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \
frameworks/base/config/hiddenapi-light-greylist.txt \
frameworks/base/config/hiddenapi-vendor-list.txt \
frameworks/base/config/hiddenapi-force-blacklist.txt \
$(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \
$(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \
$(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)
frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \
--input-public $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \
--input-private $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \
--input-whitelists $(PRIVATE_WHITELIST_INPUTS) \
--input-greylists \
frameworks/base/config/hiddenapi-light-greylist.txt \
frameworks/base/config/hiddenapi-vendor-list.txt \
<(comm -12 <(sort $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)) \
$(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)) \
$(PRIVATE_GREYLIST_INPUTS) \
--input-blacklists frameworks/base/config/hiddenapi-force-blacklist.txt \
--output-whitelist $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) \
--output-light-greylist $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \
--output-dark-greylist $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \
--output-blacklist $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)
# Include subdirectory makefiles
# ============================================================

241
tools/hiddenapi/generate_hiddenapi_lists.py Executable file
View File

@@ -0,0 +1,241 @@
#!/usr/bin/env python
#
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Generate API lists for non-SDK API enforcement.
usage: generate-hiddenapi-lists.py [-h]
--input-public INPUT_PUBLIC
--input-private INPUT_PRIVATE
[--input-whitelists [INPUT_WHITELISTS [INPUT_WHITELISTS ...]]]
[--input-greylists [INPUT_GREYLISTS [INPUT_GREYLISTS ...]]]
[--input-blacklists [INPUT_BLACKLISTS [INPUT_BLACKLISTS ...]]]
--output-whitelist OUTPUT_WHITELIST
--output-light-greylist OUTPUT_LIGHT_GREYLIST
--output-dark-greylist OUTPUT_DARK_GREYLIST
--output-blacklist OUTPUT_BLACKLIST
"""
import argparse
import os
import sys
import re
def get_args():
"""Parses command line arguments.
Returns:
Namespace: dictionary of parsed arguments
"""
parser = argparse.ArgumentParser()
parser.add_argument('--input-public', required=True, help='List of all public members')
parser.add_argument('--input-private', required=True, help='List of all private members')
parser.add_argument(
'--input-whitelists', nargs='*',
help='Lists of members to force on whitelist')
parser.add_argument(
'--input-greylists', nargs='*',
help='Lists of members to force on light greylist')
parser.add_argument(
'--input-blacklists', nargs='*',
help='Lists of members to force on blacklist')
parser.add_argument('--output-whitelist', required=True)
parser.add_argument('--output-light-greylist', required=True)
parser.add_argument('--output-dark-greylist', required=True)
parser.add_argument('--output-blacklist', required=True)
return parser.parse_args()
def read_lines(filename):
"""Reads entire file and return it as a list of lines.
Args:
filename (string): Path to the file to read from.
Returns:
list: Lines of the loaded file as a list of strings.
"""
with open(filename, 'r') as f:
return f.readlines()
def write_lines(filename, lines):
"""Writes list of lines into a file, overwriting the file it it exists.
Args:
filename (string): Path to the file to be writting into.
lines (list): List of strings to write into the file.
"""
with open(filename, 'w') as f:
f.writelines(lines)
def move_between_sets(subset, src, dst, source = "<unknown>"):
"""Removes a subset of elements from one set and add it to another.
Args:
subset (set): The subset of `src` to be moved from `src` to `dst`.
src (set): Source set. Must be a superset of `subset`.
dst (set): Destination set. Must be disjoint with `subset`.
"""
assert src.issuperset(subset), (
"Error processing: {}\n"
"The following entries were not found:\n"
"{}"
"Please visit go/hiddenapi for more information.").format(
source, "".join(map(lambda x: " " + str(x), subset.difference(src))))
assert dst.isdisjoint(subset)
# Order matters if `src` and `subset` are the same object.
dst.update(subset)
src.difference_update(subset)
def get_package_name(signature):
"""Returns the package name prefix of a class member signature.
Example: "Ljava/lang/String;->hashCode()J" --> "Ljava/lang/"
Args:
signature (string): Member signature
Returns
string: Package name of the given member
"""
class_name_end = signature.find("->")
assert class_name_end != -1, "Invalid signature: {}".format(signature)
package_name_end = signature.rfind("/", 0, class_name_end)
assert package_name_end != -1, "Invalid signature: {}".format(signature)
return signature[:package_name_end + 1]
def all_package_names(*args):
"""Returns a set of packages names in given lists of member signatures.
Example: args = [ set([ "Lpkg1/ClassA;->foo()V", "Lpkg2/ClassB;->bar()J" ]),
set([ "Lpkg1/ClassC;->baz()Z" ]) ]
return value = set([ "Lpkg1/", "Lpkg2" ])
Args:
*args (list): List of sets to iterate over and extract the package names
of its elements (member signatures)
Returns:
set: All package names extracted from the given lists of signatures.
"""
packages = set()
for arg in args:
packages = packages.union(map(get_package_name, arg))
return packages
def move_all(src, dst):
"""Moves all elements of one set to another.
Args:
src (set): Source set. Will become empty.
dst (set): Destination set. Will contain all elements of `src`.
"""
move_between_sets(src, src, dst)
def move_from_files(filenames, src, dst):
"""Loads member signatures from a list of files and moves them to a given set.
Opens files in `filenames`, reads all their lines and moves those from `src`
set to `dst` set.
Args:
filenames (list): List of paths to files to be loaded.
src (set): Set that loaded lines should be moved from.
dst (set): Set that loaded lines should be moved to.
"""
if filenames:
for filename in filenames:
move_between_sets(set(read_lines(filename)), src, dst, filename)
def move_serialization(src, dst):
"""Moves all members matching serialization API signatures between given sets.
Args:
src (set): Set that will be searched for serialization API and that API
will be removed from it.
dst (set): Set that serialization API will be moved to.
"""
serialization_patterns = [
r'readObject\(Ljava/io/ObjectInputStream;\)V',
r'readObjectNoData\(\)V',
r'readResolve\(\)Ljava/lang/Object;',
r'serialVersionUID:J',
r'serialPersistentFields:\[Ljava/io/ObjectStreamField;',
r'writeObject\(Ljava/io/ObjectOutputStream;\)V',
r'writeReplace\(\)Ljava/lang/Object;',
]
regex = re.compile(r'.*->(' + '|'.join(serialization_patterns) + r')$')
move_between_sets(filter(lambda api: regex.match(api), src), src, dst)
def move_from_packages(packages, src, dst):
"""Moves all members of given package names from one set to another.
Args:
packages (list): List of string package names.
src (set): Set that will be searched for API matching one of the given
package names. Surch API will be removed from the set.
dst (set): Set that matching API will be moved to.
"""
move_between_sets(filter(lambda api: get_package_name(api) in packages, src), src, dst)
def main(argv):
args = get_args()
# Initialize API sets by loading lists of public and private API. Public API
# are all members resolvable from SDK API stubs, other members are private.
# As an optimization, skip the step of moving public API from a full set of
# members and start with a populated whitelist.
whitelist = set(read_lines(args.input_public))
uncategorized = set(read_lines(args.input_private))
light_greylist = set()
dark_greylist = set()
blacklist = set()
# Assert that there is no overlap between public and private API.
assert whitelist.isdisjoint(uncategorized)
num_all_api = len(whitelist) + len(uncategorized)
# Read all files which manually assign members to specific lists.
move_from_files(args.input_whitelists, uncategorized, whitelist)
move_from_files(args.input_greylists, uncategorized, light_greylist)
move_from_files(args.input_blacklists, uncategorized, blacklist)
# Iterate over all uncategorized members and move serialization API to light greylist.
move_serialization(uncategorized, light_greylist)
# Extract package names of members from whitelist and light greylist, which
# are assumed to have been finalized at this point. Assign all uncategorized
# members from the same packages to the dark greylist.
dark_greylist_packages = all_package_names(whitelist, light_greylist)
move_from_packages(dark_greylist_packages, uncategorized, dark_greylist)
# Assign all uncategorized members to the blacklist.
move_all(uncategorized, blacklist)
# Assert we have not missed anything.
assert whitelist.isdisjoint(light_greylist)
assert whitelist.isdisjoint(dark_greylist)
assert whitelist.isdisjoint(blacklist)
assert light_greylist.isdisjoint(dark_greylist)
assert light_greylist.isdisjoint(blacklist)
assert dark_greylist.isdisjoint(blacklist)
assert num_all_api == len(whitelist) + len(light_greylist) + len(dark_greylist) + len(blacklist)
# Write final lists to disk.
write_lines(args.output_whitelist, whitelist)
write_lines(args.output_light_greylist, light_greylist)
write_lines(args.output_dark_greylist, dark_greylist)
write_lines(args.output_blacklist, blacklist)
if __name__ == "__main__":
main(sys.argv)

89
tools/hiddenapi/generate_hiddenapi_lists_test.py Executable file
View File

@@ -0,0 +1,89 @@
#!/usr/bin/env python
#
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Hidden API list generation."""
import unittest
from generate_hiddenapi_lists import *
class TestHiddenapiListGeneration(unittest.TestCase):
def test_move_between_sets(self):
A = set([1, 2, 3, 4])
B = set([5, 6, 7, 8])
move_between_sets(set([2, 4]), A, B)
self.assertEqual(A, set([1, 3]))
self.assertEqual(B, set([2, 4, 5, 6, 7, 8]))
def test_move_between_sets_fail_not_superset(self):
A = set([1, 2, 3, 4])
B = set([5, 6, 7, 8])
with self.assertRaises(AssertionError) as ar:
move_between_sets(set([0, 2]), A, B)
def test_move_between_sets_fail_not_disjoint(self):
A = set([1, 2, 3, 4])
B = set([4, 5, 6, 7, 8])
with self.assertRaises(AssertionError) as ar:
move_between_sets(set([1, 4]), A, B)
def test_get_package_name(self):
self.assertEqual(get_package_name("Ljava/lang/String;->clone()V"), "Ljava/lang/")
def test_get_package_name_fail_no_arrow(self):
with self.assertRaises(AssertionError) as ar:
get_package_name("Ljava/lang/String;-clone()V")
with self.assertRaises(AssertionError) as ar:
get_package_name("Ljava/lang/String;>clone()V")
with self.assertRaises(AssertionError) as ar:
get_package_name("Ljava/lang/String;__clone()V")
def test_get_package_name_fail_no_package(self):
with self.assertRaises(AssertionError) as ar:
get_package_name("LString;->clone()V")
def test_all_package_names(self):
self.assertEqual(all_package_names(), set())
self.assertEqual(all_package_names(set(["Lfoo/Bar;->baz()V"])), set(["Lfoo/"]))
self.assertEqual(
all_package_names(set(["Lfoo/Bar;->baz()V", "Lfoo/BarX;->bazx()I"])),
set(["Lfoo/"]))
self.assertEqual(
all_package_names(
set(["Lfoo/Bar;->baz()V"]),
set(["Lfoo/BarX;->bazx()I", "Labc/xyz/Mno;->ijk()J"])),
set(["Lfoo/", "Labc/xyz/"]))
def test_move_all(self):
src = set([ "abc", "xyz" ])
dst = set([ "def" ])
move_all(src, dst)
self.assertEqual(src, set())
self.assertEqual(dst, set([ "abc", "def", "xyz" ]))
def test_move_from_packages(self):
src = set([ "Lfoo/bar/ClassA;->abc()J", # will be moved
"Lfoo/bar/ClassA;->def()J", # will be moved
"Lcom/pkg/example/ClassD;->ijk:J", # not moved: different package
"Lfoo/bar/xyz/ClassC;->xyz()Z" ]) # not moved: subpackage
dst = set()
packages = set([ "Lfoo/bar/" ])
move_from_packages(packages, src, dst)
self.assertEqual(
src, set([ "Lfoo/bar/xyz/ClassC;->xyz()Z", "Lcom/pkg/example/ClassD;->ijk:J" ]))
self.assertEqual(
dst, set([ "Lfoo/bar/ClassA;->abc()J", "Lfoo/bar/ClassA;->def()J" ]))
if __name__ == '__main__':
unittest.main()