Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes from topic "hiddenapi-dark-grey"

Browse Source 
* changes:
  Ignore comment lines in hidden api lists
  Mark hidden API lists .KATI_RESTAT
  Check in P dark greylist, use it for hidden API list generation
This commit is contained in:
David Brazdil
2018-09-07 09:31:56 +00:00
committed by Gerrit Code Review
parent 4be1419838 ae88d4e21e
commit 3d93d8f323
5 changed files with 120964 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Android.mk
View File

@@ -322,6 +322,11 @@ $(OUT_DOCS)/offline-sdk-timestamp: $(OUT_DOCS)/offline-sdk-docs-docs.zip
( unzip -qo $< -d $(OUT_DOCS)/offline-sdk && touch -f $@ ) || exit 1
# ==== hiddenapi lists =======================================
.KATI_RESTAT: \
$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) \
$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \
$(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \
$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)
$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \
.KATI_IMPLICIT_OUTPUTS := \
$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \
@@ -331,6 +336,7 @@ $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \
frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \
frameworks/base/config/hiddenapi-light-greylist.txt \
frameworks/base/config/hiddenapi-vendor-list.txt \
frameworks/base/config/hiddenapi-dark-greylist.txt \
frameworks/base/config/hiddenapi-force-blacklist.txt \
$(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \
$(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \
@@ -339,17 +345,22 @@ $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \
--input-public $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \
--input-private $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \
--input-whitelists $(PRIVATE_WHITELIST_INPUTS) \
--input-greylists \
--input-light-greylists \
frameworks/base/config/hiddenapi-light-greylist.txt \
frameworks/base/config/hiddenapi-vendor-list.txt \
<(comm -12 <(sort $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)) \
$(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)) \
$(PRIVATE_GREYLIST_INPUTS) \
--input-dark-greylists frameworks/base/config/hiddenapi-dark-greylist.txt \
--input-blacklists frameworks/base/config/hiddenapi-force-blacklist.txt \
--output-whitelist $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) \
--output-light-greylist $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \
--output-dark-greylist $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \
--output-blacklist $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)
--output-whitelist $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST).tmp \
--output-light-greylist $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST).tmp \
--output-dark-greylist $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST).tmp \
--output-blacklist $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST).tmp
$(call commit-change-for-toc,$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST))
$(call commit-change-for-toc,$(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST))
$(call commit-change-for-toc,$(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST))
$(call commit-change-for-toc,$(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST))
# Include subdirectory makefiles
# ============================================================

120915
config/hiddenapi-dark-greylist.txt Normal file
View File

File diff suppressed because it is too large Load Diff

92
tools/hiddenapi/generate_hiddenapi_lists.py
View File

@@ -45,8 +45,11 @@ def get_args():
'--input-whitelists', nargs='*',
help='Lists of members to force on whitelist')
parser.add_argument(
'--input-greylists', nargs='*',
'--input-light-greylists', nargs='*',
help='Lists of members to force on light greylist')
parser.add_argument(
'--input-dark-greylists', nargs='*',
help='Lists of members to force on dark greylist')
parser.add_argument(
'--input-blacklists', nargs='*',
help='Lists of members to force on blacklist')
@@ -59,6 +62,8 @@ def get_args():
def read_lines(filename):
"""Reads entire file and return it as a list of lines.
Lines which begin with a hash are ignored.
Args:
filename (string): Path to the file to read from.
@@ -66,7 +71,7 @@ def read_lines(filename):
list: Lines of the loaded file as a list of strings.
"""
with open(filename, 'r') as f:
return f.readlines()
return filter(lambda line: not line.startswith('#'), f.readlines())
def write_lines(filename, lines):
"""Writes list of lines into a file, overwriting the file it it exists.
@@ -78,61 +83,31 @@ def write_lines(filename, lines):
with open(filename, 'w') as f:
f.writelines(lines)
def move_between_sets(subset, src, dst, source = "<unknown>"):
def move_between_sets(subset, src, dst, source = "<unknown>", ignoreMissing = False):
"""Removes a subset of elements from one set and add it to another.
Args:
subset (set): The subset of `src` to be moved from `src` to `dst`.
src (set): Source set. Must be a superset of `subset`.
dst (set): Destination set. Must be disjoint with `subset`.
source (string): Name of the data source.
ignoreMissing (bool): If true, do not check whether `src` is a superset of `subset`.
"""
assert src.issuperset(subset), (
"Error processing: {}\n"
"The following entries were not found:\n"
"{}"
"Please visit go/hiddenapi for more information.").format(
source, "".join(map(lambda x: " " + str(x), subset.difference(src))))
assert dst.isdisjoint(subset)
if ignoreMissing:
# Some entries in `subset` may not be in `src`. Remove such entries first.
subset.intersection_update(src)
else:
assert src.issuperset(subset), (
"Error processing: {}\n"
"The following entries were not found:\n"
"{}"
"Please visit go/hiddenapi for more information.").format(
source, "".join(map(lambda x: " " + str(x), subset.difference(src))))
assert dst.isdisjoint(subset)
# Order matters if `src` and `subset` are the same object.
dst.update(subset)
src.difference_update(subset)
def get_package_name(signature):
"""Returns the package name prefix of a class member signature.
Example: "Ljava/lang/String;->hashCode()J" --> "Ljava/lang/"
Args:
signature (string): Member signature
Returns
string: Package name of the given member
"""
class_name_end = signature.find("->")
assert class_name_end != -1, "Invalid signature: {}".format(signature)
package_name_end = signature.rfind("/", 0, class_name_end)
assert package_name_end != -1, "Invalid signature: {}".format(signature)
return signature[:package_name_end + 1]
def all_package_names(*args):
"""Returns a set of packages names in given lists of member signatures.
Example: args = [ set([ "Lpkg1/ClassA;->foo()V", "Lpkg2/ClassB;->bar()J" ]),
set([ "Lpkg1/ClassC;->baz()Z" ]) ]
return value = set([ "Lpkg1/", "Lpkg2" ])
Args:
*args (list): List of sets to iterate over and extract the package names
of its elements (member signatures)
Returns:
set: All package names extracted from the given lists of signatures.
"""
packages = set()
for arg in args:
packages = packages.union(map(get_package_name, arg))
return packages
def move_all(src, dst):
"""Moves all elements of one set to another.
@@ -142,7 +117,7 @@ def move_all(src, dst):
"""
move_between_sets(src, src, dst)
def move_from_files(filenames, src, dst):
def move_from_files(filenames, src, dst, ignoreMissing = False):
"""Loads member signatures from a list of files and moves them to a given set.
Opens files in `filenames`, reads all their lines and moves those from `src`
@@ -155,7 +130,7 @@ def move_from_files(filenames, src, dst):
"""
if filenames:
for filename in filenames:
move_between_sets(set(read_lines(filename)), src, dst, filename)
move_between_sets(set(read_lines(filename)), src, dst, filename, ignoreMissing)
def move_serialization(src, dst):
"""Moves all members matching serialization API signatures between given sets.
@@ -177,17 +152,6 @@ def move_serialization(src, dst):
regex = re.compile(r'.*->(' + '|'.join(serialization_patterns) + r')$')
move_between_sets(filter(lambda api: regex.match(api), src), src, dst)
def move_from_packages(packages, src, dst):
"""Moves all members of given package names from one set to another.
Args:
packages (list): List of string package names.
src (set): Set that will be searched for API matching one of the given
package names. Surch API will be removed from the set.
dst (set): Set that matching API will be moved to.
"""
move_between_sets(filter(lambda api: get_package_name(api) in packages, src), src, dst)
def main(argv):
args = get_args()
@@ -207,17 +171,15 @@ def main(argv):
# Read all files which manually assign members to specific lists.
move_from_files(args.input_whitelists, uncategorized, whitelist)
move_from_files(args.input_greylists, uncategorized, light_greylist)
move_from_files(args.input_light_greylists, uncategorized, light_greylist)
move_from_files(args.input_blacklists, uncategorized, blacklist)
# Iterate over all uncategorized members and move serialization API to light greylist.
move_serialization(uncategorized, light_greylist)
# Extract package names of members from whitelist and light greylist, which
# are assumed to have been finalized at this point. Assign all uncategorized
# members from the same packages to the dark greylist.
dark_greylist_packages = all_package_names(whitelist, light_greylist)
move_from_packages(dark_greylist_packages, uncategorized, dark_greylist)
# Read dark greylist inputs and move all entries to dark greylist.
# Note that the input is the list from P, so we will ignore missing entries.
move_from_files(args.input_dark_greylists, uncategorized, dark_greylist, ignoreMissing=True)
# Assign all uncategorized members to the blacklist.
move_all(uncategorized, blacklist)

40
tools/hiddenapi/generate_hiddenapi_lists_test.py
View File

@@ -38,33 +38,6 @@ class TestHiddenapiListGeneration(unittest.TestCase):
with self.assertRaises(AssertionError) as ar:
move_between_sets(set([1, 4]), A, B)
def test_get_package_name(self):
self.assertEqual(get_package_name("Ljava/lang/String;->clone()V"), "Ljava/lang/")
def test_get_package_name_fail_no_arrow(self):
with self.assertRaises(AssertionError) as ar:
get_package_name("Ljava/lang/String;-clone()V")
with self.assertRaises(AssertionError) as ar:
get_package_name("Ljava/lang/String;>clone()V")
with self.assertRaises(AssertionError) as ar:
get_package_name("Ljava/lang/String;__clone()V")
def test_get_package_name_fail_no_package(self):
with self.assertRaises(AssertionError) as ar:
get_package_name("LString;->clone()V")
def test_all_package_names(self):
self.assertEqual(all_package_names(), set())
self.assertEqual(all_package_names(set(["Lfoo/Bar;->baz()V"])), set(["Lfoo/"]))
self.assertEqual(
all_package_names(set(["Lfoo/Bar;->baz()V", "Lfoo/BarX;->bazx()I"])),
set(["Lfoo/"]))
self.assertEqual(
all_package_names(
set(["Lfoo/Bar;->baz()V"]),
set(["Lfoo/BarX;->bazx()I", "Labc/xyz/Mno;->ijk()J"])),
set(["Lfoo/", "Labc/xyz/"]))
def test_move_all(self):
src = set([ "abc", "xyz" ])
dst = set([ "def" ])
@@ -72,18 +45,5 @@ class TestHiddenapiListGeneration(unittest.TestCase):
self.assertEqual(src, set())
self.assertEqual(dst, set([ "abc", "def", "xyz" ]))
def test_move_from_packages(self):
src = set([ "Lfoo/bar/ClassA;->abc()J", # will be moved
"Lfoo/bar/ClassA;->def()J", # will be moved
"Lcom/pkg/example/ClassD;->ijk:J", # not moved: different package
"Lfoo/bar/xyz/ClassC;->xyz()Z" ]) # not moved: subpackage
dst = set()
packages = set([ "Lfoo/bar/" ])
move_from_packages(packages, src, dst)
self.assertEqual(
src, set([ "Lfoo/bar/xyz/ClassC;->xyz()Z", "Lcom/pkg/example/ClassD;->ijk:J" ]))
self.assertEqual(
dst, set([ "Lfoo/bar/ClassA;->abc()J", "Lfoo/bar/ClassA;->def()J" ]))
if __name__ == '__main__':
unittest.main()

6
tools/hiddenapi/sort_api.sh
View File

@@ -11,8 +11,14 @@ fi
readarray A < "$source_list"
# Sort
IFS=$'\n'
# Stash away comments
C=( $(grep -E '^#' <<< "${A[*]}") )
A=( $(grep -v -E '^#' <<< "${A[*]}") )
# Sort entries
A=( $(LC_COLLATE=C sort -f <<< "${A[*]}") )
A=( $(uniq <<< "${A[*]}") )
# Concatenate comments and entries
A=( ${C[*]} ${A[*]} )
unset IFS
# Dump array back into the file
printf '%s\n' "${A[@]}" > "$dest_list"