Merge "Automatically greylist code in 3P packages" am: 1f80714c2d

am: 8b1745650b

Change-Id: I73ba988cc378c6204a95e3f9e6931336aaf3d953
This commit is contained in:
Andrei-Valentin Onea
2019-04-02 06:40:33 -07:00
committed by android-build-merger
4 changed files with 63 additions and 143 deletions

View File

@@ -0,0 +1,2 @@
org.ccil.cowan.tagsoup
org.ccil.cowan.tagsoup.jaxp

View File

@@ -2604,108 +2604,3 @@ Lorg/apache/xpath/XPathContext;->setCurrentExpressionNodeStack(Lorg/apache/xml/u
Lorg/apache/xpath/XPathContext;->setCurrentNodeStack(Lorg/apache/xml/utils/IntStack;)V
Lorg/apache/xpath/XPathContext;->setSecureProcessing(Z)V
Lorg/apache/xpath/XPathContext;->setVarStack(Lorg/apache/xpath/VariableStack;)V
Lorg/ccil/cowan/tagsoup/AttributesImpl;-><init>(Lorg/xml/sax/Attributes;)V
Lorg/ccil/cowan/tagsoup/AttributesImpl;->addAttribute(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
Lorg/ccil/cowan/tagsoup/AttributesImpl;->data:[Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/AttributesImpl;->length:I
Lorg/ccil/cowan/tagsoup/AttributesImpl;->setAttribute(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
Lorg/ccil/cowan/tagsoup/AttributesImpl;->setValue(ILjava/lang/String;)V
Lorg/ccil/cowan/tagsoup/AutoDetector;->autoDetectingReader(Ljava/io/InputStream;)Ljava/io/Reader;
Lorg/ccil/cowan/tagsoup/Element;-><init>(Lorg/ccil/cowan/tagsoup/ElementType;Z)V
Lorg/ccil/cowan/tagsoup/Element;->anonymize()V
Lorg/ccil/cowan/tagsoup/Element;->atts()Lorg/ccil/cowan/tagsoup/AttributesImpl;
Lorg/ccil/cowan/tagsoup/Element;->canContain(Lorg/ccil/cowan/tagsoup/Element;)Z
Lorg/ccil/cowan/tagsoup/Element;->clean()V
Lorg/ccil/cowan/tagsoup/Element;->flags()I
Lorg/ccil/cowan/tagsoup/Element;->localName()Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Element;->name()Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Element;->namespace()Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Element;->next()Lorg/ccil/cowan/tagsoup/Element;
Lorg/ccil/cowan/tagsoup/Element;->parent()Lorg/ccil/cowan/tagsoup/ElementType;
Lorg/ccil/cowan/tagsoup/Element;->preclosed:Z
Lorg/ccil/cowan/tagsoup/Element;->setAttribute(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
Lorg/ccil/cowan/tagsoup/Element;->setNext(Lorg/ccil/cowan/tagsoup/Element;)V
Lorg/ccil/cowan/tagsoup/Element;->theAtts:Lorg/ccil/cowan/tagsoup/AttributesImpl;
Lorg/ccil/cowan/tagsoup/Element;->theNext:Lorg/ccil/cowan/tagsoup/Element;
Lorg/ccil/cowan/tagsoup/Element;->theType:Lorg/ccil/cowan/tagsoup/ElementType;
Lorg/ccil/cowan/tagsoup/ElementType;-><init>(Ljava/lang/String;IIILorg/ccil/cowan/tagsoup/Schema;)V
Lorg/ccil/cowan/tagsoup/ElementType;->atts()Lorg/ccil/cowan/tagsoup/AttributesImpl;
Lorg/ccil/cowan/tagsoup/ElementType;->setAttribute(Lorg/ccil/cowan/tagsoup/AttributesImpl;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
Lorg/ccil/cowan/tagsoup/ElementType;->theAtts:Lorg/ccil/cowan/tagsoup/AttributesImpl;
Lorg/ccil/cowan/tagsoup/ElementType;->theFlags:I
Lorg/ccil/cowan/tagsoup/ElementType;->theLocalName:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/ElementType;->theMemberOf:I
Lorg/ccil/cowan/tagsoup/ElementType;->theModel:I
Lorg/ccil/cowan/tagsoup/ElementType;->theName:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/ElementType;->theNamespace:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/ElementType;->theParent:Lorg/ccil/cowan/tagsoup/ElementType;
Lorg/ccil/cowan/tagsoup/ElementType;->theSchema:Lorg/ccil/cowan/tagsoup/Schema;
Lorg/ccil/cowan/tagsoup/HTMLScanner;-><init>()V
Lorg/ccil/cowan/tagsoup/HTMLSchema;-><init>()V
Lorg/ccil/cowan/tagsoup/jaxp/SAXFactoryImpl;-><init>()V
Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;-><init>()V
Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;->newInstance(Ljava/util/Map;)Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;
Lorg/ccil/cowan/tagsoup/Parser;-><init>()V
Lorg/ccil/cowan/tagsoup/Parser;->bogonsEmpty:Z
Lorg/ccil/cowan/tagsoup/Parser;->CDATAElements:Z
Lorg/ccil/cowan/tagsoup/Parser;->cleanPublicid(Ljava/lang/String;)Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->defaultAttributes:Z
Lorg/ccil/cowan/tagsoup/Parser;->etagchars:[C
Lorg/ccil/cowan/tagsoup/Parser;->expandEntities(Ljava/lang/String;)Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->getInputStream(Ljava/lang/String;Ljava/lang/String;)Ljava/io/InputStream;
Lorg/ccil/cowan/tagsoup/Parser;->ignorableWhitespace:Z
Lorg/ccil/cowan/tagsoup/Parser;->ignoreBogons:Z
Lorg/ccil/cowan/tagsoup/Parser;->lookupEntity([CII)I
Lorg/ccil/cowan/tagsoup/Parser;->makeName([CII)Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->pop()V
Lorg/ccil/cowan/tagsoup/Parser;->push(Lorg/ccil/cowan/tagsoup/Element;)V
Lorg/ccil/cowan/tagsoup/Parser;->rectify(Lorg/ccil/cowan/tagsoup/Element;)V
Lorg/ccil/cowan/tagsoup/Parser;->restart(Lorg/ccil/cowan/tagsoup/Element;)V
Lorg/ccil/cowan/tagsoup/Parser;->restartablyPop()V
Lorg/ccil/cowan/tagsoup/Parser;->rootBogons:Z
Lorg/ccil/cowan/tagsoup/Parser;->schemaProperty:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->split(Ljava/lang/String;)[Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->theAttributeName:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->theAutoDetector:Lorg/ccil/cowan/tagsoup/AutoDetector;
Lorg/ccil/cowan/tagsoup/Parser;->theContentHandler:Lorg/xml/sax/ContentHandler;
Lorg/ccil/cowan/tagsoup/Parser;->theDoctypeIsPresent:Z
Lorg/ccil/cowan/tagsoup/Parser;->theDoctypeSystemId:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->theFeatures:Ljava/util/HashMap;
Lorg/ccil/cowan/tagsoup/Parser;->theLexicalHandler:Lorg/xml/sax/ext/LexicalHandler;
Lorg/ccil/cowan/tagsoup/Parser;->theNewElement:Lorg/ccil/cowan/tagsoup/Element;
Lorg/ccil/cowan/tagsoup/Parser;->thePCDATA:Lorg/ccil/cowan/tagsoup/Element;
Lorg/ccil/cowan/tagsoup/Parser;->thePITarget:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->theSaved:Lorg/ccil/cowan/tagsoup/Element;
Lorg/ccil/cowan/tagsoup/Parser;->theScanner:Lorg/ccil/cowan/tagsoup/Scanner;
Lorg/ccil/cowan/tagsoup/Parser;->theSchema:Lorg/ccil/cowan/tagsoup/Schema;
Lorg/ccil/cowan/tagsoup/Parser;->theStack:Lorg/ccil/cowan/tagsoup/Element;
Lorg/ccil/cowan/tagsoup/Parser;->trimquotes(Ljava/lang/String;)Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Parser;->virginStack:Z
Lorg/ccil/cowan/tagsoup/PYXScanner;-><init>()V
Lorg/ccil/cowan/tagsoup/PYXWriter;-><init>(Ljava/io/Writer;)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->aname([CII)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->aval([CII)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->entity([CII)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->eof([CII)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->etag([CII)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->gi([CII)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->pcdata([CII)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->pi([CII)V
Lorg/ccil/cowan/tagsoup/ScanHandler;->stagc([CII)V
Lorg/ccil/cowan/tagsoup/Scanner;->startCDATA()V
Lorg/ccil/cowan/tagsoup/Schema;->elementType(Ljava/lang/String;III)V
Lorg/ccil/cowan/tagsoup/Schema;->getElementType(Ljava/lang/String;)Lorg/ccil/cowan/tagsoup/ElementType;
Lorg/ccil/cowan/tagsoup/Schema;->getEntity(Ljava/lang/String;)I
Lorg/ccil/cowan/tagsoup/Schema;->getPrefix()Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Schema;->getURI()Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Schema;->parent(Ljava/lang/String;Ljava/lang/String;)V
Lorg/ccil/cowan/tagsoup/Schema;->theElementTypes:Ljava/util/HashMap;
Lorg/ccil/cowan/tagsoup/Schema;->theEntities:Ljava/util/HashMap;
Lorg/ccil/cowan/tagsoup/Schema;->thePrefix:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/Schema;->theRoot:Lorg/ccil/cowan/tagsoup/ElementType;
Lorg/ccil/cowan/tagsoup/Schema;->theURI:Ljava/lang/String;
Lorg/ccil/cowan/tagsoup/XMLWriter;-><init>(Ljava/io/Writer;)V
Lorg/ccil/cowan/tagsoup/XMLWriter;->htmlMode:Z
Lorg/ccil/cowan/tagsoup/XMLWriter;->setOutput(Ljava/io/Writer;)V
Lorg/ccil/cowan/tagsoup/XMLWriter;->setOutputProperty(Ljava/lang/String;Ljava/lang/String;)V
Lorg/ccil/cowan/tagsoup/XMLWriter;->setPrefix(Ljava/lang/String;Ljava/lang/String;)V

View File

@@ -21,6 +21,7 @@ from collections import defaultdict
import os
import sys
import re
import functools
# Names of flags recognized by the `hiddenapi` tool.
FLAG_WHITELIST = "whitelist"
@@ -58,6 +59,10 @@ ALL_FLAGS_SET = set(ALL_FLAGS)
# script to skip any entries which do not exist any more.
FLAG_IGNORE_CONFLICTS_SUFFIX = "-ignore-conflicts"
# Suffix used in command line args to express that all apis within a given set
# of packages should be assign the given flag.
FLAG_PACKAGES_SUFFIX = "-packages"
# Regex patterns of fields/methods used in serialization. These are
# considered public API despite being hidden.
SERIALIZATION_PATTERNS = [
@@ -91,12 +96,16 @@ def get_args():
for flag in ALL_FLAGS:
ignore_conflicts_flag = flag + FLAG_IGNORE_CONFLICTS_SUFFIX
packages_flag = flag + FLAG_PACKAGES_SUFFIX
parser.add_argument('--' + flag, dest=flag, nargs='*', default=[], metavar='TXT_FILE',
help='lists of entries with flag "' + flag + '"')
parser.add_argument('--' + ignore_conflicts_flag, dest=ignore_conflicts_flag, nargs='*',
default=[], metavar='TXT_FILE',
help='lists of entries with flag "' + flag +
'". skip entry if missing or flag conflict.')
parser.add_argument('--' + packages_flag, dest=packages_flag, nargs='*',
default=[], metavar='TXT_FILE',
help='lists of packages to be added to ' + flag + ' list')
return parser.parse_args()
@@ -128,6 +137,19 @@ def write_lines(filename, lines):
with open(filename, 'w') as f:
f.writelines(lines)
def extract_package(signature):
"""Extracts the package from a signature.
Args:
signature (string): JNI signature of a method or field.
Returns:
The package name of the class containing the field/method.
"""
full_class_name = signature.split(";->")[0]
package_name = full_class_name[1:full_class_name.rindex("/")]
return package_name.replace('/', '.')
class FlagsDict:
def __init__(self):
self._dict_keyset = set()
@@ -206,7 +228,10 @@ class FlagsDict:
self._dict_keyset.update([ csv[0] for csv in csv_values ])
# Check that all flags are known.
csv_flags = set(reduce(lambda x, y: set(x).union(y), [ csv[1:] for csv in csv_values ], []))
csv_flags = set(functools.reduce(
lambda x, y: set(x).union(y),
[ csv[1:] for csv in csv_values ],
[]))
self._check_flags_set(csv_flags, source)
# Iterate over all CSV lines, find entry in dict and append flags to it.
@@ -273,6 +298,15 @@ def main(argv):
valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(filename))
flags.assign_flag(flag, valid_entries, filename)
# All members in the specified packages will be assigned the appropriate flag.
for flag in ALL_FLAGS:
for filename in args[flag + FLAG_PACKAGES_SUFFIX]:
packages_needing_list = set(read_lines(filename))
should_add_signature_to_list = lambda sig,lists: extract_package(
sig) in packages_needing_list and not lists
valid_entries = flags.filter_apis(should_add_signature_to_list)
flags.assign_flag(flag, valid_entries)
# Assign all remaining entries to the blacklist.
flags.assign_flag(FLAG_BLACKLIST, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))

View File

@@ -18,33 +18,23 @@ import unittest
from generate_hiddenapi_lists import *
class TestHiddenapiListGeneration(unittest.TestCase):
def test_init(self):
# Check empty lists
flags = FlagsDict([], [])
self.assertEquals(flags.generate_csv(), [])
# Check valid input - two public and two private API signatures.
flags = FlagsDict(['A', 'B'], ['C', 'D'])
self.assertEquals(flags.generate_csv(),
[ 'A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST, 'C', 'D' ])
# Check invalid input - overlapping public/private API signatures.
with self.assertRaises(AssertionError):
flags = FlagsDict(['A', 'B'], ['B', 'C', 'D'])
def test_filter_apis(self):
# Initialize flags so that A and B are put on the whitelist and
# C, D, E are left unassigned. Try filtering for the unassigned ones.
flags = FlagsDict(['A', 'B'], ['C', 'D', 'E'])
flags = FlagsDict()
flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST,
'C', 'D', 'E'])
filter_set = flags.filter_apis(lambda api, flags: not flags)
self.assertTrue(isinstance(filter_set, set))
self.assertEqual(filter_set, set([ 'C', 'D', 'E' ]))
def test_get_valid_subset_of_unassigned_keys(self):
# Create flags where only A is unassigned.
flags = FlagsDict(['A'], ['B', 'C'])
flags = FlagsDict()
flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B', 'C'])
flags.assign_flag(FLAG_GREYLIST, set(['C']))
self.assertEquals(flags.generate_csv(),
self.assertEqual(flags.generate_csv(),
[ 'A,' + FLAG_WHITELIST, 'B', 'C,' + FLAG_GREYLIST ])
# Check three things:
@@ -55,44 +45,30 @@ class TestHiddenapiListGeneration(unittest.TestCase):
flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), set([ 'B' ]))
def test_parse_and_merge_csv(self):
flags = FlagsDict(['A'], ['B'])
self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
flags = FlagsDict()
# Test empty CSV entry.
flags.parse_and_merge_csv(['B'])
self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
# Test assigning an already assigned flag.
flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST])
self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
self.assertEqual(flags.generate_csv(), [])
# Test new additions.
flags.parse_and_merge_csv([
'A,' + FLAG_GREYLIST,
'B,' + FLAG_BLACKLIST + ',' + FLAG_GREYLIST_MAX_O ])
self.assertEqual(flags.generate_csv(),
[ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST,
[ 'A,' + FLAG_GREYLIST,
'B,' + FLAG_BLACKLIST + "," + FLAG_GREYLIST_MAX_O ])
# Test unknown API signature.
with self.assertRaises(AssertionError):
flags.parse_and_merge_csv([ 'C' ])
# Test unknown flag.
with self.assertRaises(AssertionError):
flags.parse_and_merge_csv([ 'A,foo' ])
flags.parse_and_merge_csv([ 'C,foo' ])
def test_assign_flag(self):
flags = FlagsDict(['A'], ['B'])
self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
# Test assigning an already assigned flag.
flags.assign_flag(FLAG_WHITELIST, set([ 'A' ]))
self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
flags = FlagsDict()
flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B'])
# Test new additions.
flags.assign_flag(FLAG_GREYLIST, set([ 'A', 'B' ]))
self.assertEquals(flags.generate_csv(),
self.assertEqual(flags.generate_csv(),
[ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST, 'B,' + FLAG_GREYLIST ])
# Test invalid API signature.
@@ -103,5 +79,18 @@ class TestHiddenapiListGeneration(unittest.TestCase):
with self.assertRaises(AssertionError):
flags.assign_flag('foo', set([ 'A' ]))
def test_extract_package(self):
signature = 'Lcom/foo/bar/Baz;->method1()Lcom/bar/Baz;'
expected_package = 'com.foo.bar'
self.assertEqual(extract_package(signature), expected_package)
signature = 'Lcom/foo1/bar/MyClass;->method2()V'
expected_package = 'com.foo1.bar'
self.assertEqual(extract_package(signature), expected_package)
signature = 'Lcom/foo_bar/baz/MyClass;->method3()V'
expected_package = 'com.foo_bar.baz'
self.assertEqual(extract_package(signature), expected_package)
if __name__ == '__main__':
unittest.main()