Merge "Add lint database filtering tool"

api/Android.bp

@@ -24,6 +24,41 @@ package {
     default_applicable_licenses: ["frameworks_base_license"],
 }
 
+python_binary_host {
+    name: "api_versions_trimmer",
+    srcs: ["api_versions_trimmer.py"],
+    version: {
+        py2: {
+            enabled: false,
+        },
+        py3: {
+            enabled: true,
+            embedded_launcher: false,
+        },
+    },
+}
+
+python_test_host {
+    name: "api_versions_trimmer_unittests",
+    main: "api_versions_trimmer_unittests.py",
+    srcs: [
+        "api_versions_trimmer_unittests.py",
+        "api_versions_trimmer.py",
+    ],
+    test_options: {
+        unit_test: true,
+    },
+    version: {
+        py2: {
+            enabled: false,
+        },
+        py3: {
+            enabled: true,
+            embedded_launcher: false,
+        },
+    },
+}
+
 metalava_cmd = "$(location metalava)"
 // Silence reflection warnings. See b/168689341
 metalava_cmd += " -J--add-opens=java.base/java.util=ALL-UNNAMED "
@@ -401,3 +436,37 @@ genrule {
         },
     ],
 }
+
+// This rule will filter classes present in the jar files of mainline modules
+// from the lint database in api-versions.xml.
+// This is done to reduce the number of false positive NewApi findings in
+// java libraries that compile against the module SDK
+genrule {
+    name: "api-versions-xml-public-filtered",
+    srcs: [
+        // Note: order matters: first parameter is the full api-versions.xml
+        // after that the stubs files in any order
+        // stubs files are all modules that export API surfaces EXCEPT ART
+        ":framework-doc-stubs{.api_versions.xml}",
+        ":android.net.ipsec.ike.stubs{.jar}",
+        ":conscrypt.module.public.api.stubs{.jar}",
+        ":framework-connectivity.stubs{.jar}",
+        ":framework-media.stubs{.jar}",
+        ":framework-mediaprovider.stubs{.jar}",
+        ":framework-permission.stubs{.jar}",
+        ":framework-sdkextensions.stubs{.jar}",
+        ":framework-statsd.stubs{.jar}",
+        ":framework-tethering.stubs{.jar}",
+        ":framework-wifi.stubs{.jar}",
+        ":i18n.module.public.api.stubs{.jar}",
+    ],
+    out: ["api-versions-public-filtered.xml"],
+    tools: ["api_versions_trimmer"],
+    cmd: "$(location api_versions_trimmer) $(out) $(in)",
+    dist: {
+        targets: [
+            "sdk",
+            "win_sdk",
+        ],
+    },
+}

api/api_versions_trimmer.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script to remove mainline APIs from the api-versions.xml."""
+
+import argparse
+import re
+import xml.etree.ElementTree as ET
+import zipfile
+
+
+def read_classes(stubs):
+  """Read classes from the stubs file.
+
+  Args:
+    stubs: argument can be a path to a file (a string), a file-like object or a
+    path-like object
+
+  Returns:
+    a set of the classes found in the file (set of strings)
+  """
+  classes = set()
+  with zipfile.ZipFile(stubs) as z:
+    for info in z.infolist():
+      if (not info.is_dir()
+          and info.filename.endswith(".class")
+          and not info.filename.startswith("META-INF")):
+        # drop ".class" extension
+        classes.add(info.filename[:-6])
+  return classes
+
+
+def filter_method_tag(method, classes_to_remove):
+  """Updates the signature of this method by calling filter_method_signature.
+
+  Updates the method passed into this function.
+
+  Args:
+    method: xml element that represents a method
+    classes_to_remove: set of classes you to remove
+  """
+  filtered = filter_method_signature(method.get("name"), classes_to_remove)
+  method.set("name", filtered)
+
+
+def filter_method_signature(signature, classes_to_remove):
+  """Removes mentions of certain classes from this method signature.
+
+  Replaces any existing classes that need to be removed, with java/lang/Object
+
+  Args:
+    signature: string that is a java representation of a method signature
+    classes_to_remove: set of classes you to remove
+  """
+  regex = re.compile("L.*?;")
+  start = signature.find("(")
+  matches = set(regex.findall(signature[start:]))
+  for m in matches:
+    # m[1:-1] to drop the leading `L` and `;` ending
+    if m[1:-1] in classes_to_remove:
+      signature = signature.replace(m, "Ljava/lang/Object;")
+  return signature
+
+
+def filter_lint_database(database, classes_to_remove, output):
+  """Reads a lint database and writes a filtered version without some classes.
+
+  Reads database from api-versions.xml and removes any references to classes
+  in the second argument. Writes the result (another xml with the same format
+  of the database) to output.
+
+  Args:
+    database: path to xml with lint database to read
+    classes_to_remove: iterable (ideally a set or similar for quick
+    lookups) that enumerates the classes that should be removed
+    output: path to write the filtered database
+  """
+  xml = ET.parse(database)
+  root = xml.getroot()
+  for c in xml.findall("class"):
+    cname = c.get("name")
+    if cname in classes_to_remove:
+      root.remove(c)
+    else:
+      # find the <extends /> tag inside this class to see if the parent
+      # has been removed from the known classes (attribute called name)
+      super_classes = c.findall("extends")
+      for super_class in super_classes:
+        super_class_name = super_class.get("name")
+        if super_class_name in classes_to_remove:
+          super_class.set("name", "java/lang/Object")
+      interfaces = c.findall("implements")
+      for interface in interfaces:
+        interface_name = interface.get("name")
+        if interface_name in classes_to_remove:
+          c.remove(interface)
+      for method in c.findall("method"):
+        filter_method_tag(method, classes_to_remove)
+  xml.write(output)
+
+
+def main():
+  """Run the program."""
+  parser = argparse.ArgumentParser(
+      description=
+      ("Read a lint database (api-versions.xml) and many stubs jar files. "
+       "Produce another database file that doesn't include the classes present "
+       "in the stubs file(s)."))
+  parser.add_argument("output", help="Destination of the result (xml file).")
+  parser.add_argument(
+      "api_versions",
+      help="The lint database (api-versions.xml file) to read data from"
+  )
+  parser.add_argument("stubs", nargs="+", help="The stubs jar file(s)")
+  parsed = parser.parse_args()
+  classes = set()
+  for stub in parsed.stubs:
+    classes.update(read_classes(stub))
+  filter_lint_database(parsed.api_versions, classes, parsed.output)
+
+
+if __name__ == "__main__":
+  main()

api/api_versions_trimmer_unittests.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import re
+import unittest
+import xml.etree.ElementTree as ET
+import zipfile
+
+import api_versions_trimmer
+
+
+def create_in_memory_zip_file(files):
+  f = io.BytesIO()
+  with zipfile.ZipFile(f, "w") as z:
+    for fname in files:
+      with z.open(fname, mode="w") as class_file:
+        class_file.write(b"")
+  return f
+
+
+def indent(elem, level=0):
+  i = "\n" + level * "  "
+  j = "\n" + (level - 1) * "  "
+  if len(elem):
+    if not elem.text or not elem.text.strip():
+      elem.text = i + "  "
+      if not elem.tail or not elem.tail.strip():
+        elem.tail = i
+        for subelem in elem:
+          indent(subelem, level + 1)
+        if not elem.tail or not elem.tail.strip():
+          elem.tail = j
+    else:
+      if level and (not elem.tail or not elem.tail.strip()):
+        elem.tail = j
+    return elem
+
+
+def pretty_print(s):
+  tree = ET.parse(io.StringIO(s))
+  el = indent(tree.getroot())
+  res = ET.tostring(el).decode("utf-8")
+  # remove empty lines inside the result because this still breaks some
+  # comparisons
+  return re.sub(r"\n\s*\n", "\n", res, re.MULTILINE)
+
+
+class ApiVersionsTrimmerUnittests(unittest.TestCase):
+
+  def setUp(self):
+    # so it prints diffs in long strings (xml files)
+    self.maxDiff = None
+
+  def test_read_classes(self):
+    f = create_in_memory_zip_file(
+        ["a/b/C.class",
+         "a/b/D.class",
+        ]
+    )
+    res = api_versions_trimmer.read_classes(f)
+    self.assertEqual({"a/b/C", "a/b/D"}, res)
+
+  def test_read_classes_ignore_dex(self):
+    f = create_in_memory_zip_file(
+        ["a/b/C.class",
+         "a/b/D.class",
+         "a/b/E.dex",
+         "f.dex",
+        ]
+    )
+    res = api_versions_trimmer.read_classes(f)
+    self.assertEqual({"a/b/C", "a/b/D"}, res)
+
+  def test_read_classes_ignore_manifest(self):
+    f = create_in_memory_zip_file(
+        ["a/b/C.class",
+         "a/b/D.class",
+         "META-INFO/G.class"
+        ]
+    )
+    res = api_versions_trimmer.read_classes(f)
+    self.assertEqual({"a/b/C", "a/b/D"}, res)
+
+  def test_filter_method_signature(self):
+    xml = """
+    <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
+    """
+    method = ET.fromstring(xml)
+    classes_to_remove = {"android/accessibilityservice/GestureDescription"}
+    expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
+    api_versions_trimmer.filter_method_tag(method, classes_to_remove)
+    self.assertEqual(expected, method.get("name"))
+
+  def test_filter_method_signature_with_L_in_method(self):
+    xml = """
+    <method name="dispatchLeftGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
+    """
+    method = ET.fromstring(xml)
+    classes_to_remove = {"android/accessibilityservice/GestureDescription"}
+    expected = "dispatchLeftGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
+    api_versions_trimmer.filter_method_tag(method, classes_to_remove)
+    self.assertEqual(expected, method.get("name"))
+
+  def test_filter_method_signature_with_L_in_class(self):
+    xml = """
+    <method name="dispatchGesture(Landroid/accessibilityservice/LeftGestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
+    """
+    method = ET.fromstring(xml)
+    classes_to_remove = {"android/accessibilityservice/LeftGestureDescription"}
+    expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
+    api_versions_trimmer.filter_method_tag(method, classes_to_remove)
+    self.assertEqual(expected, method.get("name"))
+
+  def test_filter_method_signature_with_inner_class(self):
+    xml = """
+    <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription$Inner;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
+    """
+    method = ET.fromstring(xml)
+    classes_to_remove = {"android/accessibilityservice/GestureDescription$Inner"}
+    expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
+    api_versions_trimmer.filter_method_tag(method, classes_to_remove)
+    self.assertEqual(expected, method.get("name"))
+
+  def _run_filter_db_test(self, database_str, expected):
+    """Performs the pattern of testing the filter_lint_database method.
+
+    Filters instances of the class "a/b/C" (hard-coded) from the database string
+    and compares the result with the expected result (performs formatting of
+    the xml of both inputs)
+
+    Args:
+      database_str: string, the contents of the lint database (api-versions.xml)
+      expected: string, the expected result after filtering the original
+    database
+    """
+    database = io.StringIO(database_str)
+    classes_to_remove = {"a/b/C"}
+    output = io.BytesIO()
+    api_versions_trimmer.filter_lint_database(
+        database,
+        classes_to_remove,
+        output
+    )
+    expected = pretty_print(expected)
+    res = pretty_print(output.getvalue().decode("utf-8"))
+    self.assertEqual(expected, res)
+
+  def test_filter_lint_database_updates_method_signature_params(self):
+    self._run_filter_db_test(
+        database_str="""
+    <api version="2">
+      <!-- will be removed -->
+      <class name="a/b/C" since="1">
+        <extends name="java/lang/Object"/>
+      </class>
+
+      <class name="a/b/E" since="1">
+        <!-- extends will be modified -->
+        <extends name="a/b/C"/>
+        <!-- first parameter will be modified -->
+        <method name="dispatchGesture(La/b/C;Landroid/os/Handler;)Z" since="24"/>
+        <!-- second should remain untouched -->
+        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
+sultCallback;Landroid/os/Handler;)Z" since="24"/>
+      </class>
+    </api>
+    """,
+        expected="""
+    <api version="2">
+      <class name="a/b/E" since="1">
+        <extends name="java/lang/Object"/>
+        <method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
+        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
+sultCallback;Landroid/os/Handler;)Z" since="24"/>
+      </class>
+    </api>
+    """)
+
+  def test_filter_lint_database_updates_method_signature_return(self):
+    self._run_filter_db_test(
+        database_str="""
+    <api version="2">
+      <!-- will be removed -->
+      <class name="a/b/C" since="1">
+        <extends name="java/lang/Object"/>
+      </class>
+
+      <class name="a/b/E" since="1">
+        <!-- extends will be modified -->
+        <extends name="a/b/C"/>
+        <!-- return type should be changed -->
+        <method name="gestureIdToString(I)La/b/C;" since="24"/>
+      </class>
+    </api>
+    """,
+        expected="""
+    <api version="2">
+      <class name="a/b/E" since="1">
+
+        <extends name="java/lang/Object"/>
+
+        <method name="gestureIdToString(I)Ljava/lang/Object;" since="24"/>
+      </class>
+    </api>
+    """)
+
+  def test_filter_lint_database_removes_implements(self):
+    self._run_filter_db_test(
+        database_str="""
+    <api version="2">
+      <!-- will be removed -->
+      <class name="a/b/C" since="1">
+        <extends name="java/lang/Object"/>
+      </class>
+
+      <class name="a/b/D" since="1">
+        <extends name="java/lang/Object"/>
+        <implements name="a/b/C"/>
+        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
+sultCallback;Landroid/os/Handler;)Z" since="24"/>
+      </class>
+    </api>
+    """,
+        expected="""
+    <api version="2">
+
+      <class name="a/b/D" since="1">
+        <extends name="java/lang/Object"/>
+        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
+sultCallback;Landroid/os/Handler;)Z" since="24"/>
+      </class>
+    </api>
+    """)
+
+  def test_filter_lint_database_updates_extends(self):
+    self._run_filter_db_test(
+        database_str="""
+    <api version="2">
+      <!-- will be removed -->
+      <class name="a/b/C" since="1">
+        <extends name="java/lang/Object"/>
+      </class>
+
+      <class name="a/b/E" since="1">
+        <!-- extends will be modified -->
+        <extends name="a/b/C"/>
+        <method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
+        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
+sultCallback;Landroid/os/Handler;)Z" since="24"/>
+      </class>
+    </api>
+    """,
+        expected="""
+    <api version="2">
+      <class name="a/b/E" since="1">
+        <extends name="java/lang/Object"/>
+        <method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
+        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
+sultCallback;Landroid/os/Handler;)Z" since="24"/>
+      </class>
+    </api>
+    """)
+
+  def test_filter_lint_database_removes_class(self):
+    self._run_filter_db_test(
+        database_str="""
+    <api version="2">
+      <!-- will be removed -->
+      <class name="a/b/C" since="1">
+        <extends name="java/lang/Object"/>
+      </class>
+
+      <class name="a/b/D" since="1">
+        <extends name="java/lang/Object"/>
+        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
+sultCallback;Landroid/os/Handler;)Z" since="24"/>
+      </class>
+    </api>
+    """,
+        expected="""
+    <api version="2">
+
+      <class name="a/b/D" since="1">
+        <extends name="java/lang/Object"/>
+        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
+sultCallback;Landroid/os/Handler;)Z" since="24"/>
+      </class>
+    </api>
+    """)
+
+
+if __name__ == "__main__":
+  unittest.main()