Merge "Add onFrame methods to face auth/enroll client monitors" into sc-dev

core/java/android/hardware/face/FaceAuthenticationFrame.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.hardware.face;
+
+import android.annotation.NonNull;
+import android.os.Parcel;
+import android.os.Parcelable;
+
+/**
+ * Data model for a frame captured during face authentication.
+ *
+ * @hide
+ */
+public final class FaceAuthenticationFrame implements Parcelable {
+    @NonNull private final FaceDataFrame mData;
+
+    /**
+     * Data model for a frame captured during face authentication.
+     *
+     * @param data Information about the current frame.
+     */
+    public FaceAuthenticationFrame(@NonNull FaceDataFrame data) {
+        mData = data;
+    }
+
+    /**
+     * @return Information about the current frame.
+     */
+    @NonNull
+    public FaceDataFrame getData() {
+        return mData;
+    }
+
+    private FaceAuthenticationFrame(@NonNull Parcel source) {
+        mData = source.readParcelable(FaceDataFrame.class.getClassLoader());
+    }
+
+    @Override
+    public int describeContents() {
+        return 0;
+    }
+
+    @Override
+    public void writeToParcel(Parcel dest, int flags) {
+        dest.writeParcelable(mData, flags);
+    }
+
+    public static final Creator<FaceAuthenticationFrame> CREATOR =
+            new Creator<FaceAuthenticationFrame>() {
+
+        @Override
+        public FaceAuthenticationFrame createFromParcel(Parcel source) {
+            return new FaceAuthenticationFrame(source);
+        }
+
+        @Override
+        public FaceAuthenticationFrame[] newArray(int size) {
+            return new FaceAuthenticationFrame[size];
+        }
+    };
+}

core/java/android/hardware/face/FaceDataFrame.java

@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.hardware.face;
+
+import android.annotation.NonNull;
+import android.os.Parcel;
+import android.os.Parcelable;
+
+/**
+ * A container for data common to {@link FaceAuthenticationFrame} and {@link FaceEnrollFrame}.
+ *
+ * @hide
+ */
+public final class FaceDataFrame implements Parcelable {
+    private final int mAcquiredInfo;
+    private final int mVendorCode;
+    private final float mPan;
+    private final float mTilt;
+    private final float mDistance;
+    private final boolean mIsCancellable;
+
+    /**
+     * A container for data common to {@link FaceAuthenticationFrame} and {@link FaceEnrollFrame}.
+     *
+     * @param acquiredInfo An integer corresponding to a known acquired message.
+     * @param vendorCode An integer representing a custom vendor-specific message. Ignored unless
+     *  {@code acquiredInfo} is {@code FACE_ACQUIRED_VENDOR}.
+     * @param pan The horizontal pan of the detected face. Values in the range [-1, 1] indicate a
+     *  good capture.
+     * @param tilt The vertical tilt of the detected face. Values in the range [-1, 1] indicate a
+     *  good capture.
+     * @param distance The distance of the detected face from the device. Values in the range
+     *  [-1, 1] indicate a good capture.
+     * @param isCancellable Whether the ongoing face operation should be canceled.
+     */
+    public FaceDataFrame(
+            int acquiredInfo,
+            int vendorCode,
+            float pan,
+            float tilt,
+            float distance,
+            boolean isCancellable) {
+        mAcquiredInfo = acquiredInfo;
+        mVendorCode = vendorCode;
+        mPan = pan;
+        mTilt = tilt;
+        mDistance = distance;
+        mIsCancellable = isCancellable;
+    }
+
+    /**
+     * @return An integer corresponding to a known acquired message.
+     *
+     * @see android.hardware.biometrics.BiometricFaceConstants
+     */
+    public int getAcquiredInfo() {
+        return mAcquiredInfo;
+    }
+
+    /**
+     * @return An integer representing a custom vendor-specific message. Ignored unless
+     * {@code acquiredInfo} is {@link
+     * android.hardware.biometrics.BiometricFaceConstants#FACE_ACQUIRED_VENDOR}.
+     *
+     * @see android.hardware.biometrics.BiometricFaceConstants
+     */
+    public int getVendorCode() {
+        return mVendorCode;
+    }
+
+    /**
+     * @return The horizontal pan of the detected face. Values in the range [-1, 1] indicate a good
+     * capture.
+     */
+    public float getPan() {
+        return mPan;
+    }
+
+    /**
+     * @return The vertical tilt of the detected face. Values in the range [-1, 1] indicate a good
+     * capture.
+     */
+    public float getTilt() {
+        return mTilt;
+    }
+
+    /**
+     * @return The distance of the detected face from the device. Values in the range [-1, 1]
+     * indicate a good capture.
+     */
+    public float getDistance() {
+        return mDistance;
+    }
+
+    /**
+     * @return Whether the ongoing face operation should be canceled.
+     */
+    public boolean isCancellable() {
+        return mIsCancellable;
+    }
+
+    private FaceDataFrame(@NonNull Parcel source) {
+        mAcquiredInfo = source.readInt();
+        mVendorCode = source.readInt();
+        mPan = source.readFloat();
+        mTilt = source.readFloat();
+        mDistance = source.readFloat();
+        mIsCancellable = source.readBoolean();
+    }
+
+    @Override
+    public int describeContents() {
+        return 0;
+    }
+
+    @Override
+    public void writeToParcel(Parcel dest, int flags) {
+        dest.writeInt(mAcquiredInfo);
+        dest.writeInt(mVendorCode);
+        dest.writeFloat(mPan);
+        dest.writeFloat(mTilt);
+        dest.writeFloat(mDistance);
+        dest.writeBoolean(mIsCancellable);
+    }
+
+    public static final Creator<FaceDataFrame> CREATOR = new Creator<FaceDataFrame>() {
+        @Override
+        public FaceDataFrame createFromParcel(Parcel source) {
+            return new FaceDataFrame(source);
+        }
+
+        @Override
+        public FaceDataFrame[] newArray(int size) {
+            return new FaceDataFrame[size];
+        }
+    };
+}

core/java/android/hardware/face/FaceEnrollCell.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.hardware.face;
+
+import android.annotation.NonNull;
+import android.os.Parcel;
+import android.os.Parcelable;
+
+/**
+ * A matrix cell, corresponding to a desired face image, that may be captured during enrollment.
+ *
+ * @hide
+ */
+public final class FaceEnrollCell implements Parcelable {
+    private final int mX;
+    private final int mY;
+    private final int mZ;
+
+    /**
+     * A matrix cell, corresponding to a desired face image, that may be captured during enrollment.
+     *
+     * @param x The horizontal coordinate of this cell.
+     * @param y The vertical coordinate of this cell.
+     * @param z The depth coordinate of this cell.
+     */
+    public FaceEnrollCell(int x, int y, int z) {
+        mX = x;
+        mY = y;
+        mZ = z;
+    }
+
+    /**
+     * @return The horizontal coordinate of this cell.
+     */
+    public int getX() {
+        return mX;
+    }
+
+    /**
+     * @return The vertical coordinate of this cell.
+     */
+    public int getY() {
+        return mY;
+    }
+
+    /**
+     * @return The depth coordinate of this cell.
+     */
+    public int getZ() {
+        return mZ;
+    }
+
+    private FaceEnrollCell(@NonNull Parcel source) {
+        mX = source.readInt();
+        mY = source.readInt();
+        mZ = source.readInt();
+    }
+
+    @Override
+    public int describeContents() {
+        return 0;
+    }
+
+    @Override
+    public void writeToParcel(Parcel dest, int flags) {
+        dest.writeInt(mX);
+        dest.writeInt(mY);
+        dest.writeInt(mZ);
+    }
+
+    public static final Creator<FaceEnrollCell> CREATOR = new Creator<FaceEnrollCell>() {
+        @Override
+        public FaceEnrollCell createFromParcel(Parcel source) {
+            return new FaceEnrollCell(source);
+        }
+
+        @Override
+        public FaceEnrollCell[] newArray(int size) {
+            return new FaceEnrollCell[size];
+        }
+    };
+}

core/java/android/hardware/face/FaceEnrollFrame.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.hardware.face;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.os.Parcel;
+import android.os.Parcelable;
+
+/**
+ * Data model for a frame captured during face enrollment.
+ *
+ * @hide
+ */
+public final class FaceEnrollFrame implements Parcelable {
+    @Nullable private final FaceEnrollCell mCell;
+    @FaceEnrollStage private final int mStage;
+    @NonNull private final FaceDataFrame mData;
+
+    /**
+     * Data model for a frame captured during face enrollment.
+     *
+     * @param cell The cell captured during this frame of enrollment, if any.
+     * @param stage An integer representing the current stage of enrollment.
+     * @param data Information about the current frame.
+     */
+    public FaceEnrollFrame(
+            @Nullable FaceEnrollCell cell,
+            @FaceEnrollStage int stage,
+            @NonNull FaceDataFrame data) {
+        mCell = cell;
+        mStage = stage;
+        mData = data;
+    }
+
+    /**
+     * @return The cell captured during this frame of enrollment, if any.
+     */
+    @Nullable
+    public FaceEnrollCell getCell() {
+        return mCell;
+    }
+
+    /**
+     * @return An integer representing the current stage of enrollment.
+     */
+    @FaceEnrollStage
+    public int getStage() {
+        return mStage;
+    }
+
+    /**
+     * @return Information about the current frame.
+     */
+    @NonNull
+    public FaceDataFrame getData() {
+        return mData;
+    }
+
+    private FaceEnrollFrame(@NonNull Parcel source) {
+        mCell = source.readParcelable(FaceEnrollCell.class.getClassLoader());
+        mStage = source.readInt();
+        mData = source.readParcelable(FaceDataFrame.class.getClassLoader());
+    }
+
+    @Override
+    public int describeContents() {
+        return 0;
+    }
+
+    @Override
+    public void writeToParcel(Parcel dest, int flags) {
+        dest.writeParcelable(mCell, flags);
+        dest.writeInt(mStage);
+        dest.writeParcelable(mData, flags);
+    }
+
+    public static final Creator<FaceEnrollFrame> CREATOR = new Creator<FaceEnrollFrame>() {
+        @Override
+        public FaceEnrollFrame createFromParcel(Parcel source) {
+            return new FaceEnrollFrame(source);
+        }
+
+        @Override
+        public FaceEnrollFrame[] newArray(int size) {
+            return new FaceEnrollFrame[size];
+        }
+    };
+}

core/java/android/hardware/face/FaceEnrollStage.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.hardware.face;
+
+import android.annotation.IntDef;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * A stage that may occur during face enrollment.
+ *
+ * @hide
+ */
+@Retention(RetentionPolicy.SOURCE)
+@IntDef({
+    FaceEnrollStage.FIRST_FRAME_RECEIVED,
+    FaceEnrollStage.WAITING_FOR_CENTERING,
+    FaceEnrollStage.HOLD_STILL_IN_CENTER,
+    FaceEnrollStage.ENROLLING_MOVEMENT_1,
+    FaceEnrollStage.ENROLLING_MOVEMENT_2,
+    FaceEnrollStage.ENROLLMENT_FINISHED
+})
+public @interface FaceEnrollStage {
+    /**
+     * Enrollment has just begun. No action is needed from the user yet.
+     */
+    int FIRST_FRAME_RECEIVED = 0;
+
+    /**
+     * The user must center their face in the frame.
+     */
+    int WAITING_FOR_CENTERING = 1;
+
+    /**
+     * The user must keep their face centered in the frame.
+     */
+    int HOLD_STILL_IN_CENTER = 2;
+
+    /**
+     * The user must follow a first set of movement instructions.
+     */
+    int ENROLLING_MOVEMENT_1 = 3;
+
+    /**
+     * The user must follow a second set of movement instructions.
+     */
+    int ENROLLING_MOVEMENT_2 = 4;
+
+    /**
+     * Enrollment has completed. No more action is needed from the user.
+     */
+    int ENROLLMENT_FINISHED = 5;
+}

services/core/java/com/android/server/biometrics/sensors/face/aidl/AidlConversionUtils.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.biometrics.sensors.face.aidl;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.hardware.biometrics.face.AuthenticationFrame;
+import android.hardware.biometrics.face.BaseFrame;
+import android.hardware.biometrics.face.Cell;
+import android.hardware.biometrics.face.EnrollmentFrame;
+import android.hardware.face.FaceAuthenticationFrame;
+import android.hardware.face.FaceDataFrame;
+import android.hardware.face.FaceEnrollCell;
+import android.hardware.face.FaceEnrollFrame;
+
+/**
+ * Utilities for converting between hardware and framework-defined AIDL models.
+ */
+final class AidlConversionUtils {
+    // Prevent instantiation.
+    private AidlConversionUtils() {}
+
+    @NonNull
+    public static FaceAuthenticationFrame convert(@NonNull AuthenticationFrame frame) {
+        return new FaceAuthenticationFrame(convert(frame.data));
+    }
+
+    @NonNull
+    public static AuthenticationFrame convert(@NonNull FaceAuthenticationFrame frame) {
+        final AuthenticationFrame convertedFrame = new AuthenticationFrame();
+        convertedFrame.data = convert(frame.getData());
+        return convertedFrame;
+    }
+
+    @NonNull
+    public static FaceEnrollFrame convert(@NonNull EnrollmentFrame frame) {
+        return new FaceEnrollFrame(convert(frame.cell), frame.stage, convert(frame.data));
+    }
+
+    @NonNull
+    public static EnrollmentFrame convert(@NonNull FaceEnrollFrame frame) {
+        final EnrollmentFrame convertedFrame = new EnrollmentFrame();
+        convertedFrame.cell = convert(frame.getCell());
+        convertedFrame.stage = (byte) frame.getStage();
+        convertedFrame.data = convert(frame.getData());
+        return convertedFrame;
+    }
+
+    @NonNull
+    public static FaceDataFrame convert(@NonNull BaseFrame frame) {
+        return new FaceDataFrame(
+                frame.acquiredInfo,
+                frame.vendorCode,
+                frame.pan,
+                frame.tilt,
+                frame.distance,
+                frame.isCancellable);
+    }
+
+    @NonNull
+    public static BaseFrame convert(@NonNull FaceDataFrame frame) {
+        final BaseFrame convertedFrame = new BaseFrame();
+        convertedFrame.acquiredInfo = (byte) frame.getAcquiredInfo();
+        convertedFrame.vendorCode = frame.getVendorCode();
+        convertedFrame.pan = frame.getPan();
+        convertedFrame.tilt = frame.getTilt();
+        convertedFrame.distance = frame.getDistance();
+        convertedFrame.isCancellable = frame.isCancellable();
+        return convertedFrame;
+    }
+
+    @Nullable
+    public static FaceEnrollCell convert(@Nullable Cell cell) {
+        return cell == null ? null : new FaceEnrollCell(cell.x, cell.y, cell.z);
+    }
+
+    @Nullable
+    public static Cell convert(@Nullable FaceEnrollCell cell) {
+        if (cell == null) {
+            return null;
+        }
+
+        final Cell convertedCell = new Cell();
+        convertedCell.x = cell.getX();
+        convertedCell.y = cell.getY();
+        convertedCell.z = cell.getZ();
+        return convertedCell;
+    }
+}

services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceAuthenticationClient.java

@@ -28,6 +28,8 @@ import android.hardware.biometrics.BiometricsProtoEnums;
 import android.hardware.biometrics.common.ICancellationSignal;
 import android.hardware.biometrics.face.IFace;
 import android.hardware.biometrics.face.ISession;
+import android.hardware.face.FaceAuthenticationFrame;
+import android.hardware.face.FaceDataFrame;
 import android.hardware.face.FaceManager;
 import android.os.IBinder;
 import android.os.RemoteException;
@@ -193,6 +195,17 @@ class FaceAuthenticationClient extends AuthenticationClient<ISession> implements
         onAcquiredInternal(acquireInfo, vendorCode, shouldSend);
     }
 
+    /**
+     * Called each time a new frame is received during face authentication.
+     *
+     * @param frame Information about the current frame.
+     */
+    public void onAuthenticationFrame(@NonNull FaceAuthenticationFrame frame) {
+        // TODO(b/178414967): Send additional frame data to the client callback.
+        final FaceDataFrame data = frame.getData();
+        onAcquired(data.getAcquiredInfo(), data.getVendorCode());
+    }
+
     @Override public void onLockoutTimed(long durationMillis) {
         mLockoutCache.setLockoutModeForUser(getTargetUserId(), LockoutTracker.LOCKOUT_TIMED);
         // Lockout metrics are logged as an error code.

services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceEnrollClient.java

@@ -27,6 +27,8 @@ import android.hardware.biometrics.face.Feature;
 import android.hardware.biometrics.face.IFace;
 import android.hardware.biometrics.face.ISession;
 import android.hardware.face.Face;
+import android.hardware.face.FaceDataFrame;
+import android.hardware.face.FaceEnrollFrame;
 import android.hardware.face.FaceManager;
 import android.os.IBinder;
 import android.os.NativeHandle;
@@ -110,6 +112,17 @@ public class FaceEnrollClient extends EnrollClient<ISession> {
         onAcquiredInternal(acquireInfo, vendorCode, shouldSend);
     }
 
+    /**
+     * Called each time a new frame is received during face enrollment.
+     *
+     * @param frame Information about the current frame.
+     */
+    public void onEnrollmentFrame(@NonNull FaceEnrollFrame frame) {
+        // TODO(b/178414967): Send additional frame data to the client callback.
+        final FaceDataFrame data = frame.getData();
+        onAcquired(data.getAcquiredInfo(), data.getVendorCode());
+    }
+
     @Override
     protected void startHalOperation() {
         final ArrayList<Byte> token = new ArrayList<>();

services/core/java/com/android/server/biometrics/sensors/face/aidl/Sensor.java

@@ -45,7 +45,6 @@ import com.android.server.biometrics.SensorServiceStateProto;
 import com.android.server.biometrics.SensorStateProto;
 import com.android.server.biometrics.UserStateProto;
 import com.android.server.biometrics.Utils;
-import com.android.server.biometrics.sensors.AcquisitionClient;
 import com.android.server.biometrics.sensors.AuthenticationConsumer;
 import com.android.server.biometrics.sensors.BaseClientMonitor;
 import com.android.server.biometrics.sensors.BiometricScheduler;
@@ -170,33 +169,39 @@ public class Sensor implements IBinder.DeathRecipient {
 
         @Override
         public void onAuthenticationFrame(AuthenticationFrame frame) {
-            // TODO(b/174619156): propagate the frame to an AuthenticationClient
             mHandler.post(() -> {
                 final BaseClientMonitor client = mScheduler.getCurrentClient();
-                if (!(client instanceof AcquisitionClient)) {
-                    Slog.e(mTag, "onAcquired for non-acquisition client: "
+                if (!(client instanceof FaceAuthenticationClient)) {
+                    Slog.e(mTag, "onAuthenticationFrame for incompatible client: "
+                            + Utils.getClientName(client));
+                    return;
+
+                }
+                if (frame == null) {
+                    Slog.e(mTag, "Received null authentication frame for client: "
                             + Utils.getClientName(client));
                     return;
                 }
-
-                final AcquisitionClient<?> acquisitionClient = (AcquisitionClient<?>) client;
-                acquisitionClient.onAcquired(frame.data.acquiredInfo, frame.data.vendorCode);
+                ((FaceAuthenticationClient) client).onAuthenticationFrame(
+                        AidlConversionUtils.convert(frame));
             });
         }
 
         @Override
         public void onEnrollmentFrame(EnrollmentFrame frame) {
-            // TODO(b/174619156): propagate the frame to an EnrollmentClient
             mHandler.post(() -> {
                 final BaseClientMonitor client = mScheduler.getCurrentClient();
-                if (!(client instanceof AcquisitionClient)) {
-                    Slog.e(mTag, "onAcquired for non-acquisition client: "
+                if (!(client instanceof FaceEnrollClient)) {
+                    Slog.e(mTag, "onEnrollmentFrame for incompatible client: "
                             + Utils.getClientName(client));
                     return;
                 }
-
-                final AcquisitionClient<?> acquisitionClient = (AcquisitionClient<?>) client;
-                acquisitionClient.onAcquired(frame.data.acquiredInfo, frame.data.vendorCode);
+                if (frame == null) {
+                    Slog.e(mTag, "Received null enrollment frame for client: "
+                            + Utils.getClientName(client));
+                    return;
+                }
+                ((FaceEnrollClient) client).onEnrollmentFrame(AidlConversionUtils.convert(frame));
             });
         }