From 18ccd6c9258ed5d83585b21bbc96ea684d6f3865 Mon Sep 17 00:00:00 2001 From: Curtis Belmonte Date: Fri, 5 Feb 2021 16:01:44 -0800 Subject: [PATCH] Pass face AIDL frame data through to FaceManager Adds plumbing to send acquired frame info from the onAuthenticationFrame and onEnrollmentFrame AIDL callbacks to FaceManager. Currently, FaceManager drops most of this data and only passes the acquireInfo and vendorCode fields to appropriate authentication/enrollment callbacks. Test: atest CtsBiometricsTestCases Bug: 178414967 Change-Id: If997ddd13ce1d9bda2341df7c1f8d6347571f4e0 --- .../face/FaceAuthenticationFrame.aidl | 21 +++++ .../android/hardware/face/FaceDataFrame.java | 16 ++++ .../hardware/face/FaceEnrollFrame.aidl | 21 +++++ .../hardware/face/FaceEnrollStage.java | 6 ++ .../android/hardware/face/FaceManager.java | 87 +++++++++++++++---- .../hardware/face/IFaceServiceReceiver.aidl | 4 + .../ClientMonitorCallbackConverter.java | 32 +++++++ .../face/aidl/BiometricTestSessionImpl.java | 16 +++- .../face/aidl/FaceAuthenticationClient.java | 32 ++++--- .../sensors/face/aidl/FaceEnrollClient.java | 31 ++++--- .../face/hidl/BiometricTestSessionImpl.java | 12 +++ 11 files changed, 238 insertions(+), 40 deletions(-) create mode 100644 core/java/android/hardware/face/FaceAuthenticationFrame.aidl create mode 100644 core/java/android/hardware/face/FaceEnrollFrame.aidl diff --git a/core/java/android/hardware/face/FaceAuthenticationFrame.aidl b/core/java/android/hardware/face/FaceAuthenticationFrame.aidl new file mode 100644 index 0000000000000..4dc41f1493282 --- /dev/null +++ b/core/java/android/hardware/face/FaceAuthenticationFrame.aidl @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.face; + +/** + * @hide + */ +parcelable FaceAuthenticationFrame; diff --git a/core/java/android/hardware/face/FaceDataFrame.java b/core/java/android/hardware/face/FaceDataFrame.java index 3a0e09b70b50e..092359c991734 100644 --- a/core/java/android/hardware/face/FaceDataFrame.java +++ b/core/java/android/hardware/face/FaceDataFrame.java @@ -62,6 +62,22 @@ public final class FaceDataFrame implements Parcelable { mIsCancellable = isCancellable; } + /** + * A container for data common to {@link FaceAuthenticationFrame} and {@link FaceEnrollFrame}. + * + * @param acquiredInfo An integer corresponding to a known acquired message. + * @param vendorCode An integer representing a custom vendor-specific message. Ignored unless + * {@code acquiredInfo} is {@code FACE_ACQUIRED_VENDOR}. + */ + public FaceDataFrame(int acquiredInfo, int vendorCode) { + mAcquiredInfo = acquiredInfo; + mVendorCode = vendorCode; + mPan = 0f; + mTilt = 0f; + mDistance = 0f; + mIsCancellable = false; + } + /** * @return An integer corresponding to a known acquired message. * diff --git a/core/java/android/hardware/face/FaceEnrollFrame.aidl b/core/java/android/hardware/face/FaceEnrollFrame.aidl new file mode 100644 index 0000000000000..b8546812beeb6 --- /dev/null +++ b/core/java/android/hardware/face/FaceEnrollFrame.aidl @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.face; + +/** + * @hide + */ +parcelable FaceEnrollFrame; diff --git a/core/java/android/hardware/face/FaceEnrollStage.java b/core/java/android/hardware/face/FaceEnrollStage.java index 03dba551aae2e..de717fbe46b44 100644 --- a/core/java/android/hardware/face/FaceEnrollStage.java +++ b/core/java/android/hardware/face/FaceEnrollStage.java @@ -28,6 +28,7 @@ import java.lang.annotation.RetentionPolicy; */ @Retention(RetentionPolicy.SOURCE) @IntDef({ + FaceEnrollStage.UNKNOWN, FaceEnrollStage.FIRST_FRAME_RECEIVED, FaceEnrollStage.WAITING_FOR_CENTERING, FaceEnrollStage.HOLD_STILL_IN_CENTER, @@ -36,6 +37,11 @@ import java.lang.annotation.RetentionPolicy; FaceEnrollStage.ENROLLMENT_FINISHED }) public @interface FaceEnrollStage { + /** + * The current enrollment stage is not known. + */ + int UNKNOWN = -1; + /** * Enrollment has just begun. No action is needed from the user yet. */ diff --git a/core/java/android/hardware/face/FaceManager.java b/core/java/android/hardware/face/FaceManager.java index 588bc01d7d420..f3da6a9d4a030 100644 --- a/core/java/android/hardware/face/FaceManager.java +++ b/core/java/android/hardware/face/FaceManager.java @@ -71,17 +71,19 @@ public class FaceManager implements BiometricAuthenticator, BiometricFaceConstan private static final int MSG_FACE_DETECTED = 109; private static final int MSG_CHALLENGE_INTERRUPTED = 110; private static final int MSG_CHALLENGE_INTERRUPT_FINISHED = 111; + private static final int MSG_AUTHENTICATION_FRAME = 112; + private static final int MSG_ENROLLMENT_FRAME = 113; private final IFaceService mService; private final Context mContext; private IBinder mToken = new Binder(); - private AuthenticationCallback mAuthenticationCallback; - private FaceDetectionCallback mFaceDetectionCallback; - private EnrollmentCallback mEnrollmentCallback; - private RemovalCallback mRemovalCallback; - private SetFeatureCallback mSetFeatureCallback; - private GetFeatureCallback mGetFeatureCallback; - private GenerateChallengeCallback mGenerateChallengeCallback; + @Nullable private AuthenticationCallback mAuthenticationCallback; + @Nullable private FaceDetectionCallback mFaceDetectionCallback; + @Nullable private EnrollmentCallback mEnrollmentCallback; + @Nullable private RemovalCallback mRemovalCallback; + @Nullable private SetFeatureCallback mSetFeatureCallback; + @Nullable private GetFeatureCallback mGetFeatureCallback; + @Nullable private GenerateChallengeCallback mGenerateChallengeCallback; private CryptoObject mCryptoObject; private Face mRemovalFace; private Handler mHandler; @@ -154,6 +156,16 @@ public class FaceManager implements BiometricAuthenticator, BiometricFaceConstan public void onChallengeInterruptFinished(int sensorId) { mHandler.obtainMessage(MSG_CHALLENGE_INTERRUPT_FINISHED, sensorId).sendToTarget(); } + + @Override + public void onAuthenticationFrame(FaceAuthenticationFrame frame) { + mHandler.obtainMessage(MSG_AUTHENTICATION_FRAME, frame).sendToTarget(); + } + + @Override + public void onEnrollmentFrame(FaceEnrollFrame frame) { + mHandler.obtainMessage(MSG_ENROLLMENT_FRAME, frame).sendToTarget(); + } }; /** @@ -1248,6 +1260,12 @@ public class FaceManager implements BiometricAuthenticator, BiometricFaceConstan case MSG_CHALLENGE_INTERRUPT_FINISHED: sendChallengeInterruptFinished((int) msg.obj /* sensorId */); break; + case MSG_AUTHENTICATION_FRAME: + sendAuthenticationFrame((FaceAuthenticationFrame) msg.obj /* frame */); + break; + case MSG_ENROLLMENT_FRAME: + sendEnrollmentFrame((FaceEnrollFrame) msg.obj /* frame */); + break; default: Slog.w(TAG, "Unknown message: " + msg.what); } @@ -1349,15 +1367,52 @@ public class FaceManager implements BiometricAuthenticator, BiometricFaceConstan private void sendAcquiredResult(int acquireInfo, int vendorCode) { if (mAuthenticationCallback != null) { - mAuthenticationCallback.onAuthenticationAcquired(acquireInfo); - } - final String msg = getAcquiredString(mContext, acquireInfo, vendorCode); - final int clientInfo = acquireInfo == FACE_ACQUIRED_VENDOR - ? (vendorCode + FACE_ACQUIRED_VENDOR_BASE) : acquireInfo; - if (mEnrollmentCallback != null) { - mEnrollmentCallback.onEnrollmentHelp(clientInfo, msg); - } else if (mAuthenticationCallback != null && msg != null) { - mAuthenticationCallback.onAuthenticationHelp(clientInfo, msg); + final FaceAuthenticationFrame frame = new FaceAuthenticationFrame( + new FaceDataFrame(acquireInfo, vendorCode)); + sendAuthenticationFrame(frame); + } else if (mEnrollmentCallback != null) { + final FaceEnrollFrame frame = new FaceEnrollFrame( + null /* cell */, + FaceEnrollStage.UNKNOWN, + new FaceDataFrame(acquireInfo, vendorCode)); + sendEnrollmentFrame(frame); } } + + private void sendAuthenticationFrame(@Nullable FaceAuthenticationFrame frame) { + if (frame == null) { + Slog.w(TAG, "Received null authentication frame"); + } else if (mAuthenticationCallback != null) { + // TODO(b/178414967): Send additional frame data to callback + final int acquireInfo = frame.getData().getAcquiredInfo(); + final int vendorCode = frame.getData().getVendorCode(); + final int helpCode = getHelpCode(acquireInfo, vendorCode); + final String helpMessage = getAcquiredString(mContext, acquireInfo, vendorCode); + mAuthenticationCallback.onAuthenticationAcquired(acquireInfo); + + // Ensure that only non-null help messages are sent. + if (helpMessage != null) { + mAuthenticationCallback.onAuthenticationHelp(helpCode, helpMessage); + } + } + } + + private void sendEnrollmentFrame(@Nullable FaceEnrollFrame frame) { + if (frame == null) { + Slog.w(TAG, "Received null enrollment frame"); + } else if (mEnrollmentCallback != null) { + // TODO(b/178414967): Send additional frame data to callback + final int acquireInfo = frame.getData().getAcquiredInfo(); + final int vendorCode = frame.getData().getVendorCode(); + final int helpCode = getHelpCode(acquireInfo, vendorCode); + final String helpMessage = getAcquiredString(mContext, acquireInfo, vendorCode); + mEnrollmentCallback.onEnrollmentHelp(helpCode, helpMessage); + } + } + + private static int getHelpCode(int acquireInfo, int vendorCode) { + return acquireInfo == FACE_ACQUIRED_VENDOR + ? vendorCode + FACE_ACQUIRED_VENDOR_BASE + : acquireInfo; + } } diff --git a/core/java/android/hardware/face/IFaceServiceReceiver.aidl b/core/java/android/hardware/face/IFaceServiceReceiver.aidl index bd4d3a0b7017f..2ef1430a2f997 100644 --- a/core/java/android/hardware/face/IFaceServiceReceiver.aidl +++ b/core/java/android/hardware/face/IFaceServiceReceiver.aidl @@ -16,6 +16,8 @@ package android.hardware.face; import android.hardware.face.Face; +import android.hardware.face.FaceAuthenticationFrame; +import android.hardware.face.FaceEnrollFrame; /** * Communication channel from the FaceService back to FaceAuthenticationManager. @@ -34,4 +36,6 @@ oneway interface IFaceServiceReceiver { void onChallengeGenerated(int sensorId, long challenge); void onChallengeInterrupted(int sensorId); void onChallengeInterruptFinished(int sensorId); + void onAuthenticationFrame(in FaceAuthenticationFrame frame); + void onEnrollmentFrame(in FaceEnrollFrame frame); } diff --git a/services/core/java/com/android/server/biometrics/sensors/ClientMonitorCallbackConverter.java b/services/core/java/com/android/server/biometrics/sensors/ClientMonitorCallbackConverter.java index da76af800d3dc..25b7add0a7d8d 100644 --- a/services/core/java/com/android/server/biometrics/sensors/ClientMonitorCallbackConverter.java +++ b/services/core/java/com/android/server/biometrics/sensors/ClientMonitorCallbackConverter.java @@ -16,9 +16,12 @@ package com.android.server.biometrics.sensors; +import android.annotation.NonNull; import android.hardware.biometrics.BiometricAuthenticator; import android.hardware.biometrics.IBiometricSensorReceiver; import android.hardware.face.Face; +import android.hardware.face.FaceAuthenticationFrame; +import android.hardware.face.FaceEnrollFrame; import android.hardware.face.IFaceServiceReceiver; import android.hardware.fingerprint.Fingerprint; import android.hardware.fingerprint.IFingerprintServiceReceiver; @@ -174,4 +177,33 @@ public class ClientMonitorCallbackConverter { mFingerprintServiceReceiver.onUdfpsPointerUp(sensorId); } } + + // Face-specific callbacks for FaceManager only + + /** + * Called each time a new frame is received during face authentication. + * + * @param frame Information about the current frame. + * + * @throws RemoteException If the binder call to {@link IFaceServiceReceiver} fails. + */ + public void onAuthenticationFrame(@NonNull FaceAuthenticationFrame frame) + throws RemoteException { + if (mFaceServiceReceiver != null) { + mFaceServiceReceiver.onAuthenticationFrame(frame); + } + } + + /** + * Called each time a new frame is received during face enrollment. + * + * @param frame Information about the current frame. + * + * @throws RemoteException If the binder call to {@link IFaceServiceReceiver} fails. + */ + public void onEnrollmentFrame(@NonNull FaceEnrollFrame frame) throws RemoteException { + if (mFaceServiceReceiver != null) { + mFaceServiceReceiver.onEnrollmentFrame(frame); + } + } } diff --git a/services/core/java/com/android/server/biometrics/sensors/face/aidl/BiometricTestSessionImpl.java b/services/core/java/com/android/server/biometrics/sensors/face/aidl/BiometricTestSessionImpl.java index d2673d2969c91..897ebd719da42 100644 --- a/services/core/java/com/android/server/biometrics/sensors/face/aidl/BiometricTestSessionImpl.java +++ b/services/core/java/com/android/server/biometrics/sensors/face/aidl/BiometricTestSessionImpl.java @@ -24,6 +24,8 @@ import android.hardware.biometrics.ITestSession; import android.hardware.biometrics.face.AuthenticationFrame; import android.hardware.biometrics.face.BaseFrame; import android.hardware.face.Face; +import android.hardware.face.FaceAuthenticationFrame; +import android.hardware.face.FaceEnrollFrame; import android.hardware.face.IFaceServiceReceiver; import android.os.Binder; import android.util.Slog; @@ -117,6 +119,16 @@ public class BiometricTestSessionImpl extends ITestSession.Stub { public void onChallengeInterruptFinished(int sensorId) { } + + @Override + public void onAuthenticationFrame(FaceAuthenticationFrame frame) { + + } + + @Override + public void onEnrollmentFrame(FaceEnrollFrame frame) { + + } }; BiometricTestSessionImpl(@NonNull Context context, int sensorId, @@ -183,7 +195,7 @@ public class BiometricTestSessionImpl extends ITestSession.Stub { mSensor.getSessionForUser(userId).mHalSessionCallback.onAuthenticationFailed(); } - // TODO(b/174619156): replace with notifyAuthenticationFrame and notifyEnrollmentFrame. + // TODO(b/178414967): replace with notifyAuthenticationFrame and notifyEnrollmentFrame. @Override public void notifyAcquired(int userId, int acquireInfo) { Utils.checkPermission(mContext, TEST_BIOMETRIC); @@ -194,7 +206,7 @@ public class BiometricTestSessionImpl extends ITestSession.Stub { AuthenticationFrame authenticationFrame = new AuthenticationFrame(); authenticationFrame.data = data; - // TODO(b/174619156): Currently onAuthenticationFrame and onEnrollmentFrame are the same. + // TODO(b/178414967): Currently onAuthenticationFrame and onEnrollmentFrame are the same. // This will need to call the correct callback once the onAcquired callback is removed. mSensor.getSessionForUser(userId).mHalSessionCallback.onAuthenticationFrame( authenticationFrame); diff --git a/services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceAuthenticationClient.java b/services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceAuthenticationClient.java index 30577667e5e44..8f554028ebfd6 100644 --- a/services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceAuthenticationClient.java +++ b/services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceAuthenticationClient.java @@ -29,7 +29,6 @@ import android.hardware.biometrics.common.ICancellationSignal; import android.hardware.biometrics.face.IFace; import android.hardware.biometrics.face.ISession; import android.hardware.face.FaceAuthenticationFrame; -import android.hardware.face.FaceDataFrame; import android.hardware.face.FaceManager; import android.os.IBinder; import android.os.RemoteException; @@ -179,19 +178,16 @@ class FaceAuthenticationClient extends AuthenticationClient implements return isBiometricPrompt() ? mBiometricPromptIgnoreListVendor : mKeyguardIgnoreListVendor; } - private boolean shouldSend(int acquireInfo, int vendorCode) { - if (acquireInfo == FaceManager.FACE_ACQUIRED_VENDOR) { - return !Utils.listContains(getAcquireVendorIgnorelist(), vendorCode); - } else { - return !Utils.listContains(getAcquireIgnorelist(), acquireInfo); - } + private boolean shouldSendAcquiredMessage(int acquireInfo, int vendorCode) { + return acquireInfo == FaceManager.FACE_ACQUIRED_VENDOR + ? !Utils.listContains(getAcquireVendorIgnorelist(), vendorCode) + : !Utils.listContains(getAcquireIgnorelist(), acquireInfo); } @Override public void onAcquired(int acquireInfo, int vendorCode) { mLastAcquire = acquireInfo; - - final boolean shouldSend = shouldSend(acquireInfo, vendorCode); + final boolean shouldSend = shouldSendAcquiredMessage(acquireInfo, vendorCode); onAcquiredInternal(acquireInfo, vendorCode, shouldSend); } @@ -201,9 +197,21 @@ class FaceAuthenticationClient extends AuthenticationClient implements * @param frame Information about the current frame. */ public void onAuthenticationFrame(@NonNull FaceAuthenticationFrame frame) { - // TODO(b/178414967): Send additional frame data to the client callback. - final FaceDataFrame data = frame.getData(); - onAcquired(data.getAcquiredInfo(), data.getVendorCode()); + // Log acquisition but don't send it to the client yet, since that's handled below. + final int acquireInfo = frame.getData().getAcquiredInfo(); + final int vendorCode = frame.getData().getVendorCode(); + mLastAcquire = acquireInfo; + onAcquiredInternal(acquireInfo, vendorCode, false /* shouldSend */); + + final boolean shouldSend = shouldSendAcquiredMessage(acquireInfo, vendorCode); + if (shouldSend && getListener() != null) { + try { + getListener().onAuthenticationFrame(frame); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to send authentication frame", e); + mCallback.onClientFinished(this, false /* success */); + } + } } @Override public void onLockoutTimed(long durationMillis) { diff --git a/services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceEnrollClient.java b/services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceEnrollClient.java index da657b96afd54..898d81b0c8c46 100644 --- a/services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceEnrollClient.java +++ b/services/core/java/com/android/server/biometrics/sensors/face/aidl/FaceEnrollClient.java @@ -27,7 +27,6 @@ import android.hardware.biometrics.face.Feature; import android.hardware.biometrics.face.IFace; import android.hardware.biometrics.face.ISession; import android.hardware.face.Face; -import android.hardware.face.FaceDataFrame; import android.hardware.face.FaceEnrollFrame; import android.hardware.face.FaceManager; import android.os.IBinder; @@ -101,14 +100,15 @@ public class FaceEnrollClient extends EnrollClient { getTargetUserId()).size() >= mMaxTemplatesPerUser; } + private boolean shouldSendAcquiredMessage(int acquireInfo, int vendorCode) { + return acquireInfo == FaceManager.FACE_ACQUIRED_VENDOR + ? !Utils.listContains(mEnrollIgnoreListVendor, vendorCode) + : !Utils.listContains(mEnrollIgnoreList, acquireInfo); + } + @Override public void onAcquired(int acquireInfo, int vendorCode) { - final boolean shouldSend; - if (acquireInfo == FaceManager.FACE_ACQUIRED_VENDOR) { - shouldSend = !Utils.listContains(mEnrollIgnoreListVendor, vendorCode); - } else { - shouldSend = !Utils.listContains(mEnrollIgnoreList, acquireInfo); - } + final boolean shouldSend = shouldSendAcquiredMessage(acquireInfo, vendorCode); onAcquiredInternal(acquireInfo, vendorCode, shouldSend); } @@ -118,9 +118,20 @@ public class FaceEnrollClient extends EnrollClient { * @param frame Information about the current frame. */ public void onEnrollmentFrame(@NonNull FaceEnrollFrame frame) { - // TODO(b/178414967): Send additional frame data to the client callback. - final FaceDataFrame data = frame.getData(); - onAcquired(data.getAcquiredInfo(), data.getVendorCode()); + // Log acquisition but don't send it to the client yet, since that's handled below. + final int acquireInfo = frame.getData().getAcquiredInfo(); + final int vendorCode = frame.getData().getVendorCode(); + onAcquiredInternal(acquireInfo, vendorCode, false /* shouldSend */); + + final boolean shouldSend = shouldSendAcquiredMessage(acquireInfo, vendorCode); + if (shouldSend && getListener() != null) { + try { + getListener().onEnrollmentFrame(frame); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to send enrollment frame", e); + mCallback.onClientFinished(this, false /* success */); + } + } } @Override diff --git a/services/core/java/com/android/server/biometrics/sensors/face/hidl/BiometricTestSessionImpl.java b/services/core/java/com/android/server/biometrics/sensors/face/hidl/BiometricTestSessionImpl.java index 4142a52c9253b..d519d60881c0a 100644 --- a/services/core/java/com/android/server/biometrics/sensors/face/hidl/BiometricTestSessionImpl.java +++ b/services/core/java/com/android/server/biometrics/sensors/face/hidl/BiometricTestSessionImpl.java @@ -22,6 +22,8 @@ import android.annotation.NonNull; import android.content.Context; import android.hardware.biometrics.ITestSession; import android.hardware.face.Face; +import android.hardware.face.FaceAuthenticationFrame; +import android.hardware.face.FaceEnrollFrame; import android.hardware.face.IFaceServiceReceiver; import android.os.Binder; import android.util.Slog; @@ -106,6 +108,16 @@ public class BiometricTestSessionImpl extends ITestSession.Stub { public void onChallengeInterruptFinished(int sensorId) { } + + @Override + public void onAuthenticationFrame(FaceAuthenticationFrame frame) { + + } + + @Override + public void onEnrollmentFrame(FaceEnrollFrame frame) { + + } }; BiometricTestSessionImpl(@NonNull Context context, int sensorId, @NonNull Face10 face10,