Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Allow disk read for VisualQueryDetectionService" into udc-qpr-dev

Browse Source
This commit is contained in:
Charles Chen
2023-08-17 21:19:01 +00:00
committed by Android (Google) Code Review
parent 670ecab079 c3533b1c4f
commit 50852a0113
10 changed files with 146 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/java/android/service/voice/AlwaysOnHotwordDetector.java
View File

@@ -67,6 +67,7 @@ import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import com.android.internal.app.IVoiceInteractionManagerService;
import com.android.internal.app.IVoiceInteractionSoundTriggerSession;
import com.android.internal.infra.AndroidFuture;
import java.io.PrintWriter;
import java.lang.annotation.Retention;
@@ -1702,6 +1703,11 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {
Slog.i(TAG, "onProcessRestarted");
mHandler.sendEmptyMessage(MSG_PROCESS_RESTARTED);
}
@Override
public void onOpenFile(String filename, AndroidFuture future) throws RemoteException {
throw new UnsupportedOperationException("Hotword cannot access files from the disk.");
}
}
void onDetectorRemoteException() {

6
core/java/android/service/voice/HotwordDetectionService.java
View File

@@ -227,6 +227,12 @@ public abstract class HotwordDetectionService extends Service
public void stopDetection() {
HotwordDetectionService.this.onStopDetection();
}
@Override
public void registerRemoteStorageService(IDetectorSessionStorageService
detectorSessionStorageService) {
throw new UnsupportedOperationException("Hotword cannot access files from the disk.");
}
};
@Override

30
core/java/android/service/voice/IDetectorSessionStorageService.aidl Normal file
View File

@@ -0,0 +1,30 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.voice;
import com.android.internal.infra.AndroidFuture;
/**
* @hide
*/
oneway interface IDetectorSessionStorageService {
/**
* Called when a file open request is sent. Only files with the given names under the internal
* app storage, i.e., {@link Context#getFilesDir()} can be opened.
*/
void openFile(in String filename, in AndroidFuture future);
}

7
core/java/android/service/voice/ISandboxedDetectionService.aidl
View File

@@ -24,6 +24,7 @@ import android.os.IRemoteCallback;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.SharedMemory;
import android.service.voice.IDetectorSessionStorageService;
import android.service.voice.IDetectorSessionVisualQueryDetectionCallback;
import android.service.voice.IDspHotwordDetectionCallback;
import android.view.contentcapture.IContentCaptureManager;
@@ -71,4 +72,10 @@ oneway interface ISandboxedDetectionService {
void ping(in IRemoteCallback callback);
void stopDetection();
/**
* Registers the interface stub to talk to the voice interaction service for initialization/
* detection unrelated functionalities.
*/
void registerRemoteStorageService(in IDetectorSessionStorageService detectorSessionStorageService);
}

6
core/java/android/service/voice/SoftwareHotwordDetector.java
View File

@@ -36,6 +36,7 @@ import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import com.android.internal.app.IVoiceInteractionManagerService;
import com.android.internal.infra.AndroidFuture;
import java.io.PrintWriter;
import java.util.concurrent.Executor;
@@ -299,6 +300,11 @@ class SoftwareHotwordDetector extends AbstractDetector {
Binder.withCleanCallingIdentity(() -> mExecutor.execute(
() -> mCallback.onHotwordDetectionServiceRestarted()));
}
@Override
public void onOpenFile(String filename, AndroidFuture future) throws RemoteException {
throw new UnsupportedOperationException("Hotword cannot access files from the disk.");
}
}
/** @hide */

32
core/java/android/service/voice/VisualQueryDetectionService.java
View File

@@ -40,7 +40,12 @@ import android.util.Log;
import android.view.contentcapture.ContentCaptureManager;
import android.view.contentcapture.IContentCaptureManager;
import com.android.internal.infra.AndroidFuture;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.function.IntConsumer;
/**
@@ -86,6 +91,8 @@ public abstract class VisualQueryDetectionService extends Service
private ContentCaptureManager mContentCaptureManager;
@Nullable
private IRecognitionServiceManager mIRecognitionServiceManager;
@Nullable
private IDetectorSessionStorageService mDetectorSessionStorageService;
private final ISandboxedDetectionService mInterface = new ISandboxedDetectionService.Stub() {
@@ -154,6 +161,12 @@ public abstract class VisualQueryDetectionService extends Service
public void updateRecognitionServiceManager(IRecognitionServiceManager manager) {
mIRecognitionServiceManager = manager;
}
@Override
public void registerRemoteStorageService(IDetectorSessionStorageService
detectorSessionStorageService) {
mDetectorSessionStorageService = detectorSessionStorageService;
}
};
@Override
@@ -323,4 +336,23 @@ public abstract class VisualQueryDetectionService extends Service
}
}
/**
* Overrides {@link Context#openFileInput} to read files with the given file names under the
* internal app storage of the {@link VoiceInteractionService}, i.e., only files stored in
* {@link Context#getFilesDir()} can be opened.
*/
@Override
public @Nullable FileInputStream openFileInput(@NonNull String filename) throws
FileNotFoundException {
try {
AndroidFuture<ParcelFileDescriptor> future = new AndroidFuture<>();
mDetectorSessionStorageService.openFile(filename, future);
ParcelFileDescriptor pfd = future.get();
return new FileInputStream(pfd.getFileDescriptor());
} catch (RemoteException | ExecutionException | InterruptedException e) {
Log.w(TAG, "Cannot open file due to remote service failure");
throw new FileNotFoundException(e.getMessage());
}
}
}

33
core/java/android/service/voice/VisualQueryDetector.java
View File

@@ -25,6 +25,7 @@ import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.content.Context;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioFormat;
import android.os.Binder;
@@ -37,7 +38,10 @@ import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import com.android.internal.app.IVoiceInteractionManagerService;
import com.android.internal.infra.AndroidFuture;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.util.concurrent.Executor;
import java.util.function.Consumer;
@@ -58,17 +62,18 @@ public class VisualQueryDetector {
private final Callback mCallback;
private final Executor mExecutor;
private final Context mContext;
private final IVoiceInteractionManagerService mManagerService;
private final VisualQueryDetectorInitializationDelegate mInitializationDelegate;
VisualQueryDetector(
IVoiceInteractionManagerService managerService,
@NonNull @CallbackExecutor Executor executor,
Callback callback) {
@NonNull @CallbackExecutor Executor executor, Callback callback, Context context) {
mManagerService = managerService;
mCallback = callback;
mExecutor = executor;
mInitializationDelegate = new VisualQueryDetectorInitializationDelegate();
mContext = context;
}
/**
@@ -245,7 +250,7 @@ public class VisualQueryDetector {
@Override
void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
initAndVerifyDetector(options, sharedMemory,
new InitializationStateListener(mExecutor, mCallback),
new InitializationStateListener(mExecutor, mCallback, mContext),
DETECTOR_TYPE_VISUAL_QUERY_DETECTOR);
}
@@ -330,9 +335,12 @@ public class VisualQueryDetector {
private final Executor mExecutor;
private final Callback mCallback;
InitializationStateListener(Executor executor, Callback callback) {
private final Context mContext;
InitializationStateListener(Executor executor, Callback callback, Context context) {
this.mExecutor = executor;
this.mCallback = callback;
this.mContext = context;
}
@Override
@@ -426,5 +434,22 @@ public class VisualQueryDetector {
!TextUtils.isEmpty(errorMessage) ? errorMessage : "Error data is null");
}));
}
@Override
public void onOpenFile(String filename, AndroidFuture future) throws RemoteException {
Slog.v(TAG, "BinderCallback#onOpenFile " + filename);
Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> {
Slog.v(TAG, "onOpenFile: " + filename);
File f = new File(mContext.getFilesDir(), filename);
ParcelFileDescriptor pfd = null;
try {
Slog.d(TAG, "opened a file with ParcelFileDescriptor.");
pfd = ParcelFileDescriptor.open(f, ParcelFileDescriptor.MODE_READ_ONLY);
} catch (FileNotFoundException e) {
Slog.e(TAG, "Cannot open file. No ParcelFileDescriptor returned.");
} finally {
future.complete(pfd);
}
}));
}
}
}

2
core/java/android/service/voice/VoiceInteractionService.java
View File

@@ -965,7 +965,7 @@ public class VoiceInteractionService extends Service {
}
VisualQueryDetector visualQueryDetector =
new VisualQueryDetector(mSystemService, executor, callback);
new VisualQueryDetector(mSystemService, executor, callback, this);
HotwordDetector visualQueryDetectorInitializationDelegate =
visualQueryDetector.getInitializationDelegate();
mActiveDetectors.add(visualQueryDetectorInitializationDelegate);

6
core/java/com/android/internal/app/IHotwordRecognitionStatusCallback.aidl
View File

@@ -22,6 +22,7 @@ import android.service.voice.HotwordDetectionServiceFailure;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.SoundTriggerFailure;
import android.service.voice.VisualQueryDetectionServiceFailure;
import com.android.internal.infra.AndroidFuture;
/**
* @hide
@@ -113,4 +114,9 @@ oneway interface IHotwordRecognitionStatusCallback {
/** Called when the hotword detection process is restarted */
void onProcessRestarted();
/**
* Called when a file open request is sent.
*/
void onOpenFile(in String filename, in AndroidFuture future);
}

23
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -54,6 +54,7 @@ import android.provider.DeviceConfig;
import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetectionServiceFailure;
import android.service.voice.HotwordDetector;
import android.service.voice.IDetectorSessionStorageService;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.ISandboxedDetectionService;
import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
@@ -69,6 +70,7 @@ import android.view.contentcapture.IContentCaptureManager;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import com.android.internal.app.IVisualQueryDetectionAttentionListener;
import com.android.internal.infra.AndroidFuture;
import com.android.internal.infra.ServiceConnector;
import com.android.server.LocalServices;
import com.android.server.pm.permission.PermissionManagerServiceInternal;
@@ -157,6 +159,8 @@ final class HotwordDetectionConnection {
@NonNull private ServiceConnection mRemoteVisualQueryDetectionService;
@GuardedBy("mLock")
@Nullable private IBinder mAudioFlinger;
@Nullable private IHotwordRecognitionStatusCallback mHotwordRecognitionCallback;
@GuardedBy("mLock")
private boolean mDebugHotwordLogging = false;
@@ -694,6 +698,7 @@ final class HotwordDetectionConnection {
updateContentCaptureManager(connection);
updateSpeechService(connection);
updateServiceIdentity(connection);
updateStorageService(connection);
return connection;
}
}
@@ -910,6 +915,7 @@ final class HotwordDetectionConnection {
mVoiceInteractionServiceUid, mVoiceInteractorIdentity,
mScheduledExecutorService, mDebugHotwordLogging, mRemoteExceptionListener);
}
mHotwordRecognitionCallback = callback;
mDetectorSessions.put(detectorType, session);
session.initialize(options, sharedMemory);
}
@@ -1035,6 +1041,23 @@ final class HotwordDetectionConnection {
}));
}
private void updateStorageService(ServiceConnection connection) {
connection.run(service -> {
service.registerRemoteStorageService(new IDetectorSessionStorageService.Stub() {
@Override
public void openFile(String filename, AndroidFuture future)
throws RemoteException {
Slog.v(TAG, "BinderCallback#onFileOpen");
try {
mHotwordRecognitionCallback.onOpenFile(filename, future);
} catch (RemoteException e) {
e.rethrowFromSystemServer();
}
}
});
});
}
private void addServiceUidForAudioPolicy(int uid) {
mScheduledExecutorService.execute(() -> {
AudioManagerInternal audioManager =