From d58b04ae471188d66c21edaaebfe7075981655b7 Mon Sep 17 00:00:00 2001 From: Ytai Ben-Tsvi Date: Fri, 22 Nov 2019 12:33:09 -0800 Subject: [PATCH 1/5] Add audio.common types AIDL definition This is a subset of the types defined in hardware/interfaces/audio/common/2.0/types.hal. Specifically, the subset that sound trigger depends on. These files were auto-generated using the hidl2aidl tool, and later manually edited to workaround some missing features in the conversion tool or AIDL (mostly around enum support). Change-Id: I1c0ffe8a2ebaf09eb4516beaaf4b6fc80985bfc9 Bug: 142070343 --- media/Android.bp | 24 ++ .../media/audio/common/AudioChannelMask.aidl | 217 ++++++++++++++++++ .../media/audio/common/AudioConfig.aidl | 36 +++ .../media/audio/common/AudioFormat.aidl | 170 ++++++++++++++ .../media/audio/common/AudioOffloadInfo.aidl | 44 ++++ .../media/audio/common/AudioStreamType.aidl | 44 ++++ .../media/audio/common/AudioUsage.aidl | 40 ++++ 7 files changed, 575 insertions(+) create mode 100644 media/java/android/media/audio/common/AudioChannelMask.aidl create mode 100644 media/java/android/media/audio/common/AudioConfig.aidl create mode 100644 media/java/android/media/audio/common/AudioFormat.aidl create mode 100644 media/java/android/media/audio/common/AudioOffloadInfo.aidl create mode 100644 media/java/android/media/audio/common/AudioStreamType.aidl create mode 100644 media/java/android/media/audio/common/AudioUsage.aidl diff --git a/media/Android.bp b/media/Android.bp index 1912930f2081b..b77c701c78e88 100644 --- a/media/Android.bp +++ b/media/Android.bp @@ -109,3 +109,27 @@ java_library { srcs: [":framework-media-annotation-srcs"], installable: false, } + +aidl_interface { + name: "audio_common-aidl", + local_include_dir: "java", + srcs: [ + "java/android/media/audio/common/AudioChannelMask.aidl", + "java/android/media/audio/common/AudioConfig.aidl", + "java/android/media/audio/common/AudioFormat.aidl", + "java/android/media/audio/common/AudioOffloadInfo.aidl", + "java/android/media/audio/common/AudioStreamType.aidl", + "java/android/media/audio/common/AudioUsage.aidl", + ], + backend: + { + cpp: { + enabled: true, + }, + java: { + // Already generated as part of the entire media java library. + enabled: false, + }, + }, +} + diff --git a/media/java/android/media/audio/common/AudioChannelMask.aidl b/media/java/android/media/audio/common/AudioChannelMask.aidl new file mode 100644 index 0000000000000..b9b08e6921bce --- /dev/null +++ b/media/java/android/media/audio/common/AudioChannelMask.aidl @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + // This file has been semi-automatically generated using hidl2aidl from its counterpart in + // hardware/interfaces/audio/common/5.0/types.hal + +package android.media.audio.common; + +/** + * A channel mask per se only defines the presence or absence of a channel, not + * the order. + * + * The channel order convention is that channels are interleaved in order from + * least significant channel mask bit to most significant channel mask bit, + * with unused bits skipped. For example for stereo, LEFT would be first, + * followed by RIGHT. + * Any exceptions to this convention are noted at the appropriate API. + * + * AudioChannelMask is an opaque type and its internal layout should not be + * assumed as it may change in the future. Instead, always use functions + * to examine it. + * + * These are the current representations: + * + * REPRESENTATION_POSITION + * is a channel mask representation for position assignment. Each low-order + * bit corresponds to the spatial position of a transducer (output), or + * interpretation of channel (input). The user of a channel mask needs to + * know the context of whether it is for output or input. The constants + * OUT_* or IN_* apply to the bits portion. It is not permitted for no bits + * to be set. + * + * REPRESENTATION_INDEX + * is a channel mask representation for index assignment. Each low-order + * bit corresponds to a selected channel. There is no platform + * interpretation of the various bits. There is no concept of output or + * input. It is not permitted for no bits to be set. + * + * All other representations are reserved for future use. + * + * Warning: current representation distinguishes between input and output, but + * this will not the be case in future revisions of the platform. Wherever there + * is an ambiguity between input and output that is currently resolved by + * checking the channel mask, the implementer should look for ways to fix it + * with additional information outside of the mask. + * + * {@hide} + */ +@Backing(type="int") +enum AudioChannelMask { + /** + * must be 0 for compatibility + */ + REPRESENTATION_POSITION = 0, + /** + * 1 is reserved for future use + */ + REPRESENTATION_INDEX = 2, + /** + * 3 is reserved for future use + * + * + * These can be a complete value of AudioChannelMask + */ + NONE = 0x0, + INVALID = 0xC0000000, + /** + * These can be the bits portion of an AudioChannelMask + * with representation REPRESENTATION_POSITION. + * + * + * output channels + */ + OUT_FRONT_LEFT = 0x1, + OUT_FRONT_RIGHT = 0x2, + OUT_FRONT_CENTER = 0x4, + OUT_LOW_FREQUENCY = 0x8, + OUT_BACK_LEFT = 0x10, + OUT_BACK_RIGHT = 0x20, + OUT_FRONT_LEFT_OF_CENTER = 0x40, + OUT_FRONT_RIGHT_OF_CENTER = 0x80, + OUT_BACK_CENTER = 0x100, + OUT_SIDE_LEFT = 0x200, + OUT_SIDE_RIGHT = 0x400, + OUT_TOP_CENTER = 0x800, + OUT_TOP_FRONT_LEFT = 0x1000, + OUT_TOP_FRONT_CENTER = 0x2000, + OUT_TOP_FRONT_RIGHT = 0x4000, + OUT_TOP_BACK_LEFT = 0x8000, + OUT_TOP_BACK_CENTER = 0x10000, + OUT_TOP_BACK_RIGHT = 0x20000, + OUT_TOP_SIDE_LEFT = 0x40000, + OUT_TOP_SIDE_RIGHT = 0x80000, + /** + * Haptic channel characteristics are specific to a device and + * only used to play device specific resources (eg: ringtones). + * The HAL can freely map A and B to haptic controllers, the + * framework shall not interpret those values and forward them + * from the device audio assets. + */ + OUT_HAPTIC_A = 0x20000000, + OUT_HAPTIC_B = 0x10000000, +// TODO(ytai): Aliases not currently supported in AIDL - can inline the values. +// OUT_MONO = OUT_FRONT_LEFT, +// OUT_STEREO = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT), +// OUT_2POINT1 = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_LOW_FREQUENCY), +// OUT_2POINT0POINT2 = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_TOP_SIDE_LEFT | OUT_TOP_SIDE_RIGHT), +// OUT_2POINT1POINT2 = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_TOP_SIDE_LEFT | OUT_TOP_SIDE_RIGHT | OUT_LOW_FREQUENCY), +// OUT_3POINT0POINT2 = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_FRONT_CENTER | OUT_TOP_SIDE_LEFT | OUT_TOP_SIDE_RIGHT), +// OUT_3POINT1POINT2 = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_FRONT_CENTER | OUT_TOP_SIDE_LEFT | OUT_TOP_SIDE_RIGHT | OUT_LOW_FREQUENCY), +// OUT_QUAD = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_BACK_LEFT | OUT_BACK_RIGHT), +// OUT_QUAD_BACK = OUT_QUAD, +// /** +// * like OUT_QUAD_BACK with *_SIDE_* instead of *_BACK_* +// */ +// OUT_QUAD_SIDE = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_SIDE_LEFT | OUT_SIDE_RIGHT), +// OUT_SURROUND = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_FRONT_CENTER | OUT_BACK_CENTER), +// OUT_PENTA = (OUT_QUAD | OUT_FRONT_CENTER), +// OUT_5POINT1 = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_FRONT_CENTER | OUT_LOW_FREQUENCY | OUT_BACK_LEFT | OUT_BACK_RIGHT), +// OUT_5POINT1_BACK = OUT_5POINT1, +// /** +// * like OUT_5POINT1_BACK with *_SIDE_* instead of *_BACK_* +// */ +// OUT_5POINT1_SIDE = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_FRONT_CENTER | OUT_LOW_FREQUENCY | OUT_SIDE_LEFT | OUT_SIDE_RIGHT), +// OUT_5POINT1POINT2 = (OUT_5POINT1 | OUT_TOP_SIDE_LEFT | OUT_TOP_SIDE_RIGHT), +// OUT_5POINT1POINT4 = (OUT_5POINT1 | OUT_TOP_FRONT_LEFT | OUT_TOP_FRONT_RIGHT | OUT_TOP_BACK_LEFT | OUT_TOP_BACK_RIGHT), +// OUT_6POINT1 = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_FRONT_CENTER | OUT_LOW_FREQUENCY | OUT_BACK_LEFT | OUT_BACK_RIGHT | OUT_BACK_CENTER), +// /** +// * matches the correct AudioFormat.CHANNEL_OUT_7POINT1_SURROUND +// */ +// OUT_7POINT1 = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_FRONT_CENTER | OUT_LOW_FREQUENCY | OUT_BACK_LEFT | OUT_BACK_RIGHT | OUT_SIDE_LEFT | OUT_SIDE_RIGHT), +// OUT_7POINT1POINT2 = (OUT_7POINT1 | OUT_TOP_SIDE_LEFT | OUT_TOP_SIDE_RIGHT), +// OUT_7POINT1POINT4 = (OUT_7POINT1 | OUT_TOP_FRONT_LEFT | OUT_TOP_FRONT_RIGHT | OUT_TOP_BACK_LEFT | OUT_TOP_BACK_RIGHT), +// OUT_MONO_HAPTIC_A = (OUT_FRONT_LEFT | OUT_HAPTIC_A), +// OUT_STEREO_HAPTIC_A = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_HAPTIC_A), +// OUT_HAPTIC_AB = (OUT_HAPTIC_A | OUT_HAPTIC_B), +// OUT_MONO_HAPTIC_AB = (OUT_FRONT_LEFT | OUT_HAPTIC_A | OUT_HAPTIC_B), +// OUT_STEREO_HAPTIC_AB = (OUT_FRONT_LEFT | OUT_FRONT_RIGHT | OUT_HAPTIC_A | OUT_HAPTIC_B), + /** + * These are bits only, not complete values + * + * + * input channels + */ + IN_LEFT = 0x4, + IN_RIGHT = 0x8, + IN_FRONT = 0x10, + IN_BACK = 0x20, + IN_LEFT_PROCESSED = 0x40, + IN_RIGHT_PROCESSED = 0x80, + IN_FRONT_PROCESSED = 0x100, + IN_BACK_PROCESSED = 0x200, + IN_PRESSURE = 0x400, + IN_X_AXIS = 0x800, + IN_Y_AXIS = 0x1000, + IN_Z_AXIS = 0x2000, + IN_BACK_LEFT = 0x10000, + IN_BACK_RIGHT = 0x20000, + IN_CENTER = 0x40000, + IN_LOW_FREQUENCY = 0x100000, + IN_TOP_LEFT = 0x200000, + IN_TOP_RIGHT = 0x400000, + IN_VOICE_UPLINK = 0x4000, + IN_VOICE_DNLINK = 0x8000, +// TODO(ytai): Aliases not currently supported in AIDL - can inline the values. +// IN_MONO = IN_FRONT, +// IN_STEREO = (IN_LEFT | IN_RIGHT), +// IN_FRONT_BACK = (IN_FRONT | IN_BACK), +// IN_6 = (IN_LEFT | IN_RIGHT | IN_FRONT | IN_BACK | IN_LEFT_PROCESSED | IN_RIGHT_PROCESSED), +// IN_2POINT0POINT2 = (IN_LEFT | IN_RIGHT | IN_TOP_LEFT | IN_TOP_RIGHT), +// IN_2POINT1POINT2 = (IN_LEFT | IN_RIGHT | IN_TOP_LEFT | IN_TOP_RIGHT | IN_LOW_FREQUENCY), +// IN_3POINT0POINT2 = (IN_LEFT | IN_CENTER | IN_RIGHT | IN_TOP_LEFT | IN_TOP_RIGHT), +// IN_3POINT1POINT2 = (IN_LEFT | IN_CENTER | IN_RIGHT | IN_TOP_LEFT | IN_TOP_RIGHT | IN_LOW_FREQUENCY), +// IN_5POINT1 = (IN_LEFT | IN_CENTER | IN_RIGHT | IN_BACK_LEFT | IN_BACK_RIGHT | IN_LOW_FREQUENCY), +// IN_VOICE_UPLINK_MONO = (IN_VOICE_UPLINK | IN_MONO), +// IN_VOICE_DNLINK_MONO = (IN_VOICE_DNLINK | IN_MONO), +// IN_VOICE_CALL_MONO = (IN_VOICE_UPLINK_MONO | IN_VOICE_DNLINK_MONO), +// COUNT_MAX = 30, +// INDEX_HDR = REPRESENTATION_INDEX << COUNT_MAX, +// INDEX_MASK_1 = INDEX_HDR | ((1 << 1) - 1), +// INDEX_MASK_2 = INDEX_HDR | ((1 << 2) - 1), +// INDEX_MASK_3 = INDEX_HDR | ((1 << 3) - 1), +// INDEX_MASK_4 = INDEX_HDR | ((1 << 4) - 1), +// INDEX_MASK_5 = INDEX_HDR | ((1 << 5) - 1), +// INDEX_MASK_6 = INDEX_HDR | ((1 << 6) - 1), +// INDEX_MASK_7 = INDEX_HDR | ((1 << 7) - 1), +// INDEX_MASK_8 = INDEX_HDR | ((1 << 8) - 1), +// INDEX_MASK_9 = INDEX_HDR | ((1 << 9) - 1), +// INDEX_MASK_10 = INDEX_HDR | ((1 << 10) - 1), +// INDEX_MASK_11 = INDEX_HDR | ((1 << 11) - 1), +// INDEX_MASK_12 = INDEX_HDR | ((1 << 12) - 1), +// INDEX_MASK_13 = INDEX_HDR | ((1 << 13) - 1), +// INDEX_MASK_14 = INDEX_HDR | ((1 << 14) - 1), +// INDEX_MASK_15 = INDEX_HDR | ((1 << 15) - 1), +// INDEX_MASK_16 = INDEX_HDR | ((1 << 16) - 1), +// INDEX_MASK_17 = INDEX_HDR | ((1 << 17) - 1), +// INDEX_MASK_18 = INDEX_HDR | ((1 << 18) - 1), +// INDEX_MASK_19 = INDEX_HDR | ((1 << 19) - 1), +// INDEX_MASK_20 = INDEX_HDR | ((1 << 20) - 1), +// INDEX_MASK_21 = INDEX_HDR | ((1 << 21) - 1), +// INDEX_MASK_22 = INDEX_HDR | ((1 << 22) - 1), +// INDEX_MASK_23 = INDEX_HDR | ((1 << 23) - 1), +// INDEX_MASK_24 = INDEX_HDR | ((1 << 24) - 1), +} diff --git a/media/java/android/media/audio/common/AudioConfig.aidl b/media/java/android/media/audio/common/AudioConfig.aidl new file mode 100644 index 0000000000000..50dd796e1fa00 --- /dev/null +++ b/media/java/android/media/audio/common/AudioConfig.aidl @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + // This file has been semi-automatically generated using hidl2aidl from its counterpart in + // hardware/interfaces/audio/common/5.0/types.hal + +package android.media.audio.common; + +import android.media.audio.common.AudioFormat; +import android.media.audio.common.AudioOffloadInfo; + +/** + * Commonly used audio stream configuration parameters. + * + * {@hide} + */ +parcelable AudioConfig { + int sampleRateHz; + int channelMask; + AudioFormat format; + AudioOffloadInfo offloadInfo; + long frameCount; +} diff --git a/media/java/android/media/audio/common/AudioFormat.aidl b/media/java/android/media/audio/common/AudioFormat.aidl new file mode 100644 index 0000000000000..aadc8e26cce34 --- /dev/null +++ b/media/java/android/media/audio/common/AudioFormat.aidl @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + // This file has been semi-automatically generated using hidl2aidl from its counterpart in + // hardware/interfaces/audio/common/5.0/types.hal + +package android.media.audio.common; + +/** + * Audio format is a 32-bit word that consists of: + * main format field (upper 8 bits) + * sub format field (lower 24 bits). + * + * The main format indicates the main codec type. The sub format field indicates + * options and parameters for each format. The sub format is mainly used for + * record to indicate for instance the requested bitrate or profile. It can + * also be used for certain formats to give informations not present in the + * encoded audio stream (e.g. octet alignement for AMR). + * + * {@hide} + */ +@Backing(type="int") +enum AudioFormat { + INVALID = 0xFFFFFFFF, + DEFAULT = 0, + PCM = 0x00000000, + MP3 = 0x01000000, + AMR_NB = 0x02000000, + AMR_WB = 0x03000000, + AAC = 0x04000000, + /** + * Deprecated, Use AAC_HE_V1 + */ + HE_AAC_V1 = 0x05000000, + /** + * Deprecated, Use AAC_HE_V2 + */ + HE_AAC_V2 = 0x06000000, + VORBIS = 0x07000000, + OPUS = 0x08000000, + AC3 = 0x09000000, + E_AC3 = 0x0A000000, + DTS = 0x0B000000, + DTS_HD = 0x0C000000, + /** + * IEC61937 is encoded audio wrapped in 16-bit PCM. + */ + IEC61937 = 0x0D000000, + DOLBY_TRUEHD = 0x0E000000, + EVRC = 0x10000000, + EVRCB = 0x11000000, + EVRCWB = 0x12000000, + EVRCNW = 0x13000000, + AAC_ADIF = 0x14000000, + WMA = 0x15000000, + WMA_PRO = 0x16000000, + AMR_WB_PLUS = 0x17000000, + MP2 = 0x18000000, + QCELP = 0x19000000, + DSD = 0x1A000000, + FLAC = 0x1B000000, + ALAC = 0x1C000000, + APE = 0x1D000000, + AAC_ADTS = 0x1E000000, + SBC = 0x1F000000, + APTX = 0x20000000, + APTX_HD = 0x21000000, + AC4 = 0x22000000, + LDAC = 0x23000000, + /** + * Dolby Metadata-enhanced Audio Transmission + */ + MAT = 0x24000000, + AAC_LATM = 0x25000000, + CELT = 0x26000000, + APTX_ADAPTIVE = 0x27000000, + LHDC = 0x28000000, + LHDC_LL = 0x29000000, + APTX_TWSP = 0x2A000000, + /** + * Deprecated + */ + MAIN_MASK = 0xFF000000, + SUB_MASK = 0x00FFFFFF, + /** + * Subformats + */ + PCM_SUB_16_BIT = 0x1, + PCM_SUB_8_BIT = 0x2, + PCM_SUB_32_BIT = 0x3, + PCM_SUB_8_24_BIT = 0x4, + PCM_SUB_FLOAT = 0x5, + PCM_SUB_24_BIT_PACKED = 0x6, + MP3_SUB_NONE = 0x0, + AMR_SUB_NONE = 0x0, + AAC_SUB_MAIN = 0x1, + AAC_SUB_LC = 0x2, + AAC_SUB_SSR = 0x4, + AAC_SUB_LTP = 0x8, + AAC_SUB_HE_V1 = 0x10, + AAC_SUB_SCALABLE = 0x20, + AAC_SUB_ERLC = 0x40, + AAC_SUB_LD = 0x80, + AAC_SUB_HE_V2 = 0x100, + AAC_SUB_ELD = 0x200, + AAC_SUB_XHE = 0x300, + VORBIS_SUB_NONE = 0x0, + E_AC3_SUB_JOC = 0x1, + MAT_SUB_1_0 = 0x1, + MAT_SUB_2_0 = 0x2, + MAT_SUB_2_1 = 0x3, +// TODO(ytai): Aliases not currently supported in AIDL - can inline the values. +// /** +// * Aliases +// * +// * +// * note != AudioFormat.ENCODING_PCM_16BIT +// */ +// PCM_16_BIT = (PCM | PCM_SUB_16_BIT), +// /** +// * note != AudioFormat.ENCODING_PCM_8BIT +// */ +// PCM_8_BIT = (PCM | PCM_SUB_8_BIT), +// PCM_32_BIT = (PCM | PCM_SUB_32_BIT), +// PCM_8_24_BIT = (PCM | PCM_SUB_8_24_BIT), +// PCM_FLOAT = (PCM | PCM_SUB_FLOAT), +// PCM_24_BIT_PACKED = (PCM | PCM_SUB_24_BIT_PACKED), +// AAC_MAIN = (AAC | AAC_SUB_MAIN), +// AAC_LC = (AAC | AAC_SUB_LC), +// AAC_SSR = (AAC | AAC_SUB_SSR), +// AAC_LTP = (AAC | AAC_SUB_LTP), +// AAC_HE_V1 = (AAC | AAC_SUB_HE_V1), +// AAC_SCALABLE = (AAC | AAC_SUB_SCALABLE), +// AAC_ERLC = (AAC | AAC_SUB_ERLC), +// AAC_LD = (AAC | AAC_SUB_LD), +// AAC_HE_V2 = (AAC | AAC_SUB_HE_V2), +// AAC_ELD = (AAC | AAC_SUB_ELD), +// AAC_XHE = (AAC | AAC_SUB_XHE), +// AAC_ADTS_MAIN = (AAC_ADTS | AAC_SUB_MAIN), +// AAC_ADTS_LC = (AAC_ADTS | AAC_SUB_LC), +// AAC_ADTS_SSR = (AAC_ADTS | AAC_SUB_SSR), +// AAC_ADTS_LTP = (AAC_ADTS | AAC_SUB_LTP), +// AAC_ADTS_HE_V1 = (AAC_ADTS | AAC_SUB_HE_V1), +// AAC_ADTS_SCALABLE = (AAC_ADTS | AAC_SUB_SCALABLE), +// AAC_ADTS_ERLC = (AAC_ADTS | AAC_SUB_ERLC), +// AAC_ADTS_LD = (AAC_ADTS | AAC_SUB_LD), +// AAC_ADTS_HE_V2 = (AAC_ADTS | AAC_SUB_HE_V2), +// AAC_ADTS_ELD = (AAC_ADTS | AAC_SUB_ELD), +// AAC_ADTS_XHE = (AAC_ADTS | AAC_SUB_XHE), +// E_AC3_JOC = (E_AC3 | E_AC3_SUB_JOC), +// MAT_1_0 = (MAT | MAT_SUB_1_0), +// MAT_2_0 = (MAT | MAT_SUB_2_0), +// MAT_2_1 = (MAT | MAT_SUB_2_1), +// AAC_LATM_LC = (AAC_LATM | AAC_SUB_LC), +// AAC_LATM_HE_V1 = (AAC_LATM | AAC_SUB_HE_V1), +// AAC_LATM_HE_V2 = (AAC_LATM | AAC_SUB_HE_V2), +} diff --git a/media/java/android/media/audio/common/AudioOffloadInfo.aidl b/media/java/android/media/audio/common/AudioOffloadInfo.aidl new file mode 100644 index 0000000000000..ec10d71135aee --- /dev/null +++ b/media/java/android/media/audio/common/AudioOffloadInfo.aidl @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + // This file has been semi-automatically generated using hidl2aidl from its counterpart in + // hardware/interfaces/audio/common/5.0/types.hal + +package android.media.audio.common; + +import android.media.audio.common.AudioFormat; +import android.media.audio.common.AudioStreamType; +import android.media.audio.common.AudioUsage; + +/** + * Additional information about the stream passed to hardware decoders. + * + * {@hide} + */ +parcelable AudioOffloadInfo { + int sampleRateHz; + int channelMask; + AudioFormat format; + AudioStreamType streamType; + int bitRatePerSecond; + long durationMicroseconds; + boolean hasVideo; + boolean isStreaming; + int bitWidth; + int bufferSize; + AudioUsage usage; +} + diff --git a/media/java/android/media/audio/common/AudioStreamType.aidl b/media/java/android/media/audio/common/AudioStreamType.aidl new file mode 100644 index 0000000000000..c54566726350c --- /dev/null +++ b/media/java/android/media/audio/common/AudioStreamType.aidl @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + // This file has been semi-automatically generated using hidl2aidl from its counterpart in + // hardware/interfaces/audio/common/5.0/types.hal + +package android.media.audio.common; + +/** + * Audio streams + * + * Audio stream type describing the intended use case of a stream. + * + * {@hide} + */ +@Backing(type="int") +enum AudioStreamType { + DEFAULT = -1, + MIN = 0, + VOICE_CALL = 0, + SYSTEM = 1, + RING = 2, + MUSIC = 3, + ALARM = 4, + NOTIFICATION = 5, + BLUETOOTH_SCO = 6, + ENFORCED_AUDIBLE = 7, + DTMF = 8, + TTS = 9, + ACCESSIBILITY = 10, +} diff --git a/media/java/android/media/audio/common/AudioUsage.aidl b/media/java/android/media/audio/common/AudioUsage.aidl new file mode 100644 index 0000000000000..ef348165b22c1 --- /dev/null +++ b/media/java/android/media/audio/common/AudioUsage.aidl @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + // This file has been semi-automatically generated using hidl2aidl from its counterpart in + // hardware/interfaces/audio/common/5.0/types.hal + +package android.media.audio.common; + +/** + * {@hide} + */ +@Backing(type="int") +enum AudioUsage { + UNKNOWN = 0, + MEDIA = 1, + VOICE_COMMUNICATION = 2, + VOICE_COMMUNICATION_SIGNALLING = 3, + ALARM = 4, + NOTIFICATION = 5, + NOTIFICATION_TELEPHONY_RINGTONE = 6, + ASSISTANCE_ACCESSIBILITY = 11, + ASSISTANCE_NAVIGATION_GUIDANCE = 12, + ASSISTANCE_SONIFICATION = 13, + GAME = 14, + VIRTUAL_SOURCE = 15, + ASSISTANT = 16, +} From ab58ef6483a2943c8ee6739703cf5b8ed2abc01b Mon Sep 17 00:00:00 2001 From: Ytai Ben-Tsvi Date: Mon, 25 Nov 2019 12:26:07 -0800 Subject: [PATCH 2/5] Sound trigger middleware service definition These are the AIDL files that define the sound trigger middleware interface. This service is intended to replace the existing frameworks/av/include/soundtrigger/ISoundTriggerHwService.h using AIDL in order to avoid a large amount of hand-written parceling code and other forms of boilerplate, and provide cross-language support. Change-Id: Ia783ba4f1ea7335a984396e7024cac0411699403 Bug: 142070343 --- media/Android.bp | 38 +++++++ .../ConfidenceLevel.aidl | 35 ++++++ .../ISoundTriggerCallback.aidl | 47 ++++++++ .../ISoundTriggerMiddlewareService.aidl | 48 ++++++++ .../ISoundTriggerModule.aidl | 105 ++++++++++++++++++ .../media/soundtrigger_middleware/Phrase.aidl | 34 ++++++ .../PhraseRecognitionEvent.aidl | 31 ++++++ .../PhraseRecognitionExtra.aidl | 35 ++++++ .../PhraseSoundModel.aidl | 32 ++++++ .../RecognitionConfig.aidl | 33 ++++++ .../RecognitionEvent.aidl | 51 +++++++++ .../RecognitionMode.aidl | 32 ++++++ .../RecognitionStatus.aidl | 35 ++++++ .../soundtrigger_middleware/SoundModel.aidl | 36 ++++++ .../SoundModelType.aidl | 30 +++++ .../SoundTriggerModuleDescriptor.aidl | 31 ++++++ .../SoundTriggerModuleProperties.aidl | 53 +++++++++ .../media/soundtrigger_middleware/Status.aidl | 29 +++++ 18 files changed, 735 insertions(+) create mode 100644 media/java/android/media/soundtrigger_middleware/ConfidenceLevel.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/ISoundTriggerMiddlewareService.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/Phrase.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/PhraseRecognitionEvent.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/PhraseRecognitionExtra.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/PhraseSoundModel.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/RecognitionConfig.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/RecognitionEvent.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/RecognitionMode.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/RecognitionStatus.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/SoundModel.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/SoundModelType.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/SoundTriggerModuleDescriptor.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/SoundTriggerModuleProperties.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/Status.aidl diff --git a/media/Android.bp b/media/Android.bp index b77c701c78e88..20a9656e0479f 100644 --- a/media/Android.bp +++ b/media/Android.bp @@ -133,3 +133,41 @@ aidl_interface { }, } +aidl_interface { + name: "soundtrigger_middleware-aidl", + local_include_dir: "java", + srcs: [ + "java/android/media/soundtrigger_middleware/ConfidenceLevel.aidl", + "java/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl", + "java/android/media/soundtrigger_middleware/ISoundTriggerMiddlewareService.aidl", + "java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl", + "java/android/media/soundtrigger_middleware/Phrase.aidl", + "java/android/media/soundtrigger_middleware/PhraseRecognitionEvent.aidl", + "java/android/media/soundtrigger_middleware/PhraseRecognitionExtra.aidl", + "java/android/media/soundtrigger_middleware/PhraseSoundModel.aidl", + "java/android/media/soundtrigger_middleware/RecognitionConfig.aidl", + "java/android/media/soundtrigger_middleware/RecognitionEvent.aidl", + "java/android/media/soundtrigger_middleware/RecognitionMode.aidl", + "java/android/media/soundtrigger_middleware/RecognitionStatus.aidl", + "java/android/media/soundtrigger_middleware/SoundModel.aidl", + "java/android/media/soundtrigger_middleware/SoundModelType.aidl", + "java/android/media/soundtrigger_middleware/SoundTriggerModuleDescriptor.aidl", + "java/android/media/soundtrigger_middleware/SoundTriggerModuleProperties.aidl", + "java/android/media/soundtrigger_middleware/Status.aidl", + ], + backend: + { + cpp: { + enabled: true, + }, + java: { + // Already generated as part of the entire media java library. + enabled: false, + }, + ndk: { + // Not currently needed, and disabled because of b/146172425 + enabled: false, + }, + }, + imports: [ "audio_common-aidl" ], +} diff --git a/media/java/android/media/soundtrigger_middleware/ConfidenceLevel.aidl b/media/java/android/media/soundtrigger_middleware/ConfidenceLevel.aidl new file mode 100644 index 0000000000000..3dbc70556bd3d --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/ConfidenceLevel.aidl @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * A recognition confidence level. + * This type is used to represent either a threshold or an actual detection confidence level. + * + * {@hide} + */ +parcelable ConfidenceLevel { + /** user ID. */ + int userId; + /** + * Confidence level in percent (0 - 100). + * + */ + int levelPercent; +} diff --git a/media/java/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl b/media/java/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl new file mode 100644 index 0000000000000..149c1cd0447bc --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.RecognitionEvent; +import android.media.soundtrigger_middleware.PhraseRecognitionEvent; + +/** + * Main interface for a client to get notifications of events coming from this module. + * + * {@hide} + */ +oneway interface ISoundTriggerCallback { + /** + * Invoked whenever a recognition event is triggered (typically, on recognition, but also in + * case of external aborting of a recognition or a forced recognition event - see the status + * code in the event for determining). + */ + void onRecognition(int modelHandle, in RecognitionEvent event); + /** + * Invoked whenever a phrase recognition event is triggered (typically, on recognition, but + * also in case of external aborting of a recognition or a forced recognition event - see the + * status code in the event for determining). + */ + void onPhraseRecognition(int modelHandle, in PhraseRecognitionEvent event); + /** + * Notifies the client the recognition has become available after previously having been + * unavailable, or vice versa. This method will always be invoked once immediately after + * attachment, and then every time there is a change in availability. + * When availability changes from available to unavailable, all active recognitions are aborted, + * and this event will be sent in addition to the abort event. + */ + void onRecognitionAvailabilityChange(boolean available); +} diff --git a/media/java/android/media/soundtrigger_middleware/ISoundTriggerMiddlewareService.aidl b/media/java/android/media/soundtrigger_middleware/ISoundTriggerMiddlewareService.aidl new file mode 100644 index 0000000000000..80333070b7ce0 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/ISoundTriggerMiddlewareService.aidl @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.ISoundTriggerCallback; +import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; + +/** + * Main entry point into this module. + * + * Allows the client to enumerate the available soundtrigger devices and their capabilities, then + * attach to either one of them in order to use it. + * + * {@hide} + */ +interface ISoundTriggerMiddlewareService { + /** + * Query the available modules and their capabilities. + */ + SoundTriggerModuleDescriptor[] listModules(); + + /** + * Attach to one of the available modules. + * listModules() must be called prior to calling this method and the provided handle must be + * one of the handles from the returned list. + */ + ISoundTriggerModule attach(int handle, ISoundTriggerCallback callback); + + /** + * Notify the service that external input capture is taking place. This may cause some of the + * active recognitions to be aborted. + */ + void setExternalCaptureState(boolean active); +} \ No newline at end of file diff --git a/media/java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl b/media/java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl new file mode 100644 index 0000000000000..202595a9d89c7 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.SoundModel; +import android.media.soundtrigger_middleware.PhraseSoundModel; +import android.media.soundtrigger_middleware.RecognitionConfig; + +/** + * A sound-trigger module. + * + * This interface allows a client to operate a sound-trigger device, intended for low-power + * detection of various sound patterns, represented by a "sound model". + * + * Basic operation is to load a sound model (either a generic one or a "phrase" model), then + * initiate recognition on this model. A trigger will be delivered asynchronously via a callback + * provided by the caller earlier, when attaching to this interface. + * + * In additon to recognition events, this module will also produce abort events in cases where + * recognition has been externally preempted. + * + * {@hide} + */ +interface ISoundTriggerModule { + /** + * Load a sound model. Will return a handle to the model on success or will throw a + * ServiceSpecificException with one of the {@link Status} error codes upon a recoverable error + * (for example, lack of resources of loading a model at the time of call. + * Model must eventually be unloaded using {@link #unloadModel(int)} prior to detaching. + * + * May throw a ServiceSpecificException with an RESOURCE_CONTENTION status to indicate that + * resources required for loading the model are currently consumed by other clients. + */ + int loadModel(in SoundModel model); + + /** + * Load a phrase sound model. Will return a handle to the model on success or will throw a + * ServiceSpecificException with one of the {@link Status} error codes upon a recoverable error + * (for example, lack of resources of loading a model at the time of call. + * Model must eventually be unloaded using unloadModel prior to detaching. + * + * May throw a ServiceSpecificException with an RESOURCE_CONTENTION status to indicate that + * resources required for loading the model are currently consumed by other clients. + */ + int loadPhraseModel(in PhraseSoundModel model); + + /** + * Unload a model, previously loaded with loadModel or loadPhraseModel. After unloading, model + * can no longer be used for recognition and the resources occupied by it are released. + * Model must not be active at the time of unloading. Cient may call stopRecognition to ensure + * that. + */ + void unloadModel(int modelHandle); + + /** + * Initiate recognition on a previously loaded model. + * Recognition event would eventually be delivered via the client-provided callback, typically + * supplied during attachment to this interface. + * + * Once a recognition event is passed to the client, the recognition automatically become + * inactive, unless the event is of the RecognitionStatus.FORCED kind. Client can also shut down + * the recognition explicitly, via stopRecognition. + */ + void startRecognition(int modelHandle, in RecognitionConfig config); + + /** + * Stop a recognition of a previously active recognition. Will NOT generate a recognition event. + * This call is idempotent - calling it on an inactive model has no effect. However, it must + * only be used with a loaded model handle. + */ + void stopRecognition(int modelHandle); + + /** + * Force generation of a recognition event. Handle must be that of a loaded model. If + * recognition is inactive, will do nothing. If recognition is active, will asynchronously + * deliever an event with RecognitionStatus.FORCED status and leave recognition in active state. + * To avoid any race conditions, once an event signalling the automatic stopping of recognition + * is sent, no more forced events will get sent (even if previously requested) until recognition + * is explicitly started again. + * + * Since not all module implementations support this feature, may throw a + * ServiceSpecificException with an OPERATION_NOT_SUPPORTED status. + */ + void forceRecognitionEvent(int modelHandle); + + /** + * Detach from the module, releasing any active resources. + * This will ensure the client callback is no longer called after this call returns. + * All models must have been unloaded prior to calling this method. + */ + void detach(); +} \ No newline at end of file diff --git a/media/java/android/media/soundtrigger_middleware/Phrase.aidl b/media/java/android/media/soundtrigger_middleware/Phrase.aidl new file mode 100644 index 0000000000000..98a489f8a6a9a --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/Phrase.aidl @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * Key phrase descriptor. + * + * {@hide} + */ +parcelable Phrase { + /** Unique keyphrase ID assigned at enrollment time. */ + int id; + /** Recognition modes supported by this key phrase (bitfield of RecognitionMode enum). */ + int recognitionModes; + /** List of users IDs associated with this key phrase. */ + int[] users; + /** Locale - Java Locale style (e.g. en_US). */ + String locale; + /** Phrase text. */ + String text; +} diff --git a/media/java/android/media/soundtrigger_middleware/PhraseRecognitionEvent.aidl b/media/java/android/media/soundtrigger_middleware/PhraseRecognitionEvent.aidl new file mode 100644 index 0000000000000..6a3ec61d1ebff --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/PhraseRecognitionEvent.aidl @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.PhraseRecognitionExtra; +import android.media.soundtrigger_middleware.RecognitionEvent; + +/** + * An event that gets sent to indicate a phrase recognition (or aborting of the recognition + process). + * {@hide} + */ +parcelable PhraseRecognitionEvent { + /** Common recognition event. */ + RecognitionEvent common; + /** List of descriptors for each recognized key phrase */ + PhraseRecognitionExtra[] phraseExtras; +} diff --git a/media/java/android/media/soundtrigger_middleware/PhraseRecognitionExtra.aidl b/media/java/android/media/soundtrigger_middleware/PhraseRecognitionExtra.aidl new file mode 100644 index 0000000000000..cb96bf37a95d9 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/PhraseRecognitionExtra.aidl @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.ConfidenceLevel; + +/** + * Specialized recognition event for key phrase detection. + * {@hide} + */ +parcelable PhraseRecognitionExtra { + // TODO(ytai): Constants / enums. + + /** keyphrase ID */ + int id; + /** recognition modes used for this keyphrase */ + int recognitionModes; + /** confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER */ + int confidenceLevel; + /** number of user confidence levels */ + ConfidenceLevel[] levels; +} diff --git a/media/java/android/media/soundtrigger_middleware/PhraseSoundModel.aidl b/media/java/android/media/soundtrigger_middleware/PhraseSoundModel.aidl new file mode 100644 index 0000000000000..81028c1608ea6 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/PhraseSoundModel.aidl @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.SoundModel; +import android.media.soundtrigger_middleware.Phrase; + +/** + * Specialized sound model for key phrase detection. + * Proprietary representation of key phrases in binary data must match + * information indicated by phrases field. + * {@hide} + */ +parcelable PhraseSoundModel { + /** Common part of sound model descriptor */ + SoundModel common; + /** List of descriptors for key phrases supported by this sound model */ + Phrase[] phrases; +} diff --git a/media/java/android/media/soundtrigger_middleware/RecognitionConfig.aidl b/media/java/android/media/soundtrigger_middleware/RecognitionConfig.aidl new file mode 100644 index 0000000000000..c7642e8c12410 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/RecognitionConfig.aidl @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.PhraseRecognitionExtra; + +/** + * Configuration for tuning behavior of an active recognition process. + * {@hide} + */ +parcelable RecognitionConfig { + /* Capture and buffer audio for this recognition instance. */ + boolean captureRequested; + + /* Configuration for each key phrase. */ + PhraseRecognitionExtra[] phraseRecognitionExtras; + + /** Opaque capture configuration data. */ + byte[] data; +} diff --git a/media/java/android/media/soundtrigger_middleware/RecognitionEvent.aidl b/media/java/android/media/soundtrigger_middleware/RecognitionEvent.aidl new file mode 100644 index 0000000000000..de4d060ce4849 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/RecognitionEvent.aidl @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.audio.common.AudioConfig; +import android.media.soundtrigger_middleware.RecognitionStatus; +import android.media.soundtrigger_middleware.SoundModelType; + +/** + * An event that gets sent to indicate a recognition (or aborting of the recognition process). + * {@hide} + */ +parcelable RecognitionEvent { + /** Recognition status. */ + RecognitionStatus status; + /** Event type, same as sound model type. */ + SoundModelType type; + /** Is it possible to capture audio from this utterance buffered by the implementation. */ + boolean captureAvailable; + /* Audio session ID. framework use. */ + int captureSession; + /** + * Delay in ms between end of model detection and start of audio available for capture. + * A negative value is possible (e.g. if key phrase is also available for Capture. + */ + int captureDelayMs; + /** Duration in ms of audio captured before the start of the trigger. 0 if none. */ + int capturePreambleMs; + /** If true, the 'data' field below contains the capture of the trigger sound. */ + boolean triggerInData; + /** + * Audio format of either the trigger in event data or to use for capture of the rest of the + * utterance. + */ + AudioConfig audioConfig; + /** Additional data. */ + byte[] data; +} diff --git a/media/java/android/media/soundtrigger_middleware/RecognitionMode.aidl b/media/java/android/media/soundtrigger_middleware/RecognitionMode.aidl new file mode 100644 index 0000000000000..d8bfff4bec6f2 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/RecognitionMode.aidl @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * Recognition mode. + * {@hide} + */ +@Backing(type="int") +enum RecognitionMode { + /** Simple voice trigger. */ + VOICE_TRIGGER = 0x1, + /** Trigger only if one user in model identified. */ + USER_IDENTIFICATION = 0x2, + /** Trigger only if one user in model authenticated. */ + USER_AUTHENTICATION = 0x4, + /** Generic sound trigger. */ + GENERIC_TRIGGER = 0x8, +} diff --git a/media/java/android/media/soundtrigger_middleware/RecognitionStatus.aidl b/media/java/android/media/soundtrigger_middleware/RecognitionStatus.aidl new file mode 100644 index 0000000000000..d563edca547d2 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/RecognitionStatus.aidl @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * A status for indicating the type of a recognition event. + * {@hide} + */ +@Backing(type="int") +enum RecognitionStatus { + /** Recognition success. */ + SUCCESS = 0, + /** Recognition aborted (e.g. capture preempted by another use-case. */ + ABORTED = 1, + /** Recognition failure. */ + FAILURE = 2, + /** + * Recognition event was triggered by a forceRecognitionEvent request, not by the DSP. + * Note that forced detections *do not* stop the active recognition, unlike the other types. + */ + FORCED = 3 +} diff --git a/media/java/android/media/soundtrigger_middleware/SoundModel.aidl b/media/java/android/media/soundtrigger_middleware/SoundModel.aidl new file mode 100644 index 0000000000000..fba1ee5078361 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/SoundModel.aidl @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.SoundModelType; + +/** + * Base sound model descriptor. This struct can be extended for various specific types by way of + * aggregation. + * {@hide} + */ +parcelable SoundModel { + /** Model type. */ + SoundModelType type; + /** Unique sound model ID. */ + String uuid; + /** + * Unique vendor ID. Identifies the engine the sound model + * was build for */ + String vendorUuid; + /** Opaque data transparent to Android framework */ + byte[] data; +} diff --git a/media/java/android/media/soundtrigger_middleware/SoundModelType.aidl b/media/java/android/media/soundtrigger_middleware/SoundModelType.aidl new file mode 100644 index 0000000000000..f2abc9af7780a --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/SoundModelType.aidl @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * Sound model type. + * {@hide} + */ +@Backing(type="int") +enum SoundModelType { + /** Unspecified sound model type */ + UNKNOWN = -1, + /** Key phrase sound models */ + KEYPHRASE = 0, + /** All models other than keyphrase */ + GENERIC = 1, +} diff --git a/media/java/android/media/soundtrigger_middleware/SoundTriggerModuleDescriptor.aidl b/media/java/android/media/soundtrigger_middleware/SoundTriggerModuleDescriptor.aidl new file mode 100644 index 0000000000000..667135ff61b99 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/SoundTriggerModuleDescriptor.aidl @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger_middleware.SoundTriggerModuleProperties; + +/** + * A descriptor of an available sound trigger module, containing the handle used to reference the + * module, as well its capabilities. + * {@hide} + */ +parcelable SoundTriggerModuleDescriptor { + /** Module handle to be used for attaching to it. */ + int handle; + /** Module capabilities. */ + SoundTriggerModuleProperties properties; +} + diff --git a/media/java/android/media/soundtrigger_middleware/SoundTriggerModuleProperties.aidl b/media/java/android/media/soundtrigger_middleware/SoundTriggerModuleProperties.aidl new file mode 100644 index 0000000000000..1a3b40261a62e --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/SoundTriggerModuleProperties.aidl @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * Capabilities of a sound trigger module. + * {@hide} + */ +parcelable SoundTriggerModuleProperties { + /** Implementor name */ + String implementor; + /** Implementation description */ + String description; + /** Implementation version */ + int version; + /** + * Unique implementation ID. The UUID must change with each version of + the engine implementation */ + String uuid; + /** Maximum number of concurrent sound models loaded */ + int maxSoundModels; + /** Maximum number of key phrases */ + int maxKeyPhrases; + /** Maximum number of concurrent users detected */ + int maxUsers; + /** All supported modes. e.g RecognitionMode.VOICE_TRIGGER */ + int recognitionModes; + /** Supports seamless transition from detection to capture */ + boolean captureTransition; + /** Maximum buffering capacity in ms if captureTransition is true */ + int maxBufferMs; + /** Supports capture by other use cases while detection is active */ + boolean concurrentCapture; + /** Returns the trigger capture in event */ + boolean triggerInEvent; + /** + * Rated power consumption when detection is active with TDB + * silence/sound/speech ratio */ + int powerConsumptionMw; +} diff --git a/media/java/android/media/soundtrigger_middleware/Status.aidl b/media/java/android/media/soundtrigger_middleware/Status.aidl new file mode 100644 index 0000000000000..d8f9d8f7e891f --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/Status.aidl @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * {@hide} + */ +@Backing(type="int") +enum Status { + /** Success. */ + SUCCESS = 0, + /** Failure due to resource contention. This is typically a temporary condition. */ + RESOURCE_CONTENTION = 1, + /** Operation is not supported in this implementation. This is a permanent condition. */ + OPERATION_NOT_SUPPORTED = 2, +} From 0b0441d16c87a5104599302c180461a13a84a457 Mon Sep 17 00:00:00 2001 From: Ytai Ben-Tsvi Date: Mon, 25 Nov 2019 12:30:51 -0800 Subject: [PATCH 3/5] Add a permission for preempting sound trigger sessions Previously, the power to preempt sound trigger recognition sessions for the sake of being able to capture audio on platforms that don't support doing both concurrently, was implicitly granted based on process (audio_server) co-location with the sound trigger service. Since this service is now being migrated out of audio_server, a new permission is introduced and granted to the audio server. Change-Id: Ifcdfc2a5543d814fb0630a45cdd9bcdba4d92107 Bug: 142070343 --- core/res/AndroidManifest.xml | 6 ++++++ data/etc/platform.xml | 1 + 2 files changed, 7 insertions(+) diff --git a/core/res/AndroidManifest.xml b/core/res/AndroidManifest.xml index c925744fea3bd..cb5b4a595ac3d 100644 --- a/core/res/AndroidManifest.xml +++ b/core/res/AndroidManifest.xml @@ -4410,6 +4410,12 @@ + + + diff --git a/data/etc/platform.xml b/data/etc/platform.xml index 80098c5a81f55..0574775712a6a 100644 --- a/data/etc/platform.xml +++ b/data/etc/platform.xml @@ -157,6 +157,7 @@ + From 93c117c86fe089df0851cd024d47bba6142bd98a Mon Sep 17 00:00:00 2001 From: Ytai Ben-Tsvi Date: Mon, 25 Nov 2019 12:43:28 -0800 Subject: [PATCH 4/5] Implement the soundtrigger_middlewware service This service is intended to replace: frameworks/av/include/soundtrigger/ISoundTriggerHwService.h This change only adds the replacement service, follow up changes migrate the clients to use the new service and remove the old one. The new service is feature-equivalent to the new one, but offers the following advantages: - AIDL interface (as opposed to hand-written parceling code). - Pure Java implementation all the way to the HAL. - Better documentation. - Rigorous error handling. - Unit tests. - Reduced code complexity (less layers, better separation of concerns). - Permission-based security model (as opposed to some baked-in assumptions about process affinity). Change-Id: I79f4eff105d3e6245990be068b933d4d48c35a0d Bug: 142070343 --- core/java/android/content/Context.java | 9 + media/Android.bp | 2 + .../ISoundTriggerModule.aidl | 43 + .../ModelParameter.aidl | 38 + .../ModelParameterRange.aidl | 28 + services/core/Android.bp | 1 + .../AudioSessionProviderImpl.java | 29 + .../ConversionUtil.java | 390 +++++ .../soundtrigger_middleware/HalException.java | 49 + .../Hw2CompatUtil.java | 77 + .../ISoundTriggerHw2.java | 158 ++ .../InternalServerError.java | 40 + .../RecoverableException.java | 53 + .../SoundTriggerHw2Compat.java | 470 ++++++ .../SoundTriggerMiddlewareImpl.java | 140 ++ .../SoundTriggerMiddlewareService.java | 709 +++++++++ .../SoundTriggerModule.java | 545 +++++++ .../soundtrigger_middleware/TEST_MAPPING | 12 + .../soundtrigger_middleware/UuidUtil.java | 44 + .../ValidationUtil.java | 117 ++ services/core/jni/Android.bp | 1 + ...er_middleware_AudioSessionProviderImpl.cpp | 91 ++ services/core/jni/onload.cpp | 4 + .../java/com/android/server/SystemServer.java | 5 + .../ConversionUtilTest.java | 47 + .../SoundTriggerMiddlewareImplTest.java | 1306 +++++++++++++++++ 26 files changed, 4408 insertions(+) create mode 100644 media/java/android/media/soundtrigger_middleware/ModelParameter.aidl create mode 100644 media/java/android/media/soundtrigger_middleware/ModelParameterRange.aidl create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/AudioSessionProviderImpl.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/ConversionUtil.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/HalException.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/Hw2CompatUtil.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/ISoundTriggerHw2.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/InternalServerError.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/RecoverableException.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImpl.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareService.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/TEST_MAPPING create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/UuidUtil.java create mode 100644 services/core/java/com/android/server/soundtrigger_middleware/ValidationUtil.java create mode 100644 services/core/jni/com_android_server_soundtrigger_middleware_AudioSessionProviderImpl.cpp create mode 100644 services/tests/servicestests/src/com/android/server/soundtrigger_middleware/ConversionUtilTest.java create mode 100644 services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java diff --git a/core/java/android/content/Context.java b/core/java/android/content/Context.java index 7b580c3bde793..671b589a881a1 100644 --- a/core/java/android/content/Context.java +++ b/core/java/android/content/Context.java @@ -4324,6 +4324,15 @@ public abstract class Context { */ public static final String SOUND_TRIGGER_SERVICE = "soundtrigger"; + /** + * Use with {@link #getSystemService(String)} to access the + * {@link com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareService}. + * + * @hide + * @see #getSystemService(String) + */ + public static final String SOUND_TRIGGER_MIDDLEWARE_SERVICE = "soundtrigger_middleware"; + /** * Official published name of the (internal) permission service. * diff --git a/media/Android.bp b/media/Android.bp index 20a9656e0479f..97d3138148526 100644 --- a/media/Android.bp +++ b/media/Android.bp @@ -141,6 +141,8 @@ aidl_interface { "java/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl", "java/android/media/soundtrigger_middleware/ISoundTriggerMiddlewareService.aidl", "java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl", + "java/android/media/soundtrigger_middleware/ModelParameter.aidl", + "java/android/media/soundtrigger_middleware/ModelParameterRange.aidl", "java/android/media/soundtrigger_middleware/Phrase.aidl", "java/android/media/soundtrigger_middleware/PhraseRecognitionEvent.aidl", "java/android/media/soundtrigger_middleware/PhraseRecognitionExtra.aidl", diff --git a/media/java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl b/media/java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl index 202595a9d89c7..c4a57857dd3da 100644 --- a/media/java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl +++ b/media/java/android/media/soundtrigger_middleware/ISoundTriggerModule.aidl @@ -15,6 +15,8 @@ */ package android.media.soundtrigger_middleware; +import android.media.soundtrigger_middleware.ModelParameter; +import android.media.soundtrigger_middleware.ModelParameterRange; import android.media.soundtrigger_middleware.SoundModel; import android.media.soundtrigger_middleware.PhraseSoundModel; import android.media.soundtrigger_middleware.RecognitionConfig; @@ -96,6 +98,47 @@ interface ISoundTriggerModule { */ void forceRecognitionEvent(int modelHandle); + /** + * Set a model specific parameter with the given value. This parameter + * will keep its value for the duration the model is loaded regardless of starting and stopping + * recognition. Once the model is unloaded, the value will be lost. + * It is expected to check if the handle supports the parameter via the + * queryModelParameterSupport API prior to calling this method. + * + * @param modelHandle The sound model handle indicating which model to modify parameters + * @param modelParam Parameter to set which will be validated against the + * ModelParameter type. + * @param value The value to set for the given model parameter + */ + void setModelParameter(int modelHandle, ModelParameter modelParam, int value); + + /** + * Get a model specific parameter. This parameter will keep its value + * for the duration the model is loaded regardless of starting and stopping recognition. + * Once the model is unloaded, the value will be lost. If the value is not set, a default + * value is returned. See ModelParameter for parameter default values. + * It is expected to check if the handle supports the parameter via the + * queryModelParameterSupport API prior to calling this method. + * + * @param modelHandle The sound model associated with given modelParam + * @param modelParam Parameter to set which will be validated against the + * ModelParameter type. + * @return Value set to the requested parameter. + */ + int getModelParameter(int modelHandle, ModelParameter modelParam); + + /** + * Determine if parameter control is supported for the given model handle, and its valid value + * range if it is. + * + * @param modelHandle The sound model handle indicating which model to query + * @param modelParam Parameter to set which will be validated against the + * ModelParameter type. + * @return If parameter is supported, the return value is its valid range, otherwise null. + */ + @nullable ModelParameterRange queryModelParameterSupport(int modelHandle, + ModelParameter modelParam); + /** * Detach from the module, releasing any active resources. * This will ensure the client callback is no longer called after this call returns. diff --git a/media/java/android/media/soundtrigger_middleware/ModelParameter.aidl b/media/java/android/media/soundtrigger_middleware/ModelParameter.aidl new file mode 100644 index 0000000000000..09936278e93a7 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/ModelParameter.aidl @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * Model specific parameters to be used with parameter set and get APIs. + * + * {@hide} + */ +@Backing(type="int") +enum ModelParameter { + /** + * Placeholder for invalid model parameter used for returning error or + * passing an invalid value. + */ + INVALID = -1, + + /** + * Controls the sensitivity threshold adjustment factor for a given model. + * Negative value corresponds to less sensitive model (high threshold) and + * a positive value corresponds to a more sensitive model (low threshold). + * Default value is 0. + */ + THRESHOLD_FACTOR = 0, +} diff --git a/media/java/android/media/soundtrigger_middleware/ModelParameterRange.aidl b/media/java/android/media/soundtrigger_middleware/ModelParameterRange.aidl new file mode 100644 index 0000000000000..d6948a87dc6d7 --- /dev/null +++ b/media/java/android/media/soundtrigger_middleware/ModelParameterRange.aidl @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +/** + * Value range for a model parameter. + * + * {@hide} + */ +parcelable ModelParameterRange { + /** Minimum (inclusive) */ + int minInclusive; + /** Maximum (inclusive) */ + int maxInclusive; +} diff --git a/services/core/Android.bp b/services/core/Android.bp index 203bc61c20224..b7adfa4a3ff14 100644 --- a/services/core/Android.bp +++ b/services/core/Android.bp @@ -116,6 +116,7 @@ java_library_static { "android.hardware.oemlock-V1.0-java", "android.hardware.configstore-V1.0-java", "android.hardware.contexthub-V1.0-java", + "android.hardware.soundtrigger-V2.3-java", "android.hidl.manager-V1.2-java", "dnsresolver_aidl_interface-V2-java", "netd_event_listener_interface-java", diff --git a/services/core/java/com/android/server/soundtrigger_middleware/AudioSessionProviderImpl.java b/services/core/java/com/android/server/soundtrigger_middleware/AudioSessionProviderImpl.java new file mode 100644 index 0000000000000..3fa52301d9a0f --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/AudioSessionProviderImpl.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +/** + * An implementation of SoundTriggerMiddlewareImpl.AudioSessionProvider that ties to native + * AudioSystem module via JNI. + */ +class AudioSessionProviderImpl extends SoundTriggerMiddlewareImpl.AudioSessionProvider { + @Override + public native AudioSession acquireSession(); + + @Override + public native void releaseSession(int sessionHandle); +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/ConversionUtil.java b/services/core/java/com/android/server/soundtrigger_middleware/ConversionUtil.java new file mode 100644 index 0000000000000..9b22f33a20b0b --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/ConversionUtil.java @@ -0,0 +1,390 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.hardware.audio.common.V2_0.Uuid; +import android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback; +import android.hardware.soundtrigger.V2_3.ISoundTriggerHw; +import android.media.audio.common.AudioConfig; +import android.media.audio.common.AudioOffloadInfo; +import android.media.soundtrigger_middleware.ConfidenceLevel; +import android.media.soundtrigger_middleware.ModelParameter; +import android.media.soundtrigger_middleware.ModelParameterRange; +import android.media.soundtrigger_middleware.Phrase; +import android.media.soundtrigger_middleware.PhraseRecognitionEvent; +import android.media.soundtrigger_middleware.PhraseRecognitionExtra; +import android.media.soundtrigger_middleware.PhraseSoundModel; +import android.media.soundtrigger_middleware.RecognitionConfig; +import android.media.soundtrigger_middleware.RecognitionEvent; +import android.media.soundtrigger_middleware.RecognitionMode; +import android.media.soundtrigger_middleware.RecognitionStatus; +import android.media.soundtrigger_middleware.SoundModel; +import android.media.soundtrigger_middleware.SoundModelType; +import android.media.soundtrigger_middleware.SoundTriggerModuleProperties; +import android.os.HidlMemoryUtil; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utilities for type conversion between SoundTrigger HAL types and SoundTriggerMiddleware service + * types. + * + * @hide + */ +class ConversionUtil { + static @NonNull + SoundTriggerModuleProperties hidl2aidlProperties( + @NonNull ISoundTriggerHw.Properties hidlProperties) { + SoundTriggerModuleProperties aidlProperties = new SoundTriggerModuleProperties(); + aidlProperties.implementor = hidlProperties.implementor; + aidlProperties.description = hidlProperties.description; + aidlProperties.version = hidlProperties.version; + aidlProperties.uuid = hidl2aidlUuid(hidlProperties.uuid); + aidlProperties.maxSoundModels = hidlProperties.maxSoundModels; + aidlProperties.maxKeyPhrases = hidlProperties.maxKeyPhrases; + aidlProperties.maxUsers = hidlProperties.maxUsers; + aidlProperties.recognitionModes = hidlProperties.recognitionModes; + aidlProperties.captureTransition = hidlProperties.captureTransition; + aidlProperties.maxBufferMs = hidlProperties.maxBufferMs; + aidlProperties.concurrentCapture = hidlProperties.concurrentCapture; + aidlProperties.triggerInEvent = hidlProperties.triggerInEvent; + aidlProperties.powerConsumptionMw = hidlProperties.powerConsumptionMw; + return aidlProperties; + } + + static @NonNull + String hidl2aidlUuid(@NonNull Uuid hidlUuid) { + if (hidlUuid.node == null || hidlUuid.node.length != 6) { + throw new IllegalArgumentException("UUID.node must be of length 6."); + } + return String.format(UuidUtil.FORMAT, + hidlUuid.timeLow, + hidlUuid.timeMid, + hidlUuid.versionAndTimeHigh, + hidlUuid.variantAndClockSeqHigh, + hidlUuid.node[0], + hidlUuid.node[1], + hidlUuid.node[2], + hidlUuid.node[3], + hidlUuid.node[4], + hidlUuid.node[5]); + } + + static @NonNull + Uuid aidl2hidlUuid(@NonNull String aidlUuid) { + Matcher matcher = UuidUtil.PATTERN.matcher(aidlUuid); + if (!matcher.matches()) { + throw new IllegalArgumentException("Illegal format for UUID: " + aidlUuid); + } + Uuid hidlUuid = new Uuid(); + hidlUuid.timeLow = Integer.parseUnsignedInt(matcher.group(1), 16); + hidlUuid.timeMid = (short) Integer.parseUnsignedInt(matcher.group(2), 16); + hidlUuid.versionAndTimeHigh = (short) Integer.parseUnsignedInt(matcher.group(3), 16); + hidlUuid.variantAndClockSeqHigh = (short) Integer.parseUnsignedInt(matcher.group(4), 16); + hidlUuid.node = new byte[]{(byte) Integer.parseUnsignedInt(matcher.group(5), 16), + (byte) Integer.parseUnsignedInt(matcher.group(6), 16), + (byte) Integer.parseUnsignedInt(matcher.group(7), 16), + (byte) Integer.parseUnsignedInt(matcher.group(8), 16), + (byte) Integer.parseUnsignedInt(matcher.group(9), 16), + (byte) Integer.parseUnsignedInt(matcher.group(10), 16)}; + return hidlUuid; + } + + static int aidl2hidlSoundModelType(int aidlType) { + switch (aidlType) { + case SoundModelType.GENERIC: + return android.hardware.soundtrigger.V2_0.SoundModelType.GENERIC; + case SoundModelType.KEYPHRASE: + return android.hardware.soundtrigger.V2_0.SoundModelType.KEYPHRASE; + default: + throw new IllegalArgumentException("Unknown sound model type: " + aidlType); + } + } + + static int hidl2aidlSoundModelType(int hidlType) { + switch (hidlType) { + case android.hardware.soundtrigger.V2_0.SoundModelType.GENERIC: + return SoundModelType.GENERIC; + case android.hardware.soundtrigger.V2_0.SoundModelType.KEYPHRASE: + return SoundModelType.KEYPHRASE; + default: + throw new IllegalArgumentException("Unknown sound model type: " + hidlType); + } + } + + static @NonNull + ISoundTriggerHw.Phrase aidl2hidlPhrase(@NonNull Phrase aidlPhrase) { + ISoundTriggerHw.Phrase hidlPhrase = new ISoundTriggerHw.Phrase(); + hidlPhrase.id = aidlPhrase.id; + hidlPhrase.recognitionModes = aidl2hidlRecognitionModes(aidlPhrase.recognitionModes); + for (int aidlUser : aidlPhrase.users) { + hidlPhrase.users.add(aidlUser); + } + hidlPhrase.locale = aidlPhrase.locale; + hidlPhrase.text = aidlPhrase.text; + return hidlPhrase; + } + + static int aidl2hidlRecognitionModes(int aidlModes) { + int hidlModes = 0; + + if ((aidlModes & RecognitionMode.VOICE_TRIGGER) != 0) { + hidlModes |= android.hardware.soundtrigger.V2_0.RecognitionMode.VOICE_TRIGGER; + } + if ((aidlModes & RecognitionMode.USER_IDENTIFICATION) != 0) { + hidlModes |= android.hardware.soundtrigger.V2_0.RecognitionMode.USER_IDENTIFICATION; + } + if ((aidlModes & RecognitionMode.USER_AUTHENTICATION) != 0) { + hidlModes |= android.hardware.soundtrigger.V2_0.RecognitionMode.USER_AUTHENTICATION; + } + if ((aidlModes & RecognitionMode.GENERIC_TRIGGER) != 0) { + hidlModes |= android.hardware.soundtrigger.V2_0.RecognitionMode.GENERIC_TRIGGER; + } + return hidlModes; + } + + static int hidl2aidlRecognitionModes(int hidlModes) { + int aidlModes = 0; + if ((hidlModes & android.hardware.soundtrigger.V2_0.RecognitionMode.VOICE_TRIGGER) != 0) { + aidlModes |= RecognitionMode.VOICE_TRIGGER; + } + if ((hidlModes & android.hardware.soundtrigger.V2_0.RecognitionMode.USER_IDENTIFICATION) + != 0) { + aidlModes |= RecognitionMode.USER_IDENTIFICATION; + } + if ((hidlModes & android.hardware.soundtrigger.V2_0.RecognitionMode.USER_AUTHENTICATION) + != 0) { + aidlModes |= RecognitionMode.USER_AUTHENTICATION; + } + if ((hidlModes & android.hardware.soundtrigger.V2_0.RecognitionMode.GENERIC_TRIGGER) != 0) { + aidlModes |= RecognitionMode.GENERIC_TRIGGER; + } + return aidlModes; + } + + static @NonNull + ISoundTriggerHw.SoundModel aidl2hidlSoundModel(@NonNull SoundModel aidlModel) { + ISoundTriggerHw.SoundModel hidlModel = new ISoundTriggerHw.SoundModel(); + hidlModel.header.type = aidl2hidlSoundModelType(aidlModel.type); + hidlModel.header.uuid = aidl2hidlUuid(aidlModel.uuid); + hidlModel.header.vendorUuid = aidl2hidlUuid(aidlModel.vendorUuid); + hidlModel.data = HidlMemoryUtil.byteArrayToHidlMemory(aidlModel.data, + "SoundTrigger SoundModel"); + return hidlModel; + } + + static @NonNull + ISoundTriggerHw.PhraseSoundModel aidl2hidlPhraseSoundModel( + @NonNull PhraseSoundModel aidlModel) { + ISoundTriggerHw.PhraseSoundModel hidlModel = new ISoundTriggerHw.PhraseSoundModel(); + hidlModel.common = aidl2hidlSoundModel(aidlModel.common); + for (Phrase aidlPhrase : aidlModel.phrases) { + hidlModel.phrases.add(aidl2hidlPhrase(aidlPhrase)); + } + return hidlModel; + } + + static @NonNull + ISoundTriggerHw.RecognitionConfig aidl2hidlRecognitionConfig( + @NonNull RecognitionConfig aidlConfig) { + ISoundTriggerHw.RecognitionConfig hidlConfig = new ISoundTriggerHw.RecognitionConfig(); + hidlConfig.header.captureRequested = aidlConfig.captureRequested; + for (PhraseRecognitionExtra aidlPhraseExtra : aidlConfig.phraseRecognitionExtras) { + hidlConfig.header.phrases.add(aidl2hidlPhraseRecognitionExtra(aidlPhraseExtra)); + } + hidlConfig.data = HidlMemoryUtil.byteArrayToHidlMemory(aidlConfig.data, + "SoundTrigger RecognitionConfig"); + return hidlConfig; + } + + static @NonNull + android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra aidl2hidlPhraseRecognitionExtra( + @NonNull PhraseRecognitionExtra aidlExtra) { + android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra hidlExtra = + new android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra(); + hidlExtra.id = aidlExtra.id; + hidlExtra.recognitionModes = aidl2hidlRecognitionModes(aidlExtra.recognitionModes); + hidlExtra.confidenceLevel = aidlExtra.confidenceLevel; + hidlExtra.levels.ensureCapacity(aidlExtra.levels.length); + for (ConfidenceLevel aidlLevel : aidlExtra.levels) { + hidlExtra.levels.add(aidl2hidlConfidenceLevel(aidlLevel)); + } + return hidlExtra; + } + + static @NonNull + PhraseRecognitionExtra hidl2aidlPhraseRecognitionExtra( + @NonNull android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra hidlExtra) { + PhraseRecognitionExtra aidlExtra = new PhraseRecognitionExtra(); + aidlExtra.id = hidlExtra.id; + aidlExtra.recognitionModes = hidl2aidlRecognitionModes(hidlExtra.recognitionModes); + aidlExtra.confidenceLevel = hidlExtra.confidenceLevel; + aidlExtra.levels = new ConfidenceLevel[hidlExtra.levels.size()]; + for (int i = 0; i < hidlExtra.levels.size(); ++i) { + aidlExtra.levels[i] = hidl2aidlConfidenceLevel(hidlExtra.levels.get(i)); + } + return aidlExtra; + } + + static @NonNull + android.hardware.soundtrigger.V2_0.ConfidenceLevel aidl2hidlConfidenceLevel( + @NonNull ConfidenceLevel aidlLevel) { + android.hardware.soundtrigger.V2_0.ConfidenceLevel hidlLevel = + new android.hardware.soundtrigger.V2_0.ConfidenceLevel(); + hidlLevel.userId = aidlLevel.userId; + hidlLevel.levelPercent = aidlLevel.levelPercent; + return hidlLevel; + } + + static @NonNull + ConfidenceLevel hidl2aidlConfidenceLevel( + @NonNull android.hardware.soundtrigger.V2_0.ConfidenceLevel hidlLevel) { + ConfidenceLevel aidlLevel = new ConfidenceLevel(); + aidlLevel.userId = hidlLevel.userId; + aidlLevel.levelPercent = hidlLevel.levelPercent; + return aidlLevel; + } + + static int hidl2aidlRecognitionStatus(int hidlStatus) { + switch (hidlStatus) { + case ISoundTriggerHwCallback.RecognitionStatus.SUCCESS: + return RecognitionStatus.SUCCESS; + case ISoundTriggerHwCallback.RecognitionStatus.ABORT: + return RecognitionStatus.ABORTED; + case ISoundTriggerHwCallback.RecognitionStatus.FAILURE: + return RecognitionStatus.FAILURE; + case 3: // This doesn't have a constant in HIDL. + return RecognitionStatus.FORCED; + default: + throw new IllegalArgumentException("Unknown recognition status: " + hidlStatus); + } + } + + static @NonNull + RecognitionEvent hidl2aidlRecognitionEvent(@NonNull + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionEvent hidlEvent) { + RecognitionEvent aidlEvent = new RecognitionEvent(); + aidlEvent.status = hidl2aidlRecognitionStatus(hidlEvent.status); + aidlEvent.type = hidl2aidlSoundModelType(hidlEvent.type); + aidlEvent.captureAvailable = hidlEvent.captureAvailable; + // hidlEvent.captureSession is never a valid field. + aidlEvent.captureSession = -1; + aidlEvent.captureDelayMs = hidlEvent.captureDelayMs; + aidlEvent.capturePreambleMs = hidlEvent.capturePreambleMs; + aidlEvent.triggerInData = hidlEvent.triggerInData; + aidlEvent.audioConfig = hidl2aidlAudioConfig(hidlEvent.audioConfig); + aidlEvent.data = new byte[hidlEvent.data.size()]; + for (int i = 0; i < aidlEvent.data.length; ++i) { + aidlEvent.data[i] = hidlEvent.data.get(i); + } + return aidlEvent; + } + + static @NonNull + RecognitionEvent hidl2aidlRecognitionEvent( + @NonNull ISoundTriggerHwCallback.RecognitionEvent hidlEvent) { + RecognitionEvent aidlEvent = hidl2aidlRecognitionEvent(hidlEvent.header); + // Data needs to get overridden with 2.1 data. + aidlEvent.data = HidlMemoryUtil.hidlMemoryToByteArray(hidlEvent.data); + return aidlEvent; + } + + static @NonNull + PhraseRecognitionEvent hidl2aidlPhraseRecognitionEvent(@NonNull + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.PhraseRecognitionEvent hidlEvent) { + PhraseRecognitionEvent aidlEvent = new PhraseRecognitionEvent(); + aidlEvent.common = hidl2aidlRecognitionEvent(hidlEvent.common); + aidlEvent.phraseExtras = new PhraseRecognitionExtra[hidlEvent.phraseExtras.size()]; + for (int i = 0; i < hidlEvent.phraseExtras.size(); ++i) { + aidlEvent.phraseExtras[i] = hidl2aidlPhraseRecognitionExtra( + hidlEvent.phraseExtras.get(i)); + } + return aidlEvent; + } + + static @NonNull + PhraseRecognitionEvent hidl2aidlPhraseRecognitionEvent( + @NonNull ISoundTriggerHwCallback.PhraseRecognitionEvent hidlEvent) { + PhraseRecognitionEvent aidlEvent = new PhraseRecognitionEvent(); + aidlEvent.common = hidl2aidlRecognitionEvent(hidlEvent.common); + aidlEvent.phraseExtras = new PhraseRecognitionExtra[hidlEvent.phraseExtras.size()]; + for (int i = 0; i < hidlEvent.phraseExtras.size(); ++i) { + aidlEvent.phraseExtras[i] = hidl2aidlPhraseRecognitionExtra( + hidlEvent.phraseExtras.get(i)); + } + return aidlEvent; + } + + static @NonNull + AudioConfig hidl2aidlAudioConfig( + @NonNull android.hardware.audio.common.V2_0.AudioConfig hidlConfig) { + AudioConfig aidlConfig = new AudioConfig(); + // TODO(ytai): channelMask and format might need a more careful conversion to make sure the + // constants match. + aidlConfig.sampleRateHz = hidlConfig.sampleRateHz; + aidlConfig.channelMask = hidlConfig.channelMask; + aidlConfig.format = hidlConfig.format; + aidlConfig.offloadInfo = hidl2aidlOffloadInfo(hidlConfig.offloadInfo); + aidlConfig.frameCount = hidlConfig.frameCount; + return aidlConfig; + } + + static @NonNull + AudioOffloadInfo hidl2aidlOffloadInfo( + @NonNull android.hardware.audio.common.V2_0.AudioOffloadInfo hidlInfo) { + AudioOffloadInfo aidlInfo = new AudioOffloadInfo(); + // TODO(ytai): channelMask, format, streamType and usage might need a more careful + // conversion to make sure the constants match. + aidlInfo.sampleRateHz = hidlInfo.sampleRateHz; + aidlInfo.channelMask = hidlInfo.channelMask; + aidlInfo.format = hidlInfo.format; + aidlInfo.streamType = hidlInfo.streamType; + aidlInfo.bitRatePerSecond = hidlInfo.bitRatePerSecond; + aidlInfo.durationMicroseconds = hidlInfo.durationMicroseconds; + aidlInfo.hasVideo = hidlInfo.hasVideo; + aidlInfo.isStreaming = hidlInfo.isStreaming; + aidlInfo.bitWidth = hidlInfo.bitWidth; + aidlInfo.bufferSize = hidlInfo.bufferSize; + aidlInfo.usage = hidlInfo.usage; + return aidlInfo; + } + + @Nullable + static ModelParameterRange hidl2aidlModelParameterRange( + android.hardware.soundtrigger.V2_3.ModelParameterRange hidlRange) { + if (hidlRange == null) { + return null; + } + ModelParameterRange aidlRange = new ModelParameterRange(); + aidlRange.minInclusive = hidlRange.start; + aidlRange.maxInclusive = hidlRange.end; + return aidlRange; + } + + static int aidl2hidlModelParameter(int aidlParam) { + switch (aidlParam) { + case ModelParameter.THRESHOLD_FACTOR: + return android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR; + + default: + return android.hardware.soundtrigger.V2_3.ModelParameter.INVALID; + } + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/HalException.java b/services/core/java/com/android/server/soundtrigger_middleware/HalException.java new file mode 100644 index 0000000000000..8b3e70875183f --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/HalException.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.annotation.NonNull; + +/** + * This exception represents a non-zero status code returned by a HAL invocation. + * Depending on the operation that threw the error, the integrity of the HAL implementation and the + * client's tolerance to error, this error may or may not be recoverable. The HAL itself is expected + * to retain the state it had prior to the invocation (so, unless the error is a result of a HAL + * bug, normal operation may resume). + *

+ * The reason why this is a RuntimeException, even though the HAL interface allows returning them + * is because we expect none of them to actually occur as part of correct usage of the HAL. + * + * @hide + */ +public class HalException extends RuntimeException { + public final int errorCode; + + public HalException(int errorCode, @NonNull String message) { + super(message); + this.errorCode = errorCode; + } + + public HalException(int errorCode) { + this.errorCode = errorCode; + } + + @Override + public @NonNull String toString() { + return super.toString() + " (code " + errorCode + ")"; + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/Hw2CompatUtil.java b/services/core/java/com/android/server/soundtrigger_middleware/Hw2CompatUtil.java new file mode 100644 index 0000000000000..f0a0d8305bc6a --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/Hw2CompatUtil.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.os.HidlMemoryUtil; + +import java.util.ArrayList; + +/** + * Utilities for maintaining data compatibility between different minor versions of soundtrigger@2.x + * HAL. + * Note that some of these conversion utilities are destructive, i.e. mutate their input (for the + * sake of simplifying code and reducing copies). + */ +class Hw2CompatUtil { + static android.hardware.soundtrigger.V2_0.ISoundTriggerHw.SoundModel convertSoundModel_2_1_to_2_0( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.SoundModel soundModel) { + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.SoundModel model_2_0 = soundModel.header; + // Note: this mutates the input! + model_2_0.data = HidlMemoryUtil.hidlMemoryToByteList(soundModel.data); + return model_2_0; + } + + static android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent convertRecognitionEvent_2_0_to_2_1( + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionEvent event) { + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent event_2_1 = + new android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent(); + event_2_1.header = event; + event_2_1.data = HidlMemoryUtil.byteListToHidlMemory(event_2_1.header.data, + "SoundTrigger RecognitionEvent"); + // Note: this mutates the input! + event_2_1.header.data = new ArrayList<>(); + return event_2_1; + } + + static android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent convertPhraseRecognitionEvent_2_0_to_2_1( + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.PhraseRecognitionEvent event) { + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent + event_2_1 = + new android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent(); + event_2_1.common = convertRecognitionEvent_2_0_to_2_1(event.common); + event_2_1.phraseExtras = event.phraseExtras; + return event_2_1; + } + + static android.hardware.soundtrigger.V2_0.ISoundTriggerHw.PhraseSoundModel convertPhraseSoundModel_2_1_to_2_0( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.PhraseSoundModel soundModel) { + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.PhraseSoundModel model_2_0 = + new android.hardware.soundtrigger.V2_0.ISoundTriggerHw.PhraseSoundModel(); + model_2_0.common = convertSoundModel_2_1_to_2_0(soundModel.common); + model_2_0.phrases = soundModel.phrases; + return model_2_0; + } + + static android.hardware.soundtrigger.V2_0.ISoundTriggerHw.RecognitionConfig convertRecognitionConfig_2_1_to_2_0( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.RecognitionConfig config) { + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.RecognitionConfig config_2_0 = + config.header; + // Note: this mutates the input! + config_2_0.data = HidlMemoryUtil.hidlMemoryToByteList(config.data); + return config_2_0; + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/ISoundTriggerHw2.java b/services/core/java/com/android/server/soundtrigger_middleware/ISoundTriggerHw2.java new file mode 100644 index 0000000000000..81252c9a8c146 --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/ISoundTriggerHw2.java @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.hardware.soundtrigger.V2_3.ISoundTriggerHw; +import android.hardware.soundtrigger.V2_3.ModelParameterRange; +import android.hidl.base.V1_0.IBase; +import android.os.IHwBinder; + +/** + * This interface mimics android.hardware.soundtrigger.V2_x.ISoundTriggerHw and + * android.hardware.soundtrigger.V2_x.ISoundTriggerHwCallback, with a few key differences: + *

    + *
  • Methods in the original interface generally have a status return value and potentially a + * second return value which is the actual return value. This is reflected via a synchronous + * callback, which is not very pleasant to work with. This interface replaces that pattern with + * the convention that a HalException is thrown for non-OK status, and then we can use the + * return value for the actual return value. + *
  • This interface will always include all the methods from the latest 2.x version (and thus + * from every 2.x version) interface, with the convention that unsupported methods throw a + * {@link RecoverableException} with a + * {@link android.media.soundtrigger_middleware.Status#OPERATION_NOT_SUPPORTED} + * code. + *
  • Cases where the original interface had multiple versions of a method representing the exact + * thing, or there exists a trivial conversion between the new and old version, this interface + * represents only the latest version, without any _version suffixes. + *
  • Removes some of the obscure IBinder methods. + *
  • No RemoteExceptions are specified. Some implementations of this interface may rethrow + * RemoteExceptions as RuntimeExceptions, some can guarantee handling them somehow and never throw + * them. + *
  • soundModelCallback has been removed, since nobody cares about it. Implementations are free + * to silently discard it. + *
+ * For cases where the client wants to explicitly handle specific versions of the underlying driver + * interface, they may call {@link #interfaceDescriptor()}. + *

+ * Note to maintainers: This class must always be kept in sync with the latest 2.x version, + * so that clients have access to the entire functionality without having to burden themselves with + * compatibility, as much as possible. + */ +public interface ISoundTriggerHw2 { + /** + * @see android.hardware.soundtrigger.V2_2.ISoundTriggerHw#getProperties(android.hardware.soundtrigger.V2_0.ISoundTriggerHw.getPropertiesCallback + */ + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.Properties getProperties(); + + /** + * @see android.hardware.soundtrigger.V2_2.ISoundTriggerHw#loadSoundModel_2_1(android.hardware.soundtrigger.V2_1.ISoundTriggerHw.SoundModel, + * android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback, int, + * android.hardware.soundtrigger.V2_1.ISoundTriggerHw.loadSoundModel_2_1Callback) + */ + int loadSoundModel( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.SoundModel soundModel, + SoundTriggerHw2Compat.Callback callback, int cookie); + + /** + * @see android.hardware.soundtrigger.V2_2.ISoundTriggerHw#loadPhraseSoundModel_2_1(android.hardware.soundtrigger.V2_1.ISoundTriggerHw.PhraseSoundModel, + * android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback, int, + * android.hardware.soundtrigger.V2_1.ISoundTriggerHw.loadPhraseSoundModel_2_1Callback) + */ + int loadPhraseSoundModel( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.PhraseSoundModel soundModel, + SoundTriggerHw2Compat.Callback callback, int cookie); + + /** + * @see android.hardware.soundtrigger.V2_2.ISoundTriggerHw#unloadSoundModel(int) + */ + void unloadSoundModel(int modelHandle); + + /** + * @see android.hardware.soundtrigger.V2_2.ISoundTriggerHw#stopRecognition(int) + */ + void stopRecognition(int modelHandle); + + /** + * @see android.hardware.soundtrigger.V2_2.ISoundTriggerHw#stopAllRecognitions() + */ + void stopAllRecognitions(); + + /** + * @see android.hardware.soundtrigger.V2_2.ISoundTriggerHw#startRecognition_2_1(int, + * android.hardware.soundtrigger.V2_1.ISoundTriggerHw.RecognitionConfig, + * android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback, int) + */ + void startRecognition(int modelHandle, + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.RecognitionConfig config, + SoundTriggerHw2Compat.Callback callback, int cookie); + + /** + * @see android.hardware.soundtrigger.V2_2.ISoundTriggerHw#getModelState(int) + */ + void getModelState(int modelHandle); + + /** + * @see android.hardware.soundtrigger.V2_3.ISoundTriggerHw#getParameter(int, int, + * ISoundTriggerHw.getParameterCallback) + */ + int getModelParameter(int modelHandle, int param); + + /** + * @see android.hardware.soundtrigger.V2_3.ISoundTriggerHw#setParameter(int, int, int) + */ + void setModelParameter(int modelHandle, int param, int value); + + /** + * @return null if not supported. + * @see android.hardware.soundtrigger.V2_3.ISoundTriggerHw#queryParameter(int, int, + * ISoundTriggerHw.queryParameterCallback) + */ + ModelParameterRange queryParameter(int modelHandle, int param); + + /** + * @see IHwBinder#linkToDeath(IHwBinder.DeathRecipient, long) + */ + boolean linkToDeath(IHwBinder.DeathRecipient recipient, long cookie); + + /** + * @see IHwBinder#unlinkToDeath(IHwBinder.DeathRecipient) + */ + boolean unlinkToDeath(IHwBinder.DeathRecipient recipient); + + /** + * @see IBase#interfaceDescriptor() + */ + String interfaceDescriptor() throws android.os.RemoteException; + + interface Callback { + /** + * @see android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback#recognitionCallback_2_1(android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent, + * int) + */ + void recognitionCallback( + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent event, + int cookie); + + /** + * @see android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback#phraseRecognitionCallback_2_1(android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent, + * int) + */ + void phraseRecognitionCallback( + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent event, + int cookie); + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/InternalServerError.java b/services/core/java/com/android/server/soundtrigger_middleware/InternalServerError.java new file mode 100644 index 0000000000000..e1fb2266b7c6e --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/InternalServerError.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.annotation.NonNull; + +/** + * An internal server error. + *

+ * This exception wraps any exception thrown from a service implementation, which is a result of a + * bug in the server implementation (or any of its dependencies). + *

+ * Specifically, this type is excluded from the set of whitelisted exceptions that binder would + * tunnel to the client process, since these exceptions are ambiguous regarding whether the client + * had done something wrong or the server is buggy. For example, a client getting an + * IllegalArgumentException cannot easily determine whether they had provided illegal arguments to + * the method they were calling, or whether the method implementation provided illegal arguments to + * some method it was calling due to a bug. + * + * @hide + */ +public class InternalServerError extends RuntimeException { + public InternalServerError(@NonNull Throwable cause) { + super(cause); + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/RecoverableException.java b/services/core/java/com/android/server/soundtrigger_middleware/RecoverableException.java new file mode 100644 index 0000000000000..83618505814e4 --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/RecoverableException.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.annotation.NonNull; + +/** + * This exception represents a fault which: + *

    + *
  • Could not have been anticipated by a caller (i.e. is not a violation of any preconditions). + *
  • Is guaranteed to not have been caused any meaningful state change in the callee. The caller + * may continue operation as if the call has never been made. + *
+ *

+ * Some recoverable faults are permanent and some are transient / circumstantial, the specific error + * code can provide more information about the possible recovery options. + *

+ * The reason why this is a RuntimeException is to allow it to go through interfaces defined by + * AIDL, which we have no control over. + * + * @hide + */ +public class RecoverableException extends RuntimeException { + public final int errorCode; + + public RecoverableException(int errorCode, @NonNull String message) { + super(message); + this.errorCode = errorCode; + } + + public RecoverableException(int errorCode) { + this.errorCode = errorCode; + } + + @Override + public @NonNull String toString() { + return super.toString() + " (code " + errorCode + ")"; + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java new file mode 100644 index 0000000000000..4a852c4b68e83 --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java @@ -0,0 +1,470 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.media.soundtrigger_middleware.Status; +import android.os.IHwBinder; +import android.os.RemoteException; + +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * An implementation of {@link ISoundTriggerHw2}, on top of any + * android.hardware.soundtrigger.V2_x.ISoundTriggerHw implementation. This class hides away some of + * the details involved with retaining backward compatibility and adapts to the more pleasant syntax + * exposed by {@link ISoundTriggerHw2}, compared to the bare driver interface. + *

+ * Exception handling: + *

    + *
  • All {@link RemoteException}s get rethrown as {@link RuntimeException}. + *
  • All HAL malfunctions get thrown as {@link HalException}. + *
  • All unsupported operations get thrown as {@link RecoverableException} with a + * {@link android.media.soundtrigger_middleware.Status#OPERATION_NOT_SUPPORTED} + * code. + *
+ */ +final class SoundTriggerHw2Compat implements ISoundTriggerHw2 { + private final @NonNull + IHwBinder mBinder; + private final @NonNull + android.hardware.soundtrigger.V2_0.ISoundTriggerHw mUnderlying_2_0; + private final @Nullable + android.hardware.soundtrigger.V2_1.ISoundTriggerHw mUnderlying_2_1; + private final @Nullable + android.hardware.soundtrigger.V2_2.ISoundTriggerHw mUnderlying_2_2; + private final @Nullable + android.hardware.soundtrigger.V2_3.ISoundTriggerHw mUnderlying_2_3; + + public SoundTriggerHw2Compat( + @NonNull android.hardware.soundtrigger.V2_0.ISoundTriggerHw underlying) { + this(underlying.asBinder()); + } + + public SoundTriggerHw2Compat(IHwBinder binder) { + Objects.requireNonNull(binder); + + mBinder = binder; + + // We want to share the proxy instances rather than create a separate proxy for every + // version, so we go down the versions in descending order to find the latest one supported, + // and then simply up-cast it to obtain all the versions that are earlier. + + // Attempt 2.3 + android.hardware.soundtrigger.V2_3.ISoundTriggerHw as2_3 = + android.hardware.soundtrigger.V2_3.ISoundTriggerHw.asInterface(binder); + if (as2_3 != null) { + mUnderlying_2_0 = mUnderlying_2_1 = mUnderlying_2_2 = mUnderlying_2_3 = as2_3; + return; + } + + // Attempt 2.2 + android.hardware.soundtrigger.V2_2.ISoundTriggerHw as2_2 = + android.hardware.soundtrigger.V2_2.ISoundTriggerHw.asInterface(binder); + if (as2_2 != null) { + mUnderlying_2_0 = mUnderlying_2_1 = mUnderlying_2_2 = as2_2; + mUnderlying_2_3 = null; + return; + } + + // Attempt 2.1 + android.hardware.soundtrigger.V2_1.ISoundTriggerHw as2_1 = + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.asInterface(binder); + if (as2_1 != null) { + mUnderlying_2_0 = mUnderlying_2_1 = as2_1; + mUnderlying_2_2 = mUnderlying_2_3 = null; + return; + } + + // Attempt 2.0 + android.hardware.soundtrigger.V2_0.ISoundTriggerHw as2_0 = + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.asInterface(binder); + if (as2_0 != null) { + mUnderlying_2_0 = as2_0; + mUnderlying_2_1 = mUnderlying_2_2 = mUnderlying_2_3 = null; + return; + } + + throw new RuntimeException("Binder doesn't support ISoundTriggerHw@2.0"); + } + + private static void handleHalStatus(int status, String methodName) { + if (status != 0) { + throw new HalException(status, methodName); + } + } + + @Override + public android.hardware.soundtrigger.V2_1.ISoundTriggerHw.Properties getProperties() { + try { + AtomicInteger retval = new AtomicInteger(-1); + AtomicReference + properties = + new AtomicReference<>(); + as2_0().getProperties( + (r, p) -> { + retval.set(r); + properties.set(p); + }); + handleHalStatus(retval.get(), "getProperties"); + return properties.get(); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + } + + @Override + public int loadSoundModel( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.SoundModel soundModel, + Callback callback, int cookie) { + try { + AtomicInteger retval = new AtomicInteger(-1); + AtomicInteger handle = new AtomicInteger(0); + try { + as2_1().loadSoundModel_2_1(soundModel, new SoundTriggerCallback(callback), cookie, + (r, h) -> { + retval.set(r); + handle.set(h); + }); + } catch (NotSupported e) { + // Fall-back to the 2.0 version: + return loadSoundModel_2_0(soundModel, callback, cookie); + } + handleHalStatus(retval.get(), "loadSoundModel_2_1"); + return handle.get(); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + } + + @Override + public int loadPhraseSoundModel( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.PhraseSoundModel soundModel, + Callback callback, int cookie) { + try { + AtomicInteger retval = new AtomicInteger(-1); + AtomicInteger handle = new AtomicInteger(0); + try { + as2_1().loadPhraseSoundModel_2_1(soundModel, new SoundTriggerCallback(callback), + cookie, + (r, h) -> { + retval.set(r); + handle.set(h); + }); + } catch (NotSupported e) { + // Fall-back to the 2.0 version: + return loadPhraseSoundModel_2_0(soundModel, callback, cookie); + } + handleHalStatus(retval.get(), "loadSoundModel_2_1"); + return handle.get(); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + } + + @Override + public void unloadSoundModel(int modelHandle) { + try { + int retval = as2_0().unloadSoundModel(modelHandle); + handleHalStatus(retval, "unloadSoundModel"); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + } + + @Override + public void stopRecognition(int modelHandle) { + try { + int retval = as2_0().stopRecognition(modelHandle); + handleHalStatus(retval, "stopRecognition"); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + + } + + @Override + public void stopAllRecognitions() { + try { + int retval = as2_0().stopAllRecognitions(); + handleHalStatus(retval, "stopAllRecognitions"); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + } + + @Override + public void startRecognition(int modelHandle, + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.RecognitionConfig config, + Callback callback, int cookie) { + try { + try { + int retval = as2_1().startRecognition_2_1(modelHandle, config, + new SoundTriggerCallback(callback), cookie); + handleHalStatus(retval, "startRecognition_2_1"); + } catch (NotSupported e) { + // Fall-back to the 2.0 version: + startRecognition_2_0(modelHandle, config, callback, cookie); + } + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + } + + @Override + public void getModelState(int modelHandle) { + try { + int retval = as2_2().getModelState(modelHandle); + handleHalStatus(retval, "getModelState"); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } catch (NotSupported e) { + throw e.throwAsRecoverableException(); + } + } + + @Override + public int getModelParameter(int modelHandle, int param) { + AtomicInteger status = new AtomicInteger(-1); + AtomicInteger value = new AtomicInteger(0); + try { + as2_3().getParameter(modelHandle, param, + (s, v) -> { + status.set(s); + value.set(v); + }); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } catch (NotSupported e) { + throw e.throwAsRecoverableException(); + } + handleHalStatus(status.get(), "getParameter"); + return value.get(); + } + + @Override + public void setModelParameter(int modelHandle, int param, int value) { + try { + int retval = as2_3().setParameter(modelHandle, param, value); + handleHalStatus(retval, "setParameter"); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } catch (NotSupported e) { + throw e.throwAsRecoverableException(); + } + } + + @Override + public android.hardware.soundtrigger.V2_3.ModelParameterRange queryParameter(int modelHandle, + int param) { + AtomicInteger status = new AtomicInteger(-1); + AtomicReference + optionalRange = + new AtomicReference<>(); + try { + as2_3().queryParameter(modelHandle, param, + (s, r) -> { + status.set(s); + optionalRange.set(r); + }); + } catch (NotSupported e) { + // For older drivers, we consider no model parameter to be supported. + return null; + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + handleHalStatus(status.get(), "queryParameter"); + return (optionalRange.get().getDiscriminator() + == android.hardware.soundtrigger.V2_3.OptionalModelParameterRange.hidl_discriminator.range) + ? + optionalRange.get().range() : null; + } + + @Override + public boolean linkToDeath(IHwBinder.DeathRecipient recipient, long cookie) { + return mBinder.linkToDeath(recipient, cookie); + } + + @Override + public boolean unlinkToDeath(IHwBinder.DeathRecipient recipient) { + return mBinder.unlinkToDeath(recipient); + } + + @Override + public String interfaceDescriptor() throws RemoteException { + return as2_0().interfaceDescriptor(); + } + + private int loadSoundModel_2_0( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.SoundModel soundModel, + Callback callback, int cookie) + throws RemoteException { + // Convert the soundModel to V2.0. + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.SoundModel model_2_0 = + Hw2CompatUtil.convertSoundModel_2_1_to_2_0(soundModel); + + AtomicInteger retval = new AtomicInteger(-1); + AtomicInteger handle = new AtomicInteger(0); + as2_0().loadSoundModel(model_2_0, new SoundTriggerCallback(callback), cookie, (r, h) -> { + retval.set(r); + handle.set(h); + }); + handleHalStatus(retval.get(), "loadSoundModel"); + return handle.get(); + } + + private int loadPhraseSoundModel_2_0( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.PhraseSoundModel soundModel, + Callback callback, int cookie) + throws RemoteException { + // Convert the soundModel to V2.0. + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.PhraseSoundModel model_2_0 = + Hw2CompatUtil.convertPhraseSoundModel_2_1_to_2_0(soundModel); + + AtomicInteger retval = new AtomicInteger(-1); + AtomicInteger handle = new AtomicInteger(0); + as2_0().loadPhraseSoundModel(model_2_0, new SoundTriggerCallback(callback), cookie, + (r, h) -> { + retval.set(r); + handle.set(h); + }); + handleHalStatus(retval.get(), "loadSoundModel"); + return handle.get(); + } + + private void startRecognition_2_0(int modelHandle, + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.RecognitionConfig config, + Callback callback, int cookie) + throws RemoteException { + + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.RecognitionConfig config_2_0 = + Hw2CompatUtil.convertRecognitionConfig_2_1_to_2_0(config); + int retval = as2_0().startRecognition(modelHandle, config_2_0, + new SoundTriggerCallback(callback), cookie); + handleHalStatus(retval, "startRecognition"); + } + + private @NonNull + android.hardware.soundtrigger.V2_0.ISoundTriggerHw as2_0() { + return mUnderlying_2_0; + } + + private @NonNull + android.hardware.soundtrigger.V2_1.ISoundTriggerHw as2_1() throws NotSupported { + if (mUnderlying_2_1 == null) { + throw new NotSupported("Underlying driver version < 2.1"); + } + return mUnderlying_2_1; + } + + private @NonNull + android.hardware.soundtrigger.V2_2.ISoundTriggerHw as2_2() throws NotSupported { + if (mUnderlying_2_2 == null) { + throw new NotSupported("Underlying driver version < 2.2"); + } + return mUnderlying_2_2; + } + + private @NonNull + android.hardware.soundtrigger.V2_3.ISoundTriggerHw as2_3() throws NotSupported { + if (mUnderlying_2_3 == null) { + throw new NotSupported("Underlying driver version < 2.3"); + } + return mUnderlying_2_3; + } + + /** + * A checked exception representing the requested interface version not being supported. + * At the public interface layer, use {@link #throwAsRecoverableException()} to propagate it to + * the caller if the request cannot be fulfilled. + */ + private static class NotSupported extends Exception { + NotSupported(String message) { + super(message); + } + + /** + * Throw this as a recoverable exception. + * + * @return Never actually returns anything. Always throws. Used so that caller can write + * throw e.throwAsRecoverableException(). + */ + RecoverableException throwAsRecoverableException() { + throw new RecoverableException(Status.OPERATION_NOT_SUPPORTED, getMessage()); + } + } + + private static class SoundTriggerCallback extends + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.Stub { + private final @NonNull + Callback mDelegate; + + private SoundTriggerCallback( + @NonNull Callback delegate) { + mDelegate = Objects.requireNonNull(delegate); + } + + @Override + public void recognitionCallback_2_1( + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent event, + int cookie) { + mDelegate.recognitionCallback(event, cookie); + } + + @Override + public void phraseRecognitionCallback_2_1( + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent event, + int cookie) { + mDelegate.phraseRecognitionCallback(event, cookie); + } + + @Override + public void soundModelCallback_2_1( + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.ModelEvent event, + int cookie) { + // Nobody cares. + } + + @Override + public void recognitionCallback( + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionEvent event, + int cookie) { + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent event_2_1 = + Hw2CompatUtil.convertRecognitionEvent_2_0_to_2_1(event); + mDelegate.recognitionCallback(event_2_1, cookie); + } + + @Override + public void phraseRecognitionCallback( + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.PhraseRecognitionEvent event, + int cookie) { + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent + event_2_1 = Hw2CompatUtil.convertPhraseRecognitionEvent_2_0_to_2_1(event); + mDelegate.phraseRecognitionCallback(event_2_1, cookie); + } + + @Override + public void soundModelCallback( + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.ModelEvent event, + int cookie) { + // Nobody cares. + } + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImpl.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImpl.java new file mode 100644 index 0000000000000..9d51b65ea1527 --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.annotation.NonNull; +import android.hardware.soundtrigger.V2_0.ISoundTriggerHw; +import android.media.soundtrigger_middleware.ISoundTriggerCallback; +import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService; +import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; +import android.os.IBinder; +import android.util.Log; + +import java.util.ArrayList; +import java.util.List; + +/** + * This is an implementation of the ISoundTriggerMiddlewareService interface. + *

+ * Important conventions: + *

    + *
  • Correct usage is assumed. This implementation does not attempt to gracefully handle invalid + * usage, and such usage will result in undefined behavior. If this service is to be offered to an + * untrusted client, it must be wrapped with input and state validation. + *
  • There is no binder instance associated with this implementation. Do not call asBinder(). + *
  • The implementation may throw a {@link RecoverableException} to indicate non-fatal, + * recoverable faults. The error code would one of the + * {@link android.media.soundtrigger_middleware.Status} + * constants. Any other exception thrown should be regarded as a bug in the implementation or one + * of its dependencies (assuming correct usage). + *
  • The implementation is designed for testibility by featuring dependency injection (the + * underlying HAL driver instances are passed to the ctor) and by minimizing dependencies on + * Android runtime. + *
  • The implementation is thread-safe. + *
+ * + * @hide + */ +public class SoundTriggerMiddlewareImpl implements ISoundTriggerMiddlewareService { + static private final String TAG = "SoundTriggerMiddlewareImpl"; + private final SoundTriggerModule[] mModules; + + /** + * Interface to the audio system, which can allocate capture session handles. + * SoundTrigger uses those sessions in order to associate a recognition session with an optional + * capture from the same device that triggered the recognition. + */ + public static abstract class AudioSessionProvider { + public static final class AudioSession { + final int mSessionHandle; + final int mIoHandle; + final int mDeviceHandle; + + AudioSession(int sessionHandle, int ioHandle, int deviceHandle) { + mSessionHandle = sessionHandle; + mIoHandle = ioHandle; + mDeviceHandle = deviceHandle; + } + } + + public abstract AudioSession acquireSession(); + + public abstract void releaseSession(int sessionHandle); + } + + /** + * Most generic constructor - gets an array of HAL driver instances. + */ + public SoundTriggerMiddlewareImpl(@NonNull ISoundTriggerHw[] halServices, + @NonNull AudioSessionProvider audioSessionProvider) { + List modules = new ArrayList<>(halServices.length); + + for (int i = 0; i < halServices.length; ++i) { + ISoundTriggerHw service = halServices[i]; + try { + modules.add(new SoundTriggerModule(service, audioSessionProvider)); + } catch (Exception e) { + Log.e(TAG, "Failed to a SoundTriggerModule instance", e); + } + } + + mModules = modules.toArray(new SoundTriggerModule[modules.size()]); + } + + /** + * Convenience constructor - gets a single HAL driver instance. + */ + public SoundTriggerMiddlewareImpl(@NonNull ISoundTriggerHw halService, + @NonNull AudioSessionProvider audioSessionProvider) { + this(new ISoundTriggerHw[]{halService}, audioSessionProvider); + } + + @Override + public @NonNull + SoundTriggerModuleDescriptor[] listModules() { + SoundTriggerModuleDescriptor[] result = new SoundTriggerModuleDescriptor[mModules.length]; + + for (int i = 0; i < mModules.length; ++i) { + SoundTriggerModuleDescriptor desc = new SoundTriggerModuleDescriptor(); + desc.handle = i; + desc.properties = mModules[i].getProperties(); + result[i] = desc; + } + return result; + } + + @Override + public @NonNull + ISoundTriggerModule attach(int handle, @NonNull ISoundTriggerCallback callback) { + return mModules[handle].attach(callback); + } + + @Override + public void setExternalCaptureState(boolean active) { + for (SoundTriggerModule module : mModules) { + module.setExternalCaptureState(active); + } + } + + @Override + public @NonNull + IBinder asBinder() { + throw new UnsupportedOperationException( + "This implementation is not inteded to be used directly with Binder."); + } +} \ No newline at end of file diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareService.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareService.java new file mode 100644 index 0000000000000..a7cfe1037f11a --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareService.java @@ -0,0 +1,709 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.Manifest; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.Context; +import android.hardware.soundtrigger.V2_0.ISoundTriggerHw; +import android.media.soundtrigger_middleware.ISoundTriggerCallback; +import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService; +import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.ModelParameterRange; +import android.media.soundtrigger_middleware.PhraseRecognitionEvent; +import android.media.soundtrigger_middleware.PhraseSoundModel; +import android.media.soundtrigger_middleware.RecognitionConfig; +import android.media.soundtrigger_middleware.RecognitionEvent; +import android.media.soundtrigger_middleware.RecognitionStatus; +import android.media.soundtrigger_middleware.SoundModel; +import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; +import android.os.RemoteException; +import android.os.ServiceSpecificException; +import android.util.Log; + +import com.android.internal.util.Preconditions; +import com.android.server.SystemService; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * This is a wrapper around an {@link ISoundTriggerMiddlewareService} implementation, which exposes + * it as a Binder service and enforces permissions and correct usage by the client, as well as makes + * sure that exceptions representing a server malfunction do not get sent to the client. + *

+ * This is intended to extract the non-business logic out of the underlying implementation and thus + * make it easier to maintain each one of those separate aspects. A design trade-off is being made + * here, in that this class would need to essentially eavesdrop on all the client-server + * communication and retain all state known to the client, while the client doesn't necessarily care + * about all of it, and while the server has its own representation of this information. However, + * in this case, this is a small amount of data, and the benefits in code elegance seem worth it. + * There is also some additional cost in employing a simplistic locking mechanism here, but + * following the same line of reasoning, the benefits in code simplicity outweigh it. + *

+ * Every public method in this class, overriding an interface method, must follow the following + * pattern: + * 

+ * @Override public T method(S arg) {
+ *     // Permission check.
+ *     checkPermissions();
+ *     // Input validation.
+ *     ValidationUtil.validateS(arg);
+ *     synchronized (this) {
+ *         // State validation.
+ *         if (...state is not valid for this call...) {
+ *             throw new IllegalStateException("State is invalid because...");
+ *         }
+ *         // From here on, every exception isn't client's fault.
+ *         try {
+ *             T result = mDelegate.method(arg);
+ *             // Update state.;
+ *             ...
+ *             return result;
+ *         } catch (Exception e) {
+ *             throw handleException(e);
+ *         }
+ *     }
+ * }
+ *
+ * Following this patterns ensures a consistent and rigorous handling of all aspects associated + * with client-server separation. + *

+ * Exception handling approach:
+ * We make sure all client faults (permissions, argument and state validation) happen first, and + * would throw {@link SecurityException}, {@link IllegalArgumentException}/ + * {@link NullPointerException} or {@link + * IllegalStateException}, respectively. All those exceptions are treated specially by Binder and + * will get sent back to the client.
+ * Once this is done, any subsequent fault is considered a server fault. Only {@link + * RecoverableException}s thrown by the implementation are special-cased: they would get sent back + * to the caller as a {@link ServiceSpecificException}, which is the behavior of Binder. Any other + * exception gets wrapped with a {@link InternalServerError}, which is specifically chosen as a type + * that does NOT get forwarded by binder. Those exceptions would be handled by a high-level + * exception handler on the server side, typically resulting in rebooting the server. + *

+ * Exposing this service as a System Service:
+ * Insert this line into {@link com.android.server.SystemServer}: + * 

+ * mSystemServiceManager.startService(SoundTriggerMiddlewareService.Lifecycle.class);
+ *
+ * + * {@hide} + */ +public class SoundTriggerMiddlewareService extends ISoundTriggerMiddlewareService.Stub { + static private final String TAG = "SoundTriggerMiddlewareService"; + + final ISoundTriggerMiddlewareService mDelegate; + final Context mContext; + Set mModuleHandles; + + /** + * Constructor for internal use only. Could be exposed for testing purposes in the future. + * Users should access this class via {@link Lifecycle}. + */ + private SoundTriggerMiddlewareService( + @NonNull ISoundTriggerMiddlewareService delegate, @NonNull Context context) { + mDelegate = delegate; + mContext = context; + } + + /** + * Generic exception handling for exceptions thrown by the underlying implementation. + * + * Would throw any {@link RecoverableException} as a {@link ServiceSpecificException} (passed + * by Binder to the caller) and any other exception as {@link InternalServerError} + * (not passed by Binder to the caller). + *

+ * Typical usage: + * 

+     * try {
+     *     ... Do server operations ...
+     * } catch (Exception e) {
+     *     throw handleException(e);
+     * }
+     *
+ */ + private static @NonNull + RuntimeException handleException(@NonNull Exception e) { + if (e instanceof RecoverableException) { + throw new ServiceSpecificException(((RecoverableException) e).errorCode, + e.getMessage()); + } + throw new InternalServerError(e); + } + + @Override + public @NonNull + SoundTriggerModuleDescriptor[] listModules() { + // Permission check. + checkPermissions(); + // Input validation (always valid). + + synchronized (this) { + // State validation (always valid). + + // From here on, every exception isn't client's fault. + try { + SoundTriggerModuleDescriptor[] result = mDelegate.listModules(); + mModuleHandles = new HashSet<>(result.length); + for (SoundTriggerModuleDescriptor desc : result) { + mModuleHandles.add(desc.handle); + } + return result; + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public @NonNull + ISoundTriggerModule attach(int handle, @NonNull ISoundTriggerCallback callback) { + // Permission check. + checkPermissions(); + // Input validation. + Preconditions.checkNotNull(callback); + Preconditions.checkNotNull(callback.asBinder()); + + synchronized (this) { + // State validation. + if (mModuleHandles == null) { + throw new IllegalStateException( + "Client must call listModules() prior to attaching."); + } + if (!mModuleHandles.contains(handle)) { + throw new IllegalArgumentException("Invalid handle: " + handle); + } + + // From here on, every exception isn't client's fault. + try { + ModuleService moduleService = new ModuleService(callback); + moduleService.attach(mDelegate.attach(handle, moduleService)); + return moduleService; + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public void setExternalCaptureState(boolean active) { + // Permission check. + checkPreemptPermissions(); + // Input validation (always valid). + + synchronized (this) { + // State validation (always valid). + + // From here on, every exception isn't client's fault. + try { + mDelegate.setExternalCaptureState(active); + } catch (Exception e) { + throw handleException(e); + } + } + } + + /** + * Throws a {@link SecurityException} if caller doesn't have the right permissions to use this + * service. + */ + private void checkPermissions() { + mContext.enforceCallingOrSelfPermission(Manifest.permission.RECORD_AUDIO, + "Caller must have the android.permission.RECORD_AUDIO permission."); + mContext.enforceCallingOrSelfPermission(Manifest.permission.CAPTURE_AUDIO_HOTWORD, + "Caller must have the android.permission.CAPTURE_AUDIO_HOTWORD permission."); + } + + /** + * Throws a {@link SecurityException} if caller doesn't have the right permissions to preempt + * active sound trigger sessions. + */ + private void checkPreemptPermissions() { + mContext.enforceCallingOrSelfPermission(Manifest.permission.PREEMPT_SOUND_TRIGGER, + "Caller must have the android.permission.PREEMPT_SOUND_TRIGGER permission."); + } + + /** State of a sound model. */ + static class ModelState { + /** Activity state of a sound model. */ + enum Activity { + /** Model is loaded, recognition is inactive. */ + LOADED, + /** Model is loaded, recognition is active. */ + ACTIVE + } + + /** Activity state. */ + public Activity activityState = Activity.LOADED; + + /** + * A map of known parameter support. A missing key means we don't know yet whether the + * parameter is supported. A null value means it is known to not be supported. A non-null + * value indicates the valid value range. + */ + private Map parameterSupport = new HashMap<>(); + + /** + * Check that the given parameter is known to be supported for this model. + * + * @param modelParam The parameter key. + */ + public void checkSupported(int modelParam) { + if (!parameterSupport.containsKey(modelParam)) { + throw new IllegalStateException("Parameter has not been checked for support."); + } + ModelParameterRange range = parameterSupport.get(modelParam); + if (range == null) { + throw new IllegalArgumentException("Paramater is not supported."); + } + } + + /** + * Check that the given parameter is known to be supported for this model and that the given + * value is a valid value for it. + * + * @param modelParam The parameter key. + * @param value The value. + */ + public void checkSupported(int modelParam, int value) { + if (!parameterSupport.containsKey(modelParam)) { + throw new IllegalStateException("Parameter has not been checked for support."); + } + ModelParameterRange range = parameterSupport.get(modelParam); + if (range == null) { + throw new IllegalArgumentException("Paramater is not supported."); + } + Preconditions.checkArgumentInRange(value, range.minInclusive, range.maxInclusive, + "value"); + } + + /** + * Update support state for the given parameter for this model. + * + * @param modelParam The parameter key. + * @param range The parameter value range, or null if not supported. + */ + public void updateParameterSupport(int modelParam, @Nullable ModelParameterRange range) { + parameterSupport.put(modelParam, range); + } + } + + /** + * Entry-point to this module: exposes the module as a {@link SystemService}. + */ + public static final class Lifecycle extends SystemService { + private SoundTriggerMiddlewareService mService; + + public Lifecycle(Context context) { + super(context); + } + + @Override + public void onStart() { + ISoundTriggerHw[] services; + try { + services = new ISoundTriggerHw[]{ISoundTriggerHw.getService(true)}; + Log.d(TAG, "Connected to default ISoundTriggerHw"); + } catch (Exception e) { + Log.e(TAG, "Failed to connect to default ISoundTriggerHw", e); + services = new ISoundTriggerHw[0]; + } + + mService = new SoundTriggerMiddlewareService( + new SoundTriggerMiddlewareImpl(services, new AudioSessionProviderImpl()), + getContext()); + publishBinderService(Context.SOUND_TRIGGER_MIDDLEWARE_SERVICE, mService); + } + } + + /** + * A wrapper around an {@link ISoundTriggerModule} implementation, to address the same aspects + * mentioned in {@link SoundTriggerModule} above. This class follows the same conventions. + */ + private class ModuleService extends ISoundTriggerModule.Stub implements ISoundTriggerCallback, + DeathRecipient { + private final ISoundTriggerCallback mCallback; + private ISoundTriggerModule mDelegate; + private Map mLoadedModels = new HashMap<>(); + + ModuleService(@NonNull ISoundTriggerCallback callback) { + mCallback = callback; + try { + mCallback.asBinder().linkToDeath(this, 0); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + } + + void attach(@NonNull ISoundTriggerModule delegate) { + mDelegate = delegate; + } + + @Override + public int loadModel(@NonNull SoundModel model) { + // Permission check. + checkPermissions(); + // Input validation. + ValidationUtil.validateGenericModel(model); + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + + // From here on, every exception isn't client's fault. + try { + int handle = mDelegate.loadModel(model); + mLoadedModels.put(handle, new ModelState()); + return handle; + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public int loadPhraseModel(@NonNull PhraseSoundModel model) { + // Permission check. + checkPermissions(); + // Input validation. + ValidationUtil.validatePhraseModel(model); + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + + // From here on, every exception isn't client's fault. + try { + int handle = mDelegate.loadPhraseModel(model); + mLoadedModels.put(handle, new ModelState()); + return handle; + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public void unloadModel(int modelHandle) { + // Permission check. + checkPermissions(); + // Input validation (always valid). + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + ModelState modelState = mLoadedModels.get(modelHandle); + if (modelState == null) { + throw new IllegalStateException("Invalid handle: " + modelHandle); + } + if (modelState.activityState != ModelState.Activity.LOADED) { + throw new IllegalStateException("Model with handle: " + modelHandle + + " has invalid state for unloading: " + modelState.activityState); + } + + // From here on, every exception isn't client's fault. + try { + mDelegate.unloadModel(modelHandle); + mLoadedModels.remove(modelHandle); + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public void startRecognition(int modelHandle, @NonNull RecognitionConfig config) { + // Permission check. + checkPermissions(); + // Input validation. + ValidationUtil.validateRecognitionConfig(config); + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + ModelState modelState = mLoadedModels.get(modelHandle); + if (modelState == null) { + throw new IllegalStateException("Invalid handle: " + modelHandle); + } + if (modelState.activityState != ModelState.Activity.LOADED) { + throw new IllegalStateException("Model with handle: " + modelHandle + + " has invalid state for starting recognition: " + + modelState.activityState); + } + + // From here on, every exception isn't client's fault. + try { + mDelegate.startRecognition(modelHandle, config); + modelState.activityState = ModelState.Activity.ACTIVE; + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public void stopRecognition(int modelHandle) { + // Permission check. + checkPermissions(); + // Input validation (always valid). + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + ModelState modelState = mLoadedModels.get(modelHandle); + if (modelState == null) { + throw new IllegalStateException("Invalid handle: " + modelHandle); + } + // stopRecognition is idempotent - no need to check model state. + + // From here on, every exception isn't client's fault. + try { + mDelegate.stopRecognition(modelHandle); + modelState.activityState = ModelState.Activity.LOADED; + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public void forceRecognitionEvent(int modelHandle) { + // Permission check. + checkPermissions(); + // Input validation (always valid). + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + ModelState modelState = mLoadedModels.get(modelHandle); + if (modelState == null) { + throw new IllegalStateException("Invalid handle: " + modelHandle); + } + // forceRecognitionEvent is idempotent - no need to check model state. + + // From here on, every exception isn't client's fault. + try { + mDelegate.forceRecognitionEvent(modelHandle); + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public void setModelParameter(int modelHandle, int modelParam, int value) { + // Permission check. + checkPermissions(); + // Input validation. + ValidationUtil.validateModelParameter(modelParam); + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + ModelState modelState = mLoadedModels.get(modelHandle); + if (modelState == null) { + throw new IllegalStateException("Invalid handle: " + modelHandle); + } + modelState.checkSupported(modelParam, value); + + // From here on, every exception isn't client's fault. + try { + mDelegate.setModelParameter(modelHandle, modelParam, value); + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public int getModelParameter(int modelHandle, int modelParam) { + // Permission check. + checkPermissions(); + // Input validation. + ValidationUtil.validateModelParameter(modelParam); + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + ModelState modelState = mLoadedModels.get(modelHandle); + if (modelState == null) { + throw new IllegalStateException("Invalid handle: " + modelHandle); + } + modelState.checkSupported(modelParam); + + // From here on, every exception isn't client's fault. + try { + return mDelegate.getModelParameter(modelHandle, modelParam); + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + @Nullable + public ModelParameterRange queryModelParameterSupport(int modelHandle, int modelParam) { + // Permission check. + checkPermissions(); + // Input validation. + ValidationUtil.validateModelParameter(modelParam); + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has been detached."); + } + ModelState modelState = mLoadedModels.get(modelHandle); + if (modelState == null) { + throw new IllegalStateException("Invalid handle: " + modelHandle); + } + + // From here on, every exception isn't client's fault. + try { + ModelParameterRange result = mDelegate.queryModelParameterSupport(modelHandle, + modelParam); + modelState.updateParameterSupport(modelParam, result); + return result; + } catch (Exception e) { + throw handleException(e); + } + } + } + + @Override + public void detach() { + // Permission check. + checkPermissions(); + // Input validation (always valid). + + synchronized (this) { + // State validation. + if (mDelegate == null) { + throw new IllegalStateException("Module has already been detached."); + } + if (!mLoadedModels.isEmpty()) { + throw new IllegalStateException("Cannot detach while models are loaded."); + } + + // From here on, every exception isn't client's fault. + try { + detachInternal(); + } catch (Exception e) { + throw handleException(e); + } + } + } + + private void detachInternal() { + try { + mDelegate.detach(); + mDelegate = null; + mCallback.asBinder().unlinkToDeath(this, 0); + } catch (RemoteException e) { + throw e.rethrowAsRuntimeException(); + } + } + + //////////////////////////////////////////////////////////////////////////////////////////// + // Callbacks + + @Override + public void onRecognition(int modelHandle, @NonNull RecognitionEvent event) { + synchronized (this) { + if (event.status != RecognitionStatus.FORCED) { + mLoadedModels.get(modelHandle).activityState = ModelState.Activity.LOADED; + } + try { + mCallback.onRecognition(modelHandle, event); + } catch (RemoteException e) { + // Dead client will be handled by binderDied() - no need to handle here. + // In any case, client callbacks are considered best effort. + Log.e(TAG, "Client callback execption.", e); + } + } + } + + @Override + public void onPhraseRecognition(int modelHandle, @NonNull PhraseRecognitionEvent event) { + synchronized (this) { + if (event.common.status != RecognitionStatus.FORCED) { + mLoadedModels.get(modelHandle).activityState = ModelState.Activity.LOADED; + } + try { + mCallback.onPhraseRecognition(modelHandle, event); + } catch (RemoteException e) { + // Dead client will be handled by binderDied() - no need to handle here. + // In any case, client callbacks are considered best effort. + Log.e(TAG, "Client callback execption.", e); + } + } + } + + @Override + public void onRecognitionAvailabilityChange(boolean available) throws RemoteException { + synchronized (this) { + try { + mCallback.onRecognitionAvailabilityChange(available); + } catch (RemoteException e) { + // Dead client will be handled by binderDied() - no need to handle here. + // In any case, client callbacks are considered best effort. + Log.e(TAG, "Client callback execption.", e); + } + } + } + + @Override + public void binderDied() { + // This is called whenever our client process dies. + synchronized (this) { + try { + // Gracefully stop all active recognitions and unload the models. + for (Map.Entry entry : mLoadedModels.entrySet()) { + if (entry.getValue().activityState == ModelState.Activity.ACTIVE) { + mDelegate.stopRecognition(entry.getKey()); + } + mDelegate.unloadModel(entry.getKey()); + } + // Detach. + detachInternal(); + } catch (Exception e) { + throw handleException(e); + } + } + } + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java new file mode 100644 index 0000000000000..3444be9353951 --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java @@ -0,0 +1,545 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback; +import android.hardware.soundtrigger.V2_2.ISoundTriggerHw; +import android.media.soundtrigger_middleware.ISoundTriggerCallback; +import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.ModelParameterRange; +import android.media.soundtrigger_middleware.PhraseSoundModel; +import android.media.soundtrigger_middleware.RecognitionConfig; +import android.media.soundtrigger_middleware.SoundModel; +import android.media.soundtrigger_middleware.SoundModelType; +import android.media.soundtrigger_middleware.SoundTriggerModuleProperties; +import android.media.soundtrigger_middleware.Status; +import android.os.IBinder; +import android.os.RemoteException; +import android.util.Log; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * This is an implementation of a single module of the ISoundTriggerMiddlewareService interface, + * exposing itself through the {@link ISoundTriggerModule} interface, possibly to multiple separate + * clients. + *

+ * Typical usage is to query the module capabilities using {@link #getProperties()} and then to use + * the module through an {@link ISoundTriggerModule} instance, obtained via {@link + * #attach(ISoundTriggerCallback)}. Every such interface is its own session and state is not shared + * between sessions (i.e. cannot use a handle obtained from one session through another). + *

+ * Important conventions: + *

    + *
  • Correct usage is assumed. This implementation does not attempt to gracefully handle + * invalid usage, and such usage will result in undefined behavior. If this service is to be + * offered to an untrusted client, it must be wrapped with input and state validation. + *
  • The underlying driver is assumed to be correct. This implementation does not attempt to + * gracefully handle driver malfunction and such behavior will result in undefined behavior. If this + * service is to used with an untrusted driver, the driver must be wrapped with validation / error + * recovery code. + *
  • RemoteExceptions thrown by the driver are treated as RuntimeExceptions - they are not + * considered recoverable faults and should not occur in a properly functioning system. + *
  • There is no binder instance associated with this implementation. Do not call asBinder(). + *
  • The implementation may throw a {@link RecoverableException} to indicate non-fatal, + * recoverable faults. The error code would one of the + * {@link android.media.soundtrigger_middleware.Status} constants. Any other exception + * thrown should be regarded as a bug in the implementation or one of its dependencies + * (assuming correct usage). + *
  • The implementation is designed for testibility by featuring dependency injection (the + * underlying HAL driver instances are passed to the ctor) and by minimizing dependencies + * on Android runtime. + *
  • The implementation is thread-safe. This is achieved by a simplistic model, where all entry- + * points (both client API and driver callbacks) obtain a lock on the SoundTriggerModule instance + * for their entire scope. Any other method can be assumed to be running with the lock already + * obtained, so no further locking should be done. While this is not necessarily the most efficient + * synchronization strategy, it is very easy to reason about and this code is likely not on any + * performance-critical + * path. + *
+ * + * @hide + */ +class SoundTriggerModule { + static private final String TAG = "SoundTriggerModule"; + @NonNull private final ISoundTriggerHw2 mHalService; + @NonNull private final SoundTriggerMiddlewareImpl.AudioSessionProvider mAudioSessionProvider; + private final Set mActiveSessions = new HashSet<>(); + private int mNumLoadedModels = 0; + private SoundTriggerModuleProperties mProperties = null; + private boolean mRecognitionAvailable; + + /** + * Ctor. + * + * @param halService The underlying HAL driver. + */ + SoundTriggerModule(@NonNull android.hardware.soundtrigger.V2_0.ISoundTriggerHw halService, + @NonNull SoundTriggerMiddlewareImpl.AudioSessionProvider audioSessionProvider) { + assert halService != null; + mHalService = new SoundTriggerHw2Compat(halService); + mAudioSessionProvider = audioSessionProvider; + mProperties = ConversionUtil.hidl2aidlProperties(mHalService.getProperties()); + + // We conservatively assume that external capture is active until explicitly told otherwise. + mRecognitionAvailable = mProperties.concurrentCapture; + } + + /** + * Establish a client session with this module. + * + * This module may be shared by multiple clients, each will get its own session. While resources + * are shared between the clients, each session has its own state and data should not be shared + * across sessions. + * + * @param callback The client callback, which will be used for all messages. This is a oneway + * callback, so will never block, throw an unchecked exception or return a + * value. + * @return The interface through which this module can be controlled. + */ + synchronized @NonNull + Session attach(@NonNull ISoundTriggerCallback callback) { + Log.d(TAG, "attach()"); + Session session = new Session(callback); + mActiveSessions.add(session); + return session; + } + + /** + * Query the module's properties. + * + * @return The properties structure. + */ + synchronized @NonNull + SoundTriggerModuleProperties getProperties() { + return mProperties; + } + + /** + * Notify the module that external capture has started / finished, using the same input device + * used for recognition. + * If the underlying driver does not support recognition while capturing, capture will be + * aborted, and the recognition callback will receive and abort event. In addition, all active + * clients will be notified of the change in state. + * + * @param active true iff external capture is active. + */ + synchronized void setExternalCaptureState(boolean active) { + Log.d(TAG, String.format("setExternalCaptureState(active=%b)", active)); + if (mProperties.concurrentCapture) { + // If we support concurrent capture, we don't care about any of this. + return; + } + mRecognitionAvailable = !active; + if (!mRecognitionAvailable) { + // Our module does not support recognition while a capture is active - + // need to abort all active recognitions. + for (Session session : mActiveSessions) { + session.abortActiveRecognitions(); + } + } + for (Session session : mActiveSessions) { + session.notifyRecognitionAvailability(); + } + } + + /** + * Remove session from the list of active sessions. + * + * @param session The session to remove. + */ + private void removeSession(@NonNull Session session) { + mActiveSessions.remove(session); + } + + /** State of a single sound model. */ + private enum ModelState { + /** Initial state, until load() is called. */ + INIT, + /** Model is loaded, but recognition is not active. */ + LOADED, + /** Model is loaded and recognition is active. */ + ACTIVE + } + + /** + * A single client session with this module. + * + * This is the main interface used to interact with this module. + */ + private class Session implements ISoundTriggerModule { + private ISoundTriggerCallback mCallback; + private Map mLoadedModels = new HashMap<>(); + + /** + * Ctor. + * + * @param callback The client callback interface. + */ + private Session(@NonNull ISoundTriggerCallback callback) { + mCallback = callback; + notifyRecognitionAvailability(); + } + + @Override + public void detach() { + Log.d(TAG, "detach()"); + synchronized (SoundTriggerModule.this) { + removeSession(this); + } + } + + @Override + public int loadModel(@NonNull SoundModel model) { + Log.d(TAG, String.format("loadModel(model=%s)", model)); + synchronized (SoundTriggerModule.this) { + if (mNumLoadedModels == mProperties.maxSoundModels) { + throw new RecoverableException(Status.RESOURCE_CONTENTION, + "Maximum number of models loaded."); + } + Model loadedModel = new Model(); + int result = loadedModel.load(model); + ++mNumLoadedModels; + return result; + } + } + + @Override + public int loadPhraseModel(@NonNull PhraseSoundModel model) { + Log.d(TAG, String.format("loadPhraseModel(model=%s)", model)); + synchronized (SoundTriggerModule.this) { + if (mNumLoadedModels == mProperties.maxSoundModels) { + throw new RecoverableException(Status.RESOURCE_CONTENTION, + "Maximum number of models loaded."); + } + Model loadedModel = new Model(); + int result = loadedModel.load(model); + ++mNumLoadedModels; + Log.d(TAG, String.format("loadPhraseModel()->%d", result)); + return result; + } + } + + @Override + public void unloadModel(int modelHandle) { + Log.d(TAG, String.format("unloadModel(handle=%d)", modelHandle)); + synchronized (SoundTriggerModule.this) { + mLoadedModels.get(modelHandle).unload(); + --mNumLoadedModels; + } + } + + @Override + public void startRecognition(int modelHandle, @NonNull RecognitionConfig config) { + Log.d(TAG, + String.format("startRecognition(handle=%d, config=%s)", modelHandle, config)); + synchronized (SoundTriggerModule.this) { + mLoadedModels.get(modelHandle).startRecognition(config); + } + } + + @Override + public void stopRecognition(int modelHandle) { + Log.d(TAG, String.format("stopRecognition(handle=%d)", modelHandle)); + synchronized (SoundTriggerModule.this) { + mLoadedModels.get(modelHandle).stopRecognition(); + } + } + + @Override + public void forceRecognitionEvent(int modelHandle) { + Log.d(TAG, String.format("forceRecognitionEvent(handle=%d)", modelHandle)); + synchronized (SoundTriggerModule.this) { + mLoadedModels.get(modelHandle).forceRecognitionEvent(); + } + } + + @Override + public void setModelParameter(int modelHandle, int modelParam, int value) + throws RemoteException { + Log.d(TAG, + String.format("setModelParameter(handle=%d, param=%d, value=%d)", modelHandle, + modelParam, value)); + synchronized (SoundTriggerModule.this) { + mLoadedModels.get(modelHandle).setParameter(modelParam, value); + } + } + + @Override + public int getModelParameter(int modelHandle, int modelParam) throws RemoteException { + Log.d(TAG, String.format("getModelParameter(handle=%d, param=%d)", modelHandle, + modelParam)); + synchronized (SoundTriggerModule.this) { + return mLoadedModels.get(modelHandle).getParameter(modelParam); + } + } + + @Override + @Nullable + public ModelParameterRange queryModelParameterSupport(int modelHandle, int modelParam) { + Log.d(TAG, String.format("queryModelParameterSupport(handle=%d, param=%d)", modelHandle, + modelParam)); + synchronized (SoundTriggerModule.this) { + return mLoadedModels.get(modelHandle).queryModelParameterSupport(modelParam); + } + } + + /** + * Abort all currently active recognitions. + */ + private void abortActiveRecognitions() { + for (Model model : mLoadedModels.values()) { + model.abortActiveRecognition(); + } + } + + private void notifyRecognitionAvailability() { + try { + mCallback.onRecognitionAvailabilityChange(mRecognitionAvailable); + } catch (RemoteException e) { + // Dead client will be handled by binderDied() - no need to handle here. + // In any case, client callbacks are considered best effort. + Log.e(TAG, "Client callback execption.", e); + } + } + + @Override + public @NonNull + IBinder asBinder() { + throw new UnsupportedOperationException( + "This implementation is not intended to be used directly with Binder."); + } + + /** + * A single sound model in the system. + * + * All model-based operations are delegated to this class and implemented here. + */ + private class Model implements ISoundTriggerHw2.Callback { + public int mHandle; + private ModelState mState = ModelState.INIT; + private int mModelType = SoundModelType.UNKNOWN; + private SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession mSession; + + private @NonNull + ModelState getState() { + return mState; + } + + private void setState(@NonNull ModelState state) { + mState = state; + SoundTriggerModule.this.notifyAll(); + } + + private void waitStateChange() throws InterruptedException { + SoundTriggerModule.this.wait(); + } + + private int load(@NonNull SoundModel model) { + mModelType = model.type; + ISoundTriggerHw.SoundModel hidlModel = ConversionUtil.aidl2hidlSoundModel(model); + + mSession = mAudioSessionProvider.acquireSession(); + try { + mHandle = mHalService.loadSoundModel(hidlModel, this, 0); + } catch (Exception e) { + mAudioSessionProvider.releaseSession(mSession.mSessionHandle); + throw e; + } + + setState(ModelState.LOADED); + mLoadedModels.put(mHandle, this); + return mHandle; + } + + private int load(@NonNull PhraseSoundModel model) { + mModelType = model.common.type; + ISoundTriggerHw.PhraseSoundModel hidlModel = + ConversionUtil.aidl2hidlPhraseSoundModel(model); + + mSession = mAudioSessionProvider.acquireSession(); + try { + mHandle = mHalService.loadPhraseSoundModel(hidlModel, this, 0); + } catch (Exception e) { + mAudioSessionProvider.releaseSession(mSession.mSessionHandle); + throw e; + } + + setState(ModelState.LOADED); + mLoadedModels.put(mHandle, this); + return mHandle; + } + + private void unload() { + mAudioSessionProvider.releaseSession(mSession.mSessionHandle); + mHalService.unloadSoundModel(mHandle); + mLoadedModels.remove(mHandle); + } + + private void startRecognition(@NonNull RecognitionConfig config) { + if (!mRecognitionAvailable) { + // Recognition is unavailable - send an abort event immediately. + notifyAbort(); + return; + } + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.RecognitionConfig hidlConfig = + ConversionUtil.aidl2hidlRecognitionConfig(config); + hidlConfig.header.captureDevice = mSession.mDeviceHandle; + hidlConfig.header.captureHandle = mSession.mIoHandle; + mHalService.startRecognition(mHandle, hidlConfig, this, 0); + setState(ModelState.ACTIVE); + } + + private void stopRecognition() { + if (getState() == ModelState.LOADED) { + // This call is idempotent in order to avoid races. + return; + } + mHalService.stopRecognition(mHandle); + setState(ModelState.LOADED); + } + + /** Request a forced recognition event. Will do nothing if recognition is inactive. */ + private void forceRecognitionEvent() { + if (getState() != ModelState.ACTIVE) { + // This call is idempotent in order to avoid races. + return; + } + mHalService.getModelState(mHandle); + } + + + private void setParameter(int modelParam, int value) { + mHalService.setModelParameter(mHandle, + ConversionUtil.aidl2hidlModelParameter(modelParam), value); + } + + private int getParameter(int modelParam) { + return mHalService.getModelParameter(mHandle, + ConversionUtil.aidl2hidlModelParameter(modelParam)); + } + + @Nullable + private ModelParameterRange queryModelParameterSupport(int modelParam) { + return ConversionUtil.hidl2aidlModelParameterRange( + mHalService.queryParameter(mHandle, + ConversionUtil.aidl2hidlModelParameter(modelParam))); + } + + /** Abort the recognition, if active. */ + private void abortActiveRecognition() { + // If we're inactive, do nothing. + if (getState() != ModelState.ACTIVE) { + return; + } + // Stop recognition. + stopRecognition(); + + // Notify the client that recognition has been aborted. + notifyAbort(); + } + + /** Notify the client that recognition has been aborted. */ + private void notifyAbort() { + try { + switch (mModelType) { + case SoundModelType.GENERIC: { + android.media.soundtrigger_middleware.RecognitionEvent event = + new android.media.soundtrigger_middleware.RecognitionEvent(); + event.status = + android.media.soundtrigger_middleware.RecognitionStatus.ABORTED; + mCallback.onRecognition(mHandle, event); + } + break; + + case SoundModelType.KEYPHRASE: { + android.media.soundtrigger_middleware.PhraseRecognitionEvent event = + new android.media.soundtrigger_middleware.PhraseRecognitionEvent(); + event.common = + new android.media.soundtrigger_middleware.RecognitionEvent(); + event.common.status = + android.media.soundtrigger_middleware.RecognitionStatus.ABORTED; + mCallback.onPhraseRecognition(mHandle, event); + } + break; + + default: + Log.e(TAG, "Unknown model type: " + mModelType); + + } + } catch (RemoteException e) { + // Dead client will be handled by binderDied() - no need to handle here. + // In any case, client callbacks are considered best effort. + Log.e(TAG, "Client callback execption.", e); + } + } + + @Override + public void recognitionCallback( + @NonNull ISoundTriggerHwCallback.RecognitionEvent recognitionEvent, + int cookie) { + Log.d(TAG, String.format("recognitionCallback_2_1(event=%s, cookie=%d)", + recognitionEvent, cookie)); + synchronized (SoundTriggerModule.this) { + android.media.soundtrigger_middleware.RecognitionEvent aidlEvent = + ConversionUtil.hidl2aidlRecognitionEvent(recognitionEvent); + aidlEvent.captureSession = mSession.mSessionHandle; + try { + mCallback.onRecognition(mHandle, aidlEvent); + } catch (RemoteException e) { + // Dead client will be handled by binderDied() - no need to handle here. + // In any case, client callbacks are considered best effort. + Log.e(TAG, "Client callback execption.", e); + } + if (aidlEvent.status + != android.media.soundtrigger_middleware.RecognitionStatus.FORCED) { + setState(ModelState.LOADED); + } + } + } + + @Override + public void phraseRecognitionCallback( + @NonNull ISoundTriggerHwCallback.PhraseRecognitionEvent phraseRecognitionEvent, + int cookie) { + Log.d(TAG, String.format("phraseRecognitionCallback_2_1(event=%s, cookie=%d)", + phraseRecognitionEvent, cookie)); + synchronized (SoundTriggerModule.this) { + android.media.soundtrigger_middleware.PhraseRecognitionEvent aidlEvent = + ConversionUtil.hidl2aidlPhraseRecognitionEvent(phraseRecognitionEvent); + aidlEvent.common.captureSession = mSession.mSessionHandle; + try { + mCallback.onPhraseRecognition(mHandle, aidlEvent); + } catch (RemoteException e) { + // Dead client will be handled by binderDied() - no need to handle here. + // In any case, client callbacks are considered best effort. + Log.e(TAG, "Client callback execption.", e); + } + if (aidlEvent.common.status + != android.media.soundtrigger_middleware.RecognitionStatus.FORCED) { + setState(ModelState.LOADED); + } + } + } + } + } +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/TEST_MAPPING b/services/core/java/com/android/server/soundtrigger_middleware/TEST_MAPPING new file mode 100644 index 0000000000000..9ed894bc1ca97 --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/TEST_MAPPING @@ -0,0 +1,12 @@ +{ + "presubmit": [ + { + "name": "FrameworksServicesTests", + "options": [ + { + "include-filter": "com.android.server.soundtrigger_middleware" + } + ] + } + ] +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/UuidUtil.java b/services/core/java/com/android/server/soundtrigger_middleware/UuidUtil.java new file mode 100644 index 0000000000000..80f69d08c0898 --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/UuidUtil.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import java.util.regex.Pattern; + +/** + * Utilities for representing UUIDs as strings. + * + * @hide + */ +public class UuidUtil { + /** + * Regex pattern that can be used to validate / extract the various fields of a string-formatted + * UUID. + */ + static final Pattern PATTERN = Pattern.compile("^([a-fA-F0-9]{8})-" + + "([a-fA-F0-9]{4})-" + + "([a-fA-F0-9]{4})-" + + "([a-fA-F0-9]{4})-" + + "([a-fA-F0-9]{2})" + + "([a-fA-F0-9]{2})" + + "([a-fA-F0-9]{2})" + + "([a-fA-F0-9]{2})" + + "([a-fA-F0-9]{2})" + + "([a-fA-F0-9]{2})$"); + + /** Printf-style pattern for formatting the various fields of a UUID as a string. */ + static final String FORMAT = "%08x-%04x-%04x-%04x-%02x%02x%02x%02x%02x%02x"; +} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/ValidationUtil.java b/services/core/java/com/android/server/soundtrigger_middleware/ValidationUtil.java new file mode 100644 index 0000000000000..4898e6b59ab28 --- /dev/null +++ b/services/core/java/com/android/server/soundtrigger_middleware/ValidationUtil.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import android.annotation.Nullable; +import android.media.soundtrigger_middleware.ConfidenceLevel; +import android.media.soundtrigger_middleware.ModelParameter; +import android.media.soundtrigger_middleware.Phrase; +import android.media.soundtrigger_middleware.PhraseRecognitionExtra; +import android.media.soundtrigger_middleware.PhraseSoundModel; +import android.media.soundtrigger_middleware.RecognitionConfig; +import android.media.soundtrigger_middleware.RecognitionMode; +import android.media.soundtrigger_middleware.SoundModel; +import android.media.soundtrigger_middleware.SoundModelType; + +import com.android.internal.util.Preconditions; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utilities for asserting the validity of various data types used by this module. + * Each of the methods below would throw an {@link IllegalArgumentException} if its input is + * invalid. The input's validity is determined irrespective of any context. In cases where the valid + * value space is further limited by state, it is the caller's responsibility to assert. + * + * @hide + */ +public class ValidationUtil { + static void validateUuid(@Nullable String uuid) { + Preconditions.checkNotNull(uuid); + Matcher matcher = UuidUtil.PATTERN.matcher(uuid); + if (!matcher.matches()) { + throw new IllegalArgumentException( + "Illegal format for UUID: " + uuid); + } + } + + static void validateGenericModel(@Nullable SoundModel model) { + validateModel(model, SoundModelType.GENERIC); + } + + static void validateModel(@Nullable SoundModel model, int expectedType) { + Preconditions.checkNotNull(model); + if (model.type != expectedType) { + throw new IllegalArgumentException("Invalid type"); + } + validateUuid(model.uuid); + validateUuid(model.vendorUuid); + Preconditions.checkNotNull(model.data); + } + + static void validatePhraseModel(@Nullable PhraseSoundModel model) { + Preconditions.checkNotNull(model); + validateModel(model.common, SoundModelType.KEYPHRASE); + Preconditions.checkNotNull(model.phrases); + for (Phrase phrase : model.phrases) { + Preconditions.checkNotNull(phrase); + if ((phrase.recognitionModes & ~(RecognitionMode.VOICE_TRIGGER + | RecognitionMode.USER_IDENTIFICATION | RecognitionMode.USER_AUTHENTICATION + | RecognitionMode.GENERIC_TRIGGER)) != 0) { + throw new IllegalArgumentException("Invalid recognitionModes"); + } + Preconditions.checkNotNull(phrase.users); + Preconditions.checkNotNull(phrase.locale); + Preconditions.checkNotNull(phrase.text); + } + } + + static void validateRecognitionConfig(@Nullable RecognitionConfig config) { + Preconditions.checkNotNull(config); + Preconditions.checkNotNull(config.phraseRecognitionExtras); + for (PhraseRecognitionExtra extra : config.phraseRecognitionExtras) { + Preconditions.checkNotNull(extra); + if ((extra.recognitionModes & ~(RecognitionMode.VOICE_TRIGGER + | RecognitionMode.USER_IDENTIFICATION | RecognitionMode.USER_AUTHENTICATION + | RecognitionMode.GENERIC_TRIGGER)) != 0) { + throw new IllegalArgumentException("Invalid recognitionModes"); + } + if (extra.confidenceLevel < 0 || extra.confidenceLevel > 100) { + throw new IllegalArgumentException("Invalid confidenceLevel"); + } + Preconditions.checkNotNull(extra.levels); + for (ConfidenceLevel level : extra.levels) { + Preconditions.checkNotNull(level); + if (level.levelPercent < 0 || level.levelPercent > 100) { + throw new IllegalArgumentException("Invalid confidenceLevel"); + } + } + } + Preconditions.checkNotNull(config.data); + } + + static void validateModelParameter(int modelParam) { + switch (modelParam) { + case ModelParameter.THRESHOLD_FACTOR: + return; + + default: + throw new IllegalArgumentException("Invalid model parameter"); + } + } +} diff --git a/services/core/jni/Android.bp b/services/core/jni/Android.bp index fd8094cc43ddd..a34b7fdb911c1 100644 --- a/services/core/jni/Android.bp +++ b/services/core/jni/Android.bp @@ -35,6 +35,7 @@ cc_library_static { "com_android_server_power_PowerManagerService.cpp", "com_android_server_security_VerityUtils.cpp", "com_android_server_SerialService.cpp", + "com_android_server_soundtrigger_middleware_AudioSessionProviderImpl.cpp", "com_android_server_storage_AppFuseBridge.cpp", "com_android_server_SystemServer.cpp", "com_android_server_TestNetworkService.cpp", diff --git a/services/core/jni/com_android_server_soundtrigger_middleware_AudioSessionProviderImpl.cpp b/services/core/jni/com_android_server_soundtrigger_middleware_AudioSessionProviderImpl.cpp new file mode 100644 index 0000000000000..774534f23b8c2 --- /dev/null +++ b/services/core/jni/com_android_server_soundtrigger_middleware_AudioSessionProviderImpl.cpp @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "core_jni_helpers.h" +#include + +namespace android { + +namespace { + +#define PACKAGE "com/android/server/soundtrigger_middleware" +#define CLASSNAME PACKAGE "/AudioSessionProviderImpl" +#define SESSION_CLASSNAME PACKAGE "/SoundTriggerMiddlewareImpl$AudioSessionProvider$AudioSession" + +jobject acquireAudioSession( + JNIEnv* env, + jobject clazz) { + + audio_session_t session; + audio_io_handle_t ioHandle; + audio_devices_t device; + + status_t status = AudioSystem::acquireSoundTriggerSession(&session, + &ioHandle, + &device); + if (status != 0) { + std::ostringstream message; + message + << "AudioSystem::acquireSoundTriggerSession returned an error code: " + << status; + env->ThrowNew(FindClassOrDie(env, "java/lang/RuntimeException"), + message.str().c_str()); + return nullptr; + } + + jclass cls = FindClassOrDie(env, SESSION_CLASSNAME); + jmethodID ctor = GetMethodIDOrDie(env, cls, "", "(III)V"); + return env->NewObject(cls, + ctor, + static_cast(session), + static_cast(ioHandle), + static_cast(device)); +} + +void releaseAudioSession(JNIEnv* env, jobject clazz, jint handle) { + status_t status = + AudioSystem::releaseSoundTriggerSession(static_cast(handle)); + + if (status != 0) { + std::ostringstream message; + message + << "AudioSystem::releaseAudioSystemSession returned an error code: " + << status; + env->ThrowNew(FindClassOrDie(env, "java/lang/RuntimeException"), + message.str().c_str()); + } +} + +const JNINativeMethod g_methods[] = { + {"acquireSession", "()L" SESSION_CLASSNAME ";", + reinterpret_cast(acquireAudioSession)}, + {"releaseSession", "(I)V", + reinterpret_cast(releaseAudioSession)}, +}; + +} // namespace + +int register_com_android_server_soundtrigger_middleware_AudioSessionProviderImpl( + JNIEnv* env) { + return RegisterMethodsOrDie(env, + CLASSNAME, + g_methods, + NELEM(g_methods)); +} + +} // namespace android diff --git a/services/core/jni/onload.cpp b/services/core/jni/onload.cpp index 692c9d25baa99..165edf15ca231 100644 --- a/services/core/jni/onload.cpp +++ b/services/core/jni/onload.cpp @@ -56,6 +56,8 @@ int register_android_server_net_NetworkStatsService(JNIEnv* env); int register_android_server_security_VerityUtils(JNIEnv* env); int register_android_server_am_AppCompactor(JNIEnv* env); int register_android_server_am_LowMemDetector(JNIEnv* env); +int register_com_android_server_soundtrigger_middleware_AudioSessionProviderImpl( + JNIEnv* env); }; using namespace android; @@ -105,5 +107,7 @@ extern "C" jint JNI_OnLoad(JavaVM* vm, void* /* reserved */) register_android_server_security_VerityUtils(env); register_android_server_am_AppCompactor(env); register_android_server_am_LowMemDetector(env); + register_com_android_server_soundtrigger_middleware_AudioSessionProviderImpl( + env); return JNI_VERSION_1_4; } diff --git a/services/java/com/android/server/SystemServer.java b/services/java/com/android/server/SystemServer.java index b6e501a785efc..7c43972dfed71 100644 --- a/services/java/com/android/server/SystemServer.java +++ b/services/java/com/android/server/SystemServer.java @@ -147,6 +147,7 @@ import com.android.server.security.KeyAttestationApplicationIdProviderService; import com.android.server.security.KeyChainSystemService; import com.android.server.signedconfig.SignedConfigService; import com.android.server.soundtrigger.SoundTriggerService; +import com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareService; import com.android.server.statusbar.StatusBarManagerService; import com.android.server.storage.DeviceStorageMonitorService; import com.android.server.telecom.TelecomLoaderService; @@ -1544,6 +1545,10 @@ public final class SystemServer { } t.traceEnd(); + t.traceBegin("StartSoundTriggerMiddlewareService"); + mSystemServiceManager.startService(SoundTriggerMiddlewareService.Lifecycle.class); + t.traceEnd(); + if (mPackageManager.hasSystemFeature(PackageManager.FEATURE_BROADCAST_RADIO)) { t.traceBegin("StartBroadcastRadioService"); mSystemServiceManager.startService(BroadcastRadioService.class); diff --git a/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/ConversionUtilTest.java b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/ConversionUtilTest.java new file mode 100644 index 0000000000000..5a2ce4540b82d --- /dev/null +++ b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/ConversionUtilTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import static org.junit.Assert.assertEquals; + +import android.hardware.audio.common.V2_0.Uuid; + +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(AndroidJUnit4.class) +public class ConversionUtilTest { + private static final String TAG = "ConversionUtilTest"; + + @Test + public void testUuidRoundTrip() { + Uuid hidl = new Uuid(); + hidl.timeLow = 0xFEDCBA98; + hidl.timeMid = (short) 0xEDCB; + hidl.versionAndTimeHigh = (short) 0xDCBA; + hidl.variantAndClockSeqHigh = (short) 0xCBA9; + hidl.node = new byte[] { 0x11, 0x12, 0x13, 0x14, 0x15, 0x16 }; + + String aidl = ConversionUtil.hidl2aidlUuid(hidl); + assertEquals("fedcba98-edcb-dcba-cba9-111213141516", aidl); + + Uuid reconstructed = ConversionUtil.aidl2hidlUuid(aidl); + assertEquals(hidl, reconstructed); + } +} diff --git a/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java new file mode 100644 index 0000000000000..82f32f88d3a2e --- /dev/null +++ b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java @@ -0,0 +1,1306 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.soundtrigger_middleware; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import android.hardware.audio.common.V2_0.AudioConfig; +import android.hardware.audio.common.V2_0.Uuid; +import android.hardware.soundtrigger.V2_3.OptionalModelParameterRange; +import android.media.audio.common.AudioChannelMask; +import android.media.audio.common.AudioFormat; +import android.media.soundtrigger_middleware.ConfidenceLevel; +import android.media.soundtrigger_middleware.ISoundTriggerCallback; +import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.ModelParameter; +import android.media.soundtrigger_middleware.ModelParameterRange; +import android.media.soundtrigger_middleware.Phrase; +import android.media.soundtrigger_middleware.PhraseRecognitionEvent; +import android.media.soundtrigger_middleware.PhraseRecognitionExtra; +import android.media.soundtrigger_middleware.PhraseSoundModel; +import android.media.soundtrigger_middleware.RecognitionConfig; +import android.media.soundtrigger_middleware.RecognitionEvent; +import android.media.soundtrigger_middleware.RecognitionMode; +import android.media.soundtrigger_middleware.RecognitionStatus; +import android.media.soundtrigger_middleware.SoundModel; +import android.media.soundtrigger_middleware.SoundModelType; +import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; +import android.media.soundtrigger_middleware.SoundTriggerModuleProperties; +import android.os.HidlMemoryUtil; +import android.os.HwParcel; +import android.os.IHwBinder; +import android.os.IHwInterface; +import android.os.RemoteException; + +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; + +@RunWith(Parameterized.class) +public class SoundTriggerMiddlewareImplTest { + private static final String TAG = "SoundTriggerMiddlewareImplTest"; + + // We run the test once for every version of the underlying driver. + @Parameterized.Parameters + public static Object[] data() { + return new Object[]{ + mock(android.hardware.soundtrigger.V2_0.ISoundTriggerHw.class), + mock(android.hardware.soundtrigger.V2_1.ISoundTriggerHw.class), + mock(android.hardware.soundtrigger.V2_2.ISoundTriggerHw.class), + mock(android.hardware.soundtrigger.V2_3.ISoundTriggerHw.class), + }; + } + + @Mock + @Parameterized.Parameter + public android.hardware.soundtrigger.V2_0.ISoundTriggerHw mHalDriver; + + @Mock + private SoundTriggerMiddlewareImpl.AudioSessionProvider mAudioSessionProvider = mock( + SoundTriggerMiddlewareImpl.AudioSessionProvider.class); + + private SoundTriggerMiddlewareImpl mService; + + private static ISoundTriggerCallback createCallbackMock() { + return mock(ISoundTriggerCallback.Stub.class, Mockito.CALLS_REAL_METHODS); + } + + private static SoundModel createGenericSoundModel() { + return createSoundModel(SoundModelType.GENERIC); + } + + private static SoundModel createSoundModel(int type) { + SoundModel model = new SoundModel(); + model.type = type; + model.uuid = "12345678-2345-3456-4567-abcdef987654"; + model.vendorUuid = "87654321-5432-6543-7654-456789fedcba"; + model.data = new byte[]{91, 92, 93, 94, 95}; + return model; + } + + private static PhraseSoundModel createPhraseSoundModel() { + PhraseSoundModel model = new PhraseSoundModel(); + model.common = createSoundModel(SoundModelType.KEYPHRASE); + model.phrases = new Phrase[1]; + model.phrases[0] = new Phrase(); + model.phrases[0].id = 123; + model.phrases[0].users = new int[]{5, 6, 7}; + model.phrases[0].locale = "locale"; + model.phrases[0].text = "text"; + model.phrases[0].recognitionModes = + RecognitionMode.USER_AUTHENTICATION | RecognitionMode.USER_IDENTIFICATION; + return model; + } + + private static android.hardware.soundtrigger.V2_0.ISoundTriggerHw.Properties createDefaultProperties( + boolean supportConcurrentCapture) { + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.Properties properties = + new android.hardware.soundtrigger.V2_0.ISoundTriggerHw.Properties(); + properties.implementor = "implementor"; + properties.description = "description"; + properties.version = 123; + properties.uuid = new Uuid(); + properties.uuid.timeLow = 1; + properties.uuid.timeMid = 2; + properties.uuid.versionAndTimeHigh = 3; + properties.uuid.variantAndClockSeqHigh = 4; + properties.uuid.node = new byte[]{5, 6, 7, 8, 9, 10}; + + properties.maxSoundModels = 456; + properties.maxKeyPhrases = 567; + properties.maxUsers = 678; + properties.recognitionModes = 789; + properties.captureTransition = true; + properties.maxBufferMs = 321; + properties.concurrentCapture = supportConcurrentCapture; + properties.triggerInEvent = true; + properties.powerConsumptionMw = 432; + return properties; + } + + private static void validateDefaultProperties(SoundTriggerModuleProperties properties, + boolean supportConcurrentCapture) { + assertEquals("implementor", properties.implementor); + assertEquals("description", properties.description); + assertEquals(123, properties.version); + assertEquals("00000001-0002-0003-0004-05060708090a", properties.uuid); + assertEquals(456, properties.maxSoundModels); + assertEquals(567, properties.maxKeyPhrases); + assertEquals(678, properties.maxUsers); + assertEquals(789, properties.recognitionModes); + assertTrue(properties.captureTransition); + assertEquals(321, properties.maxBufferMs); + assertEquals(supportConcurrentCapture, properties.concurrentCapture); + assertTrue(properties.triggerInEvent); + assertEquals(432, properties.powerConsumptionMw); + } + + + private static android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionEvent createRecognitionEvent_2_0( + int hwHandle, + int status) { + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionEvent halEvent = + new android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionEvent(); + halEvent.status = status; + halEvent.type = SoundModelType.GENERIC; + halEvent.model = hwHandle; + halEvent.captureAvailable = true; + // This field is ignored. + halEvent.captureSession = 123; + halEvent.captureDelayMs = 234; + halEvent.capturePreambleMs = 345; + halEvent.triggerInData = true; + halEvent.audioConfig = new AudioConfig(); + halEvent.audioConfig.sampleRateHz = 456; + halEvent.audioConfig.channelMask = AudioChannelMask.IN_LEFT; + halEvent.audioConfig.format = AudioFormat.MP3; + // hwEvent.audioConfig.offloadInfo is irrelevant. + halEvent.data.add((byte) 31); + halEvent.data.add((byte) 32); + halEvent.data.add((byte) 33); + return halEvent; + } + + private static android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent createRecognitionEvent_2_1( + int hwHandle, + int status) { + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent halEvent = + new android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent(); + halEvent.header = createRecognitionEvent_2_0(hwHandle, status); + halEvent.header.data.clear(); + halEvent.data = HidlMemoryUtil.byteArrayToHidlMemory(new byte[]{31, 32, 33}); + return halEvent; + } + + private static void validateRecognitionEvent(RecognitionEvent event, int status) { + assertEquals(status, event.status); + assertEquals(SoundModelType.GENERIC, event.type); + assertTrue(event.captureAvailable); + assertEquals(101, event.captureSession); + assertEquals(234, event.captureDelayMs); + assertEquals(345, event.capturePreambleMs); + assertTrue(event.triggerInData); + assertEquals(456, event.audioConfig.sampleRateHz); + assertEquals(AudioChannelMask.IN_LEFT, event.audioConfig.channelMask); + assertEquals(AudioFormat.MP3, event.audioConfig.format); + } + + private static android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.PhraseRecognitionEvent createPhraseRecognitionEvent_2_0( + int hwHandle, int status) { + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.PhraseRecognitionEvent halEvent = + new android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.PhraseRecognitionEvent(); + halEvent.common = createRecognitionEvent_2_0(hwHandle, status); + + android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra halExtra = + new android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra(); + halExtra.id = 123; + halExtra.confidenceLevel = 52; + halExtra.recognitionModes = android.hardware.soundtrigger.V2_0.RecognitionMode.VOICE_TRIGGER + | android.hardware.soundtrigger.V2_0.RecognitionMode.GENERIC_TRIGGER; + android.hardware.soundtrigger.V2_0.ConfidenceLevel halLevel = + new android.hardware.soundtrigger.V2_0.ConfidenceLevel(); + halLevel.userId = 31; + halLevel.levelPercent = 43; + halExtra.levels.add(halLevel); + halEvent.phraseExtras.add(halExtra); + return halEvent; + } + + private static android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent createPhraseRecognitionEvent_2_1( + int hwHandle, int status) { + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent halEvent = + new android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent(); + halEvent.common = createRecognitionEvent_2_1(hwHandle, status); + + android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra halExtra = + new android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra(); + halExtra.id = 123; + halExtra.confidenceLevel = 52; + halExtra.recognitionModes = android.hardware.soundtrigger.V2_0.RecognitionMode.VOICE_TRIGGER + | android.hardware.soundtrigger.V2_0.RecognitionMode.GENERIC_TRIGGER; + android.hardware.soundtrigger.V2_0.ConfidenceLevel halLevel = + new android.hardware.soundtrigger.V2_0.ConfidenceLevel(); + halLevel.userId = 31; + halLevel.levelPercent = 43; + halExtra.levels.add(halLevel); + halEvent.phraseExtras.add(halExtra); + return halEvent; + } + + private static void validatePhraseRecognitionEvent(PhraseRecognitionEvent event, int status) { + validateRecognitionEvent(event.common, status); + + assertEquals(1, event.phraseExtras.length); + assertEquals(123, event.phraseExtras[0].id); + assertEquals(52, event.phraseExtras[0].confidenceLevel); + assertEquals(RecognitionMode.VOICE_TRIGGER | RecognitionMode.GENERIC_TRIGGER, + event.phraseExtras[0].recognitionModes); + assertEquals(1, event.phraseExtras[0].levels.length); + assertEquals(31, event.phraseExtras[0].levels[0].userId); + assertEquals(43, event.phraseExtras[0].levels[0].levelPercent); + } + + private void initService(boolean supportConcurrentCapture) throws RemoteException { + doAnswer(invocation -> { + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.Properties properties = + createDefaultProperties( + supportConcurrentCapture); + ((android.hardware.soundtrigger.V2_0.ISoundTriggerHw.getPropertiesCallback) invocation.getArgument( + 0)).onValues(0, + properties); + return null; + }).when(mHalDriver).getProperties(any()); + mService = new SoundTriggerMiddlewareImpl(mHalDriver, mAudioSessionProvider); + } + + private int loadGenericModel_2_0(ISoundTriggerModule module, int hwHandle) + throws RemoteException { + SoundModel model = createGenericSoundModel(); + ArgumentCaptor modelCaptor = + ArgumentCaptor.forClass( + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.SoundModel.class); + doAnswer(invocation -> { + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback callback = + invocation.getArgument(1); + int callbackCookie = invocation.getArgument(2); + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.loadSoundModelCallback + resultCallback = invocation.getArgument(3); + + // This is the return of this method. + resultCallback.onValues(0, hwHandle); + + // This is the async mCallback that comes after. + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.ModelEvent modelEvent = + new android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.ModelEvent(); + modelEvent.status = + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.SoundModelStatus.UPDATED; + modelEvent.model = hwHandle; + callback.soundModelCallback(modelEvent, callbackCookie); + return null; + }).when(mHalDriver).loadSoundModel(modelCaptor.capture(), any(), anyInt(), any()); + + when(mAudioSessionProvider.acquireSession()).thenReturn( + new SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession(101, 102, 103)); + + int handle = module.loadModel(model); + verify(mHalDriver).loadSoundModel(any(), any(), anyInt(), any()); + verify(mAudioSessionProvider).acquireSession(); + + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.SoundModel hidlModel = + modelCaptor.getValue(); + assertEquals(android.hardware.soundtrigger.V2_0.SoundModelType.GENERIC, + hidlModel.type); + assertEquals(model.uuid, ConversionUtil.hidl2aidlUuid(hidlModel.uuid)); + assertEquals(model.vendorUuid, ConversionUtil.hidl2aidlUuid(hidlModel.vendorUuid)); + assertArrayEquals(new Byte[]{91, 92, 93, 94, 95}, hidlModel.data.toArray()); + + return handle; + } + + private int loadGenericModel_2_1(ISoundTriggerModule module, int hwHandle) + throws RemoteException { + android.hardware.soundtrigger.V2_1.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_1.ISoundTriggerHw) mHalDriver; + SoundModel model = createGenericSoundModel(); + ArgumentCaptor modelCaptor = + ArgumentCaptor.forClass( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.SoundModel.class); + doAnswer(invocation -> { + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback callback = + invocation.getArgument(1); + int callbackCookie = invocation.getArgument(2); + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.loadSoundModel_2_1Callback + resultCallback = invocation.getArgument(3); + + // This is the return of this method. + resultCallback.onValues(0, hwHandle); + + // This is the async mCallback that comes after. + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.ModelEvent modelEvent = + new android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.ModelEvent(); + modelEvent.header.status = + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.SoundModelStatus.UPDATED; + modelEvent.header.model = hwHandle; + callback.soundModelCallback_2_1(modelEvent, callbackCookie); + return null; + }).when(driver).loadSoundModel_2_1(modelCaptor.capture(), any(), anyInt(), any()); + + when(mAudioSessionProvider.acquireSession()).thenReturn( + new SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession(101, 102, 103)); + + int handle = module.loadModel(model); + verify(driver).loadSoundModel_2_1(any(), any(), anyInt(), any()); + verify(mAudioSessionProvider).acquireSession(); + + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.SoundModel hidlModel = + modelCaptor.getValue(); + assertEquals(android.hardware.soundtrigger.V2_0.SoundModelType.GENERIC, + hidlModel.header.type); + assertEquals(model.uuid, ConversionUtil.hidl2aidlUuid(hidlModel.header.uuid)); + assertEquals(model.vendorUuid, ConversionUtil.hidl2aidlUuid(hidlModel.header.vendorUuid)); + assertArrayEquals(new byte[]{91, 92, 93, 94, 95}, + HidlMemoryUtil.hidlMemoryToByteArray(hidlModel.data)); + + return handle; + } + + private int loadGenericModel(ISoundTriggerModule module, int hwHandle) throws RemoteException { + if (mHalDriver instanceof android.hardware.soundtrigger.V2_1.ISoundTriggerHw) { + return loadGenericModel_2_1(module, hwHandle); + } else { + return loadGenericModel_2_0(module, hwHandle); + } + } + + private int loadPhraseModel_2_0(ISoundTriggerModule module, int hwHandle) + throws RemoteException { + PhraseSoundModel model = createPhraseSoundModel(); + ArgumentCaptor + modelCaptor = ArgumentCaptor.forClass( + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.PhraseSoundModel.class); + doAnswer(invocation -> { + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback callback = + invocation.getArgument( + 1); + int callbackCookie = invocation.getArgument(2); + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.loadPhraseSoundModelCallback + resultCallback = + invocation.getArgument( + 3); + + // This is the return of this method. + resultCallback.onValues(0, hwHandle); + + // This is the async mCallback that comes after. + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.ModelEvent modelEvent = + new android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.ModelEvent(); + modelEvent.status = + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.SoundModelStatus.UPDATED; + modelEvent.model = hwHandle; + callback.soundModelCallback(modelEvent, callbackCookie); + return null; + }).when(mHalDriver).loadPhraseSoundModel(modelCaptor.capture(), any(), anyInt(), any()); + + when(mAudioSessionProvider.acquireSession()).thenReturn( + new SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession(101, 102, 103)); + + int handle = module.loadPhraseModel(model); + verify(mHalDriver).loadPhraseSoundModel(any(), any(), anyInt(), any()); + verify(mAudioSessionProvider).acquireSession(); + + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.PhraseSoundModel hidlModel = + modelCaptor.getValue(); + + // Validate common part. + assertEquals(android.hardware.soundtrigger.V2_0.SoundModelType.KEYPHRASE, + hidlModel.common.type); + assertEquals(model.common.uuid, ConversionUtil.hidl2aidlUuid(hidlModel.common.uuid)); + assertEquals(model.common.vendorUuid, + ConversionUtil.hidl2aidlUuid(hidlModel.common.vendorUuid)); + assertArrayEquals(new Byte[]{91, 92, 93, 94, 95}, hidlModel.common.data.toArray()); + + // Validate phrase part. + assertEquals(1, hidlModel.phrases.size()); + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.Phrase hidlPhrase = + hidlModel.phrases.get(0); + assertEquals(123, hidlPhrase.id); + assertArrayEquals(new Integer[]{5, 6, 7}, hidlPhrase.users.toArray()); + assertEquals("locale", hidlPhrase.locale); + assertEquals("text", hidlPhrase.text); + assertEquals(android.hardware.soundtrigger.V2_0.RecognitionMode.USER_AUTHENTICATION + | android.hardware.soundtrigger.V2_0.RecognitionMode.USER_IDENTIFICATION, + hidlPhrase.recognitionModes); + + return handle; + } + + private int loadPhraseModel_2_1(ISoundTriggerModule module, int hwHandle) + throws RemoteException { + android.hardware.soundtrigger.V2_1.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_1.ISoundTriggerHw) mHalDriver; + + PhraseSoundModel model = createPhraseSoundModel(); + ArgumentCaptor + modelCaptor = ArgumentCaptor.forClass( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.PhraseSoundModel.class); + doAnswer(invocation -> { + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback callback = + invocation.getArgument( + 1); + int callbackCookie = invocation.getArgument(2); + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.loadPhraseSoundModel_2_1Callback + resultCallback = + invocation.getArgument( + 3); + + // This is the return of this method. + resultCallback.onValues(0, hwHandle); + + // This is the async mCallback that comes after. + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.ModelEvent modelEvent = + new android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.ModelEvent(); + modelEvent.header.status = + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.SoundModelStatus.UPDATED; + modelEvent.header.model = hwHandle; + callback.soundModelCallback_2_1(modelEvent, callbackCookie); + return null; + }).when(driver).loadPhraseSoundModel_2_1(modelCaptor.capture(), any(), anyInt(), any()); + + when(mAudioSessionProvider.acquireSession()).thenReturn( + new SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession(101, 102, 103)); + + int handle = module.loadPhraseModel(model); + verify(driver).loadPhraseSoundModel_2_1(any(), any(), anyInt(), any()); + verify(mAudioSessionProvider).acquireSession(); + + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.PhraseSoundModel hidlModel = + modelCaptor.getValue(); + + // Validate common part. + assertEquals(android.hardware.soundtrigger.V2_0.SoundModelType.KEYPHRASE, + hidlModel.common.header.type); + assertEquals(model.common.uuid, ConversionUtil.hidl2aidlUuid(hidlModel.common.header.uuid)); + assertEquals(model.common.vendorUuid, + ConversionUtil.hidl2aidlUuid(hidlModel.common.header.vendorUuid)); + assertArrayEquals(new byte[]{91, 92, 93, 94, 95}, + HidlMemoryUtil.hidlMemoryToByteArray(hidlModel.common.data)); + + // Validate phrase part. + assertEquals(1, hidlModel.phrases.size()); + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.Phrase hidlPhrase = + hidlModel.phrases.get(0); + assertEquals(123, hidlPhrase.id); + assertArrayEquals(new Integer[]{5, 6, 7}, hidlPhrase.users.toArray()); + assertEquals("locale", hidlPhrase.locale); + assertEquals("text", hidlPhrase.text); + assertEquals(android.hardware.soundtrigger.V2_0.RecognitionMode.USER_AUTHENTICATION + | android.hardware.soundtrigger.V2_0.RecognitionMode.USER_IDENTIFICATION, + hidlPhrase.recognitionModes); + + return handle; + } + + private int loadPhraseModel(ISoundTriggerModule module, int hwHandle) throws RemoteException { + if (mHalDriver instanceof android.hardware.soundtrigger.V2_1.ISoundTriggerHw) { + return loadPhraseModel_2_1(module, hwHandle); + } else { + return loadPhraseModel_2_0(module, hwHandle); + } + } + + private void unloadModel(ISoundTriggerModule module, int handle, int hwHandle) + throws RemoteException { + module.unloadModel(handle); + verify(mHalDriver).unloadSoundModel(hwHandle); + verify(mAudioSessionProvider).releaseSession(101); + } + + private SoundTriggerHwCallback startRecognition_2_0(ISoundTriggerModule module, int handle, + int hwHandle) throws RemoteException { + ArgumentCaptor + configCaptor = ArgumentCaptor.forClass( + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.RecognitionConfig.class); + ArgumentCaptor callbackCaptor = + ArgumentCaptor.forClass( + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.class); + ArgumentCaptor cookieCaptor = ArgumentCaptor.forClass(Integer.class); + + when(mHalDriver.startRecognition(eq(hwHandle), configCaptor.capture(), + callbackCaptor.capture(), cookieCaptor.capture())).thenReturn(0); + + RecognitionConfig config = createRecognitionConfig(); + + module.startRecognition(handle, config); + verify(mHalDriver).startRecognition(eq(hwHandle), any(), any(), anyInt()); + + android.hardware.soundtrigger.V2_0.ISoundTriggerHw.RecognitionConfig halConfig = + configCaptor.getValue(); + assertTrue(halConfig.captureRequested); + assertEquals(102, halConfig.captureHandle); + assertEquals(103, halConfig.captureDevice); + assertEquals(1, halConfig.phrases.size()); + android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra halPhraseExtra = + halConfig.phrases.get(0); + assertEquals(123, halPhraseExtra.id); + assertEquals(4, halPhraseExtra.confidenceLevel); + assertEquals(5, halPhraseExtra.recognitionModes); + assertEquals(1, halPhraseExtra.levels.size()); + android.hardware.soundtrigger.V2_0.ConfidenceLevel halLevel = halPhraseExtra.levels.get(0); + assertEquals(234, halLevel.userId); + assertEquals(34, halLevel.levelPercent); + assertArrayEquals(new Byte[]{5, 4, 3, 2, 1}, halConfig.data.toArray()); + + return new SoundTriggerHwCallback(callbackCaptor.getValue(), cookieCaptor.getValue()); + } + + private SoundTriggerHwCallback startRecognition_2_1(ISoundTriggerModule module, int handle, + int hwHandle) throws RemoteException { + android.hardware.soundtrigger.V2_1.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_1.ISoundTriggerHw) mHalDriver; + + ArgumentCaptor + configCaptor = ArgumentCaptor.forClass( + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.RecognitionConfig.class); + ArgumentCaptor callbackCaptor = + ArgumentCaptor.forClass( + android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.class); + ArgumentCaptor cookieCaptor = ArgumentCaptor.forClass(Integer.class); + + when(driver.startRecognition_2_1(eq(hwHandle), configCaptor.capture(), + callbackCaptor.capture(), cookieCaptor.capture())).thenReturn(0); + + RecognitionConfig config = createRecognitionConfig(); + + module.startRecognition(handle, config); + verify(driver).startRecognition_2_1(eq(hwHandle), any(), any(), anyInt()); + + android.hardware.soundtrigger.V2_1.ISoundTriggerHw.RecognitionConfig halConfig = + configCaptor.getValue(); + assertTrue(halConfig.header.captureRequested); + assertEquals(102, halConfig.header.captureHandle); + assertEquals(103, halConfig.header.captureDevice); + assertEquals(1, halConfig.header.phrases.size()); + android.hardware.soundtrigger.V2_0.PhraseRecognitionExtra halPhraseExtra = + halConfig.header.phrases.get(0); + assertEquals(123, halPhraseExtra.id); + assertEquals(4, halPhraseExtra.confidenceLevel); + assertEquals(5, halPhraseExtra.recognitionModes); + assertEquals(1, halPhraseExtra.levels.size()); + android.hardware.soundtrigger.V2_0.ConfidenceLevel halLevel = halPhraseExtra.levels.get(0); + assertEquals(234, halLevel.userId); + assertEquals(34, halLevel.levelPercent); + assertArrayEquals(new byte[]{5, 4, 3, 2, 1}, + HidlMemoryUtil.hidlMemoryToByteArray(halConfig.data)); + + return new SoundTriggerHwCallback(callbackCaptor.getValue(), cookieCaptor.getValue()); + } + + private SoundTriggerHwCallback startRecognition(ISoundTriggerModule module, int handle, + int hwHandle) throws RemoteException { + if (mHalDriver instanceof android.hardware.soundtrigger.V2_1.ISoundTriggerHw) { + return startRecognition_2_1(module, handle, hwHandle); + } else { + return startRecognition_2_0(module, handle, hwHandle); + } + } + + private RecognitionConfig createRecognitionConfig() { + RecognitionConfig config = new RecognitionConfig(); + config.captureRequested = true; + config.phraseRecognitionExtras = new PhraseRecognitionExtra[]{new PhraseRecognitionExtra()}; + config.phraseRecognitionExtras[0].id = 123; + config.phraseRecognitionExtras[0].confidenceLevel = 4; + config.phraseRecognitionExtras[0].recognitionModes = 5; + config.phraseRecognitionExtras[0].levels = new ConfidenceLevel[]{new ConfidenceLevel()}; + config.phraseRecognitionExtras[0].levels[0].userId = 234; + config.phraseRecognitionExtras[0].levels[0].levelPercent = 34; + config.data = new byte[]{5, 4, 3, 2, 1}; + return config; + } + + private void stopRecognition(ISoundTriggerModule module, int handle, int hwHandle) + throws RemoteException { + when(mHalDriver.stopRecognition(hwHandle)).thenReturn(0); + module.stopRecognition(handle); + verify(mHalDriver).stopRecognition(hwHandle); + } + + private void verifyNotStartRecognition() throws RemoteException { + verify(mHalDriver, never()).startRecognition(anyInt(), any(), any(), anyInt()); + if (mHalDriver instanceof android.hardware.soundtrigger.V2_1.ISoundTriggerHw) { + verify((android.hardware.soundtrigger.V2_1.ISoundTriggerHw) mHalDriver, + never()).startRecognition_2_1(anyInt(), any(), any(), anyInt()); + } + } + + + @Before + public void setUp() throws Exception { + clearInvocations(mHalDriver); + clearInvocations(mAudioSessionProvider); + + // This binder is associated with the mock, so it can be cast to either version of the + // HAL interface. + final IHwBinder binder = new IHwBinder() { + @Override + public void transact(int code, HwParcel request, HwParcel reply, int flags) + throws RemoteException { + // This is a little hacky, but a very easy way to gracefully reject a request for + // an unsupported interface (after queryLocalInterface() returns null, the client + // will attempt a remote transaction to obtain the interface. RemoteException will + // cause it to give up). + throw new RemoteException(); + } + + @Override + public IHwInterface queryLocalInterface(String descriptor) { + if (descriptor.equals("android.hardware.soundtrigger@2.0::ISoundTriggerHw") + || descriptor.equals("android.hardware.soundtrigger@2.1::ISoundTriggerHw") + && mHalDriver instanceof android.hardware.soundtrigger.V2_1.ISoundTriggerHw + || descriptor.equals("android.hardware.soundtrigger@2.2::ISoundTriggerHw") + && mHalDriver instanceof android.hardware.soundtrigger.V2_2.ISoundTriggerHw + || descriptor.equals("android.hardware.soundtrigger@2.3::ISoundTriggerHw") + && mHalDriver instanceof android.hardware.soundtrigger.V2_3.ISoundTriggerHw) { + return mHalDriver; + } + return null; + } + + @Override + public boolean linkToDeath(DeathRecipient recipient, long cookie) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean unlinkToDeath(DeathRecipient recipient) { + throw new UnsupportedOperationException(); + } + }; + + when(mHalDriver.asBinder()).thenReturn(binder); + } + + @Test + public void testSetUpAndTearDown() { + } + + @Test + public void testListModules() throws Exception { + initService(true); + // Note: input and output properties are NOT the same type, even though they are in any way + // equivalent. One is a type that's exposed by the HAL and one is a type that's exposed by + // the service. The service actually performs a (trivial) conversion between the two. + SoundTriggerModuleDescriptor[] allDescriptors = mService.listModules(); + assertEquals(1, allDescriptors.length); + + SoundTriggerModuleProperties properties = allDescriptors[0].properties; + + validateDefaultProperties(properties, true); + } + + @Test + public void testAttachDetach() throws Exception { + // Normal attachment / detachment. + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + verify(callback).onRecognitionAvailabilityChange(true); + assertNotNull(module); + module.detach(); + } + + @Test + public void testAttachDetachNotAvailable() throws Exception { + // Attachment / detachment during external capture, with a module not supporting concurrent + // capture. + initService(false); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + verify(callback).onRecognitionAvailabilityChange(false); + assertNotNull(module); + module.detach(); + } + + @Test + public void testAttachDetachAvailable() throws Exception { + // Attachment / detachment during external capture, with a module supporting concurrent + // capture. + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + verify(callback).onRecognitionAvailabilityChange(true); + assertNotNull(module); + module.detach(); + } + + @Test + public void testLoadUnloadModel() throws Exception { + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + + final int hwHandle = 7; + int handle = loadGenericModel(module, hwHandle); + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testLoadUnloadPhraseModel() throws Exception { + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + + final int hwHandle = 73; + int handle = loadPhraseModel(module, hwHandle); + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testStartStopRecognition() throws Exception { + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + + // Load the model. + final int hwHandle = 7; + int handle = loadGenericModel(module, hwHandle); + + // Initiate a recognition. + startRecognition(module, handle, hwHandle); + + // Stop the recognition. + stopRecognition(module, handle, hwHandle); + + // Unload the model. + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testStartStopPhraseRecognition() throws Exception { + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + + // Load the model. + final int hwHandle = 67; + int handle = loadPhraseModel(module, hwHandle); + + // Initiate a recognition. + startRecognition(module, handle, hwHandle); + + // Stop the recognition. + stopRecognition(module, handle, hwHandle); + + // Unload the model. + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testRecognition() throws Exception { + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + + // Load the model. + final int hwHandle = 7; + int handle = loadGenericModel(module, hwHandle); + + // Initiate a recognition. + SoundTriggerHwCallback hwCallback = startRecognition(module, handle, hwHandle); + + // Signal a capture from the driver. + hwCallback.sendRecognitionEvent(hwHandle, + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.SUCCESS); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + RecognitionEvent.class); + verify(callback).onRecognition(eq(handle), eventCaptor.capture()); + + // Validate the event. + validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.SUCCESS); + + // Unload the model. + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testPhraseRecognition() throws Exception { + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + + // Load the model. + final int hwHandle = 7; + int handle = loadPhraseModel(module, hwHandle); + + // Initiate a recognition. + SoundTriggerHwCallback hwCallback = startRecognition(module, handle, hwHandle); + + // Signal a capture from the driver. + hwCallback.sendPhraseRecognitionEvent(hwHandle, + android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.SUCCESS); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEvent.class); + verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture()); + + // Validate the event. + validatePhraseRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.SUCCESS); + + // Unload the model. + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testForceRecognition() throws Exception { + if (!(mHalDriver instanceof android.hardware.soundtrigger.V2_2.ISoundTriggerHw)) { + return; + } + + android.hardware.soundtrigger.V2_2.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_2.ISoundTriggerHw) mHalDriver; + + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + + // Load the model. + final int hwHandle = 17; + int handle = loadGenericModel(module, hwHandle); + + // Initiate a recognition. + SoundTriggerHwCallback hwCallback = startRecognition(module, handle, hwHandle); + + // Force a trigger. + module.forceRecognitionEvent(handle); + verify(driver).getModelState(hwHandle); + + // Signal a capture from the driver. + // '3' means 'forced', there's no constant for that in the HAL. + hwCallback.sendRecognitionEvent(hwHandle, 3); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + RecognitionEvent.class); + verify(callback).onRecognition(eq(handle), eventCaptor.capture()); + + // Validate the event. + validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.FORCED); + + // Stop the recognition. + stopRecognition(module, handle, hwHandle); + + // Unload the model. + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testForcePhraseRecognition() throws Exception { + if (!(mHalDriver instanceof android.hardware.soundtrigger.V2_2.ISoundTriggerHw)) { + return; + } + + android.hardware.soundtrigger.V2_2.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_2.ISoundTriggerHw) mHalDriver; + + initService(true); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + + // Load the model. + final int hwHandle = 17; + int handle = loadPhraseModel(module, hwHandle); + + // Initiate a recognition. + SoundTriggerHwCallback hwCallback = startRecognition(module, handle, hwHandle); + + // Force a trigger. + module.forceRecognitionEvent(handle); + verify(driver).getModelState(hwHandle); + + // Signal a capture from the driver. + // '3' means 'forced', there's no constant for that in the HAL. + hwCallback.sendPhraseRecognitionEvent(hwHandle, 3); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEvent.class); + verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture()); + + // Validate the event. + validatePhraseRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.FORCED); + + // Stop the recognition. + stopRecognition(module, handle, hwHandle); + + // Unload the model. + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testAbortRecognition() throws Exception { + // Make sure the HAL doesn't support concurrent capture. + initService(false); + mService.setExternalCaptureState(false); + + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + verify(callback).onRecognitionAvailabilityChange(true); + + // Load the model. + final int hwHandle = 11; + int handle = loadGenericModel(module, hwHandle); + + // Initiate a recognition. + startRecognition(module, handle, hwHandle); + + // Abort. + mService.setExternalCaptureState(true); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + RecognitionEvent.class); + verify(callback).onRecognition(eq(handle), eventCaptor.capture()); + + // Validate the event. + assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().status); + + // Make sure we are notified of the lost availability. + verify(callback).onRecognitionAvailabilityChange(false); + + // Attempt to start a new recognition - should get an abort event immediately, without + // involving the HAL. + clearInvocations(callback); + clearInvocations(mHalDriver); + module.startRecognition(handle, createRecognitionConfig()); + verify(callback).onRecognition(eq(handle), eventCaptor.capture()); + assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().status); + verifyNotStartRecognition(); + + // Now enable it and make sure we are notified. + mService.setExternalCaptureState(false); + verify(callback).onRecognitionAvailabilityChange(true); + + // Unload the model. + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testAbortPhraseRecognition() throws Exception { + // Make sure the HAL doesn't support concurrent capture. + initService(false); + mService.setExternalCaptureState(false); + + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + verify(callback).onRecognitionAvailabilityChange(true); + + // Load the model. + final int hwHandle = 11; + int handle = loadPhraseModel(module, hwHandle); + + // Initiate a recognition. + startRecognition(module, handle, hwHandle); + + // Abort. + mService.setExternalCaptureState(true); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEvent.class); + verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture()); + + // Validate the event. + assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().common.status); + + // Make sure we are notified of the lost availability. + verify(callback).onRecognitionAvailabilityChange(false); + + // Attempt to start a new recognition - should get an abort event immediately, without + // involving the HAL. + clearInvocations(callback); + clearInvocations(mHalDriver); + module.startRecognition(handle, createRecognitionConfig()); + verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture()); + assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().common.status); + verifyNotStartRecognition(); + + // Now enable it and make sure we are notified. + mService.setExternalCaptureState(false); + verify(callback).onRecognitionAvailabilityChange(true); + + // Unload the model. + unloadModel(module, handle, hwHandle); + module.detach(); + } + + @Test + public void testNotAbortRecognitionConcurrent() throws Exception { + // Make sure the HAL supports concurrent capture. + initService(true); + + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + verify(callback).onRecognitionAvailabilityChange(true); + clearInvocations(callback); + + // Load the model. + final int hwHandle = 13; + int handle = loadGenericModel(module, hwHandle); + + // Initiate a recognition. + startRecognition(module, handle, hwHandle); + + // Signal concurrent capture. Shouldn't abort. + mService.setExternalCaptureState(true); + verify(callback, never()).onRecognition(anyInt(), any()); + verify(callback, never()).onRecognitionAvailabilityChange(anyBoolean()); + + // Stop the recognition. + stopRecognition(module, handle, hwHandle); + + // Initiating a new one should work fine. + clearInvocations(mHalDriver); + startRecognition(module, handle, hwHandle); + verify(callback, never()).onRecognition(anyInt(), any()); + stopRecognition(module, handle, hwHandle); + + // Unload the model. + module.unloadModel(handle); + module.detach(); + } + + @Test + public void testNotAbortPhraseRecognitionConcurrent() throws Exception { + // Make sure the HAL supports concurrent capture. + initService(true); + + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + verify(callback).onRecognitionAvailabilityChange(true); + clearInvocations(callback); + + // Load the model. + final int hwHandle = 13; + int handle = loadPhraseModel(module, hwHandle); + + // Initiate a recognition. + startRecognition(module, handle, hwHandle); + + // Signal concurrent capture. Shouldn't abort. + mService.setExternalCaptureState(true); + verify(callback, never()).onPhraseRecognition(anyInt(), any()); + verify(callback, never()).onRecognitionAvailabilityChange(anyBoolean()); + + // Stop the recognition. + stopRecognition(module, handle, hwHandle); + + // Initiating a new one should work fine. + clearInvocations(mHalDriver); + startRecognition(module, handle, hwHandle); + verify(callback, never()).onRecognition(anyInt(), any()); + stopRecognition(module, handle, hwHandle); + + // Unload the model. + module.unloadModel(handle); + module.detach(); + } + + @Test + public void testParameterSupported() throws Exception { + if (!(mHalDriver instanceof android.hardware.soundtrigger.V2_3.ISoundTriggerHw)) { + return; + } + + android.hardware.soundtrigger.V2_3.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_3.ISoundTriggerHw) mHalDriver; + + initService(false); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + final int hwHandle = 12; + int modelHandle = loadGenericModel(module, hwHandle); + + doAnswer((Answer) invocation -> { + android.hardware.soundtrigger.V2_3.ISoundTriggerHw.queryParameterCallback + resultCallback = invocation.getArgument(2); + android.hardware.soundtrigger.V2_3.ModelParameterRange range = + new android.hardware.soundtrigger.V2_3.ModelParameterRange(); + range.start = 23; + range.end = 45; + OptionalModelParameterRange optionalRange = new OptionalModelParameterRange(); + optionalRange.range(range); + resultCallback.onValues(0, optionalRange); + return null; + }).when(driver).queryParameter(eq(hwHandle), + eq(android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR), any()); + + ModelParameterRange range = module.queryModelParameterSupport(modelHandle, + ModelParameter.THRESHOLD_FACTOR); + + verify(driver).queryParameter(eq(hwHandle), + eq(android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR), any()); + + assertEquals(23, range.minInclusive); + assertEquals(45, range.maxInclusive); + } + + @Test + public void testParameterNotSupportedOld() throws Exception { + if (mHalDriver instanceof android.hardware.soundtrigger.V2_3.ISoundTriggerHw) { + return; + } + + initService(false); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + final int hwHandle = 13; + int modelHandle = loadGenericModel(module, hwHandle); + + ModelParameterRange range = module.queryModelParameterSupport(modelHandle, + ModelParameter.THRESHOLD_FACTOR); + + assertNull(range); + } + + @Test + public void testParameterNotSupported() throws Exception { + if (!(mHalDriver instanceof android.hardware.soundtrigger.V2_3.ISoundTriggerHw)) { + return; + } + + android.hardware.soundtrigger.V2_3.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_3.ISoundTriggerHw) mHalDriver; + + initService(false); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + final int hwHandle = 13; + int modelHandle = loadGenericModel(module, hwHandle); + + doAnswer(invocation -> { + android.hardware.soundtrigger.V2_3.ISoundTriggerHw.queryParameterCallback + resultCallback = invocation.getArgument(2); + // This is the return of this method. + resultCallback.onValues(0, new OptionalModelParameterRange()); + return null; + }).when(driver).queryParameter(eq(hwHandle), + eq(android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR), any()); + + ModelParameterRange range = module.queryModelParameterSupport(modelHandle, + ModelParameter.THRESHOLD_FACTOR); + + verify(driver).queryParameter(eq(hwHandle), + eq(android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR), any()); + + assertNull(range); + } + + @Test + public void testGetParameter() throws Exception { + if (!(mHalDriver instanceof android.hardware.soundtrigger.V2_3.ISoundTriggerHw)) { + return; + } + + android.hardware.soundtrigger.V2_3.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_3.ISoundTriggerHw) mHalDriver; + + initService(false); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + final int hwHandle = 14; + int modelHandle = loadGenericModel(module, hwHandle); + + doAnswer(invocation -> { + android.hardware.soundtrigger.V2_3.ISoundTriggerHw.getParameterCallback + resultCallback = invocation.getArgument(2); + // This is the return of this method. + resultCallback.onValues(0, 234); + return null; + }).when(driver).getParameter(eq(hwHandle), + eq(android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR), any()); + + int value = module.getModelParameter(modelHandle, ModelParameter.THRESHOLD_FACTOR); + + verify(driver).getParameter(eq(hwHandle), + eq(android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR), any()); + + assertEquals(234, value); + } + + @Test + public void testSetParameter() throws Exception { + if (!(mHalDriver instanceof android.hardware.soundtrigger.V2_3.ISoundTriggerHw)) { + return; + } + + android.hardware.soundtrigger.V2_3.ISoundTriggerHw driver = + (android.hardware.soundtrigger.V2_3.ISoundTriggerHw) mHalDriver; + + initService(false); + ISoundTriggerCallback callback = createCallbackMock(); + ISoundTriggerModule module = mService.attach(0, callback); + final int hwHandle = 17; + int modelHandle = loadGenericModel(module, hwHandle); + + when(driver.setParameter(hwHandle, + android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR, + 456)).thenReturn(0); + + module.setModelParameter(modelHandle, ModelParameter.THRESHOLD_FACTOR, 456); + + verify(driver).setParameter(hwHandle, + android.hardware.soundtrigger.V2_3.ModelParameter.THRESHOLD_FACTOR, 456); + } + + private static class SoundTriggerHwCallback { + private final android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback mCallback; + private final int mCookie; + + SoundTriggerHwCallback(android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback callback, + int cookie) { + mCallback = callback; + mCookie = cookie; + } + + private void sendRecognitionEvent(int hwHandle, int status) throws RemoteException { + if (mCallback instanceof android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback) { + ((android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback) mCallback).recognitionCallback_2_1( + createRecognitionEvent_2_1(hwHandle, status), mCookie); + } else { + mCallback.recognitionCallback(createRecognitionEvent_2_0(hwHandle, status), + mCookie); + } + } + + private void sendPhraseRecognitionEvent(int hwHandle, int status) throws RemoteException { + if (mCallback instanceof android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback) { + ((android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback) mCallback).phraseRecognitionCallback_2_1( + createPhraseRecognitionEvent_2_1(hwHandle, status), mCookie); + } else { + mCallback.phraseRecognitionCallback( + createPhraseRecognitionEvent_2_0(hwHandle, status), mCookie); + } + } + } +} From 7d383d1a98957d46aa09ac64091e4eae73b4c662 Mon Sep 17 00:00:00 2001 From: Ytai Ben-Tsvi Date: Mon, 25 Nov 2019 12:47:40 -0800 Subject: [PATCH 5/5] Migrate SoundTrigger implementation to new service The API offered by SoundTrigger.java is now implemented on top of the new soundtrigger_middleware service. There is no longer any need for JNI - the API now talks directly with the AIDL interface of the new service. In the process, some annotations and input validation have been added to improve the overall quality of this API. Change-Id: I731ffd5a275b88f38d84dd3daa022a13f97a5ee1 Bug: 142070343 --- .../hardware/soundtrigger/ConversionUtil.java | 329 +++++ .../hardware/soundtrigger/SoundTrigger.java | 176 ++- .../soundtrigger/SoundTriggerModule.java | 334 +++-- core/jni/Android.bp | 2 - core/jni/AndroidRuntime.cpp | 2 - core/jni/android_hardware_SoundTrigger.cpp | 1071 ----------------- .../SoundTriggerModule.java | 2 +- 7 files changed, 665 insertions(+), 1251 deletions(-) create mode 100644 core/java/android/hardware/soundtrigger/ConversionUtil.java delete mode 100644 core/jni/android_hardware_SoundTrigger.cpp diff --git a/core/java/android/hardware/soundtrigger/ConversionUtil.java b/core/java/android/hardware/soundtrigger/ConversionUtil.java new file mode 100644 index 0000000000000..8231c58a105e6 --- /dev/null +++ b/core/java/android/hardware/soundtrigger/ConversionUtil.java @@ -0,0 +1,329 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.soundtrigger; + +import android.hardware.soundtrigger.ModelParams; +import android.media.AudioFormat; +import android.media.audio.common.AudioConfig; +import android.media.soundtrigger_middleware.ConfidenceLevel; +import android.media.soundtrigger_middleware.ModelParameterRange; +import android.media.soundtrigger_middleware.Phrase; +import android.media.soundtrigger_middleware.PhraseRecognitionEvent; +import android.media.soundtrigger_middleware.PhraseRecognitionExtra; +import android.media.soundtrigger_middleware.PhraseSoundModel; +import android.media.soundtrigger_middleware.RecognitionConfig; +import android.media.soundtrigger_middleware.RecognitionEvent; +import android.media.soundtrigger_middleware.RecognitionMode; +import android.media.soundtrigger_middleware.SoundModel; +import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; +import android.media.soundtrigger_middleware.SoundTriggerModuleProperties; + +import android.annotation.Nullable; + +import java.util.Arrays; +import java.util.UUID; + +/** @hide */ +class ConversionUtil { + public static SoundTrigger.ModuleProperties aidl2apiModuleDescriptor( + SoundTriggerModuleDescriptor aidlDesc) { + SoundTriggerModuleProperties properties = aidlDesc.properties; + return new SoundTrigger.ModuleProperties( + aidlDesc.handle, + properties.implementor, + properties.description, + properties.uuid, + properties.version, + properties.maxSoundModels, + properties.maxKeyPhrases, + properties.maxUsers, + aidl2apiRecognitionModes(properties.recognitionModes), + properties.captureTransition, + properties.maxBufferMs, + properties.concurrentCapture, + properties.powerConsumptionMw, + properties.triggerInEvent + ); + } + + public static int aidl2apiRecognitionModes(int aidlModes) { + int result = 0; + if ((aidlModes & RecognitionMode.VOICE_TRIGGER) != 0) { + result |= SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; + } + if ((aidlModes & RecognitionMode.USER_IDENTIFICATION) != 0) { + result |= SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; + } + if ((aidlModes & RecognitionMode.USER_AUTHENTICATION) != 0) { + result |= SoundTrigger.RECOGNITION_MODE_USER_AUTHENTICATION; + } + if ((aidlModes & RecognitionMode.GENERIC_TRIGGER) != 0) { + result |= SoundTrigger.RECOGNITION_MODE_GENERIC; + } + return result; + } + + public static int api2aidlRecognitionModes(int apiModes) { + int result = 0; + if ((apiModes & SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER) != 0) { + result |= RecognitionMode.VOICE_TRIGGER; + } + if ((apiModes & SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION) != 0) { + result |= RecognitionMode.USER_IDENTIFICATION; + } + if ((apiModes & SoundTrigger.RECOGNITION_MODE_USER_AUTHENTICATION) != 0) { + result |= RecognitionMode.USER_AUTHENTICATION; + } + if ((apiModes & SoundTrigger.RECOGNITION_MODE_GENERIC) != 0) { + result |= RecognitionMode.GENERIC_TRIGGER; + } + return result; + } + + + public static SoundModel api2aidlGenericSoundModel(SoundTrigger.GenericSoundModel apiModel) { + return api2aidlSoundModel(apiModel); + } + + public static SoundModel api2aidlSoundModel(SoundTrigger.SoundModel apiModel) { + SoundModel aidlModel = new SoundModel(); + aidlModel.type = apiModel.type; + aidlModel.uuid = api2aidlUuid(apiModel.uuid); + aidlModel.vendorUuid = api2aidlUuid(apiModel.vendorUuid); + aidlModel.data = Arrays.copyOf(apiModel.data, apiModel.data.length); + return aidlModel; + } + + public static String api2aidlUuid(UUID apiUuid) { + return apiUuid.toString(); + } + + public static PhraseSoundModel api2aidlPhraseSoundModel( + SoundTrigger.KeyphraseSoundModel apiModel) { + PhraseSoundModel aidlModel = new PhraseSoundModel(); + aidlModel.common = api2aidlSoundModel(apiModel); + aidlModel.phrases = new Phrase[apiModel.keyphrases.length]; + for (int i = 0; i < apiModel.keyphrases.length; ++i) { + aidlModel.phrases[i] = api2aidlPhrase(apiModel.keyphrases[i]); + } + return aidlModel; + } + + public static Phrase api2aidlPhrase(SoundTrigger.Keyphrase apiPhrase) { + Phrase aidlPhrase = new Phrase(); + aidlPhrase.id = apiPhrase.id; + aidlPhrase.recognitionModes = api2aidlRecognitionModes(apiPhrase.recognitionModes); + aidlPhrase.users = Arrays.copyOf(apiPhrase.users, apiPhrase.users.length); + aidlPhrase.locale = apiPhrase.locale; + aidlPhrase.text = apiPhrase.text; + return aidlPhrase; + } + + public static RecognitionConfig api2aidlRecognitionConfig( + SoundTrigger.RecognitionConfig apiConfig) { + RecognitionConfig aidlConfig = new RecognitionConfig(); + aidlConfig.captureRequested = apiConfig.captureRequested; + // apiConfig.allowMultipleTriggers is ignored by the lower layers. + aidlConfig.phraseRecognitionExtras = + new PhraseRecognitionExtra[apiConfig.keyphrases.length]; + for (int i = 0; i < apiConfig.keyphrases.length; ++i) { + aidlConfig.phraseRecognitionExtras[i] = api2aidlPhraseRecognitionExtra( + apiConfig.keyphrases[i]); + } + aidlConfig.data = Arrays.copyOf(apiConfig.data, apiConfig.data.length); + return aidlConfig; + } + + public static PhraseRecognitionExtra api2aidlPhraseRecognitionExtra( + SoundTrigger.KeyphraseRecognitionExtra apiExtra) { + PhraseRecognitionExtra aidlExtra = new PhraseRecognitionExtra(); + aidlExtra.id = apiExtra.id; + aidlExtra.recognitionModes = api2aidlRecognitionModes(apiExtra.recognitionModes); + aidlExtra.confidenceLevel = apiExtra.coarseConfidenceLevel; + aidlExtra.levels = new ConfidenceLevel[apiExtra.confidenceLevels.length]; + for (int i = 0; i < apiExtra.confidenceLevels.length; ++i) { + aidlExtra.levels[i] = api2aidlConfidenceLevel(apiExtra.confidenceLevels[i]); + } + return aidlExtra; + } + + public static SoundTrigger.KeyphraseRecognitionExtra aidl2apiPhraseRecognitionExtra( + PhraseRecognitionExtra aidlExtra) { + SoundTrigger.ConfidenceLevel[] apiLevels = + new SoundTrigger.ConfidenceLevel[aidlExtra.levels.length]; + for (int i = 0; i < aidlExtra.levels.length; ++i) { + apiLevels[i] = aidl2apiConfidenceLevel(aidlExtra.levels[i]); + } + return new SoundTrigger.KeyphraseRecognitionExtra(aidlExtra.id, + aidl2apiRecognitionModes(aidlExtra.recognitionModes), + aidlExtra.confidenceLevel, apiLevels); + } + + public static ConfidenceLevel api2aidlConfidenceLevel( + SoundTrigger.ConfidenceLevel apiLevel) { + ConfidenceLevel aidlLevel = new ConfidenceLevel(); + aidlLevel.levelPercent = apiLevel.confidenceLevel; + aidlLevel.userId = apiLevel.userId; + return aidlLevel; + } + + public static SoundTrigger.ConfidenceLevel aidl2apiConfidenceLevel( + ConfidenceLevel apiLevel) { + return new SoundTrigger.ConfidenceLevel(apiLevel.userId, apiLevel.levelPercent); + } + + public static SoundTrigger.RecognitionEvent aidl2apiRecognitionEvent( + int modelHandle, RecognitionEvent aidlEvent) { + return new SoundTrigger.GenericRecognitionEvent( + aidlEvent.status, + modelHandle, aidlEvent.captureAvailable, aidlEvent.captureSession, + aidlEvent.captureDelayMs, aidlEvent.capturePreambleMs, aidlEvent.triggerInData, + aidl2apiAudioFormat(aidlEvent.audioConfig), aidlEvent.data); + } + + public static SoundTrigger.RecognitionEvent aidl2apiPhraseRecognitionEvent( + int modelHandle, + PhraseRecognitionEvent aidlEvent) { + SoundTrigger.KeyphraseRecognitionExtra[] apiExtras = + new SoundTrigger.KeyphraseRecognitionExtra[aidlEvent.phraseExtras.length]; + for (int i = 0; i < aidlEvent.phraseExtras.length; ++i) { + apiExtras[i] = aidl2apiPhraseRecognitionExtra(aidlEvent.phraseExtras[i]); + } + return new SoundTrigger.KeyphraseRecognitionEvent(aidlEvent.common.status, modelHandle, + aidlEvent.common.captureAvailable, + aidlEvent.common.captureSession, aidlEvent.common.captureDelayMs, + aidlEvent.common.capturePreambleMs, aidlEvent.common.triggerInData, + aidl2apiAudioFormat(aidlEvent.common.audioConfig), aidlEvent.common.data, + apiExtras); + } + + public static AudioFormat aidl2apiAudioFormat(AudioConfig audioConfig) { + AudioFormat.Builder apiBuilder = new AudioFormat.Builder(); + apiBuilder.setSampleRate(audioConfig.sampleRateHz); + apiBuilder.setChannelMask(aidl2apiChannelInMask(audioConfig.channelMask)); + apiBuilder.setEncoding(aidl2apiEncoding(audioConfig.format)); + return apiBuilder.build(); + } + + public static int aidl2apiEncoding(int aidlFormat) { + switch (aidlFormat) { + case android.media.audio.common.AudioFormat.PCM + | android.media.audio.common.AudioFormat.PCM_SUB_16_BIT: + return AudioFormat.ENCODING_PCM_16BIT; + + case android.media.audio.common.AudioFormat.PCM + | android.media.audio.common.AudioFormat.PCM_SUB_8_BIT: + return AudioFormat.ENCODING_PCM_8BIT; + + case android.media.audio.common.AudioFormat.PCM + | android.media.audio.common.AudioFormat.PCM_SUB_FLOAT: + case android.media.audio.common.AudioFormat.PCM + | android.media.audio.common.AudioFormat.PCM_SUB_8_24_BIT: + case android.media.audio.common.AudioFormat.PCM + | android.media.audio.common.AudioFormat.PCM_SUB_24_BIT_PACKED: + case android.media.audio.common.AudioFormat.PCM + | android.media.audio.common.AudioFormat.PCM_SUB_32_BIT: + return AudioFormat.ENCODING_PCM_FLOAT; + + case android.media.audio.common.AudioFormat.AC3: + return AudioFormat.ENCODING_AC3; + + case android.media.audio.common.AudioFormat.E_AC3: + return AudioFormat.ENCODING_E_AC3; + + case android.media.audio.common.AudioFormat.DTS: + return AudioFormat.ENCODING_DTS; + + case android.media.audio.common.AudioFormat.DTS_HD: + return AudioFormat.ENCODING_DTS_HD; + + case android.media.audio.common.AudioFormat.MP3: + return AudioFormat.ENCODING_MP3; + + case android.media.audio.common.AudioFormat.AAC + | android.media.audio.common.AudioFormat.AAC_SUB_LC: + return AudioFormat.ENCODING_AAC_LC; + + case android.media.audio.common.AudioFormat.AAC + | android.media.audio.common.AudioFormat.AAC_SUB_HE_V1: + return AudioFormat.ENCODING_AAC_HE_V1; + + case android.media.audio.common.AudioFormat.AAC + | android.media.audio.common.AudioFormat.AAC_SUB_HE_V2: + return AudioFormat.ENCODING_AAC_HE_V2; + + case android.media.audio.common.AudioFormat.IEC61937: + return AudioFormat.ENCODING_IEC61937; + + case android.media.audio.common.AudioFormat.DOLBY_TRUEHD: + return AudioFormat.ENCODING_DOLBY_TRUEHD; + + case android.media.audio.common.AudioFormat.AAC + | android.media.audio.common.AudioFormat.AAC_SUB_ELD: + return AudioFormat.ENCODING_AAC_ELD; + + case android.media.audio.common.AudioFormat.AAC + | android.media.audio.common.AudioFormat.AAC_SUB_XHE: + return AudioFormat.ENCODING_AAC_XHE; + + case android.media.audio.common.AudioFormat.AC4: + return AudioFormat.ENCODING_AC4; + + case android.media.audio.common.AudioFormat.E_AC3 + | android.media.audio.common.AudioFormat.E_AC3_SUB_JOC: + return AudioFormat.ENCODING_E_AC3_JOC; + + case android.media.audio.common.AudioFormat.MAT: + case android.media.audio.common.AudioFormat.MAT + | android.media.audio.common.AudioFormat.MAT_SUB_1_0: + case android.media.audio.common.AudioFormat.MAT + | android.media.audio.common.AudioFormat.MAT_SUB_2_0: + case android.media.audio.common.AudioFormat.MAT + | android.media.audio.common.AudioFormat.MAT_SUB_2_1: + return AudioFormat.ENCODING_DOLBY_MAT; + + case android.media.audio.common.AudioFormat.DEFAULT: + return AudioFormat.ENCODING_DEFAULT; + + default: + return AudioFormat.ENCODING_INVALID; + } + } + + public static int api2aidlModelParameter(int apiParam) { + switch (apiParam) { + case ModelParams.THRESHOLD_FACTOR: + return android.media.soundtrigger_middleware.ModelParameter.THRESHOLD_FACTOR; + default: + return android.media.soundtrigger_middleware.ModelParameter.INVALID; + } + } + + public static int aidl2apiChannelInMask(int aidlMask) { + // We're assuming AudioFormat.CHANNEL_IN_* constants are kept in sync with + // android.media.audio.common.AudioChannelMask. + return aidlMask; + } + + public static SoundTrigger.ModelParamRange aidl2apiModelParameterRange( + @Nullable ModelParameterRange aidlRange) { + if (aidlRange == null) { + return null; + } + return new SoundTrigger.ModelParamRange(aidlRange.minInclusive, aidlRange.maxInclusive); + } +} diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java index 86f3eec4109ee..5484df416e79a 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.java +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java @@ -22,18 +22,29 @@ import static android.system.OsConstants.ENOSYS; import static android.system.OsConstants.EPERM; import static android.system.OsConstants.EPIPE; +import static java.util.Objects.requireNonNull; + import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemApi; import android.annotation.UnsupportedAppUsage; import android.app.ActivityThread; +import android.content.Context; import android.media.AudioFormat; +import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService; +import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; import android.os.Parcel; import android.os.Parcelable; +import android.os.RemoteException; +import android.os.ServiceManager; +import android.util.Log; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.UUID; /** @@ -44,6 +55,7 @@ import java.util.UUID; */ @SystemApi public class SoundTrigger { + private static final String TAG = "SoundTrigger"; private SoundTrigger() { } @@ -119,15 +131,15 @@ public class SoundTrigger { * recognition callback event */ public final boolean returnsTriggerInEvent; - ModuleProperties(int id, String implementor, String description, - String uuid, int version, int maxSoundModels, int maxKeyphrases, + ModuleProperties(int id, @NonNull String implementor, @NonNull String description, + @NonNull String uuid, int version, int maxSoundModels, int maxKeyphrases, int maxUsers, int recognitionModes, boolean supportsCaptureTransition, int maxBufferMs, boolean supportsConcurrentCapture, int powerConsumptionMw, boolean returnsTriggerInEvent) { this.id = id; - this.implementor = implementor; - this.description = description; - this.uuid = UUID.fromString(uuid); + this.implementor = requireNonNull(implementor); + this.description = requireNonNull(description); + this.uuid = UUID.fromString(requireNonNull(uuid)); this.version = version; this.maxSoundModels = maxSoundModels; this.maxKeyphrases = maxKeyphrases; @@ -231,6 +243,7 @@ public class SoundTrigger { /** Unique sound model identifier */ @UnsupportedAppUsage + @NonNull public final UUID uuid; /** Sound model type (e.g. TYPE_KEYPHRASE); */ @@ -238,17 +251,20 @@ public class SoundTrigger { /** Unique sound model vendor identifier */ @UnsupportedAppUsage + @NonNull public final UUID vendorUuid; /** Opaque data. For use by vendor implementation and enrollment application */ @UnsupportedAppUsage + @NonNull public final byte[] data; - public SoundModel(UUID uuid, UUID vendorUuid, int type, byte[] data) { - this.uuid = uuid; - this.vendorUuid = vendorUuid; + public SoundModel(@NonNull UUID uuid, @Nullable UUID vendorUuid, int type, + @Nullable byte[] data) { + this.uuid = requireNonNull(uuid); + this.vendorUuid = vendorUuid != null ? vendorUuid : new UUID(0, 0); this.type = type; - this.data = data; + this.data = data != null ? data : new byte[0]; } @Override @@ -271,8 +287,6 @@ public class SoundTrigger { if (!(obj instanceof SoundModel)) return false; SoundModel other = (SoundModel) obj; - if (!Arrays.equals(data, other.data)) - return false; if (type != other.type) return false; if (uuid == null) { @@ -285,6 +299,8 @@ public class SoundTrigger { return false; } else if (!vendorUuid.equals(other.vendorUuid)) return false; + if (!Arrays.equals(data, other.data)) + return false; return true; } } @@ -306,24 +322,28 @@ public class SoundTrigger { /** Locale of the keyphrase. JAVA Locale string e.g en_US */ @UnsupportedAppUsage + @NonNull public final String locale; /** Key phrase text */ @UnsupportedAppUsage + @NonNull public final String text; /** Users this key phrase has been trained for. countains sound trigger specific user IDs * derived from system user IDs {@link android.os.UserHandle#getIdentifier()}. */ @UnsupportedAppUsage + @NonNull public final int[] users; @UnsupportedAppUsage - public Keyphrase(int id, int recognitionModes, String locale, String text, int[] users) { + public Keyphrase(int id, int recognitionModes, @NonNull String locale, @NonNull String text, + @Nullable int[] users) { this.id = id; this.recognitionModes = recognitionModes; - this.locale = locale; - this.text = text; - this.users = users; + this.locale = requireNonNull(locale); + this.text = requireNonNull(text); + this.users = users != null ? users : new int[0]; } public static final @android.annotation.NonNull Parcelable.Creator CREATOR @@ -427,13 +447,15 @@ public class SoundTrigger { public static class KeyphraseSoundModel extends SoundModel implements Parcelable { /** Key phrases in this sound model */ @UnsupportedAppUsage + @NonNull public final Keyphrase[] keyphrases; // keyword phrases in model @UnsupportedAppUsage public KeyphraseSoundModel( - UUID uuid, UUID vendorUuid, byte[] data, Keyphrase[] keyphrases) { + @NonNull UUID uuid, @NonNull UUID vendorUuid, @Nullable byte[] data, + @Nullable Keyphrase[] keyphrases) { super(uuid, vendorUuid, TYPE_KEYPHRASE, data); - this.keyphrases = keyphrases; + this.keyphrases = keyphrases != null ? keyphrases : new Keyphrase[0]; } public static final @android.annotation.NonNull Parcelable.Creator CREATOR @@ -528,7 +550,8 @@ public class SoundTrigger { }; @UnsupportedAppUsage - public GenericSoundModel(UUID uuid, UUID vendorUuid, byte[] data) { + public GenericSoundModel(@NonNull UUID uuid, @NonNull UUID vendorUuid, + @Nullable byte[] data) { super(uuid, vendorUuid, TYPE_GENERIC_SOUND, data); } @@ -648,6 +671,12 @@ public class SoundTrigger { * @hide */ public static final int RECOGNITION_MODE_USER_AUTHENTICATION = 0x4; + /** + * Generic (non-speech) recognition. + * + * @hide + */ + public static final int RECOGNITION_MODE_GENERIC = 0x8; /** * Status codes for {@link RecognitionEvent} @@ -739,6 +768,7 @@ public class SoundTrigger { * * @hide */ + @NonNull public final AudioFormat captureFormat; /** * Opaque data for use by system applications who know about voice engine internals, @@ -747,13 +777,14 @@ public class SoundTrigger { * @hide */ @UnsupportedAppUsage + @NonNull public final byte[] data; /** @hide */ @UnsupportedAppUsage public RecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, - boolean triggerInData, AudioFormat captureFormat, byte[] data) { + boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data) { this.status = status; this.soundModelHandle = soundModelHandle; this.captureAvailable = captureAvailable; @@ -761,8 +792,8 @@ public class SoundTrigger { this.captureDelayMs = captureDelayMs; this.capturePreambleMs = capturePreambleMs; this.triggerInData = triggerInData; - this.captureFormat = captureFormat; - this.data = data; + this.captureFormat = requireNonNull(captureFormat); + this.data = data != null ? data : new byte[0]; } /** @@ -965,19 +996,21 @@ public class SoundTrigger { /** List of all keyphrases in the sound model for which recognition should be performed with * options for each keyphrase. */ @UnsupportedAppUsage + @NonNull public final KeyphraseRecognitionExtra keyphrases[]; /** Opaque data for use by system applications who know about voice engine internals, * typically during enrollment. */ @UnsupportedAppUsage + @NonNull public final byte[] data; @UnsupportedAppUsage public RecognitionConfig(boolean captureRequested, boolean allowMultipleTriggers, - KeyphraseRecognitionExtra[] keyphrases, byte[] data) { + @Nullable KeyphraseRecognitionExtra[] keyphrases, @Nullable byte[] data) { this.captureRequested = captureRequested; this.allowMultipleTriggers = allowMultipleTriggers; - this.keyphrases = keyphrases; - this.data = data; + this.keyphrases = keyphrases != null ? keyphrases : new KeyphraseRecognitionExtra[0]; + this.data = data != null ? data : new byte[0]; } public static final @android.annotation.NonNull Parcelable.Creator CREATOR @@ -1126,15 +1159,17 @@ public class SoundTrigger { /** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to * be recognized (RecognitionConfig) */ @UnsupportedAppUsage + @NonNull public final ConfidenceLevel[] confidenceLevels; @UnsupportedAppUsage public KeyphraseRecognitionExtra(int id, int recognitionModes, int coarseConfidenceLevel, - ConfidenceLevel[] confidenceLevels) { + @Nullable ConfidenceLevel[] confidenceLevels) { this.id = id; this.recognitionModes = recognitionModes; this.coarseConfidenceLevel = coarseConfidenceLevel; - this.confidenceLevels = confidenceLevels; + this.confidenceLevels = + confidenceLevels != null ? confidenceLevels : new ConfidenceLevel[0]; } public static final @android.annotation.NonNull Parcelable.Creator CREATOR @@ -1217,16 +1252,18 @@ public class SoundTrigger { public static class KeyphraseRecognitionEvent extends RecognitionEvent implements Parcelable { /** Indicates if the key phrase is present in the buffered audio available for capture */ @UnsupportedAppUsage + @NonNull public final KeyphraseRecognitionExtra[] keyphraseExtras; @UnsupportedAppUsage public KeyphraseRecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, - boolean triggerInData, AudioFormat captureFormat, byte[] data, - KeyphraseRecognitionExtra[] keyphraseExtras) { + boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, + @Nullable KeyphraseRecognitionExtra[] keyphraseExtras) { super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, capturePreambleMs, triggerInData, captureFormat, data); - this.keyphraseExtras = keyphraseExtras; + this.keyphraseExtras = + keyphraseExtras != null ? keyphraseExtras : new KeyphraseRecognitionExtra[0]; } public static final @android.annotation.NonNull Parcelable.Creator CREATOR @@ -1343,8 +1380,8 @@ public class SoundTrigger { @UnsupportedAppUsage public GenericRecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, - int capturePreambleMs, boolean triggerInData, AudioFormat captureFormat, - byte[] data) { + int capturePreambleMs, boolean triggerInData, @NonNull AudioFormat captureFormat, + @Nullable byte[] data) { super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, capturePreambleMs, triggerInData, captureFormat, data); @@ -1408,13 +1445,14 @@ public class SoundTrigger { /** The updated sound model handle */ public final int soundModelHandle; /** New sound model data */ + @NonNull public final byte[] data; @UnsupportedAppUsage - SoundModelEvent(int status, int soundModelHandle, byte[] data) { + SoundModelEvent(int status, int soundModelHandle, @Nullable byte[] data) { this.status = status; this.soundModelHandle = soundModelHandle; - this.data = data; + this.data = data != null ? data : new byte[0]; } public static final @android.annotation.NonNull Parcelable.Creator CREATOR @@ -1498,8 +1536,9 @@ public class SoundTrigger { * @hide */ public static final int SERVICE_STATE_DISABLED = 1; - - /** + private static Object mServiceLock = new Object(); + private static ISoundTriggerMiddlewareService mService; + /** * @return returns current package name. */ static String getCurrentOpPackageName() { @@ -1523,24 +1562,21 @@ public class SoundTrigger { * @hide */ @UnsupportedAppUsage - public static int listModules(ArrayList modules) { - return listModules(getCurrentOpPackageName(), modules); + public static int listModules(@NonNull ArrayList modules) { + try { + SoundTriggerModuleDescriptor[] descs = getService().listModules(); + modules.clear(); + modules.ensureCapacity(descs.length); + for (SoundTriggerModuleDescriptor desc : descs) { + modules.add(ConversionUtil.aidl2apiModuleDescriptor(desc)); + } + return STATUS_OK; + } catch (RemoteException e) { + Log.e(TAG, "Exception caught", e); + return STATUS_DEAD_OBJECT; + } } - /** - * Returns a list of descriptors for all hardware modules loaded. - * @param opPackageName - * @param modules A ModuleProperties array where the list will be returned. - * @return - {@link #STATUS_OK} in case of success - * - {@link #STATUS_ERROR} in case of unspecified error - * - {@link #STATUS_PERMISSION_DENIED} if the caller does not have system permission - * - {@link #STATUS_NO_INIT} if the native service cannot be reached - * - {@link #STATUS_BAD_VALUE} if modules is null - * - {@link #STATUS_DEAD_OBJECT} if the binder transaction to the native service fails - */ - private static native int listModules(String opPackageName, - ArrayList modules); - /** * Get an interface on a hardware module to control sound models and recognition on * this module. @@ -1553,14 +1589,40 @@ public class SoundTrigger { * @hide */ @UnsupportedAppUsage - public static SoundTriggerModule attachModule(int moduleId, - StatusListener listener, - Handler handler) { - if (listener == null) { + public static @NonNull SoundTriggerModule attachModule(int moduleId, + @NonNull StatusListener listener, + @Nullable Handler handler) { + Looper looper = handler != null ? handler.getLooper() : Looper.getMainLooper(); + try { + return new SoundTriggerModule(getService(), moduleId, listener, looper); + } catch (RemoteException e) { + Log.e(TAG, "", e); return null; } - SoundTriggerModule module = new SoundTriggerModule(moduleId, listener, handler); - return module; + } + + private static ISoundTriggerMiddlewareService getService() { + synchronized (mServiceLock) { + while (true) { + IBinder binder = null; + try { + binder = + ServiceManager.getServiceOrThrow( + Context.SOUND_TRIGGER_MIDDLEWARE_SERVICE); + binder.linkToDeath(() -> { + synchronized (mServiceLock) { + mService = null; + } + }, 0); + mService = ISoundTriggerMiddlewareService.Stub.asInterface(binder); + break; + } catch (Exception e) { + Log.e(TAG, "Failed to bind to soundtrigger service", e); + } + } + return mService; + } + } /** diff --git a/core/java/android/hardware/soundtrigger/SoundTriggerModule.java b/core/java/android/hardware/soundtrigger/SoundTriggerModule.java index b16ef5c433463..7cf5600192394 100644 --- a/core/java/android/hardware/soundtrigger/SoundTriggerModule.java +++ b/core/java/android/hardware/soundtrigger/SoundTriggerModule.java @@ -16,14 +16,23 @@ package android.hardware.soundtrigger; +import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.UnsupportedAppUsage; -import android.hardware.soundtrigger.SoundTrigger.ModelParamRange; +import android.media.soundtrigger_middleware.ISoundTriggerCallback; +import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService; +import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.ModelParameterRange; +import android.media.soundtrigger_middleware.PhraseRecognitionEvent; +import android.media.soundtrigger_middleware.PhraseSoundModel; +import android.media.soundtrigger_middleware.RecognitionEvent; +import android.media.soundtrigger_middleware.SoundModel; import android.os.Handler; +import android.os.IBinder; import android.os.Looper; import android.os.Message; - -import java.lang.ref.WeakReference; +import android.os.RemoteException; +import android.util.Log; /** * The SoundTriggerModule provides APIs to control sound models and sound detection @@ -32,39 +41,47 @@ import java.lang.ref.WeakReference; * @hide */ public class SoundTriggerModule { - @UnsupportedAppUsage - private long mNativeContext; + private static final String TAG = "SoundTriggerModule"; - @UnsupportedAppUsage - private int mId; - private NativeEventHandlerDelegate mEventHandlerDelegate; - - // to be kept in sync with core/jni/android_hardware_SoundTrigger.cpp private static final int EVENT_RECOGNITION = 1; private static final int EVENT_SERVICE_DIED = 2; - private static final int EVENT_SOUNDMODEL = 3; - private static final int EVENT_SERVICE_STATE_CHANGE = 4; + private static final int EVENT_SERVICE_STATE_CHANGE = 3; + @UnsupportedAppUsage + private int mId; + private EventHandlerDelegate mEventHandlerDelegate; + private ISoundTriggerModule mService; - SoundTriggerModule(int moduleId, SoundTrigger.StatusListener listener, Handler handler) { + SoundTriggerModule(@NonNull ISoundTriggerMiddlewareService service, + int moduleId, @NonNull SoundTrigger.StatusListener listener, @NonNull Looper looper) + throws RemoteException { mId = moduleId; - mEventHandlerDelegate = new NativeEventHandlerDelegate(listener, handler); - native_setup(SoundTrigger.getCurrentOpPackageName(), - new WeakReference(this)); + mEventHandlerDelegate = new EventHandlerDelegate(listener, looper); + mService = service.attach(moduleId, mEventHandlerDelegate); + mService.asBinder().linkToDeath(mEventHandlerDelegate, 0); } - private native void native_setup(String opPackageName, Object moduleThis); @Override protected void finalize() { - native_finalize(); + detach(); } - private native void native_finalize(); /** * Detach from this module. The {@link SoundTrigger.StatusListener} callback will not be called * anymore and associated resources will be released. - * */ + * All models must have been unloaded prior to detaching. + */ @UnsupportedAppUsage - public native void detach(); + public synchronized void detach() { + try { + if (mService != null) { + mService.asBinder().unlinkToDeath(mEventHandlerDelegate, 0); + mService.detach(); + mService = null; + } + } catch (Exception e) { + handleException(e); + } + } /** * Load a {@link SoundTrigger.SoundModel} to the hardware. A sound model must be loaded in @@ -82,7 +99,26 @@ public class SoundTriggerModule { * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence */ @UnsupportedAppUsage - public native int loadSoundModel(SoundTrigger.SoundModel model, int[] soundModelHandle); + public synchronized int loadSoundModel(@NonNull SoundTrigger.SoundModel model, + @NonNull int[] soundModelHandle) { + try { + if (model instanceof SoundTrigger.GenericSoundModel) { + SoundModel aidlModel = ConversionUtil.api2aidlGenericSoundModel( + (SoundTrigger.GenericSoundModel) model); + soundModelHandle[0] = mService.loadModel(aidlModel); + return SoundTrigger.STATUS_OK; + } + if (model instanceof SoundTrigger.KeyphraseSoundModel) { + PhraseSoundModel aidlModel = ConversionUtil.api2aidlPhraseSoundModel( + (SoundTrigger.KeyphraseSoundModel) model); + soundModelHandle[0] = mService.loadPhraseModel(aidlModel); + return SoundTrigger.STATUS_OK; + } + return SoundTrigger.STATUS_BAD_VALUE; + } catch (Exception e) { + return handleException(e); + } + } /** * Unload a {@link SoundTrigger.SoundModel} and abort any pendiong recognition @@ -97,7 +133,14 @@ public class SoundTriggerModule { * service fails */ @UnsupportedAppUsage - public native int unloadSoundModel(int soundModelHandle); + public synchronized int unloadSoundModel(int soundModelHandle) { + try { + mService.unloadModel(soundModelHandle); + return SoundTrigger.STATUS_OK; + } catch (Exception e) { + return handleException(e); + } + } /** * Start listening to all key phrases in a {@link SoundTrigger.SoundModel}. @@ -117,7 +160,16 @@ public class SoundTriggerModule { * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence */ @UnsupportedAppUsage - public native int startRecognition(int soundModelHandle, SoundTrigger.RecognitionConfig config); + public synchronized int startRecognition(int soundModelHandle, + SoundTrigger.RecognitionConfig config) { + try { + mService.startRecognition(soundModelHandle, + ConversionUtil.api2aidlRecognitionConfig(config)); + return SoundTrigger.STATUS_OK; + } catch (Exception e) { + return handleException(e); + } + } /** * Stop listening to all key phrases in a {@link SoundTrigger.SoundModel} @@ -133,12 +185,20 @@ public class SoundTriggerModule { * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence */ @UnsupportedAppUsage - public native int stopRecognition(int soundModelHandle); + public synchronized int stopRecognition(int soundModelHandle) { + try { + mService.stopRecognition(soundModelHandle); + return SoundTrigger.STATUS_OK; + } catch (Exception e) { + return handleException(e); + } + } /** * Get the current state of a {@link SoundTrigger.SoundModel}. - * The state will be returned asynchronously as a {@link SoundTrigger#RecognitionEvent} - * in the callback registered in the {@link SoundTrigger.startRecognition} method. + * The state will be returned asynchronously as a {@link SoundTrigger.RecognitionEvent} + * in the callback registered in the + * {@link SoundTrigger#attachModule(int, SoundTrigger.StatusListener, Handler)} method. * @param soundModelHandle The sound model handle indicating which model's state to return * @return - {@link SoundTrigger#STATUS_OK} in case of success * - {@link SoundTrigger#STATUS_ERROR} in case of unspecified error @@ -150,46 +210,71 @@ public class SoundTriggerModule { * service fails * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence */ - public native int getModelState(int soundModelHandle); + public synchronized int getModelState(int soundModelHandle) { + try { + mService.forceRecognitionEvent(soundModelHandle); + return SoundTrigger.STATUS_OK; + } catch (Exception e) { + return handleException(e); + } + } /** * Set a model specific {@link ModelParams} with the given value. This - * parameter will keep its value for the duration the model is loaded regardless of starting and + * parameter will keep its value for the duration the model is loaded regardless of starting + * and * stopping recognition. Once the model is unloaded, the value will be lost. - * {@link SoundTriggerModule#isParameterSupported} should be checked first before calling this - * method. + * {@link SoundTriggerModule#queryParameter(int, int)} should be checked first before calling + * this method. * * @param soundModelHandle handle of model to apply parameter - * @param modelParam {@link ModelParams} - * @param value Value to set + * @param modelParam {@link ModelParams} + * @param value Value to set * @return - {@link SoundTrigger#STATUS_OK} in case of success - * - {@link SoundTrigger#STATUS_NO_INIT} if the native service cannot be reached - * - {@link SoundTrigger#STATUS_BAD_VALUE} invalid input parameter - * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence or - * if API is not supported by HAL + * - {@link SoundTrigger#STATUS_NO_INIT} if the native service cannot be reached + * - {@link SoundTrigger#STATUS_BAD_VALUE} invalid input parameter + * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence or + * if API is not supported by HAL */ - public native int setParameter(int soundModelHandle, - @ModelParams int modelParam, int value); + public synchronized int setParameter(int soundModelHandle, @ModelParams int modelParam, + int value) { + try { + mService.setModelParameter(soundModelHandle, + ConversionUtil.api2aidlModelParameter(modelParam), value); + return SoundTrigger.STATUS_OK; + } catch (Exception e) { + return handleException(e); + } + } /** * Get a model specific {@link ModelParams}. This parameter will keep its value * for the duration the model is loaded regardless of starting and stopping recognition. * Once the model is unloaded, the value will be lost. If the value is not set, a default * value is returned. See {@link ModelParams} for parameter default values. - * {@link SoundTriggerModule#isParameterSupported} should be checked first before + * {@link SoundTriggerModule#queryParameter(int, int)} should be checked first before * calling this method. Otherwise, an exception can be thrown. * * @param soundModelHandle handle of model to get parameter - * @param modelParam {@link ModelParams} + * @param modelParam {@link ModelParams} * @return value of parameter * @throws UnsupportedOperationException if hal or model do not support this API. - * {@link SoundTriggerModule#isParameterSupported} should be checked first. - * @throws IllegalArgumentException if invalid model handle or parameter is passed. - * {@link SoundTriggerModule#isParameterSupported} should be checked first. + * {@link SoundTriggerModule#queryParameter(int, int)} + * should + * be checked first. + * @throws IllegalArgumentException if invalid model handle or parameter is passed. + * {@link SoundTriggerModule#queryParameter(int, int)} + * should be checked first. */ - public native int getParameter(int soundModelHandle, - @ModelParams int modelParam) - throws UnsupportedOperationException, IllegalArgumentException; + public synchronized int getParameter(int soundModelHandle, @ModelParams int modelParam) + throws UnsupportedOperationException, IllegalArgumentException { + try { + return mService.getModelParameter(soundModelHandle, + ConversionUtil.api2aidlModelParameter(modelParam)); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } /** * Determine if parameter control is supported for the given model handle. @@ -197,85 +282,98 @@ public class SoundTriggerModule { * {@link SoundTriggerModule#getParameter}. * * @param soundModelHandle handle of model to get parameter - * @param modelParam {@link ModelParams} + * @param modelParam {@link ModelParams} * @return supported range of parameter, null if not supported */ @Nullable - public native ModelParamRange queryParameter(int soundModelHandle, @ModelParams int modelParam); - - private class NativeEventHandlerDelegate { - private final Handler mHandler; - - NativeEventHandlerDelegate(final SoundTrigger.StatusListener listener, - Handler handler) { - // find the looper for our new event handler - Looper looper; - if (handler != null) { - looper = handler.getLooper(); - } else { - looper = Looper.getMainLooper(); - } - - // construct the event handler with this looper - if (looper != null) { - // implement the event handler delegate - mHandler = new Handler(looper) { - @Override - public void handleMessage(Message msg) { - switch(msg.what) { - case EVENT_RECOGNITION: - if (listener != null) { - listener.onRecognition( - (SoundTrigger.RecognitionEvent)msg.obj); - } - break; - case EVENT_SOUNDMODEL: - if (listener != null) { - listener.onSoundModelUpdate( - (SoundTrigger.SoundModelEvent)msg.obj); - } - break; - case EVENT_SERVICE_STATE_CHANGE: - if (listener != null) { - listener.onServiceStateChange(msg.arg1); - } - break; - case EVENT_SERVICE_DIED: - if (listener != null) { - listener.onServiceDied(); - } - break; - default: - break; - } - } - }; - } else { - mHandler = null; - } - } - - Handler handler() { - return mHandler; + public synchronized SoundTrigger.ModelParamRange queryParameter(int soundModelHandle, + @ModelParams int modelParam) { + try { + return ConversionUtil.aidl2apiModelParameterRange(mService.queryModelParameterSupport( + soundModelHandle, + ConversionUtil.api2aidlModelParameter(modelParam))); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); } } - @SuppressWarnings("unused") - @UnsupportedAppUsage - private static void postEventFromNative(Object module_ref, - int what, int arg1, int arg2, Object obj) { - SoundTriggerModule module = (SoundTriggerModule)((WeakReference)module_ref).get(); - if (module == null) { - return; + private int handleException(Exception e) { + Log.e(TAG, "", e); + if (e instanceof NullPointerException) { + return SoundTrigger.STATUS_NO_INIT; + } + if (e instanceof RemoteException) { + return SoundTrigger.STATUS_DEAD_OBJECT; + } + if (e instanceof IllegalArgumentException) { + return SoundTrigger.STATUS_BAD_VALUE; + } + if (e instanceof IllegalStateException) { + return SoundTrigger.STATUS_INVALID_OPERATION; + } + return SoundTrigger.STATUS_ERROR; + } + + private class EventHandlerDelegate extends ISoundTriggerCallback.Stub implements + IBinder.DeathRecipient { + private final Handler mHandler; + + EventHandlerDelegate(@NonNull final SoundTrigger.StatusListener listener, + @NonNull Looper looper) { + + // construct the event handler with this looper + // implement the event handler delegate + mHandler = new Handler(looper) { + @Override + public void handleMessage(Message msg) { + switch (msg.what) { + case EVENT_RECOGNITION: + listener.onRecognition( + (SoundTrigger.RecognitionEvent) msg.obj); + break; + case EVENT_SERVICE_STATE_CHANGE: + listener.onServiceStateChange(msg.arg1); + break; + case EVENT_SERVICE_DIED: + listener.onServiceDied(); + break; + default: + Log.e(TAG, "Unknown message: " + msg.toString()); + break; + } + } + }; } - NativeEventHandlerDelegate delegate = module.mEventHandlerDelegate; - if (delegate != null) { - Handler handler = delegate.handler(); - if (handler != null) { - Message m = handler.obtainMessage(what, arg1, arg2, obj); - handler.sendMessage(m); - } + @Override + public synchronized void onRecognition(int handle, RecognitionEvent event) + throws RemoteException { + Message m = mHandler.obtainMessage(EVENT_RECOGNITION, + ConversionUtil.aidl2apiRecognitionEvent(handle, event)); + mHandler.sendMessage(m); + } + + @Override + public synchronized void onPhraseRecognition(int handle, PhraseRecognitionEvent event) + throws RemoteException { + Message m = mHandler.obtainMessage(EVENT_RECOGNITION, + ConversionUtil.aidl2apiPhraseRecognitionEvent(handle, event)); + mHandler.sendMessage(m); + } + + @Override + public synchronized void onRecognitionAvailabilityChange(boolean available) + throws RemoteException { + Message m = mHandler.obtainMessage(EVENT_SERVICE_STATE_CHANGE, + available ? SoundTrigger.SERVICE_STATE_ENABLED + : SoundTrigger.SERVICE_STATE_DISABLED); + mHandler.sendMessage(m); + } + + @Override + public synchronized void binderDied() { + Message m = mHandler.obtainMessage(EVENT_SERVICE_DIED); + mHandler.sendMessage(m); } } } diff --git a/core/jni/Android.bp b/core/jni/Android.bp index b91d3595fe6aa..a2f6a62157ed5 100644 --- a/core/jni/Android.bp +++ b/core/jni/Android.bp @@ -177,7 +177,6 @@ cc_library_shared { "android_hardware_HardwareBuffer.cpp", "android_hardware_SensorManager.cpp", "android_hardware_SerialPort.cpp", - "android_hardware_SoundTrigger.cpp", "android_hardware_UsbDevice.cpp", "android_hardware_UsbDeviceConnection.cpp", "android_hardware_UsbRequest.cpp", @@ -259,7 +258,6 @@ cc_library_shared { "libpdfium", "libimg_utils", "libnetd_client", - "libsoundtrigger", "libprocessgroup", "libnativebridge_lazy", "libnativeloader_lazy", diff --git a/core/jni/AndroidRuntime.cpp b/core/jni/AndroidRuntime.cpp index b8fd3ad43ce56..f7a994f7071fb 100644 --- a/core/jni/AndroidRuntime.cpp +++ b/core/jni/AndroidRuntime.cpp @@ -80,7 +80,6 @@ extern int register_android_hardware_camera2_DngCreator(JNIEnv *env); extern int register_android_hardware_HardwareBuffer(JNIEnv *env); extern int register_android_hardware_SensorManager(JNIEnv *env); extern int register_android_hardware_SerialPort(JNIEnv *env); -extern int register_android_hardware_SoundTrigger(JNIEnv *env); extern int register_android_hardware_UsbDevice(JNIEnv *env); extern int register_android_hardware_UsbDeviceConnection(JNIEnv *env); extern int register_android_hardware_UsbRequest(JNIEnv *env); @@ -1508,7 +1507,6 @@ static const RegJNIRec gRegJNI[] = { REG_JNI(register_android_hardware_HardwareBuffer), REG_JNI(register_android_hardware_SensorManager), REG_JNI(register_android_hardware_SerialPort), - REG_JNI(register_android_hardware_SoundTrigger), REG_JNI(register_android_hardware_UsbDevice), REG_JNI(register_android_hardware_UsbDeviceConnection), REG_JNI(register_android_hardware_UsbRequest), diff --git a/core/jni/android_hardware_SoundTrigger.cpp b/core/jni/android_hardware_SoundTrigger.cpp deleted file mode 100644 index 4376b0b4ac428..0000000000000 --- a/core/jni/android_hardware_SoundTrigger.cpp +++ /dev/null @@ -1,1071 +0,0 @@ -/* -** -** Copyright 2014, The Android Open Source Project -** -** Licensed under the Apache License, Version 2.0 (the "License"); -** you may not use this file except in compliance with the License. -** You may obtain a copy of the License at -** -** http://www.apache.org/licenses/LICENSE-2.0 -** -** Unless required by applicable law or agreed to in writing, software -** distributed under the License is distributed on an "AS IS" BASIS, -** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -** See the License for the specific language governing permissions and -** limitations under the License. -*/ - -//#define LOG_NDEBUG 0 -#define LOG_TAG "SoundTrigger-JNI" -#include - -#include "jni.h" -#include -#include -#include "core_jni_helpers.h" -#include -#include -#include -#include -#include -#include -#include -#include "android_media_AudioFormat.h" - -using namespace android; - -static jclass gArrayListClass; -static struct { - jmethodID add; -} gArrayListMethods; - -static jclass gUUIDClass; -static struct { - jmethodID toString; -} gUUIDMethods; - -static const char* const kUnsupportedOperationExceptionClassPathName = - "java/lang/UnsupportedOperationException"; -static jclass gUnsupportedOperationExceptionClass; -static const char* const kIllegalArgumentExceptionClassPathName = - "java/lang/IllegalArgumentException"; -static jclass gIllegalArgumentExceptionClass; - -static const char* const kSoundTriggerClassPathName = "android/hardware/soundtrigger/SoundTrigger"; -static jclass gSoundTriggerClass; - -static const char* const kModuleClassPathName = "android/hardware/soundtrigger/SoundTriggerModule"; -static jclass gModuleClass; -static struct { - jfieldID mNativeContext; - jfieldID mId; -} gModuleFields; -static jmethodID gPostEventFromNative; - -static const char* const kModulePropertiesClassPathName = - "android/hardware/soundtrigger/SoundTrigger$ModuleProperties"; -static jclass gModulePropertiesClass; -static jmethodID gModulePropertiesCstor; - -static const char* const kSoundModelClassPathName = - "android/hardware/soundtrigger/SoundTrigger$SoundModel"; -static jclass gSoundModelClass; -static struct { - jfieldID uuid; - jfieldID vendorUuid; - jfieldID data; -} gSoundModelFields; - -static const char* const kGenericSoundModelClassPathName = - "android/hardware/soundtrigger/SoundTrigger$GenericSoundModel"; -static jclass gGenericSoundModelClass; - -static const char* const kKeyphraseClassPathName = - "android/hardware/soundtrigger/SoundTrigger$Keyphrase"; -static jclass gKeyphraseClass; -static struct { - jfieldID id; - jfieldID recognitionModes; - jfieldID locale; - jfieldID text; - jfieldID users; -} gKeyphraseFields; - -static const char* const kKeyphraseSoundModelClassPathName = - "android/hardware/soundtrigger/SoundTrigger$KeyphraseSoundModel"; -static jclass gKeyphraseSoundModelClass; -static struct { - jfieldID keyphrases; -} gKeyphraseSoundModelFields; - -static const char* const kModelParamRangeClassPathName = - "android/hardware/soundtrigger/SoundTrigger$ModelParamRange"; -static jclass gModelParamRangeClass; -static jmethodID gModelParamRangeCstor; - -static const char* const kRecognitionConfigClassPathName = - "android/hardware/soundtrigger/SoundTrigger$RecognitionConfig"; -static jclass gRecognitionConfigClass; -static struct { - jfieldID captureRequested; - jfieldID keyphrases; - jfieldID data; -} gRecognitionConfigFields; - -static const char* const kRecognitionEventClassPathName = - "android/hardware/soundtrigger/SoundTrigger$RecognitionEvent"; -static jclass gRecognitionEventClass; -static jmethodID gRecognitionEventCstor; - -static const char* const kKeyphraseRecognitionEventClassPathName = - "android/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionEvent"; -static jclass gKeyphraseRecognitionEventClass; -static jmethodID gKeyphraseRecognitionEventCstor; - -static const char* const kGenericRecognitionEventClassPathName = - "android/hardware/soundtrigger/SoundTrigger$GenericRecognitionEvent"; -static jclass gGenericRecognitionEventClass; -static jmethodID gGenericRecognitionEventCstor; - -static const char* const kKeyphraseRecognitionExtraClassPathName = - "android/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra"; -static jclass gKeyphraseRecognitionExtraClass; -static jmethodID gKeyphraseRecognitionExtraCstor; -static struct { - jfieldID id; - jfieldID recognitionModes; - jfieldID coarseConfidenceLevel; - jfieldID confidenceLevels; -} gKeyphraseRecognitionExtraFields; - -static const char* const kConfidenceLevelClassPathName = - "android/hardware/soundtrigger/SoundTrigger$ConfidenceLevel"; -static jclass gConfidenceLevelClass; -static jmethodID gConfidenceLevelCstor; -static struct { - jfieldID userId; - jfieldID confidenceLevel; -} gConfidenceLevelFields; - -static const char* const kAudioFormatClassPathName = - "android/media/AudioFormat"; -static jclass gAudioFormatClass; -static jmethodID gAudioFormatCstor; - -static const char* const kSoundModelEventClassPathName = - "android/hardware/soundtrigger/SoundTrigger$SoundModelEvent"; -static jclass gSoundModelEventClass; -static jmethodID gSoundModelEventCstor; - -static Mutex gLock; - -enum { - SOUNDTRIGGER_STATUS_OK = 0, - SOUNDTRIGGER_STATUS_ERROR = INT_MIN, - SOUNDTRIGGER_PERMISSION_DENIED = -1, - SOUNDTRIGGER_STATUS_NO_INIT = -19, - SOUNDTRIGGER_STATUS_BAD_VALUE = -22, - SOUNDTRIGGER_STATUS_DEAD_OBJECT = -32, - SOUNDTRIGGER_INVALID_OPERATION = -38, -}; - -enum { - SOUNDTRIGGER_EVENT_RECOGNITION = 1, - SOUNDTRIGGER_EVENT_SERVICE_DIED = 2, - SOUNDTRIGGER_EVENT_SOUNDMODEL = 3, - SOUNDTRIGGER_EVENT_SERVICE_STATE_CHANGE = 4, -}; - -static jint throwUnsupportedOperationException(JNIEnv *env) -{ - return env->ThrowNew(gUnsupportedOperationExceptionClass, nullptr); -} - -static jint throwIllegalArgumentException(JNIEnv *env) -{ - return env->ThrowNew(gIllegalArgumentExceptionClass, nullptr); -} - -// ---------------------------------------------------------------------------- -// ref-counted object for callbacks -class JNISoundTriggerCallback: public SoundTriggerCallback -{ -public: - JNISoundTriggerCallback(JNIEnv* env, jobject thiz, jobject weak_thiz); - ~JNISoundTriggerCallback(); - - virtual void onRecognitionEvent(struct sound_trigger_recognition_event *event); - virtual void onSoundModelEvent(struct sound_trigger_model_event *event); - virtual void onServiceStateChange(sound_trigger_service_state_t state); - virtual void onServiceDied(); - -private: - jclass mClass; // Reference to SoundTrigger class - jobject mObject; // Weak ref to SoundTrigger Java object to call on -}; - -JNISoundTriggerCallback::JNISoundTriggerCallback(JNIEnv* env, jobject thiz, jobject weak_thiz) -{ - - // Hold onto the SoundTriggerModule class for use in calling the static method - // that posts events to the application thread. - jclass clazz = env->GetObjectClass(thiz); - if (clazz == NULL) { - ALOGE("Can't find class %s", kModuleClassPathName); - return; - } - mClass = (jclass)env->NewGlobalRef(clazz); - - // We use a weak reference so the SoundTriggerModule object can be garbage collected. - // The reference is only used as a proxy for callbacks. - mObject = env->NewGlobalRef(weak_thiz); -} - -JNISoundTriggerCallback::~JNISoundTriggerCallback() -{ - // remove global references - JNIEnv *env = AndroidRuntime::getJNIEnv(); - env->DeleteGlobalRef(mObject); - env->DeleteGlobalRef(mClass); -} - -void JNISoundTriggerCallback::onRecognitionEvent(struct sound_trigger_recognition_event *event) -{ - JNIEnv *env = AndroidRuntime::getJNIEnv(); - jobject jEvent = NULL; - jbyteArray jData = NULL; - - if (event->data_size) { - jData = env->NewByteArray(event->data_size); - jbyte *nData = env->GetByteArrayElements(jData, NULL); - memcpy(nData, (char *)event + event->data_offset, event->data_size); - env->ReleaseByteArrayElements(jData, nData, 0); - } - - jobject jAudioFormat = NULL; - if (event->trigger_in_data || event->capture_available) { - jint channelMask = (jint)audio_channel_mask_get_bits(event->audio_config.channel_mask); - jint channelIndexMask = (jint)AUDIO_CHANNEL_NONE; - - switch (audio_channel_mask_get_representation(event->audio_config.channel_mask)) { - case AUDIO_CHANNEL_REPRESENTATION_INDEX: - channelIndexMask = channelMask; - channelMask = (jint)AUDIO_CHANNEL_NONE; - break; - default: - break; - } - jAudioFormat = env->NewObject(gAudioFormatClass, - gAudioFormatCstor, - audioFormatFromNative(event->audio_config.format), - event->audio_config.sample_rate, - channelMask, - channelIndexMask); - - } - if (event->type == SOUND_MODEL_TYPE_KEYPHRASE) { - struct sound_trigger_phrase_recognition_event *phraseEvent = - (struct sound_trigger_phrase_recognition_event *)event; - - jobjectArray jExtras = env->NewObjectArray(phraseEvent->num_phrases, - gKeyphraseRecognitionExtraClass, NULL); - if (jExtras == NULL) { - return; - } - - for (size_t i = 0; i < phraseEvent->num_phrases; i++) { - jobjectArray jConfidenceLevels = env->NewObjectArray( - phraseEvent->phrase_extras[i].num_levels, - gConfidenceLevelClass, NULL); - - if (jConfidenceLevels == NULL) { - return; - } - for (size_t j = 0; j < phraseEvent->phrase_extras[i].num_levels; j++) { - jobject jConfidenceLevel = env->NewObject(gConfidenceLevelClass, - gConfidenceLevelCstor, - phraseEvent->phrase_extras[i].levels[j].user_id, - phraseEvent->phrase_extras[i].levels[j].level); - env->SetObjectArrayElement(jConfidenceLevels, j, jConfidenceLevel); - env->DeleteLocalRef(jConfidenceLevel); - } - - jobject jNewExtra = env->NewObject(gKeyphraseRecognitionExtraClass, - gKeyphraseRecognitionExtraCstor, - phraseEvent->phrase_extras[i].id, - phraseEvent->phrase_extras[i].recognition_modes, - phraseEvent->phrase_extras[i].confidence_level, - jConfidenceLevels); - - if (jNewExtra == NULL) { - return; - } - env->SetObjectArrayElement(jExtras, i, jNewExtra); - env->DeleteLocalRef(jNewExtra); - env->DeleteLocalRef(jConfidenceLevels); - } - jEvent = env->NewObject(gKeyphraseRecognitionEventClass, gKeyphraseRecognitionEventCstor, - event->status, event->model, event->capture_available, - event->capture_session, event->capture_delay_ms, - event->capture_preamble_ms, event->trigger_in_data, - jAudioFormat, jData, jExtras); - env->DeleteLocalRef(jExtras); - } else if (event->type == SOUND_MODEL_TYPE_GENERIC) { - jEvent = env->NewObject(gGenericRecognitionEventClass, gGenericRecognitionEventCstor, - event->status, event->model, event->capture_available, - event->capture_session, event->capture_delay_ms, - event->capture_preamble_ms, event->trigger_in_data, - jAudioFormat, jData); - } else { - jEvent = env->NewObject(gRecognitionEventClass, gRecognitionEventCstor, - event->status, event->model, event->capture_available, - event->capture_session, event->capture_delay_ms, - event->capture_preamble_ms, event->trigger_in_data, - jAudioFormat, jData); - } - - if (jAudioFormat != NULL) { - env->DeleteLocalRef(jAudioFormat); - } - if (jData != NULL) { - env->DeleteLocalRef(jData); - } - - env->CallStaticVoidMethod(mClass, gPostEventFromNative, mObject, - SOUNDTRIGGER_EVENT_RECOGNITION, 0, 0, jEvent); - - env->DeleteLocalRef(jEvent); - if (env->ExceptionCheck()) { - ALOGW("An exception occurred while notifying an event."); - env->ExceptionClear(); - } -} - -void JNISoundTriggerCallback::onSoundModelEvent(struct sound_trigger_model_event *event) -{ - JNIEnv *env = AndroidRuntime::getJNIEnv(); - jobject jEvent = NULL; - jbyteArray jData = NULL; - - if (event->data_size) { - jData = env->NewByteArray(event->data_size); - jbyte *nData = env->GetByteArrayElements(jData, NULL); - memcpy(nData, (char *)event + event->data_offset, event->data_size); - env->ReleaseByteArrayElements(jData, nData, 0); - } - - jEvent = env->NewObject(gSoundModelEventClass, gSoundModelEventCstor, - event->status, event->model, jData); - - env->DeleteLocalRef(jData); - env->CallStaticVoidMethod(mClass, gPostEventFromNative, mObject, - SOUNDTRIGGER_EVENT_SOUNDMODEL, 0, 0, jEvent); - env->DeleteLocalRef(jEvent); - if (env->ExceptionCheck()) { - ALOGW("An exception occurred while notifying an event."); - env->ExceptionClear(); - } -} - -void JNISoundTriggerCallback::onServiceStateChange(sound_trigger_service_state_t state) -{ - JNIEnv *env = AndroidRuntime::getJNIEnv(); - env->CallStaticVoidMethod(mClass, gPostEventFromNative, mObject, - SOUNDTRIGGER_EVENT_SERVICE_STATE_CHANGE, state, 0, NULL); - if (env->ExceptionCheck()) { - ALOGW("An exception occurred while notifying an event."); - env->ExceptionClear(); - } -} - -void JNISoundTriggerCallback::onServiceDied() -{ - JNIEnv *env = AndroidRuntime::getJNIEnv(); - - env->CallStaticVoidMethod(mClass, gPostEventFromNative, mObject, - SOUNDTRIGGER_EVENT_SERVICE_DIED, 0, 0, NULL); - if (env->ExceptionCheck()) { - ALOGW("An exception occurred while notifying an event."); - env->ExceptionClear(); - } -} - -// ---------------------------------------------------------------------------- - -static sp getSoundTrigger(JNIEnv* env, jobject thiz) -{ - Mutex::Autolock l(gLock); - SoundTrigger* const st = (SoundTrigger*)env->GetLongField(thiz, - gModuleFields.mNativeContext); - return sp(st); -} - -static sp setSoundTrigger(JNIEnv* env, jobject thiz, const sp& module) -{ - Mutex::Autolock l(gLock); - sp old = (SoundTrigger*)env->GetLongField(thiz, - gModuleFields.mNativeContext); - if (module.get()) { - module->incStrong((void*)setSoundTrigger); - } - if (old != 0) { - old->decStrong((void*)setSoundTrigger); - } - env->SetLongField(thiz, gModuleFields.mNativeContext, (jlong)module.get()); - return old; -} - - -static jint -android_hardware_SoundTrigger_listModules(JNIEnv *env, jobject clazz, - jstring opPackageName, jobject jModules) -{ - ALOGV("listModules"); - - if (jModules == NULL) { - ALOGE("listModules NULL AudioPatch ArrayList"); - return SOUNDTRIGGER_STATUS_BAD_VALUE; - } - if (!env->IsInstanceOf(jModules, gArrayListClass)) { - ALOGE("listModules not an arraylist"); - return SOUNDTRIGGER_STATUS_BAD_VALUE; - } - - unsigned int numModules = 0; - struct sound_trigger_module_descriptor *nModules = NULL; - - ScopedUtfChars opPackageNameStr(env, opPackageName); - const String16 opPackageNameString16 = String16(opPackageNameStr.c_str()); - - status_t status = SoundTrigger::listModules(opPackageNameString16, nModules, &numModules); - if (status != NO_ERROR || numModules == 0) { - return (jint)status; - } - - nModules = (struct sound_trigger_module_descriptor *) - calloc(numModules, sizeof(struct sound_trigger_module_descriptor)); - - status = SoundTrigger::listModules(opPackageNameString16, nModules, &numModules); - ALOGV("listModules SoundTrigger::listModules status %d numModules %d", status, numModules); - - if (status != NO_ERROR) { - numModules = 0; - } - - for (size_t i = 0; i < numModules; i++) { - char str[SOUND_TRIGGER_MAX_STRING_LEN]; - - jstring implementor = env->NewStringUTF(nModules[i].properties.implementor); - jstring description = env->NewStringUTF(nModules[i].properties.description); - SoundTrigger::guidToString(&nModules[i].properties.uuid, - str, - SOUND_TRIGGER_MAX_STRING_LEN); - jstring uuid = env->NewStringUTF(str); - - ALOGV("listModules module %zu id %d description %s maxSoundModels %d", - i, nModules[i].handle, nModules[i].properties.description, - nModules[i].properties.max_sound_models); - - jobject newModuleDesc = env->NewObject(gModulePropertiesClass, gModulePropertiesCstor, - nModules[i].handle, - implementor, description, uuid, - nModules[i].properties.version, - nModules[i].properties.max_sound_models, - nModules[i].properties.max_key_phrases, - nModules[i].properties.max_users, - nModules[i].properties.recognition_modes, - nModules[i].properties.capture_transition, - nModules[i].properties.max_buffer_ms, - nModules[i].properties.concurrent_capture, - nModules[i].properties.power_consumption_mw, - nModules[i].properties.trigger_in_event); - - env->DeleteLocalRef(implementor); - env->DeleteLocalRef(description); - env->DeleteLocalRef(uuid); - if (newModuleDesc == NULL) { - status = SOUNDTRIGGER_STATUS_ERROR; - goto exit; - } - env->CallBooleanMethod(jModules, gArrayListMethods.add, newModuleDesc); - } - -exit: - free(nModules); - return (jint) status; -} - -static void -android_hardware_SoundTrigger_setup(JNIEnv *env, jobject thiz, - jstring opPackageName, jobject weak_this) -{ - ALOGV("setup"); - - ScopedUtfChars opPackageNameStr(env, opPackageName); - const String16 opPackageNameString16 = String16(opPackageNameStr.c_str()); - - sp callback = new JNISoundTriggerCallback(env, thiz, weak_this); - - sound_trigger_module_handle_t handle = - (sound_trigger_module_handle_t)env->GetIntField(thiz, gModuleFields.mId); - - sp module = SoundTrigger::attach(opPackageNameString16, handle, callback); - if (module == 0) { - return; - } - - setSoundTrigger(env, thiz, module); -} - -static void -android_hardware_SoundTrigger_detach(JNIEnv *env, jobject thiz) -{ - ALOGV("detach"); - sp module = setSoundTrigger(env, thiz, 0); - ALOGV("detach module %p", module.get()); - if (module != 0) { - ALOGV("detach module->detach()"); - module->detach(); - } -} - -static void -android_hardware_SoundTrigger_finalize(JNIEnv *env, jobject thiz) -{ - ALOGV("finalize"); - sp module = getSoundTrigger(env, thiz); - if (module != 0) { - ALOGW("SoundTrigger finalized without being detached"); - } - android_hardware_SoundTrigger_detach(env, thiz); -} - -static jint -android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, - jobject jSoundModel, jintArray jHandle) -{ - jint status = SOUNDTRIGGER_STATUS_OK; - jbyte *nData = NULL; - struct sound_trigger_sound_model *nSoundModel; - jbyteArray jData; - sp memoryDealer; - sp memory; - size_t size; - sound_model_handle_t handle = 0; - jobject jUuid; - jstring jUuidString; - const char *nUuidString; - - ALOGV("loadSoundModel"); - sp module = getSoundTrigger(env, thiz); - if (module == NULL) { - return SOUNDTRIGGER_STATUS_ERROR; - } - if (jHandle == NULL) { - return SOUNDTRIGGER_STATUS_BAD_VALUE; - } - jsize jHandleLen = env->GetArrayLength(jHandle); - if (jHandleLen == 0) { - return SOUNDTRIGGER_STATUS_BAD_VALUE; - } - jint *nHandle = env->GetIntArrayElements(jHandle, NULL); - if (nHandle == NULL) { - return SOUNDTRIGGER_STATUS_ERROR; - } - if (!env->IsInstanceOf(jSoundModel, gSoundModelClass)) { - status = SOUNDTRIGGER_STATUS_BAD_VALUE; - goto exit; - } - size_t offset; - sound_trigger_sound_model_type_t type; - if (env->IsInstanceOf(jSoundModel, gKeyphraseSoundModelClass)) { - offset = sizeof(struct sound_trigger_phrase_sound_model); - type = SOUND_MODEL_TYPE_KEYPHRASE; - } else if (env->IsInstanceOf(jSoundModel, gGenericSoundModelClass)) { - offset = sizeof(struct sound_trigger_generic_sound_model); - type = SOUND_MODEL_TYPE_GENERIC; - } else { - offset = sizeof(struct sound_trigger_sound_model); - type = SOUND_MODEL_TYPE_UNKNOWN; - } - - jUuid = env->GetObjectField(jSoundModel, gSoundModelFields.uuid); - jUuidString = (jstring)env->CallObjectMethod(jUuid, gUUIDMethods.toString); - nUuidString = env->GetStringUTFChars(jUuidString, NULL); - sound_trigger_uuid_t nUuid; - SoundTrigger::stringToGuid(nUuidString, &nUuid); - env->ReleaseStringUTFChars(jUuidString, nUuidString); - env->DeleteLocalRef(jUuidString); - - sound_trigger_uuid_t nVendorUuid; - jUuid = env->GetObjectField(jSoundModel, gSoundModelFields.vendorUuid); - if (jUuid != NULL) { - jUuidString = (jstring)env->CallObjectMethod(jUuid, gUUIDMethods.toString); - nUuidString = env->GetStringUTFChars(jUuidString, NULL); - SoundTrigger::stringToGuid(nUuidString, &nVendorUuid); - env->ReleaseStringUTFChars(jUuidString, nUuidString); - env->DeleteLocalRef(jUuidString); - } else { - SoundTrigger::stringToGuid("00000000-0000-0000-0000-000000000000", &nVendorUuid); - } - - jData = (jbyteArray)env->GetObjectField(jSoundModel, gSoundModelFields.data); - if (jData == NULL) { - status = SOUNDTRIGGER_STATUS_BAD_VALUE; - goto exit; - } - size = env->GetArrayLength(jData); - - nData = env->GetByteArrayElements(jData, NULL); - if (jData == NULL) { - status = SOUNDTRIGGER_STATUS_ERROR; - goto exit; - } - - memoryDealer = new MemoryDealer(offset + size, "SoundTrigge-JNI::LoadModel"); - if (memoryDealer == 0) { - status = SOUNDTRIGGER_STATUS_ERROR; - goto exit; - } - memory = memoryDealer->allocate(offset + size); - if (memory == 0 || memory->unsecurePointer() == NULL) { - status = SOUNDTRIGGER_STATUS_ERROR; - goto exit; - } - - nSoundModel = (struct sound_trigger_sound_model *)memory->unsecurePointer(); - - nSoundModel->type = type; - nSoundModel->uuid = nUuid; - nSoundModel->vendor_uuid = nVendorUuid; - nSoundModel->data_size = size; - nSoundModel->data_offset = offset; - memcpy((char *)nSoundModel + offset, nData, size); - if (type == SOUND_MODEL_TYPE_KEYPHRASE) { - struct sound_trigger_phrase_sound_model *phraseModel = - (struct sound_trigger_phrase_sound_model *)nSoundModel; - - jobjectArray jPhrases = - (jobjectArray)env->GetObjectField(jSoundModel, gKeyphraseSoundModelFields.keyphrases); - if (jPhrases == NULL) { - status = SOUNDTRIGGER_STATUS_BAD_VALUE; - goto exit; - } - - size_t numPhrases = env->GetArrayLength(jPhrases); - phraseModel->num_phrases = numPhrases; - ALOGV("loadSoundModel numPhrases %zu", numPhrases); - for (size_t i = 0; i < numPhrases; i++) { - jobject jPhrase = env->GetObjectArrayElement(jPhrases, i); - phraseModel->phrases[i].id = - env->GetIntField(jPhrase,gKeyphraseFields.id); - phraseModel->phrases[i].recognition_mode = - env->GetIntField(jPhrase,gKeyphraseFields.recognitionModes); - - jintArray jUsers = (jintArray)env->GetObjectField(jPhrase, gKeyphraseFields.users); - phraseModel->phrases[i].num_users = env->GetArrayLength(jUsers); - jint *nUsers = env->GetIntArrayElements(jUsers, NULL); - memcpy(phraseModel->phrases[i].users, - nUsers, - phraseModel->phrases[i].num_users * sizeof(int)); - env->ReleaseIntArrayElements(jUsers, nUsers, 0); - env->DeleteLocalRef(jUsers); - - jstring jLocale = (jstring)env->GetObjectField(jPhrase, gKeyphraseFields.locale); - const char *nLocale = env->GetStringUTFChars(jLocale, NULL); - strncpy(phraseModel->phrases[i].locale, - nLocale, - SOUND_TRIGGER_MAX_LOCALE_LEN); - jstring jText = (jstring)env->GetObjectField(jPhrase, gKeyphraseFields.text); - const char *nText = env->GetStringUTFChars(jText, NULL); - strncpy(phraseModel->phrases[i].text, - nText, - SOUND_TRIGGER_MAX_STRING_LEN); - - env->ReleaseStringUTFChars(jLocale, nLocale); - env->DeleteLocalRef(jLocale); - env->ReleaseStringUTFChars(jText, nText); - env->DeleteLocalRef(jText); - ALOGV("loadSoundModel phrases %zu text %s locale %s", - i, phraseModel->phrases[i].text, phraseModel->phrases[i].locale); - env->DeleteLocalRef(jPhrase); - } - env->DeleteLocalRef(jPhrases); - } else if (type == SOUND_MODEL_TYPE_GENERIC) { - /* No initialization needed */ - } - status = module->loadSoundModel(memory, &handle); - ALOGV("loadSoundModel status %d handle %d", status, handle); - -exit: - if (nHandle != NULL) { - nHandle[0] = (jint)handle; - env->ReleaseIntArrayElements(jHandle, nHandle, NULL); - } - if (nData != NULL) { - env->ReleaseByteArrayElements(jData, nData, NULL); - } - return status; -} - -static jint -android_hardware_SoundTrigger_unloadSoundModel(JNIEnv *env, jobject thiz, - jint jHandle) -{ - jint status = SOUNDTRIGGER_STATUS_OK; - ALOGV("unloadSoundModel"); - sp module = getSoundTrigger(env, thiz); - if (module == NULL) { - return SOUNDTRIGGER_STATUS_ERROR; - } - status = module->unloadSoundModel((sound_model_handle_t)jHandle); - - return status; -} - -static jint -android_hardware_SoundTrigger_startRecognition(JNIEnv *env, jobject thiz, - jint jHandle, jobject jConfig) -{ - jint status = SOUNDTRIGGER_STATUS_OK; - ALOGV("startRecognition"); - sp module = getSoundTrigger(env, thiz); - if (module == NULL) { - return SOUNDTRIGGER_STATUS_ERROR; - } - - if (!env->IsInstanceOf(jConfig, gRecognitionConfigClass)) { - return SOUNDTRIGGER_STATUS_BAD_VALUE; - } - - jbyteArray jData = (jbyteArray)env->GetObjectField(jConfig, gRecognitionConfigFields.data); - jsize dataSize = 0; - jbyte *nData = NULL; - if (jData != NULL) { - dataSize = env->GetArrayLength(jData); - if (dataSize == 0) { - return SOUNDTRIGGER_STATUS_BAD_VALUE; - } - nData = env->GetByteArrayElements(jData, NULL); - if (nData == NULL) { - return SOUNDTRIGGER_STATUS_ERROR; - } - } - - size_t totalSize = sizeof(struct sound_trigger_recognition_config) + dataSize; - sp memoryDealer = - new MemoryDealer(totalSize, "SoundTrigge-JNI::StartRecognition"); - if (memoryDealer == 0) { - return SOUNDTRIGGER_STATUS_ERROR; - } - sp memory = memoryDealer->allocate(totalSize); - if (memory == 0 || memory->unsecurePointer() == NULL) { - return SOUNDTRIGGER_STATUS_ERROR; - } - if (dataSize != 0) { - memcpy((char *)memory->unsecurePointer() + sizeof(struct sound_trigger_recognition_config), - nData, - dataSize); - env->ReleaseByteArrayElements(jData, nData, 0); - } - env->DeleteLocalRef(jData); - struct sound_trigger_recognition_config *config = - (struct sound_trigger_recognition_config *)memory->unsecurePointer(); - config->data_size = dataSize; - config->data_offset = sizeof(struct sound_trigger_recognition_config); - config->capture_requested = env->GetBooleanField(jConfig, - gRecognitionConfigFields.captureRequested); - - config->num_phrases = 0; - jobjectArray jPhrases = - (jobjectArray)env->GetObjectField(jConfig, gRecognitionConfigFields.keyphrases); - if (jPhrases != NULL) { - config->num_phrases = env->GetArrayLength(jPhrases); - } - ALOGV("startRecognition num phrases %d", config->num_phrases); - for (size_t i = 0; i < config->num_phrases; i++) { - jobject jPhrase = env->GetObjectArrayElement(jPhrases, i); - config->phrases[i].id = env->GetIntField(jPhrase, - gKeyphraseRecognitionExtraFields.id); - config->phrases[i].recognition_modes = env->GetIntField(jPhrase, - gKeyphraseRecognitionExtraFields.recognitionModes); - config->phrases[i].confidence_level = env->GetIntField(jPhrase, - gKeyphraseRecognitionExtraFields.coarseConfidenceLevel); - config->phrases[i].num_levels = 0; - jobjectArray jConfidenceLevels = (jobjectArray)env->GetObjectField(jPhrase, - gKeyphraseRecognitionExtraFields.confidenceLevels); - if (jConfidenceLevels != NULL) { - config->phrases[i].num_levels = env->GetArrayLength(jConfidenceLevels); - } - ALOGV("startRecognition phrase %zu num_levels %d", i, config->phrases[i].num_levels); - for (size_t j = 0; j < config->phrases[i].num_levels; j++) { - jobject jConfidenceLevel = env->GetObjectArrayElement(jConfidenceLevels, j); - config->phrases[i].levels[j].user_id = env->GetIntField(jConfidenceLevel, - gConfidenceLevelFields.userId); - config->phrases[i].levels[j].level = env->GetIntField(jConfidenceLevel, - gConfidenceLevelFields.confidenceLevel); - env->DeleteLocalRef(jConfidenceLevel); - } - ALOGV("startRecognition phrases %zu", i); - env->DeleteLocalRef(jConfidenceLevels); - env->DeleteLocalRef(jPhrase); - } - env->DeleteLocalRef(jPhrases); - - status = module->startRecognition(jHandle, memory); - return status; -} - -static jint -android_hardware_SoundTrigger_stopRecognition(JNIEnv *env, jobject thiz, - jint jHandle) -{ - jint status = SOUNDTRIGGER_STATUS_OK; - ALOGV("stopRecognition"); - sp module = getSoundTrigger(env, thiz); - if (module == NULL) { - return SOUNDTRIGGER_STATUS_ERROR; - } - status = module->stopRecognition(jHandle); - return status; -} - -static jint -android_hardware_SoundTrigger_getModelState(JNIEnv *env, jobject thiz, - jint jHandle) -{ - jint status = SOUNDTRIGGER_STATUS_OK; - ALOGV("getModelState"); - sp module = getSoundTrigger(env, thiz); - if (module == NULL) { - return SOUNDTRIGGER_STATUS_ERROR; - } - status = module->getModelState(jHandle); - return status; -} - -static jint -android_hardware_SoundTrigger_setParameter(JNIEnv *env, jobject thiz, - jint jHandle, jint jModelParam, jint jValue) -{ - ALOGV("setParameter"); - sp module = getSoundTrigger(env, thiz); - if (module == NULL) { - return SOUNDTRIGGER_STATUS_NO_INIT; - } - return module->setParameter(jHandle, (sound_trigger_model_parameter_t) jModelParam, jValue); -} - -static jint -android_hardware_SoundTrigger_getParameter(JNIEnv *env, jobject thiz, - jint jHandle, jint jModelParam) -{ - ALOGV("getParameter"); - sp module = getSoundTrigger(env, thiz); - if (module == NULL) { - throwUnsupportedOperationException(env); - return -1; - } - - jint nValue; - jint status = module->getParameter(jHandle, - (sound_trigger_model_parameter_t) jModelParam, &nValue); - - switch (status) { - case 0: - return nValue; - case -EINVAL: - throwIllegalArgumentException(env); - break; - default: - throwUnsupportedOperationException(env); - break; - } - - return -1; -} - -static jobject -android_hardware_SoundTrigger_queryParameter(JNIEnv *env, jobject thiz, - jint jHandle, jint jModelParam) -{ - ALOGV("queryParameter"); - sp module = getSoundTrigger(env, thiz); - if (module == nullptr) { - return nullptr; - } - - sound_trigger_model_parameter_range_t nRange; - jint nValue = module->queryParameter(jHandle, - (sound_trigger_model_parameter_t) jModelParam, &nRange); - - if (nValue != 0) { - ALOGE("failed to query parameter error code: %d", nValue); - return nullptr; - } - - return env->NewObject(gModelParamRangeClass, gModelParamRangeCstor, nRange.start, nRange.end); -} - -static const JNINativeMethod gMethods[] = { - {"listModules", - "(Ljava/lang/String;Ljava/util/ArrayList;)I", - (void *)android_hardware_SoundTrigger_listModules}, -}; - - -static const JNINativeMethod gModuleMethods[] = { - {"native_setup", - "(Ljava/lang/String;Ljava/lang/Object;)V", - (void *)android_hardware_SoundTrigger_setup}, - {"native_finalize", - "()V", - (void *)android_hardware_SoundTrigger_finalize}, - {"detach", - "()V", - (void *)android_hardware_SoundTrigger_detach}, - {"loadSoundModel", - "(Landroid/hardware/soundtrigger/SoundTrigger$SoundModel;[I)I", - (void *)android_hardware_SoundTrigger_loadSoundModel}, - {"unloadSoundModel", - "(I)I", - (void *)android_hardware_SoundTrigger_unloadSoundModel}, - {"startRecognition", - "(ILandroid/hardware/soundtrigger/SoundTrigger$RecognitionConfig;)I", - (void *)android_hardware_SoundTrigger_startRecognition}, - {"stopRecognition", - "(I)I", - (void *)android_hardware_SoundTrigger_stopRecognition}, - {"getModelState", - "(I)I", - (void *)android_hardware_SoundTrigger_getModelState}, - {"setParameter", - "(III)I", - (void *)android_hardware_SoundTrigger_setParameter}, - {"getParameter", - "(II)I", - (void *)android_hardware_SoundTrigger_getParameter}, - {"queryParameter", - "(II)Landroid/hardware/soundtrigger/SoundTrigger$ModelParamRange;", - (void *)android_hardware_SoundTrigger_queryParameter} -}; - -int register_android_hardware_SoundTrigger(JNIEnv *env) -{ - jclass arrayListClass = FindClassOrDie(env, "java/util/ArrayList"); - gArrayListClass = MakeGlobalRefOrDie(env, arrayListClass); - gArrayListMethods.add = GetMethodIDOrDie(env, arrayListClass, "add", "(Ljava/lang/Object;)Z"); - - jclass uuidClass = FindClassOrDie(env, "java/util/UUID"); - gUUIDClass = MakeGlobalRefOrDie(env, uuidClass); - gUUIDMethods.toString = GetMethodIDOrDie(env, uuidClass, "toString", "()Ljava/lang/String;"); - - jclass exUClass = FindClassOrDie(env, kUnsupportedOperationExceptionClassPathName); - gUnsupportedOperationExceptionClass = MakeGlobalRefOrDie(env, exUClass); - - jclass exIClass = FindClassOrDie(env, kIllegalArgumentExceptionClassPathName); - gIllegalArgumentExceptionClass = MakeGlobalRefOrDie(env, exIClass); - - jclass lClass = FindClassOrDie(env, kSoundTriggerClassPathName); - gSoundTriggerClass = MakeGlobalRefOrDie(env, lClass); - - jclass moduleClass = FindClassOrDie(env, kModuleClassPathName); - gModuleClass = MakeGlobalRefOrDie(env, moduleClass); - gPostEventFromNative = GetStaticMethodIDOrDie(env, moduleClass, "postEventFromNative", - "(Ljava/lang/Object;IIILjava/lang/Object;)V"); - gModuleFields.mNativeContext = GetFieldIDOrDie(env, moduleClass, "mNativeContext", "J"); - gModuleFields.mId = GetFieldIDOrDie(env, moduleClass, "mId", "I"); - - jclass modulePropertiesClass = FindClassOrDie(env, kModulePropertiesClassPathName); - gModulePropertiesClass = MakeGlobalRefOrDie(env, modulePropertiesClass); - gModulePropertiesCstor = GetMethodIDOrDie(env, modulePropertiesClass, "", - "(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;IIIIIZIZIZ)V"); - - jclass soundModelClass = FindClassOrDie(env, kSoundModelClassPathName); - gSoundModelClass = MakeGlobalRefOrDie(env, soundModelClass); - gSoundModelFields.uuid = GetFieldIDOrDie(env, soundModelClass, "uuid", "Ljava/util/UUID;"); - gSoundModelFields.vendorUuid = GetFieldIDOrDie(env, soundModelClass, "vendorUuid", - "Ljava/util/UUID;"); - gSoundModelFields.data = GetFieldIDOrDie(env, soundModelClass, "data", "[B"); - - jclass genericSoundModelClass = FindClassOrDie(env, kGenericSoundModelClassPathName); - gGenericSoundModelClass = MakeGlobalRefOrDie(env, genericSoundModelClass); - - jclass keyphraseClass = FindClassOrDie(env, kKeyphraseClassPathName); - gKeyphraseClass = MakeGlobalRefOrDie(env, keyphraseClass); - gKeyphraseFields.id = GetFieldIDOrDie(env, keyphraseClass, "id", "I"); - gKeyphraseFields.recognitionModes = GetFieldIDOrDie(env, keyphraseClass, "recognitionModes", - "I"); - gKeyphraseFields.locale = GetFieldIDOrDie(env, keyphraseClass, "locale", "Ljava/lang/String;"); - gKeyphraseFields.text = GetFieldIDOrDie(env, keyphraseClass, "text", "Ljava/lang/String;"); - gKeyphraseFields.users = GetFieldIDOrDie(env, keyphraseClass, "users", "[I"); - - jclass keyphraseSoundModelClass = FindClassOrDie(env, kKeyphraseSoundModelClassPathName); - gKeyphraseSoundModelClass = MakeGlobalRefOrDie(env, keyphraseSoundModelClass); - gKeyphraseSoundModelFields.keyphrases = GetFieldIDOrDie(env, keyphraseSoundModelClass, - "keyphrases", - "[Landroid/hardware/soundtrigger/SoundTrigger$Keyphrase;"); - - jclass modelParamRangeClass = FindClassOrDie(env, kModelParamRangeClassPathName); - gModelParamRangeClass = MakeGlobalRefOrDie(env, modelParamRangeClass); - gModelParamRangeCstor = GetMethodIDOrDie(env, modelParamRangeClass, "", "(II)V"); - - jclass recognitionEventClass = FindClassOrDie(env, kRecognitionEventClassPathName); - gRecognitionEventClass = MakeGlobalRefOrDie(env, recognitionEventClass); - gRecognitionEventCstor = GetMethodIDOrDie(env, recognitionEventClass, "", - "(IIZIIIZLandroid/media/AudioFormat;[B)V"); - - jclass keyphraseRecognitionEventClass = FindClassOrDie(env, - kKeyphraseRecognitionEventClassPathName); - gKeyphraseRecognitionEventClass = MakeGlobalRefOrDie(env, keyphraseRecognitionEventClass); - gKeyphraseRecognitionEventCstor = GetMethodIDOrDie(env, keyphraseRecognitionEventClass, "", - "(IIZIIIZLandroid/media/AudioFormat;[B[Landroid/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra;)V"); - - jclass genericRecognitionEventClass = FindClassOrDie(env, - kGenericRecognitionEventClassPathName); - gGenericRecognitionEventClass = MakeGlobalRefOrDie(env, genericRecognitionEventClass); - gGenericRecognitionEventCstor = GetMethodIDOrDie(env, genericRecognitionEventClass, "", - "(IIZIIIZLandroid/media/AudioFormat;[B)V"); - - jclass keyRecognitionConfigClass = FindClassOrDie(env, kRecognitionConfigClassPathName); - gRecognitionConfigClass = MakeGlobalRefOrDie(env, keyRecognitionConfigClass); - gRecognitionConfigFields.captureRequested = GetFieldIDOrDie(env, keyRecognitionConfigClass, - "captureRequested", "Z"); - gRecognitionConfigFields.keyphrases = GetFieldIDOrDie(env, keyRecognitionConfigClass, - "keyphrases", "[Landroid/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra;"); - gRecognitionConfigFields.data = GetFieldIDOrDie(env, keyRecognitionConfigClass, "data", "[B"); - - jclass keyphraseRecognitionExtraClass = FindClassOrDie(env, - kKeyphraseRecognitionExtraClassPathName); - gKeyphraseRecognitionExtraClass = MakeGlobalRefOrDie(env, keyphraseRecognitionExtraClass); - gKeyphraseRecognitionExtraCstor = GetMethodIDOrDie(env, keyphraseRecognitionExtraClass, - "", "(III[Landroid/hardware/soundtrigger/SoundTrigger$ConfidenceLevel;)V"); - gKeyphraseRecognitionExtraFields.id = GetFieldIDOrDie(env, gKeyphraseRecognitionExtraClass, - "id", "I"); - gKeyphraseRecognitionExtraFields.recognitionModes = GetFieldIDOrDie(env, - gKeyphraseRecognitionExtraClass, "recognitionModes", "I"); - gKeyphraseRecognitionExtraFields.coarseConfidenceLevel = GetFieldIDOrDie(env, - gKeyphraseRecognitionExtraClass, "coarseConfidenceLevel", "I"); - gKeyphraseRecognitionExtraFields.confidenceLevels = GetFieldIDOrDie(env, - gKeyphraseRecognitionExtraClass, "confidenceLevels", - "[Landroid/hardware/soundtrigger/SoundTrigger$ConfidenceLevel;"); - - jclass confidenceLevelClass = FindClassOrDie(env, kConfidenceLevelClassPathName); - gConfidenceLevelClass = MakeGlobalRefOrDie(env, confidenceLevelClass); - gConfidenceLevelCstor = GetMethodIDOrDie(env, confidenceLevelClass, "", "(II)V"); - gConfidenceLevelFields.userId = GetFieldIDOrDie(env, confidenceLevelClass, "userId", "I"); - gConfidenceLevelFields.confidenceLevel = GetFieldIDOrDie(env, confidenceLevelClass, - "confidenceLevel", "I"); - - jclass audioFormatClass = FindClassOrDie(env, kAudioFormatClassPathName); - gAudioFormatClass = MakeGlobalRefOrDie(env, audioFormatClass); - gAudioFormatCstor = GetMethodIDOrDie(env, audioFormatClass, "", "(IIII)V"); - - jclass soundModelEventClass = FindClassOrDie(env, kSoundModelEventClassPathName); - gSoundModelEventClass = MakeGlobalRefOrDie(env, soundModelEventClass); - gSoundModelEventCstor = GetMethodIDOrDie(env, soundModelEventClass, "", "(II[B)V"); - - - RegisterMethodsOrDie(env, kSoundTriggerClassPathName, gMethods, NELEM(gMethods)); - return RegisterMethodsOrDie(env, kModuleClassPathName, gModuleMethods, NELEM(gModuleMethods)); -} diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java index 3444be9353951..81789e1362c0d 100644 --- a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java @@ -65,7 +65,7 @@ import java.util.Set; * {@link android.media.soundtrigger_middleware.Status} constants. Any other exception * thrown should be regarded as a bug in the implementation or one of its dependencies * (assuming correct usage). - *
  • The implementation is designed for testibility by featuring dependency injection (the + *
  • The implementation is designed for testability by featuring dependency injection (the * underlying HAL driver instances are passed to the ctor) and by minimizing dependencies * on Android runtime. *
  • The implementation is thread-safe. This is achieved by a simplistic model, where all entry-