Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Implement time markers for TTS."

Browse Source
This commit is contained in:
Niels Egberts
2017-01-24 17:19:30 +00:00
committed by Android (Google) Code Review
parent f779f91cd2 65c5078456
commit f549a5c3f5
11 changed files with 232 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/current.txt
View File

@@ -36245,6 +36245,7 @@ package android.speech.tts {
method public abstract int getMaxBufferSize();
method public abstract boolean hasFinished();
method public abstract boolean hasStarted();
method public default void rangeStart(int, int, int);
method public abstract int start(int, int, int);
}
@@ -36397,6 +36398,7 @@ package android.speech.tts {
method public void onError(java.lang.String, int);
method public abstract void onStart(java.lang.String);
method public void onStop(java.lang.String, boolean);
method public void onUtteranceRangeStart(java.lang.String, int, int);
}
public class Voice implements android.os.Parcelable {

2
api/system-current.txt
View File

@@ -39235,6 +39235,7 @@ package android.speech.tts {
method public abstract int getMaxBufferSize();
method public abstract boolean hasFinished();
method public abstract boolean hasStarted();
method public default void rangeStart(int, int, int);
method public abstract int start(int, int, int);
}
@@ -39387,6 +39388,7 @@ package android.speech.tts {
method public void onError(java.lang.String, int);
method public abstract void onStart(java.lang.String);
method public void onStop(java.lang.String, boolean);
method public void onUtteranceRangeStart(java.lang.String, int, int);
}
public class Voice implements android.os.Parcelable {

2
api/test-current.txt
View File

@@ -36366,6 +36366,7 @@ package android.speech.tts {
method public abstract int getMaxBufferSize();
method public abstract boolean hasFinished();
method public abstract boolean hasStarted();
method public default void rangeStart(int, int, int);
method public abstract int start(int, int, int);
}
@@ -36518,6 +36519,7 @@ package android.speech.tts {
method public void onError(java.lang.String, int);
method public abstract void onStart(java.lang.String);
method public void onStop(java.lang.String, boolean);
method public void onUtteranceRangeStart(java.lang.String, int, int);
}
public class Voice implements android.os.Parcelable {

23
core/java/android/speech/tts/BlockingAudioTrack.java
View File

@@ -164,7 +164,7 @@ class BlockingAudioTrack {
// all data from the audioTrack has been sent to the mixer, so
// it's safe to release at this point.
if (DBG) Log.d(TAG, "Releasing audio track [" + track.hashCode() + "]");
synchronized(mAudioTrackLock) {
synchronized (mAudioTrackLock) {
mAudioTrack = null;
}
track.release();
@@ -340,4 +340,25 @@ class BlockingAudioTrack {
return value < min ? min : (value < max ? value : max);
}
/**
* @see
* AudioTrack#setPlaybackPositionUpdateListener(AudioTrack.OnPlaybackPositionUpdateListener).
*/
public void setPlaybackPositionUpdateListener(
AudioTrack.OnPlaybackPositionUpdateListener listener) {
synchronized (mAudioTrackLock) {
if (mAudioTrack != null) {
mAudioTrack.setPlaybackPositionUpdateListener(listener);
}
}
}
/** @see AudioTrack#setNotificationMarkerPosition(int). */
public void setNotificationMarkerPosition(int frames) {
synchronized (mAudioTrackLock) {
if (mAudioTrack != null) {
mAudioTrack.setNotificationMarkerPosition(frames);
}
}
}
}

15
core/java/android/speech/tts/ITextToSpeechCallback.aidl
View File

@@ -83,4 +83,19 @@ oneway interface ITextToSpeechCallback {
* callback.
*/
void onAudioAvailable(String utteranceId, in byte[] audio);
/**
* Tells the client that the engine is about to speak the specified range of the utterance.
*
* <p>
* Only called if the engine supplies timing information by calling
* {@link SynthesisCallback#rangeStart(int, int, int)} and only when the request is played back
* by the service, not when using {@link android.speech.tts.TextToSpeech#synthesizeToFile}.
* </p>
*
* @param utteranceId Unique id identifying the synthesis request.
* @param start The start character index of the range in the utterance text.
* @param end The end character index of the range (exclusive) in the utterance text.
*/
void onUtteranceRangeStart(String utteranceId, int start, int end);
}

8
core/java/android/speech/tts/PlaybackSynthesisCallback.java
View File

@@ -271,4 +271,12 @@ class PlaybackSynthesisCallback extends AbstractSynthesisCallback {
mStatusCode = errorCode;
}
}
public void rangeStart(int markerInFrames, int start, int end) {
if (mItem == null) {
Log.e(TAG, "mItem is null");
return;
}
mItem.rangeStart(markerInFrames, start, end);
}
}

22
core/java/android/speech/tts/SynthesisCallback.java
View File

@@ -142,4 +142,26 @@ public interface SynthesisCallback {
* <p>Useful for checking if a fallback from network request is possible.
*/
boolean hasFinished();
/**
* The service may call this method to provide timing information about the spoken text.
*
* <p>Calling this method means that at the given audio frame, the given range of the input is
* about to be spoken. If this method is called the client will receive a callback on the
* listener ({@link UtteranceProgressListener#onUtteranceRangeStart}) at the moment that frame
* has been reached by the playback head.
*
* <p>The markerInFrames is a frame index into the audio for this synthesis request, i.e. into
* the concatenation of the audio bytes sent to audioAvailable for this synthesis request. The
* definition of a frame depends on the format given by {@link #start}. See {@link AudioFormat}
* for more information.
*
* <p>This method should only be called on the synthesis thread, while in {@link
* TextToSpeechService#onSynthesizeText}.
*
* @param markerInFrames The position in frames in the audio where this range is spoken.
* @param start The start index of the range in the input text.
* @param end The end index (exclusive) of the range in the input text.
*/
default void rangeStart(int markerInFrames, int start, int end) {}
}

64
core/java/android/speech/tts/SynthesisPlaybackQueueItem.java
View File

@@ -17,18 +17,21 @@ package android.speech.tts;
import android.speech.tts.TextToSpeechService.AudioOutputParams;
import android.speech.tts.TextToSpeechService.UtteranceProgressDispatcher;
import android.media.AudioTrack;
import android.util.Log;
import java.util.LinkedList;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
* Manages the playback of a list of byte arrays representing audio data
* that are queued by the engine to an audio track.
* Manages the playback of a list of byte arrays representing audio data that are queued by the
* engine to an audio track.
*/
final class SynthesisPlaybackQueueItem extends PlaybackQueueItem {
final class SynthesisPlaybackQueueItem extends PlaybackQueueItem
implements AudioTrack.OnPlaybackPositionUpdateListener {
private static final String TAG = "TTS.SynthQueueItem";
private static final boolean DBG = false;
@@ -63,6 +66,10 @@ final class SynthesisPlaybackQueueItem extends PlaybackQueueItem {
private final BlockingAudioTrack mAudioTrack;
private final AbstractEventLogger mLogger;
// Stores a queue of markers. When the marker in front is reached the client is informed and we
// wait for the next one.
private ConcurrentLinkedQueue<ProgressMarker> markerList = new ConcurrentLinkedQueue<>();
SynthesisPlaybackQueueItem(AudioOutputParams audioParams, int sampleRate,
int audioFormat, int channelCount, UtteranceProgressDispatcher dispatcher,
Object callerIdentity, AbstractEventLogger logger) {
@@ -89,6 +96,8 @@ final class SynthesisPlaybackQueueItem extends PlaybackQueueItem {
return;
}
mAudioTrack.setPlaybackPositionUpdateListener(this);
try {
byte[] buffer = null;
@@ -172,6 +181,55 @@ final class SynthesisPlaybackQueueItem extends PlaybackQueueItem {
}
}
/** Convenience class for passing around TTS markers. */
private class ProgressMarker {
// The index in frames of this marker.
public final int frames;
// The start index in the text of the utterance.
public final int start;
// The end index (exclusive) in the text of the utterance.
public final int end;
public ProgressMarker(int frames, int start, int end) {
this.frames = frames;
this.start = start;
this.end = end;
}
}
/** Set a callback for the first marker in the queue. */
void updateMarker() {
ProgressMarker marker = markerList.peek();
if (marker != null) {
// Zero is used to disable the marker. The documentation recommends to use a non-zero
// position near zero such as 1.
int markerInFrames = marker.frames == 0 ? 1 : marker.frames;
mAudioTrack.setNotificationMarkerPosition(markerInFrames);
}
}
/** Informs us that at markerInFrames, the range between start and end is about to be spoken. */
void rangeStart(int markerInFrames, int start, int end) {
markerList.add(new ProgressMarker(markerInFrames, start, end));
updateMarker();
}
@Override
public void onMarkerReached(AudioTrack track) {
ProgressMarker marker = markerList.poll();
if (marker == null) {
Log.e(TAG, "onMarkerReached reached called but no marker in queue");
return;
}
// Inform the client.
getDispatcher().dispatchOnUtteranceRangeStart(marker.start, marker.end);
// Listen for the next marker.
// It's ok if this marker is in the past, in that case onMarkerReached will be called again.
updateMarker();
}
@Override
public void onPeriodicNotification(AudioTrack track) {}
void put(byte[] buffer) throws InterruptedException {
try {

102
core/java/android/speech/tts/TextToSpeech.java
View File

@@ -2103,55 +2103,69 @@ public class TextToSpeech {
private boolean mEstablished;
private final ITextToSpeechCallback.Stub mCallback = new ITextToSpeechCallback.Stub() {
public void onStop(String utteranceId, boolean isStarted) throws RemoteException {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onStop(utteranceId, isStarted);
}
};
private final ITextToSpeechCallback.Stub mCallback =
new ITextToSpeechCallback.Stub() {
public void onStop(String utteranceId, boolean isStarted)
throws RemoteException {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onStop(utteranceId, isStarted);
}
};
@Override
public void onSuccess(String utteranceId) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onDone(utteranceId);
}
}
@Override
public void onSuccess(String utteranceId) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onDone(utteranceId);
}
}
@Override
public void onError(String utteranceId, int errorCode) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onError(utteranceId);
}
}
@Override
public void onError(String utteranceId, int errorCode) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onError(utteranceId);
}
}
@Override
public void onStart(String utteranceId) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onStart(utteranceId);
}
}
@Override
public void onStart(String utteranceId) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onStart(utteranceId);
}
}
@Override
public void onBeginSynthesis(String utteranceId, int sampleRateInHz, int audioFormat,
int channelCount) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onBeginSynthesis(utteranceId, sampleRateInHz, audioFormat, channelCount);
}
}
@Override
public void onBeginSynthesis(
String utteranceId,
int sampleRateInHz,
int audioFormat,
int channelCount) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onBeginSynthesis(
utteranceId, sampleRateInHz, audioFormat, channelCount);
}
}
@Override
public void onAudioAvailable(String utteranceId, byte[] audio) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onAudioAvailable(utteranceId, audio);
}
}
};
@Override
public void onAudioAvailable(String utteranceId, byte[] audio) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onAudioAvailable(utteranceId, audio);
}
}
@Override
public void onUtteranceRangeStart(String utteranceId, int start, int end) {
UtteranceProgressListener listener = mUtteranceProgressListener;
if (listener != null) {
listener.onUtteranceRangeStart(utteranceId, start, end);
}
}
};
private class SetupConnectionAsyncTask extends AsyncTask<Void, Void, Integer> {
private final ComponentName mName;

22
core/java/android/speech/tts/TextToSpeechService.java
View File

@@ -663,6 +663,8 @@ public abstract class TextToSpeechService extends Service {
void dispatchOnBeginSynthesis(int sampleRateInHz, int audioFormat, int channelCount);
void dispatchOnAudioAvailable(byte[] audio);
public void dispatchOnUtteranceRangeStart(int start, int end);
}
/** Set of parameters affecting audio output. */
@@ -882,6 +884,15 @@ public abstract class TextToSpeechService extends Service {
}
}
@Override
public void dispatchOnUtteranceRangeStart(int start, int end) {
final String utteranceId = getUtteranceId();
if (utteranceId != null) {
mCallbacks.dispatchOnUtteranceRangeStart(
getCallerIdentity(), utteranceId, start, end);
}
}
abstract public String getUtteranceId();
String getStringParam(Bundle params, String key, String defaultValue) {
@@ -1559,6 +1570,17 @@ public abstract class TextToSpeechService extends Service {
}
}
public void dispatchOnUtteranceRangeStart(
Object callerIdentity, String utteranceId, int start, int end) {
ITextToSpeechCallback cb = getCallbackFor(callerIdentity);
if (cb == null) return;
try {
cb.onUtteranceRangeStart(utteranceId, start, end);
} catch (RemoteException e) {
Log.e(TAG, "Callback dispatchOnUtteranceRangeStart(String, int, int) failed: " + e);
}
}
@Override
public void onCallbackDied(ITextToSpeechCallback callback, Object cookie) {
IBinder caller = (IBinder) cookie;

20
core/java/android/speech/tts/UtteranceProgressListener.java
View File

@@ -122,8 +122,24 @@ public abstract class UtteranceProgressListener {
}
/**
* Wraps an old deprecated OnUtteranceCompletedListener with a shiny new
* progress listener.
* This is called when the TTS service is about to speak the specified range of the utterance
* with the given utteranceId.
*
* <p>This method is called when the audio is expected to start playing on the speaker. Note
* that this is different from {@link #onAudioAvailable} which is called as soon as the audio is
* generated.
*
* <p>Only called if the engine supplies timing information by calling {@link
* SynthesisCallback#rangeStart(int, int, int)}.
*
* @param utteranceId Unique id identifying the synthesis request.
* @param start The start index of the range in the utterance text.
* @param end The end index of the range (exclusive) in the utterance text.
*/
public void onUtteranceRangeStart(String utteranceId, int start, int end) {}
/**
* Wraps an old deprecated OnUtteranceCompletedListener with a shiny new progress listener.
*
* @hide
*/