Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix issue #22013372: Assist should take translationX and friends...

Browse Source 
...into account when calculating the position information

Actually what we need here is the full transformation matrix, if it
is available.  And that means actually computing the location of
views on the screen requires doing this all through transformations,
so the AssistVisualizer has been changed to do this (while still
also keeping the old mechanism for comparison to verify that things
are working correctly).

Also added new properties for elevation and alpha.

And optimized the parcelling of AssistStructure to not write things
that aren't needed; this reduces the parcelled size by about half.

Change-Id: I50b0dd2e6599c74701a5d188617a3eff64b07d03
This commit is contained in:
Dianne Hackborn
2015-06-23 19:33:02 +00:00
parent f207285d8d
commit 70d8be7616
9 changed files with 407 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/current.txt
View File

@@ -5846,10 +5846,12 @@ package android.app.assist {
}
public static class AssistStructure.ViewNode {
method public float getAlpha();
method public android.app.assist.AssistStructure.ViewNode getChildAt(int);
method public int getChildCount();
method public java.lang.String getClassName();
method public java.lang.CharSequence getContentDescription();
method public float getElevation();
method public android.os.Bundle getExtras();
method public int getHeight();
method public java.lang.String getHint();
@@ -5868,6 +5870,7 @@ package android.app.assist {
method public float getTextSize();
method public int getTextStyle();
method public int getTop();
method public android.graphics.Matrix getTransformation();
method public int getVisibility();
method public int getWidth();
method public boolean isAccessibilityFocused();
@@ -36962,6 +36965,7 @@ package android.view {
method public abstract android.view.ViewStructure newChild(int);
method public abstract void setAccessibilityFocused(boolean);
method public abstract void setActivated(boolean);
method public abstract void setAlpha(float);
method public abstract void setCheckable(boolean);
method public abstract void setChecked(boolean);
method public abstract void setChildCount(int);
@@ -36970,6 +36974,7 @@ package android.view {
method public abstract void setContentDescription(java.lang.CharSequence);
method public abstract void setContextClickable(boolean);
method public abstract void setDimens(int, int, int, int, int, int);
method public abstract void setElevation(float);
method public abstract void setEnabled(boolean);
method public abstract void setFocusable(boolean);
method public abstract void setFocused(boolean);
@@ -36980,6 +36985,7 @@ package android.view {
method public abstract void setText(java.lang.CharSequence);
method public abstract void setText(java.lang.CharSequence, int, int);
method public abstract void setTextStyle(float, int, int, int);
method public abstract void setTransformation(android.graphics.Matrix);
method public abstract void setVisibility(int);
}

6
api/system-current.txt
View File

@@ -5964,10 +5964,12 @@ package android.app.assist {
}
public static class AssistStructure.ViewNode {
method public float getAlpha();
method public android.app.assist.AssistStructure.ViewNode getChildAt(int);
method public int getChildCount();
method public java.lang.String getClassName();
method public java.lang.CharSequence getContentDescription();
method public float getElevation();
method public android.os.Bundle getExtras();
method public int getHeight();
method public java.lang.String getHint();
@@ -5986,6 +5988,7 @@ package android.app.assist {
method public float getTextSize();
method public int getTextStyle();
method public int getTop();
method public android.graphics.Matrix getTransformation();
method public int getVisibility();
method public int getWidth();
method public boolean isAccessibilityFocused();
@@ -39243,6 +39246,7 @@ package android.view {
method public abstract android.view.ViewStructure newChild(int);
method public abstract void setAccessibilityFocused(boolean);
method public abstract void setActivated(boolean);
method public abstract void setAlpha(float);
method public abstract void setCheckable(boolean);
method public abstract void setChecked(boolean);
method public abstract void setChildCount(int);
@@ -39251,6 +39255,7 @@ package android.view {
method public abstract void setContentDescription(java.lang.CharSequence);
method public abstract void setContextClickable(boolean);
method public abstract void setDimens(int, int, int, int, int, int);
method public abstract void setElevation(float);
method public abstract void setEnabled(boolean);
method public abstract void setFocusable(boolean);
method public abstract void setFocused(boolean);
@@ -39261,6 +39266,7 @@ package android.view {
method public abstract void setText(java.lang.CharSequence);
method public abstract void setText(java.lang.CharSequence, int, int);
method public abstract void setTextStyle(float, int, int, int);
method public abstract void setTransformation(android.graphics.Matrix);
method public abstract void setVisibility(int);
}

292
core/java/android/app/assist/AssistStructure.java
View File

@@ -2,6 +2,7 @@ package android.app.assist;
import android.app.Activity;
import android.content.ComponentName;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.os.Binder;
import android.os.Bundle;
@@ -128,24 +129,24 @@ public class AssistStructure implements Parcelable {
view.dispatchProvideStructure(builder);
}
WindowNode(Parcel in, PooledStringReader preader) {
WindowNode(Parcel in, PooledStringReader preader, float[] tmpMatrix) {
mX = in.readInt();
mY = in.readInt();
mWidth = in.readInt();
mHeight = in.readInt();
mTitle = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(in);
mDisplayId = in.readInt();
mRoot = new ViewNode(in, preader);
mRoot = new ViewNode(in, preader, tmpMatrix);
}
void writeToParcel(Parcel out, PooledStringWriter pwriter) {
int writeToParcel(Parcel out, PooledStringWriter pwriter, float[] tmpMatrix) {
out.writeInt(mX);
out.writeInt(mY);
out.writeInt(mWidth);
out.writeInt(mHeight);
TextUtils.writeToParcel(mTitle, out, 0);
out.writeInt(mDisplayId);
mRoot.writeToParcel(out, pwriter);
return mRoot.writeToParcel(out, pwriter, tmpMatrix);
}
/**
@@ -216,7 +217,7 @@ public class AssistStructure implements Parcelable {
public static final int TEXT_STYLE_UNDERLINE = 1<<2;
public static final int TEXT_STYLE_STRIKE_THRU = 1<<3;
int mId;
int mId = View.NO_ID;
String mIdPackage;
String mIdType;
String mIdEntry;
@@ -226,20 +227,35 @@ public class AssistStructure implements Parcelable {
int mScrollY;
int mWidth;
int mHeight;
Matrix mMatrix;
float mElevation;
float mAlpha = 1.0f;
static final int FLAGS_DISABLED = 0x00000001;
static final int FLAGS_VISIBILITY_MASK = View.VISIBLE|View.INVISIBLE|View.GONE;
static final int FLAGS_FOCUSABLE = 0x00000010;
static final int FLAGS_FOCUSED = 0x00000020;
static final int FLAGS_ACCESSIBILITY_FOCUSED = 0x04000000;
static final int FLAGS_SELECTED = 0x00000040;
static final int FLAGS_ASSIST_BLOCKED = 0x00000080;
static final int FLAGS_ACTIVATED = 0x40000000;
static final int FLAGS_CHECKABLE = 0x00000100;
static final int FLAGS_CHECKED = 0x00000200;
static final int FLAGS_CLICKABLE = 0x00004000;
static final int FLAGS_LONG_CLICKABLE = 0x00200000;
static final int FLAGS_CONTEXT_CLICKABLE = 0x00400000;
static final int FLAGS_CLICKABLE = 0x00000400;
static final int FLAGS_LONG_CLICKABLE = 0x00000800;
static final int FLAGS_ACCESSIBILITY_FOCUSED = 0x00001000;
static final int FLAGS_ACTIVATED = 0x00002000;
static final int FLAGS_CONTEXT_CLICKABLE = 0x00004000;
static final int FLAGS_HAS_MATRIX = 0x40000000;
static final int FLAGS_HAS_ALPHA = 0x20000000;
static final int FLAGS_HAS_ELEVATION = 0x10000000;
static final int FLAGS_HAS_SCROLL = 0x08000000;
static final int FLAGS_HAS_LARGE_COORDS = 0x04000000;
static final int FLAGS_HAS_CONTENT_DESCRIPTION = 0x02000000;
static final int FLAGS_HAS_TEXT = 0x01000000;
static final int FLAGS_HAS_EXTRAS = 0x00800000;
static final int FLAGS_HAS_ID = 0x00400000;
static final int FLAGS_HAS_CHILDREN = 0x00200000;
static final int FLAGS_ALL_CONTROL = 0xfff00000;
int mFlags;
@@ -254,79 +270,153 @@ public class AssistStructure implements Parcelable {
ViewNode() {
}
ViewNode(Parcel in, PooledStringReader preader) {
mId = in.readInt();
if (mId != 0) {
mIdEntry = preader.readString();
if (mIdEntry != null) {
mIdType = preader.readString();
mIdPackage = preader.readString();
} else {
mIdPackage = mIdType = null;
}
} else {
mIdPackage = mIdType = mIdEntry = null;
}
mX = in.readInt();
mY = in.readInt();
mScrollX = in.readInt();
mScrollY = in.readInt();
mWidth = in.readInt();
mHeight = in.readInt();
mFlags = in.readInt();
ViewNode(Parcel in, PooledStringReader preader, float[] tmpMatrix) {
mClassName = preader.readString();
mContentDescription = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(in);
if (in.readInt() != 0) {
mText = new ViewNodeText(in);
} else {
mText = null;
mFlags = in.readInt();
final int flags = mFlags;
if ((flags&FLAGS_HAS_ID) != 0) {
mId = in.readInt();
if (mId != 0) {
mIdEntry = preader.readString();
if (mIdEntry != null) {
mIdType = preader.readString();
mIdPackage = preader.readString();
}
}
}
mExtras = in.readBundle();
final int NCHILDREN = in.readInt();
if (NCHILDREN > 0) {
if ((flags&FLAGS_HAS_LARGE_COORDS) != 0) {
mX = in.readInt();
mY = in.readInt();
mWidth = in.readInt();
mHeight = in.readInt();
} else {
int val = in.readInt();
mX = val&0x7fff;
mY = (val>>16)&0x7fff;
val = in.readInt();
mWidth = val&0x7fff;
mHeight = (val>>16)&0x7fff;
}
if ((flags&FLAGS_HAS_SCROLL) != 0) {
mScrollX = in.readInt();
mScrollY = in.readInt();
}
if ((flags&FLAGS_HAS_MATRIX) != 0) {
mMatrix = new Matrix();
in.readFloatArray(tmpMatrix);
mMatrix.setValues(tmpMatrix);
}
if ((flags&FLAGS_HAS_ELEVATION) != 0) {
mElevation = in.readFloat();
}
if ((flags&FLAGS_HAS_ALPHA) != 0) {
mAlpha = in.readFloat();
}
if ((flags&FLAGS_HAS_CONTENT_DESCRIPTION) != 0) {
mContentDescription = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(in);
}
if ((flags&FLAGS_HAS_TEXT) != 0) {
mText = new ViewNodeText(in);
}
if ((flags&FLAGS_HAS_EXTRAS) != 0) {
mExtras = in.readBundle();
}
if ((flags&FLAGS_HAS_CHILDREN) != 0) {
final int NCHILDREN = in.readInt();
mChildren = new ViewNode[NCHILDREN];
for (int i=0; i<NCHILDREN; i++) {
mChildren[i] = new ViewNode(in, preader);
mChildren[i] = new ViewNode(in, preader, tmpMatrix);
}
} else {
mChildren = null;
}
}
void writeToParcel(Parcel out, PooledStringWriter pwriter) {
out.writeInt(mId);
if (mId != 0) {
pwriter.writeString(mIdEntry);
if (mIdEntry != null) {
pwriter.writeString(mIdType);
pwriter.writeString(mIdPackage);
int writeToParcel(Parcel out, PooledStringWriter pwriter, float[] tmpMatrix) {
int flags = mFlags & ~FLAGS_ALL_CONTROL;
if (mId != View.NO_ID) {
flags |= FLAGS_HAS_ID;
}
if ((mX&~0x7fff) != 0 || (mY&~0x7fff) != 0
|| (mWidth&~0x7fff) != 0 | (mHeight&~0x7fff) != 0) {
flags |= FLAGS_HAS_LARGE_COORDS;
}
if (mScrollX != 0 || mScrollY != 0) {
flags |= FLAGS_HAS_SCROLL;
}
if (mMatrix != null) {
flags |= FLAGS_HAS_MATRIX;
}
if (mElevation != 0) {
flags |= FLAGS_HAS_ELEVATION;
}
if (mAlpha != 1.0f) {
flags |= FLAGS_HAS_ALPHA;
}
if (mContentDescription != null) {
flags |= FLAGS_HAS_CONTENT_DESCRIPTION;
}
if (mText != null) {
flags |= FLAGS_HAS_TEXT;
}
if (mExtras != null) {
flags |= FLAGS_HAS_EXTRAS;
}
if (mChildren != null) {
flags |= FLAGS_HAS_CHILDREN;
}
pwriter.writeString(mClassName);
out.writeInt(flags);
if ((flags&FLAGS_HAS_ID) != 0) {
out.writeInt(mId);
if (mId != 0) {
pwriter.writeString(mIdEntry);
if (mIdEntry != null) {
pwriter.writeString(mIdType);
pwriter.writeString(mIdPackage);
}
}
}
out.writeInt(mX);
out.writeInt(mY);
out.writeInt(mScrollX);
out.writeInt(mScrollY);
out.writeInt(mWidth);
out.writeInt(mHeight);
out.writeInt(mFlags);
pwriter.writeString(mClassName);
TextUtils.writeToParcel(mContentDescription, out, 0);
if (mText != null) {
out.writeInt(1);
mText.writeToParcel(out);
if ((flags&FLAGS_HAS_LARGE_COORDS) != 0) {
out.writeInt(mX);
out.writeInt(mY);
out.writeInt(mWidth);
out.writeInt(mHeight);
} else {
out.writeInt(0);
out.writeInt((mY<<16) | mX);
out.writeInt((mHeight<<16) | mWidth);
}
out.writeBundle(mExtras);
if (mChildren != null) {
if ((flags&FLAGS_HAS_SCROLL) != 0) {
out.writeInt(mScrollX);
out.writeInt(mScrollY);
}
if ((flags&FLAGS_HAS_MATRIX) != 0) {
mMatrix.getValues(tmpMatrix);
out.writeFloatArray(tmpMatrix);
}
if ((flags&FLAGS_HAS_ELEVATION) != 0) {
out.writeFloat(mElevation);
}
if ((flags&FLAGS_HAS_ALPHA) != 0) {
out.writeFloat(mAlpha);
}
if ((flags&FLAGS_HAS_CONTENT_DESCRIPTION) != 0) {
TextUtils.writeToParcel(mContentDescription, out, 0);
}
if ((flags&FLAGS_HAS_TEXT) != 0) {
mText.writeToParcel(out);
}
if ((flags&FLAGS_HAS_EXTRAS) != 0) {
out.writeBundle(mExtras);
}
int N = 1;
if ((flags&FLAGS_HAS_CHILDREN) != 0) {
final int NCHILDREN = mChildren.length;
out.writeInt(NCHILDREN);
for (int i=0; i<NCHILDREN; i++) {
mChildren[i].writeToParcel(out, pwriter);
N += mChildren[i].writeToParcel(out, pwriter, tmpMatrix);
}
} else {
out.writeInt(0);
}
return N;
}
/**
@@ -407,6 +497,33 @@ public class AssistStructure implements Parcelable {
return mHeight;
}
/**
* Returns the transformation that has been applied to this view, such as a translation
* or scaling. The returned Matrix object is owned by ViewNode; do not modify it.
* Returns null if there is no transformation applied to the view.
*/
public Matrix getTransformation() {
return mMatrix;
}
/**
* Returns the visual elevation of the view, used for shadowing and other visual
* characterstics, as set by {@link ViewStructure#setElevation
* ViewStructure.setElevation(float)}.
*/
public float getElevation() {
return mElevation;
}
/**
* Returns the alpha transformation of the view, used to reduce the overall opacity
* of the view's contents, as set by {@link ViewStructure#setAlpha
* ViewStructure.setAlpha(float)}.
*/
public float getAlpha() {
return mAlpha;
}
/**
* Returns the visibility mode of this view, as per
* {@link android.view.View#getVisibility() View.getVisibility()}.
@@ -645,6 +762,25 @@ public class AssistStructure implements Parcelable {
mNode.mHeight = height;
}
@Override
public void setTransformation(Matrix matrix) {
if (matrix == null) {
mNode.mMatrix = null;
} else {
mNode.mMatrix = new Matrix(matrix);
}
}
@Override
public void setElevation(float elevation) {
mNode.mElevation = elevation;
}
@Override
public void setAlpha(float alpha) {
mNode.mAlpha = alpha;
}
@Override
public void setVisibility(int visibility) {
mNode.mFlags = (mNode.mFlags&~ViewNode.FLAGS_VISIBILITY_MASK) | visibility;
@@ -919,6 +1055,18 @@ public class AssistStructure implements Parcelable {
if (scrollX != 0 || scrollY != 0) {
Log.i(TAG, prefix + " Scroll: " + scrollX + "," + scrollY);
}
Matrix matrix = node.getTransformation();
if (matrix != null) {
Log.i(TAG, prefix + " Transformation: " + matrix);
}
float elevation = node.getElevation();
if (elevation != 0) {
Log.i(TAG, prefix + " Elevation: " + elevation);
}
float alpha = node.getAlpha();
if (alpha != 0) {
Log.i(TAG, prefix + " Alpha: " + elevation);
}
CharSequence contentDescription = node.getContentDescription();
if (contentDescription != null) {
Log.i(TAG, prefix + " Content description: " + contentDescription);
@@ -1010,27 +1158,33 @@ public class AssistStructure implements Parcelable {
}
if (mPendingAsyncChildren.size() > 0) {
// We waited too long, assume none of the assist structure is valid.
Log.w(TAG, "Skipping assist structure, waiting too long for async children (have "
+ mPendingAsyncChildren.size() + " remaining");
skipStructure = true;
}
}
int start = out.dataPosition();
PooledStringWriter pwriter = new PooledStringWriter(out);
float[] tmpMatrix = new float[9];
ComponentName.writeToParcel(mActivityComponent, out);
final int N = skipStructure ? 0 : mWindowNodes.size();
out.writeInt(N);
int NV = 0;
for (int i=0; i<N; i++) {
mWindowNodes.get(i).writeToParcel(out, pwriter);
NV += mWindowNodes.get(i).writeToParcel(out, pwriter, tmpMatrix);
}
pwriter.finish();
Log.i(TAG, "Flattened assist data: " + (out.dataPosition() - start) + " bytes");
Log.i(TAG, "Flattened assist data: " + (out.dataPosition() - start) + " bytes, containing "
+ N + " windows, " + NV + " views");
}
void readContentFromParcel(Parcel in) {
PooledStringReader preader = new PooledStringReader(in);
float[] tmpMatrix = new float[9];
mActivityComponent = ComponentName.readFromParcel(in);
final int N = in.readInt();
for (int i=0; i<N; i++) {
mWindowNodes.add(new WindowNode(in, preader));
mWindowNodes.add(new WindowNode(in, preader, tmpMatrix));
}
//dump();
}

21
core/java/android/view/View.java
View File

@@ -6179,6 +6179,10 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
structure.setId(id, null, null, null);
}
structure.setDimens(mLeft, mTop, mScrollX, mScrollY, mRight - mLeft, mBottom - mTop);
if (!hasIdentityMatrix()) {
structure.setTransformation(getMatrix());
}
structure.setElevation(getZ());
structure.setVisibility(getVisibility());
structure.setEnabled(isEnabled());
if (isClickable()) {
@@ -6215,11 +6219,6 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
structure.setContentDescription(getContentDescription());
}
/** @hide */
public void onProvideAssistStructure(ViewStructure structure) {
onProvideStructure(structure);
}
/**
* Called when assist structure is being retrieved from a view as part of
* {@link android.app.Activity#onProvideAssistData Activity.onProvideAssistData} to
@@ -6232,7 +6231,6 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
AccessibilityNodeProvider provider = getAccessibilityNodeProvider();
if (provider != null) {
AccessibilityNodeInfo info = createAccessibilityNodeInfo();
Log.i("View", "Provider of " + this + ": children=" + info.getChildCount());
structure.setChildCount(1);
ViewStructure root = structure.newChild(0);
populateVirtualStructure(root, provider, info);
@@ -6240,11 +6238,6 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
}
}
/** @hide */
public void onProvideVirtualAssistStructure(ViewStructure structure) {
onProvideVirtualStructure(structure);
}
private void populateVirtualStructure(ViewStructure structure,
AccessibilityNodeProvider provider, AccessibilityNodeInfo info) {
structure.setId(AccessibilityNodeInfo.getVirtualDescendantId(info.getSourceNodeId()),
@@ -6284,8 +6277,6 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
CharSequence cname = info.getClassName();
structure.setClassName(cname != null ? cname.toString() : null);
structure.setContentDescription(info.getContentDescription());
Log.i("View", "vassist " + cname + " @ " + rect.toShortString()
+ " text=" + info.getText() + " cd=" + info.getContentDescription());
if (info.getText() != null || info.getError() != null) {
structure.setText(info.getText(), info.getTextSelectionStart(),
info.getTextSelectionEnd());
@@ -6310,8 +6301,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
*/
public void dispatchProvideStructure(ViewStructure structure) {
if (!isAssistBlocked()) {
onProvideAssistStructure(structure);
onProvideVirtualAssistStructure(structure);
onProvideStructure(structure);
onProvideVirtualStructure(structure);
} else {
structure.setClassName(getAccessibilityClassName().toString());
structure.setAssistBlocked(true);

23
core/java/android/view/ViewStructure.java
View File

@@ -16,6 +16,7 @@
package android.view;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.os.Bundle;
@@ -49,6 +50,28 @@ public abstract class ViewStructure {
public abstract void setDimens(int left, int top, int scrollX, int scrollY, int width,
int height);
/**
* Set the transformation matrix associated with this view, as per
* {@link View#getMatrix View.getMatrix()}, or null if there is none.
*/
public abstract void setTransformation(Matrix matrix);
/**
* Set the visual elevation (shadow) of the view, as per
* {@link View#getZ View.getZ()}. Note this is <em>not</em> related
* to the physical Z-ordering of this view relative to its other siblings (that is how
* they overlap when drawing), it is only the visual representation for shadowing.
*/
public abstract void setElevation(float elevation);
/**
* Set an alpha transformation that is applied to this view, as per
* {@link View#getAlpha View.getAlpha()}. Value ranges from 0
* (completely transparent) to 1 (completely opaque); the default is 1, which means
* no transformation.
*/
public abstract void setAlpha(float alpha);
/**
* Set the visibility state of this view, as per
* {@link View#getVisibility View.getVisibility()}.

41
tests/VoiceInteraction/res/layout/voice_interaction_session.xml
View File

@@ -18,6 +18,11 @@
android:layout_width="match_parent"
android:layout_height="match_parent">
<ImageView android:id="@+id/full_screenshot"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:visibility="invisible"/>
<com.android.test.voiceinteraction.AssistVisualizer android:id="@+id/assist_visualizer"
android:layout_width="match_parent"
android:layout_height="match_parent" />
@@ -30,19 +35,29 @@
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_gravity="top"
android:orientation="vertical"
android:orientation="horizontal"
android:background="#ffffffff"
android:elevation="8dp"
>
<ImageView android:id="@+id/screenshot"
android:layout_width="wrap_content"
android:layout_height="46dp"
android:adjustViewBounds="true" />
<View android:layout_width="0dp"
android:layout_height="0dp"
android:layout_weight="1" />
<Button android:id="@+id/do_tree"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/tree" />
<Button android:id="@+id/do_text"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/text" />
<Button android:id="@+id/start"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="top|right"
android:text="@string/start"
/>
<ImageView android:id="@+id/screenshot"
android:layout_width="wrap_content"
android:layout_height="wrap_content"/>
android:text="@string/start" />
</LinearLayout>
<LinearLayout android:id="@+id/bottom_content"
@@ -58,26 +73,22 @@
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginBottom="16dp"
android:textAppearance="?android:attr/textAppearanceMedium"
/>
android:textAppearance="?android:attr/textAppearanceMedium" />
<LinearLayout android:layout_width="match_parent" android:layout_height="wrap_content"
android:orientation="horizontal">
<Button android:id="@+id/confirm"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/confirm"
/>
android:text="@string/confirm" />
<Button android:id="@+id/complete"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/complete"
/>
android:text="@string/complete" />
<Button android:id="@+id/abort"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/abort"
/>
android:text="@string/abort" />
</LinearLayout>
</LinearLayout>

2
tests/VoiceInteraction/res/values/strings.xml
View File

@@ -17,6 +17,8 @@
<resources>
<string name="start">Start</string>
<string name="tree">Tree</string>
<string name="text">Text</string>
<string name="asyncStructure">(Async structure goes here)</string>
<string name="confirm">Confirm</string>
<string name="abort">Abort</string>

103
tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java
View File

@@ -20,6 +20,7 @@ import android.annotation.Nullable;
import android.app.assist.AssistStructure;
import android.content.Context;
import android.graphics.Canvas;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.Rect;
import android.util.AttributeSet;
@@ -31,10 +32,32 @@ import java.util.ArrayList;
public class AssistVisualizer extends View {
static final String TAG = "AssistVisualizer";
static class TextEntry {
final Rect bounds;
final int parentLeft, parentTop;
final Matrix matrix;
final String className;
final CharSequence text;
TextEntry(AssistStructure.ViewNode node, int parentLeft, int parentTop, Matrix matrix) {
int left = parentLeft+node.getLeft();
int top = parentTop+node.getTop();
bounds = new Rect(left, top, left+node.getWidth(), top+node.getHeight());
this.parentLeft = parentLeft;
this.parentTop = parentTop;
this.matrix = new Matrix(matrix);
this.className = node.getClassName();
this.text = node.getText() != null ? node.getText() : node.getContentDescription();
}
}
AssistStructure mAssistStructure;
final Paint mFramePaint = new Paint();
final ArrayList<Rect> mTextRects = new ArrayList<>();
final Paint mFrameNoTransformPaint = new Paint();
final ArrayList<Matrix> mMatrixStack = new ArrayList<>();
final ArrayList<TextEntry> mTextRects = new ArrayList<>();
final int[] mTmpLocation = new int[2];
final float[] mTmpMatrixPoint = new float[2];
public AssistVisualizer(Context context, @Nullable AttributeSet attrs) {
super(context, attrs);
@@ -42,17 +65,26 @@ public class AssistVisualizer extends View {
mFramePaint.setColor(0xffff0000);
mFramePaint.setStyle(Paint.Style.STROKE);
mFramePaint.setStrokeWidth(0);
float density = getResources().getDisplayMetrics().density;
mFramePaint.setShadowLayer(density, density, density, 0xff000000);
mFrameNoTransformPaint.setColor(0xff0000ff);
mFrameNoTransformPaint.setStyle(Paint.Style.STROKE);
mFrameNoTransformPaint.setStrokeWidth(0);
mFrameNoTransformPaint.setShadowLayer(density, density, density, 0xff000000);
}
public void setAssistStructure(AssistStructure as) {
mAssistStructure = as;
mAssistStructure.dump();
mTextRects.clear();
final int N = as.getWindowNodeCount();
if (N > 0) {
for (int i=0; i<N; i++) {
AssistStructure.WindowNode windowNode = as.getWindowNodeAt(i);
buildTextRects(windowNode.getRootViewNode(), windowNode.getLeft(),
mMatrixStack.clear();
Matrix matrix = new Matrix();
matrix.setTranslate(windowNode.getLeft(), windowNode.getTop());
mMatrixStack.add(matrix);
buildTextRects(windowNode.getRootViewNode(), 0, windowNode.getLeft(),
windowNode.getTop());
}
}
@@ -60,31 +92,62 @@ public class AssistVisualizer extends View {
invalidate();
}
public void logTree() {
if (mAssistStructure != null) {
mAssistStructure.dump();
}
}
public void logText() {
final int N = mTextRects.size();
for (int i=0; i<N; i++) {
TextEntry te = mTextRects.get(i);
Log.d(TAG, "View " + te.className + " " + te.bounds.toShortString()
+ " in " + te.parentLeft + "," + te.parentTop
+ " matrix=" + te.matrix.toShortString() + ": "
+ te.text);
}
}
public void clearAssistData() {
mAssistStructure = null;
mTextRects.clear();
}
void buildTextRects(AssistStructure.ViewNode root, int parentLeft, int parentTop) {
void buildTextRects(AssistStructure.ViewNode root, int matrixStackIndex,
int parentLeft, int parentTop) {
if (root.getVisibility() != View.VISIBLE) {
return;
}
int left = parentLeft+root.getLeft();
int top = parentTop+root.getTop();
Matrix parentMatrix = mMatrixStack.get(matrixStackIndex);
matrixStackIndex++;
Matrix matrix;
if (mMatrixStack.size() > matrixStackIndex) {
matrix = mMatrixStack.get(matrixStackIndex);
matrix.set(parentMatrix);
} else {
matrix = new Matrix(parentMatrix);
mMatrixStack.add(matrix);
}
matrix.preTranslate(root.getLeft(), root.getTop());
int left = parentLeft + root.getLeft();
int top = parentTop + root.getTop();
Matrix transform = root.getTransformation();
if (transform != null) {
matrix.preConcat(transform);
}
if (root.getText() != null || root.getContentDescription() != null) {
Rect r = new Rect(left, top, left+root.getWidth(), top+root.getHeight());
Log.d(TAG, "View " + root.getClassName() + " " + left + "," + top + " tr "
+ r.toShortString() + ": "
+ (root.getText() != null ? root.getText() : root.getContentDescription()));
mTextRects.add(r);
TextEntry te = new TextEntry(root, parentLeft, parentTop, matrix);
mTextRects.add(te);
}
final int N = root.getChildCount();
if (N > 0) {
left -= root.getScrollX();
top -= root.getScrollY();
matrix.preTranslate(-root.getScrollX(), -root.getScrollY());
for (int i=0; i<N; i++) {
AssistStructure.ViewNode child = root.getChildAt(i);
buildTextRects(child, left, top);
buildTextRects(child, matrixStackIndex, left, top);
}
}
}
@@ -96,9 +159,19 @@ public class AssistVisualizer extends View {
final int N = mTextRects.size();
Log.d(TAG, "Drawing text rects in " + this + ": found " + mTextRects.size());
for (int i=0; i<N; i++) {
Rect r = mTextRects.get(i);
canvas.drawRect(r.left-mTmpLocation[0], r.top-mTmpLocation[1],
r.right-mTmpLocation[0], r.bottom-mTmpLocation[1], mFramePaint);
TextEntry te = mTextRects.get(i);
canvas.drawRect(te.bounds.left - mTmpLocation[0], te.bounds.top - mTmpLocation[1],
te.bounds.right - mTmpLocation[0], te.bounds.bottom - mTmpLocation[1],
mFrameNoTransformPaint);
}
for (int i=0; i<N; i++) {
TextEntry te = mTextRects.get(i);
canvas.save();
canvas.translate(-mTmpLocation[0], -mTmpLocation[1]);
canvas.concat(te.matrix);
canvas.drawRect(0, 0, te.bounds.right - te.bounds.left, te.bounds.bottom - te.bounds.top,
mFramePaint);
canvas.restore();
}
}
}

29
tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java
View File

@@ -42,8 +42,11 @@ public class MainInteractionSession extends VoiceInteractionSession
View mTopContent;
View mBottomContent;
TextView mText;
Button mTreeButton;
Button mTextButton;
Button mStartButton;
ImageView mScreenshot;
ImageView mFullScreenshot;
Button mConfirmButton;
Button mCompleteButton;
Button mAbortButton;
@@ -110,9 +113,15 @@ public class MainInteractionSession extends VoiceInteractionSession
mTopContent = mContentView.findViewById(R.id.top_content);
mBottomContent = mContentView.findViewById(R.id.bottom_content);
mText = (TextView)mContentView.findViewById(R.id.text);
mTreeButton = (Button)mContentView.findViewById(R.id.do_tree);
mTreeButton.setOnClickListener(this);
mTextButton = (Button)mContentView.findViewById(R.id.do_text);
mTextButton.setOnClickListener(this);
mStartButton = (Button)mContentView.findViewById(R.id.start);
mStartButton.setOnClickListener(this);
mScreenshot = (ImageView)mContentView.findViewById(R.id.screenshot);
mScreenshot.setOnClickListener(this);
mFullScreenshot = (ImageView)mContentView.findViewById(R.id.full_screenshot);
mConfirmButton = (Button)mContentView.findViewById(R.id.confirm);
mConfirmButton.setOnClickListener(this);
mCompleteButton = (Button)mContentView.findViewById(R.id.complete);
@@ -156,8 +165,10 @@ public class MainInteractionSession extends VoiceInteractionSession
mScreenshot.setAdjustViewBounds(true);
mScreenshot.setMaxWidth(screenshot.getWidth()/3);
mScreenshot.setMaxHeight(screenshot.getHeight()/3);
mFullScreenshot.setImageBitmap(screenshot);
} else {
mScreenshot.setImageDrawable(null);
mFullScreenshot.setImageDrawable(null);
}
}
@@ -183,7 +194,15 @@ public class MainInteractionSession extends VoiceInteractionSession
}
public void onClick(View v) {
if (v == mStartButton) {
if (v == mTreeButton) {
if (mAssistVisualizer != null) {
mAssistVisualizer.logTree();
}
} else if (v == mTextButton) {
if (mAssistVisualizer != null) {
mAssistVisualizer.logText();
}
} else if (v == mStartButton) {
mState = STATE_LAUNCHING;
updateState();
startVoiceActivity(mStartIntent);
@@ -219,9 +238,15 @@ public class MainInteractionSession extends VoiceInteractionSession
} else if (v == mAbortButton) {
mPendingRequest.sendAbortVoiceResult(null);
mPendingRequest = null;
} else if (v== mCompleteButton) {
} else if (v == mCompleteButton) {
mPendingRequest.sendCompleteVoiceResult(null);
mPendingRequest = null;
} else if (v == mScreenshot) {
if (mFullScreenshot.getVisibility() != View.VISIBLE) {
mFullScreenshot.setVisibility(View.VISIBLE);
} else {
mFullScreenshot.setVisibility(View.INVISIBLE);
}
}
updateState();
}