Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DL lifecycle: handle slow DL binding.

Browse Source 
Bug: 182214420
Test: atest PackageManagerShellCommandTest PackageManagerShellCommandIncrementalTest IncrementalServiceTest PackageManagerServiceTest ChecksumsTest
Change-Id: I5959e01177ab702de1f754f4ba433004925ce98b
This commit is contained in:
Alex Buynytskyy
2021-03-09 19:24:23 -08:00
parent 6bf2762532
commit 7e06d712d2
8 changed files with 282 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
core/java/android/content/pm/IDataLoaderStatusListener.aidl
View File

@@ -23,32 +23,34 @@ package android.content.pm;
oneway interface IDataLoaderStatusListener {
/** The DataLoader process died, binder disconnected or class destroyed. */
const int DATA_LOADER_DESTROYED = 0;
/** The system is in process of binding to the DataLoader. */
const int DATA_LOADER_BINDING = 1;
/** DataLoader process is running and bound to. */
const int DATA_LOADER_BOUND = 1;
const int DATA_LOADER_BOUND = 2;
/** DataLoader has handled onCreate(). */
const int DATA_LOADER_CREATED = 2;
const int DATA_LOADER_CREATED = 3;
/** DataLoader can receive missing pages and read pages notifications,
* and ready to provide data. */
const int DATA_LOADER_STARTED = 3;
const int DATA_LOADER_STARTED = 4;
/** DataLoader no longer ready to provide data and is not receiving
* any notifications from IncFS. */
const int DATA_LOADER_STOPPED = 4;
const int DATA_LOADER_STOPPED = 5;
/** DataLoader streamed everything necessary to continue installation. */
const int DATA_LOADER_IMAGE_READY = 5;
const int DATA_LOADER_IMAGE_READY = 6;
/** Installation can't continue as DataLoader failed to stream necessary data. */
const int DATA_LOADER_IMAGE_NOT_READY = 6;
const int DATA_LOADER_IMAGE_NOT_READY = 7;
/** DataLoader instance can't run at the moment, but might recover later.
* It's up to system to decide if the app is still usable. */
const int DATA_LOADER_UNAVAILABLE = 7;
const int DATA_LOADER_UNAVAILABLE = 8;
/** DataLoader reports that this instance is invalid and can never be restored.
* Warning: this is a terminal status that data loader should use carefully and
* the system should almost never use - e.g. only if all recovery attempts
* fail and all retry limits are exceeded. */
const int DATA_LOADER_UNRECOVERABLE = 8;
const int DATA_LOADER_UNRECOVERABLE = 9;
/** There are no known issues with the data stream. */
const int STREAM_HEALTHY = 0;

2
services/core/java/com/android/server/pm/DataLoaderManagerService.java
View File

@@ -180,6 +180,8 @@ public class DataLoaderManagerService extends SystemService {
mId = id;
mListener = listener;
mDataLoader = null;
callListener(IDataLoaderStatusListener.DATA_LOADER_BINDING);
}
@Override

10
services/core/java/com/android/server/pm/PackageInstallerSession.java
View File

@@ -1011,8 +1011,9 @@ public class PackageInstallerSession extends IPackageInstallerSession.Stub {
"DataLoader installation of APEX modules is not allowed.");
}
if (this.params.dataLoaderParams.getComponentName().getPackageName()
== SYSTEM_DATA_LOADER_PACKAGE && mContext.checkCallingOrSelfPermission(
boolean systemDataLoader = SYSTEM_DATA_LOADER_PACKAGE.equals(
this.params.dataLoaderParams.getComponentName().getPackageName());
if (systemDataLoader && mContext.checkCallingOrSelfPermission(
Manifest.permission.USE_SYSTEM_DATA_LOADERS)
!= PackageManager.PERMISSION_GRANTED) {
throw new SecurityException("You need the "
@@ -3653,6 +3654,7 @@ public class PackageInstallerSession extends IPackageInstallerSession.Stub {
@Override
public void onStatusChanged(int dataLoaderId, int status) {
switch (status) {
case IDataLoaderStatusListener.DATA_LOADER_BINDING:
case IDataLoaderStatusListener.DATA_LOADER_STOPPED:
case IDataLoaderStatusListener.DATA_LOADER_DESTROYED:
return;
@@ -3763,8 +3765,8 @@ public class PackageInstallerSession extends IPackageInstallerSession.Stub {
healthCheckParams.unhealthyTimeoutMs = INCREMENTAL_STORAGE_UNHEALTHY_TIMEOUT_MS;
healthCheckParams.unhealthyMonitoringMs = INCREMENTAL_STORAGE_UNHEALTHY_MONITORING_MS;
final boolean systemDataLoader =
params.getComponentName().getPackageName() == SYSTEM_DATA_LOADER_PACKAGE;
final boolean systemDataLoader = SYSTEM_DATA_LOADER_PACKAGE.equals(
params.getComponentName().getPackageName());
final IStorageHealthListener healthListener = new IStorageHealthListener.Stub() {
@Override

90
services/incremental/IncrementalService.cpp
View File

@@ -74,6 +74,13 @@ struct Constants {
// If DL was up and not crashing for 10mins, we consider it healthy and reset all delays.
static constexpr auto healthyDataLoaderUptime = 10min;
// For healthy DLs, we'll retry every ~5secs for ~10min
static constexpr auto bindRetryInterval = 5s;
static constexpr auto bindGracePeriod = 10min;
static constexpr auto bindingTimeout = 1min;
// 10s, 100s (~2min), 1000s (~15min), 10000s (~3hrs)
static constexpr auto minBindDelay = 10s;
static constexpr auto maxBindDelay = 10000s;
@@ -293,6 +300,7 @@ IncrementalService::IncrementalService(ServiceManagerWrapper&& sm, std::string_v
mTimedQueue(sm.getTimedQueue()),
mProgressUpdateJobQueue(sm.getProgressUpdateJobQueue()),
mFs(sm.getFs()),
mClock(sm.getClock()),
mIncrementalDir(rootDir) {
CHECK(mVold) << "Vold service is unavailable";
CHECK(mDataLoaderManager) << "DataLoaderManagerService is unavailable";
@@ -302,6 +310,7 @@ IncrementalService::IncrementalService(ServiceManagerWrapper&& sm, std::string_v
CHECK(mTimedQueue) << "TimedQueue is unavailable";
CHECK(mProgressUpdateJobQueue) << "mProgressUpdateJobQueue is unavailable";
CHECK(mFs) << "Fs is unavailable";
CHECK(mClock) << "Clock is unavailable";
mJobQueue.reserve(16);
mJobProcessor = std::thread([this]() {
@@ -2241,17 +2250,44 @@ void IncrementalService::DataLoaderStub::setTargetStatusLocked(int status) {
<< status << " (current " << mCurrentStatus << ")";
}
Milliseconds IncrementalService::DataLoaderStub::updateBindDelay() {
std::optional<Milliseconds> IncrementalService::DataLoaderStub::needToBind() {
std::unique_lock lock(mMutex);
const auto now = mService.mClock->now();
const bool healthy = (mPreviousBindDelay == 0ms);
if (mCurrentStatus == IDataLoaderStatusListener::DATA_LOADER_BINDING &&
now - mCurrentStatusTs <= Constants::bindingTimeout) {
LOG(INFO) << "Binding still in progress. "
<< (healthy ? "The DL is healthy/freshly bound, ok to retry for a few times."
: "Already unhealthy, don't do anything.");
// Binding still in progress.
if (!healthy) {
// Already unhealthy, don't do anything.
return {};
}
// The DL is healthy/freshly bound, ok to retry for a few times.
if (now - mPreviousBindTs <= Constants::bindGracePeriod) {
// Still within grace period.
if (now - mCurrentStatusTs >= Constants::bindRetryInterval) {
// Retry interval passed, retrying.
mCurrentStatusTs = now;
mPreviousBindDelay = 0ms;
return 0ms;
}
return {};
}
// fallthrough, mark as unhealthy, and retry with delay
}
const auto previousBindTs = mPreviousBindTs;
const auto now = Clock::now();
mPreviousBindTs = now;
const auto nonCrashingInterval = std::max(castToMs(now - previousBindTs), 100ms);
if (previousBindTs.time_since_epoch() == Clock::duration::zero() ||
nonCrashingInterval > Constants::healthyDataLoaderUptime) {
mPreviousBindDelay = 0ms;
return mPreviousBindDelay;
return 0ms;
}
constexpr auto minBindDelayMs = castToMs(Constants::minBindDelay);
@@ -2264,12 +2300,16 @@ Milliseconds IncrementalService::DataLoaderStub::updateBindDelay() {
const auto bindDelayJitterRangeMs = bindDelayMs / Constants::bindDelayJitterDivider;
const auto bindDelayJitterMs = rand() % (bindDelayJitterRangeMs * 2) - bindDelayJitterRangeMs;
mPreviousBindDelay = std::chrono::milliseconds(bindDelayMs + bindDelayJitterMs);
return mPreviousBindDelay;
}
bool IncrementalService::DataLoaderStub::bind() {
const auto bindDelay = updateBindDelay();
const auto maybeBindDelay = needToBind();
if (!maybeBindDelay) {
LOG(DEBUG) << "Skipping bind to " << mParams.packageName << " because of pending bind.";
return true;
}
const auto bindDelay = *maybeBindDelay;
if (bindDelay > 1s) {
LOG(INFO) << "Delaying bind to " << mParams.packageName << " by "
<< bindDelay.count() / 1000 << "s";
@@ -2279,7 +2319,21 @@ bool IncrementalService::DataLoaderStub::bind() {
auto status = mService.mDataLoaderManager->bindToDataLoader(id(), mParams, bindDelay.count(),
this, &result);
if (!status.isOk() || !result) {
LOG(ERROR) << "Failed to bind a data loader for mount " << id();
const bool healthy = (bindDelay == 0ms);
LOG(ERROR) << "Failed to bind a data loader for mount " << id()
<< (healthy ? ", retrying." : "");
// Internal error, retry for healthy/new DLs.
// Let needToBind migrate it to unhealthy after too many retries.
if (healthy) {
if (mService.addTimedJob(*mService.mTimedQueue, id(), Constants::bindRetryInterval,
[this]() { fsmStep(); })) {
// Mark as binding so that we know it's not the DL's fault.
setCurrentStatus(IDataLoaderStatusListener::DATA_LOADER_BINDING);
return true;
}
}
return false;
}
return true;
@@ -2339,7 +2393,14 @@ bool IncrementalService::DataLoaderStub::fsmStep() {
// Do nothing, this is a reset state.
break;
case IDataLoaderStatusListener::DATA_LOADER_DESTROYED: {
return destroy();
switch (currentStatus) {
case IDataLoaderStatusListener::DATA_LOADER_BINDING:
setCurrentStatus(IDataLoaderStatusListener::DATA_LOADER_DESTROYED);
return true;
default:
return destroy();
}
break;
}
case IDataLoaderStatusListener::DATA_LOADER_STARTED: {
switch (currentStatus) {
@@ -2353,6 +2414,7 @@ bool IncrementalService::DataLoaderStub::fsmStep() {
switch (currentStatus) {
case IDataLoaderStatusListener::DATA_LOADER_DESTROYED:
case IDataLoaderStatusListener::DATA_LOADER_UNAVAILABLE:
case IDataLoaderStatusListener::DATA_LOADER_BINDING:
return bind();
case IDataLoaderStatusListener::DATA_LOADER_BOUND:
return create();
@@ -2372,7 +2434,8 @@ binder::Status IncrementalService::DataLoaderStub::onStatusChanged(MountId mount
fromServiceSpecificError(-EINVAL, "onStatusChange came to invalid DataLoaderStub");
}
if (id() != mountId) {
LOG(ERROR) << "Mount ID mismatch: expected " << id() << ", but got: " << mountId;
LOG(ERROR) << "onStatusChanged: mount ID mismatch: expected " << id()
<< ", but got: " << mountId;
return binder::Status::fromServiceSpecificError(-EPERM, "Mount ID mismatch.");
}
if (newStatus == IDataLoaderStatusListener::DATA_LOADER_UNRECOVERABLE) {
@@ -2396,11 +2459,13 @@ void IncrementalService::DataLoaderStub::setCurrentStatus(int newStatus) {
}
oldStatus = mCurrentStatus;
mCurrentStatus = newStatus;
targetStatus = mTargetStatus;
listener = mStatusListener;
// Change the status.
mCurrentStatus = newStatus;
mCurrentStatusTs = mService.mClock->now();
if (mCurrentStatus == IDataLoaderStatusListener::DATA_LOADER_UNAVAILABLE ||
mCurrentStatus == IDataLoaderStatusListener::DATA_LOADER_UNRECOVERABLE) {
// For unavailable, unbind from DataLoader to ensure proper re-commit.
@@ -2428,7 +2493,8 @@ binder::Status IncrementalService::DataLoaderStub::reportStreamHealth(MountId mo
"reportStreamHealth came to invalid DataLoaderStub");
}
if (id() != mountId) {
LOG(ERROR) << "Mount ID mismatch: expected " << id() << ", but got: " << mountId;
LOG(ERROR) << "reportStreamHealth: mount ID mismatch: expected " << id()
<< ", but got: " << mountId;
return binder::Status::fromServiceSpecificError(-EPERM, "Mount ID mismatch.");
}
{
@@ -2694,6 +2760,8 @@ static std::string toHexString(const RawMetadata& metadata) {
void IncrementalService::DataLoaderStub::onDump(int fd) {
dprintf(fd, " dataLoader: {\n");
dprintf(fd, " currentStatus: %d\n", mCurrentStatus);
dprintf(fd, " currentStatusTs: %lldmcs\n",
(long long)(elapsedMcs(mCurrentStatusTs, Clock::now())));
dprintf(fd, " targetStatus: %d\n", mTargetStatus);
dprintf(fd, " targetStatusTs: %lldmcs\n",
(long long)(elapsedMcs(mTargetStatusTs, Clock::now())));

7
services/incremental/IncrementalService.h
View File

@@ -267,7 +267,10 @@ private:
BootClockTsUs getOldestTsFromLastPendingReads();
Milliseconds elapsedMsSinceKernelTs(TimePoint now, BootClockTsUs kernelTsUs);
Milliseconds updateBindDelay();
// If the stub has to bind to the DL.
// Returns {} if bind operation is already in progress.
// Or bind delay in ms.
std::optional<Milliseconds> needToBind();
void registerForPendingReads();
void unregisterFromPendingReads();
@@ -283,6 +286,7 @@ private:
std::condition_variable mStatusCondition;
int mCurrentStatus = content::pm::IDataLoaderStatusListener::DATA_LOADER_DESTROYED;
TimePoint mCurrentStatusTs = {};
int mTargetStatus = content::pm::IDataLoaderStatusListener::DATA_LOADER_DESTROYED;
TimePoint mTargetStatusTs = {};
@@ -443,6 +447,7 @@ private:
const std::unique_ptr<TimedQueueWrapper> mTimedQueue;
const std::unique_ptr<TimedQueueWrapper> mProgressUpdateJobQueue;
const std::unique_ptr<FsWrapper> mFs;
const std::unique_ptr<ClockWrapper> mClock;
const std::string mIncrementalDir;
mutable std::mutex mLock;

12
services/incremental/ServiceWrappers.cpp
View File

@@ -329,6 +329,14 @@ public:
}
};
class RealClockWrapper final : public ClockWrapper {
public:
RealClockWrapper() = default;
~RealClockWrapper() = default;
TimePoint now() const final { return Clock::now(); }
};
RealServiceManager::RealServiceManager(sp<IServiceManager> serviceManager, JNIEnv* env)
: mServiceManager(std::move(serviceManager)), mJvm(RealJniWrapper::getJvm(env)) {}
@@ -388,6 +396,10 @@ std::unique_ptr<FsWrapper> RealServiceManager::getFs() {
return std::make_unique<RealFsWrapper>();
}
std::unique_ptr<ClockWrapper> RealServiceManager::getClock() {
return std::make_unique<RealClockWrapper>();
}
static JavaVM* getJavaVm(JNIEnv* env) {
CHECK(env);
JavaVM* jvm = nullptr;

8
services/incremental/ServiceWrappers.h
View File

@@ -158,6 +158,12 @@ public:
virtual void listFilesRecursive(std::string_view directoryPath, FileCallback onFile) const = 0;
};
class ClockWrapper {
public:
virtual ~ClockWrapper() = default;
virtual TimePoint now() const = 0;
};
class ServiceManagerWrapper {
public:
virtual ~ServiceManagerWrapper() = default;
@@ -170,6 +176,7 @@ public:
virtual std::unique_ptr<TimedQueueWrapper> getTimedQueue() = 0;
virtual std::unique_ptr<TimedQueueWrapper> getProgressUpdateJobQueue() = 0;
virtual std::unique_ptr<FsWrapper> getFs() = 0;
virtual std::unique_ptr<ClockWrapper> getClock() = 0;
};
// --- Real stuff ---
@@ -187,6 +194,7 @@ public:
std::unique_ptr<TimedQueueWrapper> getTimedQueue() final;
std::unique_ptr<TimedQueueWrapper> getProgressUpdateJobQueue() final;
std::unique_ptr<FsWrapper> getFs() final;
std::unique_ptr<ClockWrapper> getClock() final;
private:
template <class INTERFACE>

162
services/incremental/test/IncrementalServiceTest.cpp
View File

@@ -248,6 +248,27 @@ public:
}
return binder::Status::ok();
}
binder::Status bindToDataLoaderNotOkWithNoDelay(int32_t mountId,
const DataLoaderParamsParcel& params,
int bindDelayMs,
const sp<IDataLoaderStatusListener>& listener,
bool* _aidl_return) {
CHECK(bindDelayMs == 0) << bindDelayMs;
*_aidl_return = false;
return binder::Status::ok();
}
binder::Status bindToDataLoaderBindingWithNoDelay(int32_t mountId,
const DataLoaderParamsParcel& params,
int bindDelayMs,
const sp<IDataLoaderStatusListener>& listener,
bool* _aidl_return) {
CHECK(bindDelayMs == 0) << bindDelayMs;
*_aidl_return = true;
if (listener) {
listener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_BINDING);
}
return binder::Status::ok();
}
binder::Status bindToDataLoaderOkWith10sDelay(int32_t mountId,
const DataLoaderParamsParcel& params,
int bindDelayMs,
@@ -557,6 +578,21 @@ public:
}
};
class MockClockWrapper : public ClockWrapper {
public:
MOCK_CONST_METHOD0(now, TimePoint());
void start() { ON_CALL(*this, now()).WillByDefault(Invoke(this, &MockClockWrapper::getClock)); }
template <class Delta>
void advance(Delta delta) {
mClock += delta;
}
TimePoint getClock() const { return mClock; }
TimePoint mClock = Clock::now();
};
class MockStorageHealthListener : public os::incremental::BnStorageHealthListener {
public:
MOCK_METHOD2(onHealthStatus, binder::Status(int32_t storageId, int32_t status));
@@ -594,7 +630,7 @@ public:
std::unique_ptr<MockLooperWrapper> looper,
std::unique_ptr<MockTimedQueueWrapper> timedQueue,
std::unique_ptr<MockTimedQueueWrapper> progressUpdateJobQueue,
std::unique_ptr<MockFsWrapper> fs)
std::unique_ptr<MockFsWrapper> fs, std::unique_ptr<MockClockWrapper> clock)
: mVold(std::move(vold)),
mDataLoaderManager(std::move(dataLoaderManager)),
mIncFs(std::move(incfs)),
@@ -603,7 +639,8 @@ public:
mLooper(std::move(looper)),
mTimedQueue(std::move(timedQueue)),
mProgressUpdateJobQueue(std::move(progressUpdateJobQueue)),
mFs(std::move(fs)) {}
mFs(std::move(fs)),
mClock(std::move(clock)) {}
std::unique_ptr<VoldServiceWrapper> getVoldService() final { return std::move(mVold); }
std::unique_ptr<DataLoaderManagerWrapper> getDataLoaderManager() final {
return std::move(mDataLoaderManager);
@@ -619,6 +656,7 @@ public:
return std::move(mProgressUpdateJobQueue);
}
std::unique_ptr<FsWrapper> getFs() final { return std::move(mFs); }
std::unique_ptr<ClockWrapper> getClock() final { return std::move(mClock); }
private:
std::unique_ptr<MockVoldService> mVold;
@@ -630,6 +668,7 @@ private:
std::unique_ptr<MockTimedQueueWrapper> mTimedQueue;
std::unique_ptr<MockTimedQueueWrapper> mProgressUpdateJobQueue;
std::unique_ptr<MockFsWrapper> mFs;
std::unique_ptr<MockClockWrapper> mClock;
};
// --- IncrementalServiceTest ---
@@ -657,6 +696,8 @@ public:
mProgressUpdateJobQueue = progressUpdateJobQueue.get();
auto fs = std::make_unique<NiceMock<MockFsWrapper>>();
mFs = fs.get();
auto clock = std::make_unique<NiceMock<MockClockWrapper>>();
mClock = clock.get();
mIncrementalService = std::make_unique<
IncrementalService>(MockServiceManager(std::move(vold),
std::move(dataloaderManager),
@@ -664,12 +705,13 @@ public:
std::move(jni), std::move(looper),
std::move(timedQueue),
std::move(progressUpdateJobQueue),
std::move(fs)),
std::move(fs), std::move(clock)),
mRootDir.path);
mDataLoaderParcel.packageName = "com.test";
mDataLoaderParcel.arguments = "uri";
mDataLoaderManager->unbindFromDataLoaderSuccess();
mIncrementalService->onSystemReady();
mClock->start();
setupSuccess();
}
@@ -724,6 +766,7 @@ protected:
NiceMock<MockTimedQueueWrapper>* mTimedQueue = nullptr;
NiceMock<MockTimedQueueWrapper>* mProgressUpdateJobQueue = nullptr;
NiceMock<MockFsWrapper>* mFs = nullptr;
NiceMock<MockClockWrapper>* mClock = nullptr;
NiceMock<MockDataLoader>* mDataLoader = nullptr;
std::unique_ptr<IncrementalService> mIncrementalService;
TemporaryDir mRootDir;
@@ -853,6 +896,119 @@ TEST_F(IncrementalServiceTest, testDataLoaderDestroyedAndDelayed) {
mDataLoaderManager->setDataLoaderStatusDestroyed();
}
TEST_F(IncrementalServiceTest, testDataLoaderOnRestart) {
mIncFs->waitForPendingReadsSuccess();
mIncFs->openMountSuccess();
constexpr auto bindRetryInterval = 5s;
EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(10);
EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(6);
EXPECT_CALL(*mDataLoader, start(_)).Times(6);
EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(2);
TemporaryDir tempDir;
int storageId =
mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
IncrementalService::CreateOptions::CreateNew);
ASSERT_GE(storageId, 0);
// First binds to DataLoader fails... because it's restart.
ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
.WillByDefault(Invoke(mDataLoaderManager,
&MockDataLoaderManager::bindToDataLoaderNotOkWithNoDelay));
// Request DL start.
mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {}, {});
// Retry callback present.
ASSERT_EQ(storageId, mTimedQueue->mId);
ASSERT_EQ(mTimedQueue->mAfter, bindRetryInterval);
auto retryCallback = mTimedQueue->mWhat;
mTimedQueue->clearJob(storageId);
// Expecting the same bindToDataLoaderNotOkWithNoDelay call.
mClock->advance(5s);
retryCallback();
// Retry callback present.
ASSERT_EQ(storageId, mTimedQueue->mId);
ASSERT_EQ(mTimedQueue->mAfter, bindRetryInterval);
retryCallback = mTimedQueue->mWhat;
mTimedQueue->clearJob(storageId);
// Returning "binding" so that we can retry.
ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
.WillByDefault(Invoke(mDataLoaderManager,
&MockDataLoaderManager::bindToDataLoaderBindingWithNoDelay));
// Expecting bindToDataLoaderBindingWithNoDelay call.
mClock->advance(5s);
retryCallback();
// No retry callback.
ASSERT_EQ(mTimedQueue->mAfter, 0ms);
ASSERT_EQ(mTimedQueue->mWhat, nullptr);
// Should not change the bindToDataLoader call count
ASSERT_NE(nullptr, mLooper->mCallback);
ASSERT_NE(nullptr, mLooper->mCallbackData);
auto looperCb = mLooper->mCallback;
auto looperCbData = mLooper->mCallbackData;
looperCb(-1, -1, looperCbData);
// Expecting the same bindToDataLoaderBindingWithNoDelay call.
mClock->advance(5s);
// Use pending reads callback to trigger binding.
looperCb(-1, -1, looperCbData);
// No retry callback.
ASSERT_EQ(mTimedQueue->mAfter, 0ms);
ASSERT_EQ(mTimedQueue->mWhat, nullptr);
// Now we are out of 10m "retry" budget, let's finally bind.
ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
.WillByDefault(Invoke(mDataLoaderManager, &MockDataLoaderManager::bindToDataLoaderOk));
mClock->advance(11min);
// Use pending reads callback to trigger binding.
looperCb(-1, -1, looperCbData);
// No retry callback.
ASSERT_EQ(mTimedQueue->mAfter, 0ms);
ASSERT_EQ(mTimedQueue->mWhat, nullptr);
// And test the rest of the backoff.
// Simulated crash/other connection breakage.
ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
.WillByDefault(Invoke(mDataLoaderManager,
&MockDataLoaderManager::bindToDataLoaderOkWith10sDelay));
mDataLoaderManager->setDataLoaderStatusDestroyed();
ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
.WillByDefault(Invoke(mDataLoaderManager,
&MockDataLoaderManager::bindToDataLoaderOkWith100sDelay));
mDataLoaderManager->setDataLoaderStatusDestroyed();
ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
.WillByDefault(Invoke(mDataLoaderManager,
&MockDataLoaderManager::bindToDataLoaderOkWith1000sDelay));
mDataLoaderManager->setDataLoaderStatusDestroyed();
ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
.WillByDefault(Invoke(mDataLoaderManager,
&MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
mDataLoaderManager->setDataLoaderStatusDestroyed();
ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
.WillByDefault(Invoke(mDataLoaderManager,
&MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
mDataLoaderManager->setDataLoaderStatusDestroyed();
}
TEST_F(IncrementalServiceTest, testStartDataLoaderCreate) {
mDataLoader->initializeCreateOkNoStatus();
EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);