@Test
public void synchronizationTaskRetryOnFailure() throws Throwable {

    int testSynchRetryCount = 3;
    this.host.waitForNodeGroupConvergence();
    // Create test factory service
    this.host.waitForReplicatedFactoryServiceAvailable(
            UriUtils.buildUri(this.host, ExampleService.FACTORY_LINK),
            ServiceUriPaths.DEFAULT_NODE_SELECTOR);
    String synchTaskServicePath = UriUtils.buildUriPath(
            SynchronizationTaskService.FACTORY_LINK,
            UriUtils.convertPathCharsFromLink(ExampleService.FACTORY_LINK));
    URI synchTaskServiceUri = UriUtils.buildUri(this.host.getUri(), synchTaskServicePath);
    // Delete the original synchronization Task associated with test factory service. We will later
    // replace self-link of this Task with our test Task service to control the success/failure of synch task.
    this.host.sendAndWaitExpectSuccess(Operation
            .createDelete(synchTaskServiceUri)
            .setCompletion(this.host.getCompletion()));
    // Create synchronization Task with same link, but new Task service.
    ControlledStateSynchTaskService.State state = new ControlledStateSynchTaskService.State();
    state.taskInfo = new TaskState();
    state.maxFailureCount = testSynchRetryCount;
    state.documentSelfLink = synchTaskServicePath;
    Operation post = Operation
            .createPost(this.host, synchTaskServicePath)
            .setBody(state);
    this.host.startService(post, new ControlledStateSynchTaskService());
    this.host.waitForServiceAvailable(synchTaskServiceUri);
    // Trigger node group maintenance. This will call handleNodeGroupMaintenance in test factory service,
    // which will trigger the synchronization Task, but this time, our injected synch Task will be called.
    this.host.scheduleNodeGroupChangeMaintenance(ServiceUriPaths.DEFAULT_NODE_SELECTOR);
    // Verify that synchronization was retried at-least 1 time.
    waitForSynchRetries(Service.STAT_NAME_SYNCH_TASK_RETRY_COUNT,
            (synchRetryCount) -> synchRetryCount.latestValue >= 1);
    // Wait for synchronization gets completed after retries, and retry counter is reset to 0.
    waitForSynchRetries(Service.STAT_NAME_SYNCH_TASK_RETRY_COUNT,
            (synchRetryCount) -> synchRetryCount.latestValue == 0);
    // Make test synch Task to be in FAILED state forever, to test maximum retry limit.
    state = new ControlledStateSynchTaskService.State();
    state.maxFailureCount = Integer.MAX_VALUE;
    state.documentSelfLink = synchTaskServicePath;
    this.host.sendAndWaitExpectSuccess(
            Operation.createPatch(this.host, synchTaskServicePath)
                .setBody(state)
                .setCompletion(this.host.getCompletion()));
    this.host.scheduleNodeGroupChangeMaintenance(ServiceUriPaths.DEFAULT_NODE_SELECTOR);
    // Speed up the retries.
    this.host.setMaintenanceIntervalMicros(TimeUnit.MILLISECONDS.toMicros(1));
    // Verify that synchronization was retried maximum times.
    waitForSynchRetries(Service.STAT_NAME_SYNCH_TASK_RETRY_COUNT,
            (synchRetryCount) -> synchRetryCount.latestValue >= FactoryService.MAX_SYNCH_RETRY_COUNT);
    // Verify overall synch failure count
    waitForSynchRetries(Service.STAT_NAME_CHILD_SYNCH_FAILURE_COUNT,
            (synchFailureCount) -> synchFailureCount.latestValue >= 1);
}