@Test
public void testAddShardReplica() throws Exception {

    String name = "testAddShardReplica";
    String moduleShardsConfig = "module-shards-cars-member-1.conf";
    MemberNode leaderNode1 = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
            .moduleShardsConfig(moduleShardsConfig).waitForShardLeader("cars").build();
    MemberNode newReplicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
            .moduleShardsConfig(moduleShardsConfig).build();
    leaderNode1.waitForMembersUp("member-2");
    doAddShardReplica(newReplicaNode2, "cars", "member-1");
    MemberNode newReplicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
            .moduleShardsConfig(moduleShardsConfig).build();
    leaderNode1.waitForMembersUp("member-3");
    newReplicaNode2.waitForMembersUp("member-3");
    doAddShardReplica(newReplicaNode3, "cars", "member-1", "member-2");
    verifyRaftPeersPresent(newReplicaNode2.configDataStore(), "cars", "member-1", "member-3");
    verifyRaftPeersPresent(newReplicaNode2.operDataStore(), "cars", "member-1", "member-3");
    // Write data to member-2's config datastore and read/verify via member-3
    final NormalizedNode<?, ?> configCarsNode = writeCarsNodeAndVerify(newReplicaNode2.configDataStore(),
            newReplicaNode3.configDataStore());
    // Write data to member-3's oper datastore and read/verify via member-2
    writeCarsNodeAndVerify(newReplicaNode3.operDataStore(), newReplicaNode2.operDataStore());
    // Verify all data has been replicated. We expect 4 log entries and thus last applied index of 3 -
    // 2 ServerConfigurationPayload entries,  the transaction payload entry plus a purge payload.
    RaftStateVerifier verifier = raftState -> {
        assertEquals("Commit index", 4, raftState.getCommitIndex());
        assertEquals("Last applied index", 4, raftState.getLastApplied());
    };
    verifyRaftState(leaderNode1.configDataStore(), "cars", verifier);
    verifyRaftState(leaderNode1.operDataStore(), "cars", verifier);
    verifyRaftState(newReplicaNode2.configDataStore(), "cars", verifier);
    verifyRaftState(newReplicaNode2.operDataStore(), "cars", verifier);
    verifyRaftState(newReplicaNode3.configDataStore(), "cars", verifier);
    verifyRaftState(newReplicaNode3.operDataStore(), "cars", verifier);
    // Restart member-3 and verify the cars config shard is re-instated.
    Cluster.get(leaderNode1.kit().getSystem()).down(Cluster.get(newReplicaNode3.kit().getSystem()).selfAddress());
    newReplicaNode3.cleanup();
    newReplicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
            .moduleShardsConfig(moduleShardsConfig).createOperDatastore(false).build();
    verifyRaftState(newReplicaNode3.configDataStore(), "cars", verifier);
    readCarsNodeAndVerify(newReplicaNode3.configDataStore(), configCarsNode);
}