@Test
public void testRemoveShardReplica() throws Exception {

    String name = "testRemoveShardReplica";
    String moduleShardsConfig = "module-shards-member1-and-2-and-3.conf";
    final MemberNode leaderNode1 = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
            .moduleShardsConfig(moduleShardsConfig).datastoreContextBuilder(
                    DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(300).shardElectionTimeoutFactor(1))
            .build();
    final MemberNode replicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
            .moduleShardsConfig(moduleShardsConfig).build();
    final MemberNode replicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
            .moduleShardsConfig(moduleShardsConfig).build();
    leaderNode1.configDataStore().waitTillReady();
    replicaNode3.configDataStore().waitTillReady();
    verifyRaftPeersPresent(leaderNode1.configDataStore(), "cars", "member-2", "member-3");
    verifyRaftPeersPresent(replicaNode2.configDataStore(), "cars", "member-1", "member-3");
    verifyRaftPeersPresent(replicaNode3.configDataStore(), "cars", "member-1", "member-2");
    // Invoke RPC service on member-3 to remove it's local shard
    ClusterAdminRpcService service3 = new ClusterAdminRpcService(replicaNode3.configDataStore(),
            replicaNode3.operDataStore(), null);
    RpcResult<RemoveShardReplicaOutput> rpcResult = service3.removeShardReplica(new RemoveShardReplicaInputBuilder()
            .setShardName("cars").setMemberName("member-3").setDataStoreType(DataStoreType.Config).build())
            .get(10, TimeUnit.SECONDS);
    verifySuccessfulRpcResult(rpcResult);
    verifyRaftPeersPresent(leaderNode1.configDataStore(), "cars", "member-2");
    verifyRaftPeersPresent(replicaNode2.configDataStore(), "cars", "member-1");
    verifyNoShardPresent(replicaNode3.configDataStore(), "cars");
    // Restart member-2 and verify member-3 isn't present.
    Cluster.get(leaderNode1.kit().getSystem()).down(Cluster.get(replicaNode2.kit().getSystem()).selfAddress());
    replicaNode2.cleanup();
    MemberNode newPeplicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
            .moduleShardsConfig(moduleShardsConfig).build();
    newPeplicaNode2.configDataStore().waitTillReady();
    verifyRaftPeersPresent(newPeplicaNode2.configDataStore(), "cars", "member-1");
    // Invoke RPC service on member-1 to remove member-2
    ClusterAdminRpcService service1 = new ClusterAdminRpcService(leaderNode1.configDataStore(),
            leaderNode1.operDataStore(), null);
    rpcResult = service1.removeShardReplica(new RemoveShardReplicaInputBuilder().setShardName("cars")
            .setMemberName("member-2").setDataStoreType(DataStoreType.Config).build()).get(10, TimeUnit.SECONDS);
    verifySuccessfulRpcResult(rpcResult);
    verifyRaftPeersPresent(leaderNode1.configDataStore(), "cars");
    verifyNoShardPresent(newPeplicaNode2.configDataStore(), "cars");
}