@Test
public void testGetValidFiles() throws IOException, MetadataException {

  Map<String, Map<Long, Set<File>>> allFileList = new HashMap<>();
  Random r = new Random(0);
  for (int i = 0; i < 3; i++) {
    IoTDB.metaManager.setStorageGroup(new PartialPath(getSgName(i)));
  }
  for (int i = 0; i < 3; i++) {
    for (int j = 0; j < 5; j++) {
      allFileList.computeIfAbsent(getSgName(i), k -> new HashMap<>())
          .computeIfAbsent(0L, k -> new HashSet<>());
      String rand = r.nextInt(10000) + TSFILE_SUFFIX;
      String fileName = FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME
          + File.separator + getSgName(i) + File.separator + "0" + File.separator + rand;
      File file = new File(fileName);
      allFileList.get(getSgName(i)).get(0L).add(file);
      if (!file.getParentFile().exists()) {
        file.getParentFile().mkdirs();
      }
      if (!file.exists() && !file.createNewFile()) {
        logger.error("Can not create new file {}", file.getPath());
      }
      if (!new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists()
          && !new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile()) {
        logger.error("Can not create new file {}", file.getPath());
      }
    }
  }
  Map<String, Map<Long, Set<File>>> lastFileMap;
  Map<String, Map<Long, Set<File>>> curFileMap;
  Map<String, Map<Long, Set<File>>> deletedFilesMap;
  Map<String, Map<Long, Set<File>>> toBeSyncedFilesMap;
  // lastFileList is empty
  manager.getValidFiles(dataDir);
  assertTrue(SyncUtils.isEmpty(manager.getLastLocalFilesMap()));
  updateLastLocalFiles(allFileList);
  manager.getValidFiles(dataDir);
  lastFileMap = manager.getLastLocalFilesMap();
  assertFileMap(allFileList, lastFileMap);
  // add some files
  Map<String, Map<Long, Set<File>>> correctToBeSyncedFiles = new HashMap<>();
  r = new Random(1);
  for (int i = 0; i < 3; i++) {
    for (int j = 0; j < 5; j++) {
      allFileList.computeIfAbsent(getSgName(i), k -> new HashMap<>())
          .computeIfAbsent(0L, k -> new HashSet<>());
      correctToBeSyncedFiles.computeIfAbsent(getSgName(i), k -> new HashMap<>())
          .computeIfAbsent(0L, k -> new HashSet<>());
      String rand = r.nextInt(10000) + TSFILE_SUFFIX;
      String fileName =
          FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME
              + File.separator + getSgName(i) + File.separator + "0" + File.separator + rand;
      File file = new File(fileName);
      allFileList.get(getSgName(i)).get(0L).add(file);
      correctToBeSyncedFiles.get(getSgName(i)).get(0L).add(file);
      if (!file.getParentFile().exists()) {
        file.getParentFile().mkdirs();
      }
      if (!file.exists() && !file.createNewFile()) {
        logger.error("Can not create new file {}", file.getPath());
      }
      if (!new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists()
          && !new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile()) {
        logger.error("Can not create new file {}", file.getPath());
      }
    }
  }
  manager.getValidFiles(dataDir);
  curFileMap = manager.getCurrentSealedLocalFilesMap();
  toBeSyncedFilesMap = manager.getToBeSyncedFilesMap();
  assertFileMap(allFileList, curFileMap);
  assertFileMap(correctToBeSyncedFiles, toBeSyncedFilesMap);
  updateLastLocalFiles(allFileList);
  manager.getValidFiles(dataDir);
  lastFileMap = manager.getLastLocalFilesMap();
  assertFileMap(allFileList, lastFileMap);
  // add some files and delete some files
  correctToBeSyncedFiles.clear();
  r = new Random(2);
  for (int i = 0; i < 3; i++) {
    for (int j = 0; j < 5; j++) {
      allFileList.computeIfAbsent(getSgName(i), k -> new HashMap<>())
          .computeIfAbsent(0L, k -> new HashSet<>());
      correctToBeSyncedFiles.computeIfAbsent(getSgName(i), k -> new HashMap<>())
          .computeIfAbsent(0L, k -> new HashSet<>());
      String rand = r.nextInt(10000) + TSFILE_SUFFIX;
      String fileName =
          FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME
              + File.separator + getSgName(i) + File.separator + "0" + File.separator
              + File.separator + rand;
      File file = new File(fileName);
      allFileList.get(getSgName(i)).get(0L).add(file);
      correctToBeSyncedFiles.get(getSgName(i)).get(0L).add(file);
      if (!file.getParentFile().exists()) {
        file.getParentFile().mkdirs();
      }
      if (!file.exists() && !file.createNewFile()) {
        logger.error("Can not create new file {}", file.getPath());
      }
      if (!new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists()
          && !new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile()) {
        logger.error("Can not create new file {}", file.getPath());
      }
    }
  }
  int count = 0;
  Map<String, Map<Long, Set<File>>> correctDeleteFile = new HashMap<>();
  for (Entry<String, Map<Long, Set<File>>> entry : allFileList.entrySet()) {
    correctDeleteFile.put(entry.getKey(), new HashMap<>());
    for (Entry<Long, Set<File>> innerEntry : entry.getValue().entrySet()) {
      Set<File> files = innerEntry.getValue();
      correctDeleteFile.get(entry.getKey()).putIfAbsent(innerEntry.getKey(), new HashSet<>());
      for (File file : files) {
        count++;
        if (count % 3 == 0 && lastFileMap.get(entry.getKey()).get(0L).contains(file)) {
          correctDeleteFile.get(entry.getKey()).get(0L).add(file);
        }
      }
    }
  }
  for (Entry<String, Map<Long, Set<File>>> entry : correctDeleteFile.entrySet()) {
    correctDeleteFile.put(entry.getKey(), new HashMap<>());
    for (Entry<Long, Set<File>> innerEntry : entry.getValue().entrySet()) {
      Set<File> files = innerEntry.getValue();
      correctDeleteFile.get(entry.getKey()).putIfAbsent(innerEntry.getKey(), new HashSet<>());
      for (File file : innerEntry.getValue()) {
        file.delete();
        new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).delete();
        allFileList.get(entry.getKey()).get(0L).remove(file);
      }
    }
  }
  manager.getValidFiles(dataDir);
  curFileMap = manager.getCurrentSealedLocalFilesMap();
  deletedFilesMap = manager.getDeletedFilesMap();
  toBeSyncedFilesMap = manager.getToBeSyncedFilesMap();
  assertFileMap(allFileList, curFileMap);
  assertFileMap(correctDeleteFile, deletedFilesMap);
  assertFileMap(correctToBeSyncedFiles, toBeSyncedFilesMap);
  // add some invalid files
  r = new Random(3);
  for (int i = 0; i < 3; i++) {
    for (int j = 0; j < 5; j++) {
      allFileList.computeIfAbsent(getSgName(i), k -> new HashMap<>())
          .computeIfAbsent(0L, k -> new HashSet<>());
      String rand = String.valueOf(r.nextInt(10000));
      String fileName =
          FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME
              + File.separator + getSgName(i) + File.separator + "0" + File.separator
              + File.separator + rand;
      File file = new File(fileName);
      allFileList.get(getSgName(i)).get(0L).add(file);
      if (!file.getParentFile().exists()) {
        file.getParentFile().mkdirs();
      }
      if (!file.exists() && !file.createNewFile()) {
        logger.error("Can not create new file {}", file.getPath());
      }
    }
  }
  manager.getValidFiles(dataDir);
  curFileMap = manager.getCurrentSealedLocalFilesMap();
  deletedFilesMap = manager.getDeletedFilesMap();
  toBeSyncedFilesMap = manager.getToBeSyncedFilesMap();
  assertFileMap(curFileMap, allFileList);
  assertFileMap(curFileMap, allFileList);
  assertFileMap(correctDeleteFile, deletedFilesMap);
  assertFileMap(correctToBeSyncedFiles, toBeSyncedFilesMap);
}