@Test
public void should_tokenize_for_cpd() throws Exception {

  File file = TestUtils.getResource("/org/sonar/plugins/groovy/foundation/Greet.groovy");
  SensorContextTester context = SensorContextTester.create(file.getParentFile());
  DefaultInputFile inputFile = new DefaultInputFile("", "Greet.groovy")
    .setLanguage(Groovy.KEY)
    .setType(Type.MAIN)
    .initMetadata(new String(Files.readAllBytes(file.toPath()), "UTF-8"));
  context.fileSystem().add(inputFile);
  GroovyHighlighterAndTokenizer highlighter = new GroovyHighlighterAndTokenizer(inputFile);
  context = Mockito.spy(context);
  highlighter.processFile(context);
  assertThat(context.cpdTokens(":Greet.groovy")).extracting("value").containsExactly("classGreet{",
    "defname",
    "Greet(who){name=who}",
    "defsalute(){printlnLITERALnameLITERALnameLITERAL}",
    "}",
    "/** * Javadoc style */",
    "@groovy.beans.Bindable",
    "classCool{",
    "doublex=1.4// Comment",
    "}");
  Mockito.verify(context, Mockito.times(1)).newCpdTokens();
}