@Test
public void testDehyphenizationWithLayoutTokens() throws Exception {

    List<String> tokens = GrobidAnalyzer.getInstance().tokenize("This is hype-\n nized.");
    List<LayoutToken> layoutTokens = new ArrayList<>();
    for (String token : tokens) {
        if (token.equals("\n")) {
            layoutTokens.get(layoutTokens.size() - 1).setNewLineAfter(true);
        }
        layoutTokens.add(new LayoutToken(token));
    }
    List<LayoutToken> output = TextUtilities.dehyphenize(layoutTokens);
    assertNotNull(output);
    assertThat(LayoutTokensUtil.toText(output), is("This is hypenized."));
}