class ShingleExtractor implements Iterable<Shingle> {

	private void addText(String text) {
		if (text == null || text.isEmpty())
			return;
		List<Sentence> sentences = ss.splitPos(text);

		// int start = 0;
		for (Sentence sentence : sentences) {
			String currSentence = sentence.getText();
			int startSentence = sentence.getStart();

			// System.out.println("SENTENCE [" + currSentence + "]");
			// //List<Token> textShingles = new LinkedList<Token>();
			// FIXME CLEAN SHOULD NO CHANGE THE OFFSETS OF THE TOKENS
			// currSentence = sm.clean(currSentence);
			List<Token> tokens = ts.tokenizePos(currSentence);
			List<Token> cleanTokens = new LinkedList<Token>();
			// experimental
			for (Token t : tokens) {

				t.setStart(t.getStart() + startSentence);
				t.setEnd(t.getEnd() + startSentence);
				String token = text.substring(t.getStart(), t.getEnd());
				String cleanToken = sm.clean(token);
				if (cleanToken.isEmpty())
					continue;

				// System.out.println(token + "-> " + cleanToken);
				// System.out.println("token in text: " + token);

				// Skip empty token (or tokens made only of chars cleaned
				// above)

				t.setText(cleanToken);
				cleanTokens.add(t);
			}
			cleanedSentences.add(cleanTokens);
		}

	}

	private  ShingleExtractor();
	public  ShingleExtractor(Article a);
	public  ShingleExtractor(String text);

}