学步园

public class AnalyzerUtil {
	public static void main(String[] args) {
		StandardAnalyzer sa=new StandardAnalyzer(Version.LUCENE_45);
		String str="I am come from jiangxi fengcheng tongtian . this is a dog";
		TokenStream ts=null;
		try {
			ts=sa.tokenStream("content", new StringReader(str));
			CharTermAttribute cta=ts.addAttribute(CharTermAttribute.class);
			ts.reset();

			while(ts.incrementToken())
			{
				//CharTermAttribute cta=ts.getAttribute(CharTermAttribute.class);
				System.out.println(cta);
			}
			ts.end();
			/*while(ts.incrementToken()){
				System.out.println(cta);
			}*/
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally{
			if(ts!=null)
				try {
					ts.close();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
		}
	}
}

输出：

i
am
come
from
beijing
dog

reset():This method is called by a consumer before it begins consumption using incrementToken(). 

如果没有加上ts.reset();会报空指针异常。如：

Exception in thread "main" java.lang.NullPointerException
	at org.apache.lucene.analysis.standard.StandardTokenizerImpl.zzRefill(StandardTokenizerImpl.java:921)
	at org.apache.lucene.analysis.standard.StandardTokenizerImpl.getNextToken(StandardTokenizerImpl.java:1128)
	at org.apache.lucene.analysis.standard.StandardTokenizer.incrementToken(StandardTokenizer.java:173)
	at org.apache.lucene.analysis.standard.StandardFilter.incrementToken(StandardFilter.java:49)
	at org.apache.lucene.analysis.core.LowerCaseFilter.incrementToken(LowerCaseFilter.java:54)
	at org.apache.lucene.analysis.util.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:82)
	at com.lucene.analyzer.AnalyzerUtil.main(AnalyzerUtil.java:24)