学步园

随便搞搞搜索引擎技术，刚开始分词而已，综合考察了几个比较有名的分词方法，决定还是用中科院的分词程序。其中C#有个开源的，且网上已经有用SharpICTCLAS为lucene.net写的分词接口了，不过想试试用好一点的分词程序的效果，所以选了2009共享版的。本人编程技术还是非常菜的，如有不对请大家指出。

分词接口的代码我是综合这两篇博客：http://ythzjk.javaeye.com/blog/334194，http://www.cnblogs.com/birdshover/archive/2009/04/03/1122305.html#1494633。

代码都不长也不难，仔细阅读后很容易根据把自己的分词程序放进lucene。据我观察似乎只要修改Tokenizer接口中next()方法就够了。不过他们的代码我觉得有个问题，每次初始化分词程序都是在Tokenizer的构造方法中，这就影响了分词效率，每分一句话（标题、正文）就要加载一次字典。于是我把加载词典的部分写成静态的。

调用ICTCLAS30.dll的方法，其实只用到了几个方法而已

ICTCLASAnalyzer:

using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using System.Collections;

namespace TestLucene
{

    public class ICTCLASAnalyzer : Analyzer
    {
        //定义要过滤的词
        public string[] CHINESE_ENGLISH_STOP_WORDS;
        public string StopPath = @"E:/MyCsProj/TestLucene/Stopwords.txt";

        public ICTCLASAnalyzer()
        {
            ArrayList StopWords= new ArrayList();
            StreamReader reader = new StreamReader(StopPath, System.Text.Encoding.UTF8);
            string noise = reader.ReadLine();
            int i = 0;
            while (!string.IsNullOrEmpty(noise))
            {
                StopWords.Add(noise);
                noise = reader.ReadLine();
                i++;
            }
            CHINESE_ENGLISH_STOP_WORDS = new String[i];

            while (i>0)
            {
                i--;
                CHINESE_ENGLISH_STOP_WORDS[i] = (string)StopWords[i];
            }
            StopWords.Clear();
        }

        /**//// Constructs a
{@link StandardTokenizer} filtered by a {@link
        /// StandardFilter}, a {@link LowerCaseFilter} and a
{@link StopFilter}.
        ///
        public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
        {

            TokenStream result = new ICTCLASTokenizer(reader);
            result = new StandardFilter(result);
            result = new LowerCaseFilter(result);
            result = new StopFilter(result, CHINESE_ENGLISH_STOP_WORDS);
            return result;
        }

    }

Tokenizer:

using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using Lucene.Net.Analysis;
using System.IO;

namespace TestLucene
{
    class ICTCLASTokenizer : Tokenizer
    {
        //result_t[] result; //本来我想直接使用这个结构的结果 但发现中英文识别字符长度不一样，分词之后原句子对应分词的位置不到了
        int startIndex = 0;
        int endIndex = 0;
        int i = 0;
        string[] pResult;
        /**/
        ///
        /// 待分词的句子
        ///
        private string sentence;
        /**/
        /// Constructs a tokenizer for this Reader.
        ///

        public static bool flag = ICTCLAS.ICTCLAS_Init(@"E:/MyCsProj/ttt/bin/Debug");

        public static int userdic = ICTCLAS.ICTCLAS_ImportUserDict("userdict.txt");

        public ICTCLASTokenizer(System.IO.TextReader reader)
        {
            this.input = reader;

            sentence = input.ReadToEnd();
           

            if(!flag)
            {
                System.Console.WriteLine("Init ICTCLAS failed!");
                return;
            }

            string sResult= ICTCLAS.ICTCLAS_ParagraphProcess(sentence,1);
            pResult = Regex.Split(sResult,@"(?<=/w) ");//直接获取分词结果字符串，在结果上分出词。
            //Console.WriteLine(sResult);
        }

        /**/
        /// 进行切词，返回数据流中下一个token或者数据流为空时返回null
        ///
        public override Token Next()
        {
            Token token = null;
            while (i < pResult.Length-1)
            {

                string word = pResult[i].Split('/')[0];
                MatchCollection rw = Regex.Matches(word, @"/s");
                int space = rw.Count;
               
                startIndex=sentence.IndexOf(word,endIndex);
                endIndex =startIndex+word.Length;

                token = new Token(sentence.Substring(startIndex+space, word.Length-space),startIndex+space,endIndex);

                i++;
               // Console.WriteLine("word: {0},({1},{2})", sentence.Substring(startIndex + space, word.Length - space), startIndex + space, endIndex);
                return token;
            }
            return null;
        }
    }
}