学步园

 建立搜索引擎四步： 

   抓取数据---》解析数据----》创建索引-----》执行搜索

第一部分：体会

今天做了一个简单示例，用的版本是lucune 2.4.1,跟教科书上的有些不同，但教科书上的示例都能正常执行。

第二部分：遇到的问题

刚把程序写出来的时候，出现了搜索不到结果的情况，后来发现是在添加Field的时候没有把搜索的字段设置成

Field.Index.ANALYZED，

如果你在搜索器中设定的搜索字段是 text,那么在创建索引的时候应该把这个字段设置成可以被索引。

也就是

  field = new Field("text",text,Field.Store.YES,Field.Index.ANALYZED);

而不是

field = new Field("text",text,Field.Store.YES,Field.Index.NO);

第三部分：新老版本变更的问题

以下是lucune2.4.1版本与老版本的不同之处。

创建索引时的不同

老版本

IndexWriter writer = new IndexWriter(indexPath,new StandardAnalyzer(),true);

新版本

IndexWriter writer = new IndexWriter(indexPath,new StandardAnalyzer(),true,IndexWriter.MaxFieldLength.UNLIMITED);

在执行索引时的不同，注意看下 search和searchNew两个方法的不同就可以了

第四部分：成果

LuceneIndexer.java

package creatindex;</p> <p>import java.awt.BorderLayout;<br /> import java.awt.Container;<br /> import java.awt.GridLayout;<br /> import java.awt.event.ActionEvent;<br /> import java.awt.event.ActionListener;<br /> import java.io.File;<br /> import java.io.IOException;<br /> import java.text.SimpleDateFormat;<br /> import java.util.Date;</p> <p>import javax.swing.JButton;<br /> import javax.swing.JFileChooser;<br /> import javax.swing.JFrame;<br /> import javax.swing.JLabel;<br /> import javax.swing.JOptionPane;<br /> import javax.swing.JPanel;<br /> import javax.swing.JScrollPane;<br /> import javax.swing.JTextArea;<br /> import javax.swing.JTextField;<br /> import javax.swing.SwingUtilities;<br /> import javax.swing.UIManager;</p> <p>import org.apache.lucene.analysis.standard.StandardAnalyzer;<br /> import org.apache.lucene.document.Document;<br /> import org.apache.lucene.document.Field;<br /> import org.apache.lucene.index.IndexWriter;</p> <p>public class LuceneIndexer {</p> <p> /**<br /> * @param args<br /> */<br /> private JTextField jtfa;<br /> private JButton jba;<br /> private JTextField jtfb;<br /> private JButton jbb;<br /> private JButton jbc;<br /> private static JTextArea jta;</p> <p> private void createAndShowGUI(){</p> <p> String lf = "com.sun.java.swing.plaf.windows.WindowsLookAndFeel";<br /> try{<br /> UIManager.setLookAndFeel(lf);<br /> }<br /> catch(Exception ce){<br /> JOptionPane.showMessageDialog(null,"无法设定外观感觉");<br /> }</p> <p> JFrame.setDefaultLookAndFeelDecorated(true);</p> <p> JFrame frame = new JFrame("dogdog Indexer!dogdog7788@foxmail.com");<br /> frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);</p> <p> final JFileChooser fc = new JFileChooser();</p> <p> fc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);</p> <p> Container con = frame.getContentPane();</p> <p> con.setLayout(new BorderLayout());</p> <p> JPanel jpup = new JPanel();</p> <p> jpup.setLayout(new GridLayout(3,2));</p> <p> jtfa = new JTextField(30);</p> <p> jba = new JButton("选择被缩影的文件存放路径");</p> <p> jba.addActionListener(<br /> new ActionListener(){</p> <p> public void actionPerformed(ActionEvent a) {<br /> // TODO Auto-generated method stub<br /> int r=fc.showOpenDialog(null);<br /> if(r==JFileChooser.APPROVE_OPTION){<br /> jtfa.setText(fc.getSelectedFile().getPath());<br /> jbc.setEnabled(true);<br /> }<br /> }</p> <p> } </p> <p> );</p> <p> jtfb = new JTextField(30);<br /> JButton jbb = new JButton("选择索引存放的路径");</p> <p> jbb.addActionListener(<br /> new ActionListener(){</p> <p> public void actionPerformed(ActionEvent a) {<br /> // TODO Auto-generated method stub<br /> int r= fc.showOpenDialog(null);<br /> if(r==JFileChooser.APPROVE_OPTION){<br /> jtfb.setText(fc.getSelectedFile().getPath());<br /> jbc.setEnabled(true);<br /> }<br /> }}<br /> );</p> <p> JLabel jl = new JLabel("");<br /> jbc = new JButton("建立索引");<br /> jbc.addActionListener(<br /> new ActionListener(){</p> <p> public void actionPerformed(ActionEvent arg0) {<br /> // TODO Auto-generated method stub<br /> try{<br /> LuceneIndexerTool.index(jtfa.getText(),jtfb.getText());<br /> }<br /> catch(Exception e){<br /> e.printStackTrace();<br /> jbc.setEnabled(true);<br /> JOptionPane.showMessageDialog(null, "索引创建失败！");<br /> System.out.println(e.getMessage());<br /> }<br /> }</p> <p> }<br /> );<br /> jpup.add(jtfa);<br /> jpup.add(jba);<br /> jpup.add(jbb);<br /> jpup.add(jl);<br /> jpup.add(jbc);</p> <p> jta = new JTextArea(10,60);<br /> JScrollPane jsp = new JScrollPane(jta);</p> <p> con.add(jpup,BorderLayout.NORTH);<br /> con.add(jsp,BorderLayout.CENTER);</p> <p> frame.setSize(200,100);<br /> frame.pack();<br /> frame.setVisible(true);<br /> }</p> <p> public static void main(String[] args) {<br /> // TODO Auto-generated method stub<br /> SwingUtilities.invokeLater(<br /> new Runnable(){</p> <p> public void run() {<br /> // TODO Auto-generated method stub<br /> try{<br /> new LuceneIndexer().createAndShowGUI();<br /> }<br /> catch(Exception e){<br /> JOptionPane.showMessageDialog(null, "程序加载失败");<br /> }<br /> }</p> <p> }<br /> );<br /> }<br /> static class LuceneIndexerTool {</p> <p> public static void index(String filesPath,String indexPath) throws IOException{<br /> IndexWriter writer = new IndexWriter(indexPath,new StandardAnalyzer(),true,IndexWriter.MaxFieldLength.UNLIMITED);</p> <p> String s[] = FileList.getFiles(filesPath);<br /> int len = s.length;<br /> String ext = "";<br /> for(int i= 0;i<len;i++){<br /> File f = new File(s[i]);<br /> ext = getExt(f);<br /> if(ext.equalsIgnoreCase("html")||ext.equalsIgnoreCase("htm")||ext.equalsIgnoreCase("txt")){<br /> Document doc = new Document();</p> <p> //filename字段<br /> String filename = f.getName();<br /> Field field = new Field("filename",filename,Field.Store.YES,Field.Index.ANALYZED);<br /> doc.add(field);</p> <p> //uri字段<br /> String filepath = f.getPath();<br /> field = new Field("uri",filepath,Field.Store.YES,Field.Index.NO);<br /> doc.add(field);</p> <p> //cdate字段</p> <p> Date d = new Date(f.lastModified());<br /> SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd E");<br /> String cdate = sdf.format(d);<br /> field = new Field("cdate",cdate,Field.Store.YES,Field.Index.NO);<br /> doc.add(field);</p> <p> //size字段<br /> double si = f.length();<br /> String size = "";<br /> if(si>1024){<br /> size = String.valueOf(Math.floor(si/1024))+"K";<br /> }<br /> else{<br /> size = String.valueOf(Math.floor(si))+"Bytes";<br /> }<br /> field = new Field("size",size,Field.Store.YES,Field.Index.NO);<br /> doc.add(field);</p> <p> //text字段<br /> String text = FileText.getText(f);<br /> field = new Field("text",text,Field.Store.YES,Field.Index.ANALYZED);<br /> doc.add(field);</p> <p> //digest字段<br /> String digest="";<br /> if(text.length()>200){<br /> digest = text.substring(0, 201);<br /> }<br /> else{<br /> digest =text;<br /> }<br /> field = new Field("digest",digest,Field.Store.YES,Field.Index.ANALYZED);<br /> doc.add(field);</p> <p> writer.addDocument(doc);</p> <p> jta.setText(jta.getText()+"已经归入索引："+f.getName()+"___"+cdate+"/n");<br /> }</p> <p> }<br /> writer.close();<br /> JOptionPane.showMessageDialog(null, "索引建立完毕","dogdog advice",JOptionPane.INFORMATION_MESSAGE);<br /> }</p> <p> public static String getExt(File f)<br /> {<br /> String s = f.getName();<br /> try{<br /> s = s.substring(s.lastIndexOf(".")+1);<br /> //System.out.println(s.lastIndexOf("s"));<br /> }<br /> catch(Exception e){<br /> s="";<br /> }<br /> return s;</p> <p> }</p> <p> }</p> <p>}<br />

LuceneSearcher.java

package creatsearch;</p> <p>import java.awt.BorderLayout;<br /> import java.awt.Container;<br /> import java.awt.FlowLayout;<br /> import java.awt.GridLayout;<br /> import java.awt.event.ActionEvent;<br /> import java.awt.event.ActionListener;<br /> import java.io.File;<br /> import java.io.FileWriter;<br /> import java.io.IOException;<br /> import java.io.PrintWriter;<br /> import java.util.ArrayList;<br /> import java.util.List;</p> <p>import javax.swing.JButton;<br /> import javax.swing.JFileChooser;<br /> import javax.swing.JFrame;<br /> import javax.swing.JOptionPane;<br /> import javax.swing.JPanel;<br /> import javax.swing.JScrollPane;<br /> import javax.swing.JTextArea;<br /> import javax.swing.JTextField;<br /> import javax.swing.SwingUtilities;<br /> import javax.swing.UIManager;</p> <p>import org.apache.lucene.analysis.standard.StandardAnalyzer;<br /> import org.apache.lucene.document.Document;<br /> import org.apache.lucene.document.Field;<br /> import org.apache.lucene.index.Term;<br /> import org.apache.lucene.queryParser.MultiFieldQueryParser;<br /> import org.apache.lucene.queryParser.ParseException;<br /> import org.apache.lucene.queryParser.QueryParser;<br /> import org.apache.lucene.search.BooleanClause;<br /> import org.apache.lucene.search.Hits;<br /> import org.apache.lucene.search.IndexSearcher;<br /> import org.apache.lucene.search.Query;<br /> import org.apache.lucene.search.ScoreDoc;<br /> import org.apache.lucene.search.TermQuery;<br /> import org.apache.lucene.search.TopDocCollector;<br /> import org.apache.lucene.search.TopDocs;</p> <p>import creatindex.LuceneIndexer;</p> <p>public class LuceneSearcher {</p> <p> /**<br /> * @param args<br /> */<br /> private JTextField jtfa;<br /> private JButton jba;<br /> private JTextField jtfb;<br /> private JButton jbb;<br /> private JButton jbc;<br /> private static JTextArea jta;<br /> private JTextField jtfc;<br /> private JButton jbd;<br /> private JButton jbe;</p> <p> private void createAndShowGUI(){<br /> String lf = "com.sun.java.swing.plaf.windows.WindowsLookAndFeel";</p> <p> try{<br /> UIManager.setLookAndFeel(lf);<br /> }<br /> catch(Exception e){<br /> JOptionPane.showMessageDialog(null,"无法设定外观感觉");<br /> }<br /> JFrame.setDefaultLookAndFeelDecorated(true);<br /> JFrame frame = new JFrame("dogdog searcher");<br /> frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);<br /> final JFileChooser fc = new JFileChooser();<br /> fc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);</p> <p> fc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);</p> <p> Container con = frame.getContentPane();</p> <p> con.setLayout(new BorderLayout());</p> <p> JPanel jpup = new JPanel();</p> <p> jpup.setLayout(new GridLayout(2,2));</p> <p> jtfa = new JTextField(30);</p> <p> jba = new JButton("选择索引的存放路径");</p> <p> jba.addActionListener(<br /> new ActionListener(){</p> <p> public void actionPerformed(ActionEvent a) {<br /> // TODO Auto-generated method stub<br /> int r=fc.showOpenDialog(null);<br /> if(r==JFileChooser.APPROVE_OPTION){<br /> jtfa.setText(fc.getSelectedFile().getPath());<br /> }<br /> }</p> <p> } </p> <p> );</p> <p> jtfb = new JTextField(30);<br /> JButton jbb = new JButton("搜索");<br /> jbb.addActionListener(<br /> new ActionListener(){<br /> public void actionPerformed(ActionEvent arg0) {<br /> // TODO Auto-generated method stub<br /> try{<br /> String indexPath=jtfa.getText();<br /> String phrase = jtfb.getText();<br /> //LuceneSearcherTool.search(phrase, indexPath);<br /> LuceneSearcherTool.searchNew(phrase, indexPath);<br /> }<br /> catch(Exception e){<br /> JOptionPane.showMessageDialog(null, "搜索失败！","提示",JOptionPane.ERROR_MESSAGE);<br /> e.printStackTrace();<br /> }</p> <p> }</p> <p> }<br /> );</p> <p> jpup.add(jtfa);<br /> jpup.add(jba);<br /> jpup.add(jtfb);<br /> jpup.add(jbb);</p> <p> jta = new JTextArea(10,30);<br /> JScrollPane jsp = new JScrollPane(jta);</p> <p> JPanel jpdown = new JPanel();<br /> jpdown.setLayout(new FlowLayout());<br /> jtfc=new JTextField(35);<br /> jbd = new JButton("设定导出路径");<br /> fc.setFileSelectionMode(JFileChooser.FILES_AND_DIRECTORIES);<br /> jbd.addActionListener(new ActionListener(){</p> <p> public void actionPerformed(ActionEvent arg0) {<br /> // TODO Auto-generated method stub<br /> int r = fc.showOpenDialog(null);<br /> if(r==JFileChooser.APPROVE_OPTION){<br /> jtfc.setText(fc.getSelectedFile().getPath());<br /> }<br /> }</p> <p> });</p> <p> jbe = new JButton("导出搜索结果");<br /> jbe.addActionListener(new ActionListener(){</p> <p> public void actionPerformed(ActionEvent arg0) {<br /> // TODO Auto-generated method stub<br /> try{<br /> File f = new File(jtfc.getText());<br /> FileWriter fw = new FileWriter(f);<br /> PrintWriter pw = new PrintWriter(fw);</p> <p> pw.write(jta.getText());<br /> pw.flush();<br /> pw.close();<br /> JOptionPane.showMessageDialog(null, "成功写入文件");<br /> }<br /> catch(Exception e){<br /> JOptionPane.showMessageDialog(null, "写入文件失败");<br /> e.printStackTrace();<br /> }<br /> }</p> <p> });</p> <p> jpdown.add(jtfc);<br /> jpdown.add(jbd);<br /> jpdown.add(jbe);</p> <p> con.add(jpup,BorderLayout.NORTH);<br /> con.add(jsp,BorderLayout.CENTER);<br /> con.add(jpdown,BorderLayout.SOUTH);</p> <p> frame.setSize(200,100);<br /> frame.pack();<br /> frame.setVisible(true);</p> <p> }</p> <p> static class LuceneSearcherTool{<br /> @SuppressWarnings("deprecation")<br /> public static void search(String phrase,String indexPath) throws IOException, Exception{<br /> System.out.println(phrase.toString());<br /> //建立搜索器<br /> IndexSearcher searcher = new IndexSearcher(indexPath);<br /> //搜索text字段<br /> Term t = new Term("text",phrase);<br /> //生成Query对象<br /> Query q = new TermQuery(t);</p> <p> Hits hs = searcher.search(q);</p> <p> //搜索到的结果数量<br /> int num = hs.length();<br /> jta.setText("检索到的记录数量："+num+"/n");<br /> jta.setText(jta.getText()+"***************"+"/n/n");</p> <p> //输出结果</p> <p> for(int i=0;i<num;i++){<br /> Document doc = hs.doc(i);<br /> if(doc == null){<br /> continue;<br /> }</p> <p> //获得FILENAME字段<br /> Field field = doc.getField("filename");<br /> String filename = field.stringValue();</p> <p> //uri字段<br /> field = doc.getField("uri");<br /> String uri = field.stringValue();</p> <p> //cdate字段<br /> field = doc.getField("cdate");<br /> String cdate = field.stringValue();<br /> //String cdate = "cdate";</p> <p> //digest字段<br /> field = doc.getField("digest");<br /> String digest = field.stringValue();</p> <p> StringBuffer sb = new StringBuffer();</p> <p> sb.append("URI:"+uri+"/n");<br /> sb.append("filename:"+filename+"/n");<br /> sb.append("cdate:"+cdate+"/n");<br /> sb.append("digest:"+digest+"/n");<br /> sb.append("******************"+"/n");</p> <p> jta.setText(jta.getText()+sb.toString());<br /> }<br /> searcher.close();<br /> }</p> <p> //2.4.1版本有所改动<br /> public static void searchNew(String phrase,String indexPath) throws IOException, Exception{</p> <p> IndexSearcher newSearcher = new IndexSearcher(indexPath);</p> <p> QueryParser parser = new QueryParser("text",new StandardAnalyzer());<br /> Query q = parser.parse(phrase);</p> <p> //也可以这样生成Query<br /> //Term t = new Term("text",phrase);<br /> //Query q = new TermQuery(t);</p> <p> TopDocs topDocs = newSearcher.search(q, 100);//100是显示队列的Size<br /> ScoreDoc[] hits = topDocs.scoreDocs;</p> <p> jta.setText("共有"+newSearcher.maxDoc()+"条记录,其中"+hits.length+"条符合搜索条件/n");<br /> jta.setText(jta.getText()+"************/n");</p> <p>//输出结果</p> <p> for(int i=0;i<hits.length;i++){<br /> int docId = hits[i].doc;<br /> Document doc = newSearcher.doc(docId);<br /> if(doc == null){<br /> continue;<br /> }</p> <p> //获得FILENAME字段<br /> Field field = doc.getField("filename");<br /> String filename = field.stringValue();</p> <p> //uri字段<br /> field = doc.getField("uri");<br /> String uri = field.stringValue();</p> <p> //cdate字段<br /> field = doc.getField("cdate");<br /> String cdate = field.stringValue();<br /> //String cdate = "cdate";</p> <p> //digest字段<br /> field = doc.getField("digest");<br /> String digest = field.stringValue();</p> <p> StringBuffer sb = new StringBuffer();</p> <p> sb.append("URI:"+uri+"/n");<br /> sb.append("filename:"+filename+"/n");<br /> sb.append("cdate:"+cdate+"/n");<br /> sb.append("digest:"+digest+"/n");<br /> sb.append("******************"+"/n");</p> <p> jta.setText(jta.getText()+sb.toString());<br /> }<br /> newSearcher.close();</p> <p> }<br /> }<br /> public static void main(String[] args) {<br /> // TODO Auto-generated method stub</p> <p> // TODO Auto-generated method stub<br /> SwingUtilities.invokeLater(<br /> new Runnable(){</p> <p> public void run() {<br /> // TODO Auto-generated method stub<br /> try{<br /> new LuceneSearcher().createAndShowGUI();<br /> }<br /> catch(Exception e){<br /> JOptionPane.showMessageDialog(null, "程序加载失败");<br /> }<br /> }</p> <p> }<br /> );</p> <p> }</p> <p>}<br />

另有两个辅助类

FileList和FileText

package creatindex;</p> <p>import java.io.File;<br /> import java.io.IOException;</p> <p>public class FileList {<br /> private static final String SEP = "/";<br /> private static StringBuffer sb = new StringBuffer("");<br /> //返回目录下的文件名的数组<br /> public static String[] getFiles(File f) throws IOException{<br /> if(f.isDirectory()){<br /> File[] fs = f.listFiles();<br /> for(int i=0;i<fs.length;i++){<br /> getFiles(fs[i]);<br /> }<br /> }<br /> else{<br /> sb.append(f.getPath()+SEP);<br /> }<br /> String s = sb.toString();<br /> String[] ss = s.split(SEP);<br /> return ss;<br /> }<br /> //返回目录下的文件名的数组--重载<br /> public static String[] getFiles(String t) throws IOException{<br /> File f = new File(t);<br /> return getFiles(f);<br /> }</p> <p> public static void main(String[] args) throws IOException{<br /> String s[] = FileList.getFiles("D:/apache-tomcat-5.5.26/bin");<br /> for(int i=0;i<s.length;i++){<br /> System.out.println(s[i]);<br /> }<br /> }<br /> }<br />

 package creatindex;</p> <p>import java.io.BufferedReader;<br /> import java.io.File;<br /> import java.io.FileReader;</p> <p>public class FileText {</p> <p> //读取一个文件的所有内容<br /> public static String getText(File f){<br /> StringBuffer sb = new StringBuffer("");<br /> try{<br /> FileReader fr = new FileReader(f);<br /> BufferedReader br = new BufferedReader(fr);<br /> String s = br.readLine();</p> <p> while(s!=null){<br /> sb.append(s);<br /> s=br.readLine();<br /> }<br /> br.close();<br /> }<br /> catch(Exception e){<br /> sb.append("");<br /> }<br /> return sb.toString();<br /> }</p> <p> //读取一个文件的所有内容--重载<br /> public static String getText(String s){<br /> String t = "";<br /> try{<br /> File f = new File(s);<br /> t = getText(f);<br /> }<br /> catch(Exception e){<br /> t="";<br /> }<br /> return null;<br /> }<br /> }<br />