学步园

     如何从网页上抓取有价值的东西？看懂了下面的程序(非常简单)，想从网页上抓取什么信息(标题、内容、Email、价格等)就能抓取什么信息。

package catchhtml;<br /> import java.io.BufferedReader;<br /> import java.io.IOException;<br /> import java.io.InputStreamReader;<br /> import java.net.MalformedURLException;<br /> import java.net.URL;<br /> import java.util.ArrayList;<br /> import java.util.List;<br /> import java.util.regex.Matcher;<br /> import java.util.regex.Pattern;</p> <p>public class GetHtmlTitle {</p> <p> public GetHtmlTitle(String htmlUrl){<br /> System.out.println("/n------------开始读取网页(" + htmlUrl + ")-----------");<br /> String htmlSource = "";<br /> htmlSource = getHtmlSource(htmlUrl);//获取htmlUrl网址网页的源码<br /> System.out.println("------------读取网页(" + htmlUrl + ")结束-----------/n");<br /> System.out.println("------------分析(" + htmlUrl + ")结果如下-----------/n");<br /> String title = getTitle(htmlSource);<br /> System.out.println("网站标题： " + title);<br /> }</p> <p> /**<br /> * 根据网址返回网页的源码<br /> * @param htmlUrl<br /> * @return<br /> */<br /> public String getHtmlSource(String htmlUrl){<br /> URL url;<br /> StringBuffer sb = new StringBuffer();<br /> try{<br /> url = new URL(htmlUrl);<br /> BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), "UTF-8"));//读取网页全部内容<br /> String temp;<br /> while ((temp = in.readLine()) != null)<br /> {<br /> sb.append(temp);<br /> }<br /> in.close();<br /> }catch (MalformedURLException e) {<br /> System.out.println("你输入的URL格式有问题！请仔细输入");<br /> }catch (IOException e) {<br /> e.printStackTrace();<br /> }<br /> return sb.toString();<br /> }</p> <p> /**<br /> * 从html源码(字符串)中去掉标题<br /> * @param htmlSource<br /> * @return<br /> */<br /> public String getTitle(String htmlSource){<br /> List<String> list = new ArrayList<String>();<br /> String title = "";</p> <p> //Pattern pa = Pattern.compile("<title>.*?</title>", Pattern.CANON_EQ);也可以<br /> Pattern pa = Pattern.compile("<title>.*?</title>");//源码中标题正则表达式<br /> Matcher ma = pa.matcher(htmlSource);<br /> while (ma.find())//寻找符合el的字串<br /> {<br /> list.add(ma.group());//将符合el的字串加入到list中<br /> }<br /> for (int i = 0; i < list.size(); i++)<br /> {<br /> title = title + list.get(i);<br /> }<br /> return outTag(title);<br /> }</p> <p> /**<br /> * 去掉html源码中的标签<br /> * @param s<br /> * @return<br /> */<br /> public String outTag(String s)<br /> {<br /> return s.replaceAll("<.*?>", "");<br /> }</p> <p> public static void main(String[] args) {<br /> String htmlUrl = "http://www.157buy.com";<br /> new GetHtmlTitle(htmlUrl);</p> <p> }<br /> }<br />