HttpConnectionManager.java
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.zip.GZIPInputStream;
import javax.net.ssl.SSLHandshakeException;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.NoHttpResponseException;
import org.apache.http.ParseException;
import org.apache.http.StatusLine;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
/**
* http连接、抓取管理类
* @author lidongyang
* @createtime Oct 18, 2012 1:55:18 PM
*
* @note 基本测试版
*/
public class HttpConnectionManager {
/**
* 连接池里的最大连接数
*/
public static final int MAX_TOTAL_CONNECTIONS = 100;
/**
* 每个路由的默认最大连接数
*/
public static final int MAX_ROUTE_CONNECTIONS = 50;
/**
* 连接超时时间
*/
public static final int CONNECT_TIMEOUT = 50000;
/**
* 套接字超时时间
*/
public static final int SOCKET_TIMEOUT = 50000;
/**
* 连接池中 连接请求执行被阻塞的超时时间
*/
public static final long CONN_MANAGER_TIMEOUT = 60000;
/**
* http连接相关参数
*/
private static HttpParams parentParams;
/**
* http线程池管理器
*/
private static PoolingClientConnectionManager cm;
/**
* http客户端
*/
private static DefaultHttpClient httpClient;
/**
* 默认目标主机
*/
private static final HttpHost DEFAULT_TARGETHOST = new HttpHost("http://www.qq.com", 80);
/**
* 初始化http连接池，设置参数、http头等等信息
*/
static {
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(
new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(
new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
cm = new PoolingClientConnectionManager(schemeRegistry);
cm.setMaxTotal(MAX_TOTAL_CONNECTIONS);
cm.setDefaultMaxPerRoute(MAX_ROUTE_CONNECTIONS);
cm.setMaxPerRoute(new HttpRoute(DEFAULT_TARGETHOST), 20); //设置对目标主机的最大连接数
parentParams = new BasicHttpParams();
parentParams.setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
parentParams.setParameter(ClientPNames.DEFAULT_HOST, DEFAULT_TARGETHOST); //设置默认targetHost
parentParams.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY);
parentParams.setParameter(ClientPNames.CONN_MANAGER_TIMEOUT, CONN_MANAGER_TIMEOUT);
parentParams.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, CONNECT_TIMEOUT);
parentParams.setParameter(CoreConnectionPNames.SO_TIMEOUT, SOCKET_TIMEOUT);
parentParams.setParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true);
parentParams.setParameter(ClientPNames.HANDLE_REDIRECTS, true);
//设置头信息,模拟浏览器
Collection
collection = new ArrayList
();
collection.add(new BasicHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)"));
collection.add(new BasicHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
collection.add(new BasicHeader("Accept-Language", "zh-cn,zh,en-US,en;q=0.5"));
collection.add(new BasicHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7"));
collection.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
parentParams.setParameter(ClientPNames.DEFAULT_HEADERS, collection);
//请求重试处理
HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 5) {
// 如果超过最大重试次数，那么就不要继续了
return false;
}
if (exception instanceof NoHttpResponseException) {
// 如果服务器丢掉了连接，那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {
// 不要重试SSL握手异常
return false;
}
HttpRequest request = (HttpRequest) context.getAttribute(ExecutionContext.HTTP_REQUEST);
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {
// 如果请求被认为是幂等的，那么就重试
return true;
}
return false;
}
};
httpClient = new DefaultHttpClient(cm, parentParams);
httpClient.setHttpRequestRetryHandler(httpRequestRetryHandler);
}
/**
* 抓取页面代码
* @param url 目标页面的url
* @return 页面代码
*/
public String getHtml(String url) {
HttpHost proxyHost = new HttpHost("211.142.236.137", 8080);//代理
String html = getHtml(url, proxyHost);
int count = 0;
while(StringUtils.isEmpty(html)){
proxyHost = new HttpHost("211.142.236.137", 80);//更换代理
html = getHtml(url, proxyHost);
count++;
if(count > 3){
System.out.println("抓取失败");
break;
}
}
System.out.println(html.length());
return html;
}
/**
* 抓取url所指的页面代码
* @param url 目标页面的url
* @return 页面代码
*/
public String getHtml(String url, HttpHost proxyHost) {
String html = "";
HttpGet httpGet = new HttpGet(url);
httpGet.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);//设置代理
HttpResponse httpResponse;
HttpEntity httpEntity;
try {
httpResponse = httpClient.execute(httpGet);
StatusLine statusLine = httpResponse.getStatusLine();
int statusCode = statusLine.getStatusCode();
System.out.println(statusCode);
if(200 != statusCode) {
return html;
}
httpEntity = httpResponse.getEntity();
if(httpEntity != null){
html = readHtmlContentFromEntity(httpEntity);
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
if(httpGet != null){
httpGet.releaseConnection();
}
}
return html;
}
/**
* 从response返回的实体中读取页面代码
* @param httpEntity Http实体
* @return 页面代码
* @throws ParseException
* @throws IOException
*/
private String readHtmlContentFromEntity(HttpEntity httpEntity) throws ParseException, IOException {
String html = "";
Header header = httpEntity.getContentEncoding();
if(httpEntity.getContentLength() < 2147483647L){ //EntityUtils无法处理ContentLength超过2147483647L的Entity
if(header != null && "gzip".equals(header.getValue())){
html = EntityUtils.toString(new GzipDecompressingEntity(httpEntity));
} else {
html = EntityUtils.toString(httpEntity);
}
} else {
InputStream in = httpEntity.getContent();
if(header != null && "gzip".equals(header.getValue())){
html = unZip(in, ContentType.getOrDefault(httpEntity).getCharset().toString());
} else {
html = readInStreamToString(in, ContentType.getOrDefault(httpEntity).getCharset().toString());
}
if(in != null){
in.close();
}
}
return html;
}
/**
* 测试代理是否可用（其实和getHtml(String url, HttpHost proxyHost)的代码差不多，为了从功能上区别，暂时这样）
* @param httpHost 封装了代理的ip地址和端口
* @param url 用来测试的页面
* @return true 可用 false 不可用
*/
public boolean isProxyUsable(HttpHost proxyHost, String url) {
HttpGet httpGet = new HttpGet(url);
httpGet.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
try {
HttpResponse httpResponse = httpClient.execute(httpGet);
StatusLine statusLine = httpResponse.getStatusLine();
int statusCode = statusLine.getStatusCode();
System.out.println(statusCode);
if(200 != statusCode) {
return false;
}
HttpEntity httpEntity = httpResponse.getEntity();
if(httpEntity != null) {
String html = readHtmlContentFromEntity(httpEntity);
System.out.println(html.length());
if(StringUtils.isEmpty(html)){
return false;
}
} else {
return false;
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return false;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return false;
}
return true;
}
/**
* 解压服务器返回的gzip流
* @param in 抓取返回的InputStream流
* @param charSet 页面内容编码
* @return 页面内容的String格式
* @throws IOException
*/
private String unZip(InputStream in, String charSet) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPInputStream gis = null;
try {
gis = new GZIPInputStream(in);
byte[] _byte = new byte[1024];
int len = 0;
while ((len = gis.read(_byte)) != -1) {
baos.write(_byte, 0, len);
}
String unzipString = new String(baos.toByteArray(), charSet);
return unzipString;
} finally {
if (gis != null) {
gis.close();
}
if(baos != null){
baos.close();
}
}
}
/**
* 读取InputStream流
* @param in InputStream流
* @return 从流中读取的String
* @throws IOException
*/
private String readInStreamToString(InputStream in, String charSet) throws IOException {
StringBuilder str = new StringBuilder();
String line;
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in, charSet));
while((line = bufferedReader.readLine()) != null){
str.append(line);
str.append("\n");
}
if(bufferedReader != null) {
bufferedReader.close();
}
return str.toString();
}
/**
* for test
* @author lidongyang
* @createtime Oct 18, 2012 2:35:09 PM
*/
public class Test implements Runnable {
String url;
int threadNum;
public Test() {
}
public Test(String url, int threadNum) {
this.url = url;
this.threadNum = threadNum;
}
@Override
public void run() {
getHtml(url);
}
}
/**
* for test
* @param args
* @throws InterruptedException
*/
public static void main(String[] args) throws InterruptedException{
HttpConnectionManager httpConnectionManager = new HttpConnectionManager();
Date start = new Date();
httpConnectionManager.getHtml("http://www.qq.com");
Date end = new Date();
System.out.println((end.getTime() - start.getTime())/1000.0 + " 秒");
}
}
GetQqNews.java
package parser;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import spider.HttpConnectionManager;
/** test
* @author lidongyang
* @createtime Oct 23, 2012 11:05:33 AM
*/
public class GetQqNews {
public static void main(String[] args){
HttpConnectionManager httpConnectionManager = new HttpConnectionManager();
String html = httpConnectionManager.getHtml("http://www.qq.com");
Document doc = Jsoup.parse(html);
Elements newsList = doc.select("[class=ft fl]").select("ul").select("li").select("a");
for (Element element : newsList) {
System.out.println(element.attr("href") + "----" + element.text());
}
}
}