学步园

      最近下载网络数据时,遇到一个问题,就是C#封装的WebRequest每次请求默认的cookie都是新添加的,所以到需要cookie支持的页面的数据就不能下载,在网上找了很久,终于找到了解决方案,现在我把这个解决的代码贴下来,以供以后使用。

保持请求cookie的详细代码:

using System;<br /> using System.Collections.Generic;<br /> using System.Linq;<br /> using System.Text;<br /> using System.IO;<br /> using System.Net;</p> <p>namespace FileDownLoad<br /> {<br /> public class DownloadHelper<br /> {<br /> /// <summary><br /> /// 网站Cookies<br /> /// </summary><br /> private string _cookieHeader = string.Empty;<br /> /// <summary><br /> /// 网站编码<br /> /// </summary><br /> private string _code = string.Empty;</p> <p> private Dictionary<string, string> _para = new Dictionary<string, string>();</p> <p> private string _pageContent = string.Empty;</p> <p> public string CookieHeader<br /> {<br /> get<br /> {<br /> return _cookieHeader;<br /> }<br /> set<br /> {<br /> _cookieHeader = value;<br /> }<br /> }</p> <p> public string Code<br /> {<br /> get { return _code; }<br /> set { _code = value; }<br /> }</p> <p> public string PageContent<br /> {<br /> get { return _pageContent; }<br /> set { _pageContent = value; }<br /> }</p> <p> public Dictionary<string, string> Para<br /> {<br /> get { return _para; }<br /> set { _para = value; }<br /> }</p> <p> public DownloadHelper() { }</p> <p> public DownloadHelper(string code)<br /> {<br /> this._code = code;<br /> }</p> <p> /// <summary><br /> /// 功能描述：模拟登录页面，提交登录数据进行登录，并记录Header中的cookie<br /> /// </summary><br /> /// <param name="strURL">登录数据提交的页面地址</param><br /> /// <param name="strArgs">用户登录数据</param><br /> /// <param name="strReferer">引用地址</param><br /> /// <param name="code">网站编码</param><br /> /// <returns>可以返回页面内容或不返回</returns><br /> public string PostData(string strURL, string strArgs, string strReferer, string code, string method)<br /> {<br /> return PostData(strURL, strArgs, strReferer, code, method, string.Empty);<br /> }</p> <p> public string PostData(string strURL, string strArgs, string strReferer, string code, string method, string contentType)<br /> {<br /> try<br /> {<br /> string strResult = "";</p> <p> HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(strURL);</p> <p> myHttpWebRequest.AllowAutoRedirect = true;</p> <p> myHttpWebRequest.KeepAlive = true;</p> <p> myHttpWebRequest.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd.ms-excel, application/msword, application/x-shockwave-flash, */*";</p> <p> myHttpWebRequest.Referer = strReferer;</p> <p> myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 2.0.50727)";</p> <p> if (string.IsNullOrEmpty(contentType))<br /> {</p> <p> myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";</p> <p> }</p> <p> else<br /> {</p> <p> myHttpWebRequest.ContentType = contentType;</p> <p> }</p> <p> myHttpWebRequest.Method = method;</p> <p> myHttpWebRequest.Headers.Add("Accept-Encoding", "gzip, deflate");</p> <p> if (myHttpWebRequest.CookieContainer == null)<br /> {</p> <p> myHttpWebRequest.CookieContainer = new CookieContainer();</p> <p> }</p> <p> if (this.CookieHeader.Length > 0)<br /> {</p> <p> myHttpWebRequest.Headers.Add("cookie:" + this.CookieHeader);</p> <p> myHttpWebRequest.CookieContainer.SetCookies(new Uri(strURL), this.CookieHeader);</p> <p> }</p> <p> byte[] postData = Encoding.GetEncoding(code).GetBytes(strArgs);</p> <p> myHttpWebRequest.ContentLength = postData.Length;</p> <p> System.IO.Stream PostStream = myHttpWebRequest.GetRequestStream();</p> <p> PostStream.Write(postData, 0, postData.Length);</p> <p> PostStream.Close();</p> <p> HttpWebResponse response = null;</p> <p> System.IO.StreamReader sr = null;</p> <p> response = (HttpWebResponse)myHttpWebRequest.GetResponse();</p> <p> if (myHttpWebRequest.CookieContainer != null)<br /> {</p> <p> this.CookieHeader = myHttpWebRequest.CookieContainer.GetCookieHeader(new Uri(strURL));</p> <p> }</p> <p> sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(code)); // //utf-8 </p> <p> strResult = sr.ReadToEnd();</p> <p> sr.Close();</p> <p> response.Close();</p> <p> return strResult;</p> <p> }</p> <p> catch (Exception ex)<br /> {<br /> //Utilities.Document.Create("C://error.log", strArgs, true, Encoding.UTF8);<br /> }</p> <p> return string.Empty;<br /> }</p> <p> /// <summary><br /> /// 功能描述：在PostLogin成功登录后记录下Headers中的cookie，然后获取此网站上其他页面的内容<br /> /// </summary><br /> /// <param name="strURL">获取网站的某页面的地址</param><br /> /// <param name="strReferer">引用的地址</param><br /> /// <returns>返回页面内容</returns><br /> public string GetPage(string strURL, string strReferer, string code)<br /> {<br /> return GetPage(strURL, strReferer, code, string.Empty);<br /> }</p> <p> public string GetPage(string strURL, string strReferer, string code, string contentType)<br /> {<br /> string strResult = "";</p> <p> HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(strURL);</p> <p> myHttpWebRequest.AllowAutoRedirect = true;</p> <p> myHttpWebRequest.KeepAlive = false;</p> <p> myHttpWebRequest.Accept = "*/*";</p> <p> myHttpWebRequest.Referer = strReferer;</p> <p> myHttpWebRequest.Headers.Add("Accept-Encoding", "gzip, deflate");</p> <p> myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 7.0; MSIE 8.0 Windows NT 5.1; SV1; Maxthon; .NET CLR 2.0.50727)";</p> <p> if (string.IsNullOrEmpty(contentType))<br /> {<br /> myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";<br /> }</p> <p> else<br /> {<br /> myHttpWebRequest.ContentType = contentType;<br /> }</p> <p> myHttpWebRequest.Method = "GET";</p> <p> if (myHttpWebRequest.CookieContainer == null)<br /> {<br /> myHttpWebRequest.CookieContainer = new CookieContainer();<br /> }</p> <p> if (this.CookieHeader.Length > 0)<br /> {<br /> myHttpWebRequest.Headers.Add("cookie:" + this.CookieHeader);</p> <p> myHttpWebRequest.CookieContainer.SetCookies(new Uri(strURL), this.CookieHeader);<br /> }</p> <p> HttpWebResponse response = null;</p> <p> System.IO.StreamReader sr = null;</p> <p> response = (HttpWebResponse)myHttpWebRequest.GetResponse();</p> <p> Stream streamReceive;</p> <p> string gzip = response.ContentEncoding;</p> <p> if (string.IsNullOrEmpty(gzip) || gzip.ToLower() != "gzip")<br /> {<br /> streamReceive = response.GetResponseStream();<br /> }</p> <p> else<br /> {<br /> streamReceive = new System.IO.Compression.GZipStream(response.GetResponseStream(), System.IO.Compression.CompressionMode.Decompress);<br /> }</p> <p> sr = new System.IO.StreamReader(streamReceive, Encoding.GetEncoding(code));</p> <p> if (response.ContentLength > 1)<br /> {<br /> strResult = sr.ReadToEnd();<br /> }</p> <p> else<br /> {<br /> char[] buffer = new char[256];</p> <p> int count = 0;</p> <p> StringBuilder sb = new StringBuilder();</p> <p> while ((count = sr.Read(buffer, 0, buffer.Length)) > 0)<br /> {<br /> sb.Append(new string(buffer));<br /> }</p> <p> strResult = sb.ToString();<br /> }</p> <p> sr.Close();</p> <p> response.Close();</p> <p> return strResult;<br /> }</p> <p> public void SaveLink(string strURL, string strReferer, string code)<br /> {<br /> HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(strURL);</p> <p> myHttpWebRequest.AllowAutoRedirect = true;</p> <p> myHttpWebRequest.KeepAlive = false;</p> <p> myHttpWebRequest.Accept = "*/*";</p> <p> myHttpWebRequest.Referer = strReferer;</p> <p> myHttpWebRequest.Headers.Add("Accept-Encoding", "gzip, deflate");</p> <p> myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 7.0; MSIE 8.0 Windows NT 5.1; SV1; Maxthon; .NET CLR 2.0.50727)";</p> <p> myHttpWebRequest.ContentType = "application/x-www-form-urlencoded"; </p> <p> myHttpWebRequest.Method = "GET";</p> <p> if (myHttpWebRequest.CookieContainer == null)<br /> {<br /> myHttpWebRequest.CookieContainer = new CookieContainer();<br /> }</p> <p> if (this.CookieHeader.Length > 0)<br /> {<br /> myHttpWebRequest.Headers.Add("cookie:" + this.CookieHeader);</p> <p> myHttpWebRequest.CookieContainer.SetCookies(new Uri(strURL), this.CookieHeader);<br /> }</p> <p> HttpWebResponse response = null;</p> <p> response = (HttpWebResponse)myHttpWebRequest.GetResponse();</p> <p> long count = response.ContentLength;<br /> }</p> <p> public void DownloadPage(string strURL, string strReferer, string code, string contentType,string path)<br /> {<br /> HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(strURL);</p> <p> myHttpWebRequest.AllowAutoRedirect = true;</p> <p> myHttpWebRequest.KeepAlive = false;</p> <p> myHttpWebRequest.Accept = "*/*";</p> <p> myHttpWebRequest.Referer = strReferer;</p> <p> myHttpWebRequest.Headers.Add("Accept-Encoding", "gzip, deflate,");</p> <p> myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 7.0; MSIE 8.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 2.0.50727)";</p> <p> if (string.IsNullOrEmpty(contentType))<br /> {<br /> myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";<br /> }<br /> else<br /> {<br /> myHttpWebRequest.ContentType = contentType;<br /> }</p> <p> myHttpWebRequest.Method = "GET";</p> <p> if (myHttpWebRequest.CookieContainer == null)<br /> {<br /> myHttpWebRequest.CookieContainer = new CookieContainer();<br /> }</p> <p> if (this.CookieHeader.Length > 0)<br /> {<br /> myHttpWebRequest.Headers.Add("cookie:" + this.CookieHeader);</p> <p> myHttpWebRequest.CookieContainer.SetCookies(new Uri(strURL), this.CookieHeader);<br /> }</p> <p> HttpWebResponse response = null;</p> <p> response = (HttpWebResponse)myHttpWebRequest.GetResponse();</p> <p> Stream stream = response.GetResponseStream();<br /> byte[] arrByte = new byte[1024];<br /> int startPos = 0;</p> <p> FileStream fStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write);<br /> while (true)<br /> {<br /> int readCnt = stream.Read(arrByte, 0, 1024);</p> <p> if (readCnt == 0)<br /> break;</p> <p> fStream.Write(arrByte, 0, readCnt);</p> <p> startPos += readCnt;<br /> }</p> <p> stream.Close();<br /> fStream.Close();<br /> }<br /> }<br /> }</p> <p>

代码中有重复的部分,这里我就懒得再提取一次公共代码了。