现在的位置: 首页 > 综合 > 正文

使用HttpWebRequest的POST取得网页内容(异步操作)2篇集合

2018年03月19日 ⁄ 综合 ⁄ 共 11380字 ⁄ 字号 评论关闭
上次说了,如何通过post来取得网页内容。可是有一个问题出现了。当时用的方法是同步操作,如果我其中的一个IP或是在进行转化的过程中,出现了问题,
哪么这个程序就会停下来,当然了,有的朋友可能会说,用try...catch也可以啊。因为我是循环取值,所以在catch里加一个continue就
行了。可是以前没有搞过异步操作,所以想用这个机会搞一下。就看了一下。
这一看不要紧,搞得我一头雾水。上网上去问,大家给出的方法都是用多线程来搞定。这个在下一个帖子中我会发上来。这里主要说一下用BeginGetResponse,EndGetResponse来完成异步操作。
先说一下,我的要求:
一,从数据库中取出所有的IP
二,利用httpwebrequest的post方法在网上取得它的相应地址。当然了,是用post还是用get主要是看网上的网站是用什么方法来传值。
三,将所得到的值写入数据库

下面我把我的代码放上来,前台没有什么东西,运行的结果是直接写入数据库,
数据库名为DBCT_Dev
省份表S_Province,其中ProvinceName为省分名。ProvinceCode为省分相对的Code
市级表S_City,其中CityName为市名。ZipCode为市相对应的Code。
这二个表是在网上找到的,有很多中国省市关联表,大家可以去下一个。
用户IP表UserCode,其中Code为其IP所对应的省市Code,IP为用户的IP地址。

好了,下面是代码

  1. using System;
  2. using System.Collections.Generic;
  3. using System.Linq;
  4. using System.Web;
  5. using System.Web.UI;
  6. using System.Web.UI.WebControls;
  7. using System.IO;
  8. using System.Text;
  9. using System.Net;
  10. using System.Threading;
  11. using System.Data;
  12. using System.Collections;
  13. namespace getPageValue
  14. {
  15.     public partial class two : System.Web.UI.Page
  16.     {
  17.         public DBClass db = new DBClass();//我自己写的一个类,主要是用来做一些数据库操作的,你可以使用你自己的
  18.         public static Hashtable ht = new Hashtable();
  19.         private static ManualResetEvent allDone = new ManualResetEvent(false);
  20.         public static string singleid;
  21.         protected void Page_Load(object sender, EventArgs e)
  22.         {
  23.             if (!Page.IsPostBack)
  24.             {
  25.                 //将数据库中Code为空ip不为空的记录存入,放到一个DataTable中。
  26.                 string strsql = "select * from UserCode where UserIP<>'' and (Code is null or code = '')";
  27.                 
  28.                 DataTable dt = db.GetDataTable(strsql);
  29.                 for (int i = 0; i < dt.Rows.Count; i++)
  30.                 {
  31.                     //调用转化方法
  32.                     getPost(dt.Rows[i]["id"].ToString());
  33.                     //每次调用间隔一秒,通过实验得到,如果你的数据量大,最好间隔5秒
  34.                     Thread.Sleep(1000);
  35.                 }
  36.                 Thread.Sleep(5000);
  37.                 dt.Dispose();
  38.                 //调用将转化信息存入数据库的方法
  39.                 ChangeUserCode();
  40.             }
  41.         }
  42.         public void getPost(string id)
  43.         {
  44.             //定义变量
  45.             //singleid 传入的用户信息ID
  46.             //singleip 根据用户信息ID,而取到的IP
  47.             //strAction post必传值之一
  48.             //strCode 传入用户信息表中的code值
  49.             //
  50.             singleid = id;
  51.             //通过GetSingleValue方法,得到相应的值,这里GetSingleValue(表名,要得到的值,条件语句)
  52.             string singleip = db.GetSingleValue("UserCode""UserIP""id=" + singleid).Trim();
  53.             string strAction = "2";
  54.             string strCode = "";
  55.             //将得到的post值,转化为byte型
  56.             ASCIIEncoding encoding = new ASCIIEncoding();
  57.             string postData = "ip=" + singleip;
  58.             postData += "&action=" + strAction;
  59.             byte[] data = encoding.GetBytes(postData);
  60.             //建立HttpWebRequest
  61.             HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create("http://www2.ip138.com/ips8.asp");
  62.             myRequest.Method = "POST";
  63.             myRequest.ContentType = "application/x-www-form-urlencoded";
  64.             myRequest.ContentLength = data.Length;
  65.             Stream myStream = myRequest.GetRequestStream();
  66.             myStream.Write(data, 0, data.Length);
  67.             myStream.Close();
  68.             //开始调用异步操作
  69.             myRequest.BeginGetResponse(new AsyncCallback(ReadCallback), myRequest);
  70.             allDone.WaitOne();
  71.         }
  72.         private void ReadCallback(IAsyncResult asynchronousResult)
  73.         {
  74.             HttpWebRequest request = (HttpWebRequest)asynchronousResult.AsyncState;
  75.             //Stream postStream = request.EndGetResponse(asynchronousResult);
  76.             //异步回调方法使用 EndGetResponse 方法返回实际的 WebResponse。 
  77.             HttpWebResponse response = (HttpWebResponse)request.EndGetResponse(asynchronousResult);
  78.             Stream streamResponse = response.GetResponseStream();
  79.             StreamReader streamRead = new StreamReader(streamResponse, Encoding.Default);
  80.             string content = streamRead.ReadToEnd();
  81.             //string con = responseString.Substring(responseString.IndexOf("
    本站主数据") + 6, responseString.IndexOf("</li><li>参考数据
    一") - responseString.IndexOf("本站主数据") - 1);
  82.             //Response.Write(con + "<br>");
  83.             //判断是否是有省市的信息,如果信息中有省,市,则进行存入数据库,如为127.0.0.1,是直接给本省code。其它信息,则不给记录
  84.             string singleip = "";
  85.             string strCode = "";
  86.             
  87.             if (singleip == "127.0.0.1")
  88.             {
  89.                 strCode = "0008";
  90.             }
  91.             else
  92.             {
  93.                 if (content.IndexOf("省") == -1 && content.IndexOf("市") == -1 && singleid != "127.0.0.1")
  94.                 {
  95.                     if (content.IndexOf("查询太频繁") != -1)
  96.                     {
  97.                         strCode = "查询太频繁";
  98.                     }
  99.                     else
  100.                     {
  101.                         strCode = "0035";
  102.                     }
  103.                 }
  104.                 else
  105.                 {
  106.                     if (content.IndexOf("省") != -1)
  107.                     {
  108.                         string con = content.Substring(content.IndexOf("本站主数据") + 6, content.IndexOf("</li><li>参考数据一") - content.IndexOf("本站主数据") - 1);
  109.                         string strpro = con.Substring(0, con.IndexOf("省") + 1);
  110.                         strCode = db.GetSingleValue("S_Province""ProvinceCode""ProvinceName='" + strpro + "'").Trim();
  111.                         //strCode = strpro;
  112.                     }
  113.                     if (content.IndexOf("市") != -1)
  114.                     {
  115.                         string con = content.Substring(content.IndexOf("本站主数据") + 6, content.IndexOf("</li><li>参考数据一") - content.IndexOf("本站主数据") - 1);
  116.                         string strpro = con.Substring(con.IndexOf("省") + 1, con.IndexOf("市") - con.IndexOf("省"));
  117.                         strCode += db.GetSingleValue("S_City""ZipCode""CityName='" + strpro + "'").Trim(); ;
  118.                     }
  119.                 }
  120.             }
  121.             if (strCode == "")
  122.             {
  123.                 strCode = "0035";
  124.             }
  125.             //将信息存入hashtable
  126.             ht.Add(singleid, strCode);
  127.             streamResponse.Close();
  128.             streamRead.Close();
  129.             response.Close();
  130.             allDone.Set();
  131.         }
  132.         public void ChangeUserCode()
  133.         {
  134.             if (ht.Count > 0)
  135.             {
  136.                 foreach (DictionaryEntry objDE in ht)
  137.                 {
  138.                     string strsql = "update UserCode set Code='" + objDE.Value.ToString() + "' where id=" + objDE.Key.ToString() + "";
  139.                     try
  140.                     {
  141.                         db.ExecuteSql(strsql);
  142.                     }
  143.                     catch
  144.                     {
  145.                         continue;
  146.                     }
  147.                 }
  148.             }
  149.         }
  150.     }
  151. }

下面的是周公在CSDN上一个帖子上给出的代码是用BeginGetRequestStream,GetResponseStream来完成的

  1. using System;
  2. using System.Net;
  3. using System.IO;
  4. using System.Text;
  5. using System.Threading;
  6. class HttpWebRequestBeginGetRequest
  7. {
  8.     public static ManualResetEvent allDone = new ManualResetEvent(false);
  9.     public static void Main()
  10.     {
  11.         
  12.             // Create a new HttpWebRequest object.
  13.             HttpWebRequest request=(HttpWebRequest) WebRequest.Create("http://www.contoso.com/example.aspx");    
  14.     
  15.             // Set the ContentType property. 
  16.             request.ContentType="application/x-www-form-urlencoded";
  17.             // Set the Method property to 'POST' to post data to the URI.
  18.             request.Method = "POST";
  19.             // Start the asynchronous operation.    
  20.             request.BeginGetRequestStream(new AsyncCallback(ReadCallback), request);    
  21.             
  22.             // Keep the main thread from continuing while the asynchronous
  23.             // operation completes. A real world application
  24.             // could do something useful such as updating its user interface. 
  25.             allDone.WaitOne();
  26.             // Get the response.
  27.             HttpWebResponse response = (HttpWebResponse)request.GetResponse();
  28.             Stream streamResponse = response.GetResponseStream();
  29.             StreamReader streamRead = new StreamReader(streamResponse);
  30.             string responseString = streamRead.ReadToEnd();
  31.             Console.WriteLine(responseString);
  32.             // Close the stream object.
  33.             streamResponse.Close();
  34.             streamRead.Close();
  35.     
  36.             // Release the HttpWebResponse.
  37.             response.Close();
  38.         }
  39.     
  40.     private static void ReadCallback(IAsyncResult asynchronousResult)
  41.     {    
  42.             HttpWebRequest request = (HttpWebRequest)asynchronousResult.AsyncState;
  43.             // End the operation.
  44.             Stream postStream = request.EndGetRequestStream(asynchronousResult);
  45.             Console.WriteLine("Please enter the input data to be posted:");
  46.             string postData = Console.ReadLine ();
  47.             
  48.             // Convert the string into a byte array.
  49.             byte[] byteArray = Encoding.UTF8.GetBytes(postData);
  50.             // Write to the request stream.
  51.             postStream.Write(byteArray, 0, postData.Length);
  52.             postStream.Close ();
  53.             allDone.Set();    
  54.     }
  55. }
  56. ----------------------------------------------------------------------------------------------------------
  57.  
    人气:44  分类: C#编程 来源:互联网

    ----------------------------Un Test-------------------------
    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Text;
    using System.Net;
    using System.Threading;
    using System.IO;

    namespace WebReqHtml
    {
        internal class WebReqState
        {
            public byte[] Buffer;
            public MemoryStream ms;
            public const int BufferSize = 1024;
            public Stream OrginalStream;
            public HttpWebResponse WebResponse;
        
            public WebReqState()
            {
                Buffer = new byte[1024];
                ms = new MemoryStream();
            }
        }
        public delegate  void  ReadDataComplectedEventHandler(byte[] data);

        class HtmlFromWebReq
        {

            private Uri url;

            public event ReadDataComplectedEventHandler OnReadDataComplected;

            public HtmlFromWebReq(string absoluteUrl)
            {
                url = new Uri(absoluteUrl);
            }
            protected void readDataCallback(IAsyncResult ar)
            {

                WebReqState rs = ar.AsyncState as WebReqState;
                int read = rs.OrginalStream.EndRead(ar);
                if (read > 0)
                {
                    rs.ms.Write(rs.Buffer, 0, read);
                    rs.OrginalStream.BeginRead(rs.Buffer, 0, WebReqState.BufferSize, new AsyncCallback(readDataCallback), rs);
                }
                else

                {
                    rs.OrginalStream.Close();
                    rs.WebResponse.Close();
                    if (OnReadDataComplected != null)
                    {
                        OnReadDataComplected(rs.ms.ToArray());
                    }
                }

            }

            protected void responseCallback(IAsyncResult ar)
            {
                HttpWebRequest req = ar.AsyncState as HttpWebRequest;
                if (req == null)
                    return;
                HttpWebResponse response = req.EndGetResponse(ar) as HttpWebResponse;
                if (response.StatusCode!= HttpStatusCode.OK)
                {
                    response.Close();
                    return;
                }
                WebReqState st=new WebReqState();
                st.WebResponse = response;
                Stream repsonseStream = response.GetResponseStream();
                st.OrginalStream = repsonseStream;
                repsonseStream.BeginRead(st.Buffer, 0, WebReqState.BufferSize, new AsyncCallback(readDataCallback), st);

            }

            public void BeginCreateHtml()
            {
                HttpWebRequest req = HttpWebRequest.Create(url.AbsoluteUri) as HttpWebRequest;
                req.BeginGetResponse( new AsyncCallback(responseCallback), req);
                
            }
             
        }
    }

抱歉!评论已关闭.