学步园

免积分源码地址

      个人比较喜欢小旭和月之门出品的游戏音乐。发现月之门不知道什么时候换了个漂亮的网站。而且也提供了音乐在线试听的页面。如图所示。

地址是  http://www.cyberweaver.net/demo.asp

查看了一下网页源代码。发现所有的音乐地址都存在于百度网盘中。于是整了一个爬虫将地址都抓取出来了。

因为很简单。没什么多解释的。就直接上代码。是C#的。

using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Xml;

namespace GetUrl
{
    public partial class Form1 : Form
    {

        string strCode;
        ArrayList alLinks;

        public Form1()
        {
            InitializeComponent();
        }


        private void button1_Click(object sender, EventArgs e)
        {
            if (textBox1.Text == "")
            {
                MessageBox.Show("请输入网址");
                return;
            }
            string strURL = textBox1.Text.ToString().Trim();
            if (strURL.Substring(0, 7) != @"http://")
            {
                strURL = @"http://" + strURL;
            }
            MessageBox.Show("正在获取页面代码，请稍后...");
            strCode = GetPageSource(strURL);
            MessageBox.Show("正在提取超链接，请稍侯...");
            alLinks = GetHyperLinks(strCode);
            MessageBox.Show("正在写入文件，请稍侯...");
            WriteToXml(strURL, alLinks);
        }

        // 获取指定网页的HTML代码 
        public static string GetPageSource(string URL)
        {
            Uri uri = new Uri(URL);

            HttpWebRequest hwReq = (HttpWebRequest)WebRequest.Create(uri);
            HttpWebResponse hwRes = (HttpWebResponse)hwReq.GetResponse();

            hwReq.Method = "Get";

            hwReq.KeepAlive = false;

            StreamReader reader = new StreamReader(hwRes.GetResponseStream(), System.Text.Encoding.GetEncoding("GB2312"));

            return reader.ReadToEnd();
        }
        // 提取HTML代码中的网址 
        public static ArrayList GetHyperLinks(string htmlCode)
        {
            ArrayList al = new ArrayList();            
            string strRegex = @"demo_class.asp?sort=([^a-zA-Z0-9])+";
            strRegex = @"demo_class.asp\?sort=([^x00-xff]{4})&id=606";
            strRegex = @"demo_class.asp\?sort=([^x00-xff]{4})&id\=([a-z0-9]+)";

            Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
            MatchCollection m = r.Matches(htmlCode);

            for (int i = 0; i <= m.Count - 1; i++)
            {
                bool rep = false;
                string strNew = m[i].ToString();

                // 过滤重复的URL 
                foreach (string str in al)
                {
                    if (strNew == str)
                    {
                        rep = true;
                        break;
                    }
                }

                if (!rep) al.Add(strNew);
            }

            al.Sort();

            return al;
        }
      
     // 提取HTML代码中的网址 
        public static string GetHyperLinks_Song(string htmlCode)
        {
            //http://pan.baidu.com/share/link?shareid=112744&uk=455690558
            string strRegex;
            strRegex = @"http://pan.baidu.com/share/link\?shareid=([a-z0-9]+)&uk=([a-z0-9]+)";

            Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
            MatchCollection m = r.Matches(htmlCode);

            for (int i = 0; i <= m.Count - 1; i++)
            {
                bool rep = false;
                string strNew = m[i].ToString();   
                return strNew;
            }
            return null;
        }

        // 把网址写入xml文件 
        static void WriteToXml(string strURL, ArrayList alHyperLinks)
        {
            XmlTextWriter writer = new XmlTextWriter("HyperLinks.xml", Encoding.UTF8);

            writer.Formatting = Formatting.Indented;
            writer.WriteStartDocument(false);
            writer.WriteDocType("HyperLinks", null, "urls.dtd", null);
            writer.WriteComment("提取自" + strURL + "的超链接");
            writer.WriteStartElement("HyperLinks");
            writer.WriteStartElement("HyperLinks", null);
            writer.WriteAttributeString("DateTime", DateTime.Now.ToString());


            foreach (string str in alHyperLinks)
            {
                string title = GetDomain(str);
                string body = str;
                writer.WriteElementString(title, null, body);
            }

            writer.WriteEndElement();
            writer.WriteEndElement();

            writer.Flush();
            writer.Close();
        }

        static void WriteToTxt(string strURL)
        {
            string strAddrTxt = "月之门游戏音乐地址.txt";
            if (File.Exists(strAddrTxt))
            {
                StreamWriter sw = new StreamWriter(strAddrTxt, true, Encoding.Default);
                //该编码类型不会改变已有文件的编码类型
                sw.WriteLine(strURL);
                sw.Close();
                return ;
            }
            else
            {
                try
                {
                    FileStream fs = new FileStream(strAddrTxt, FileMode.CreateNew);
                    fs.Close();
                    StreamWriter sw = new StreamWriter(strAddrTxt, true, Encoding.Default);
                    //该编码类型不会改变已有文件的编码类型
                    sw.WriteLine(strURL);
                    sw.Close();
                    return ;
                }
                catch (Exception e)
                {
                    MessageBox.Show(e.Message.ToString());
                    return ;
                }
            }
        }

        // 获取网址的域名后缀 
        static string GetDomain(string strURL)
        {
            string retVal;

            string strRegex = @"(\.com/|\.net/|\.cn/|\.org/|\.gov/)";

            Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
            Match m = r.Match(strURL);
            retVal = m.ToString();

            strRegex = @"\.|/$";
            retVal = Regex.Replace(retVal, strRegex, "").ToString();

            if (retVal == "")
                retVal = "other";

            return retVal;
        }

        //直接分析超链接
        private void button1_Click_1(object sender, EventArgs e)
        {
            string strContent = null;
            ArrayList arry_Link2_DemoPage;

            //获取网页
            if (textBox1.Text == "")
            {
                MessageBox.Show("请输入网址");
                return;
            }
            string strURL = textBox1.Text.ToString().Trim();
            if (strURL.Substring(0, 7) != @"http://")
            {
                strURL = @"http://" + strURL;
            }
            MessageBox.Show("正在获取页面代码，请稍后...");
            strContent = GetPageSource(strURL);

            //分析 http://www.cyberweaver.net/demo_class.asp 的超链接
            arry_Link2_DemoPage = GetHyperLinks(strContent);


            //下载 Demo 分页内容并分析其中的 http://pan.baidu.com/share/link?shareid=112744&uk=455690558 这种超链接
            for (int i = 0; i < arry_Link2_DemoPage.Count;i++ )
            {
                string strDemoContent = null;
                string strUrlDemo = "http://www.cyberweaver.net/" + arry_Link2_DemoPage[i];
                strDemoContent = GetPageSource(strUrlDemo);
                string strSongAddr = GetHyperLinks_Song(strDemoContent);

                //这里获取到了文件地址写入xml
                WriteToTxt(strSongAddr);
                
            }

        }




        //下载网页到本地
        private void button2_Click(object sender, EventArgs e)
        {
            if (textBox1.Text == "")
            {
                MessageBox.Show("请输入网址");
                return;
            }

            SaveFileDialog sfg = new SaveFileDialog();
            sfg.Filter = "网页文件(*.html)|*.html;*.htm";
            if (sfg.ShowDialog() == DialogResult.OK)
            {
                string strURL = textBox1.Text.ToString().Trim();
                if (strURL.Substring(0, 7) != @"http://")
                {
                    strURL = @"http://" + strURL;
                }

                //这里也可以填入一张图片的地址然后下载该图片
                WebRequest request = WebRequest.Create(strURL);
                WebResponse response = request.GetResponse();
                Stream reader = response.GetResponseStream();
                FileStream writer = new FileStream(sfg.FileName, FileMode.OpenOrCreate, FileAccess.Write);
                byte[] buff = new byte[512];
                int c = 0; //实际读取的字节数
                while ((c = reader.Read(buff, 0, buff.Length)) > 0)
                {
                    writer.Write(buff, 0, c);
                }
                writer.Close();
                writer.Dispose();
                reader.Close();
                reader.Dispose();
                response.Close();
            }
        }




        //分析地址 http://www.cyberweaver.net/demo.asp  中的超链接
        private void button3_Click(object sender, EventArgs e)
        {

        }







        //测试正则表达式
        private void button3_Click_1(object sender, EventArgs e)
        {
            string strContent = null;
            try
            {
                using (StreamReader sr = new StreamReader("月之门在线试听网页.html", Encoding.GetEncoding("gb2312")))
                {                   
                    String line = null;
                    while ((line = sr.ReadLine()) != null)
                    {
                        strContent += line.ToString();
                    }
                }
            }
            catch (Exception e2)
            {
                Console.WriteLine(e2.Message);
            }



            ArrayList al = new ArrayList();

            //string strRegex = @"http://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?";
            //<a href="demo_class.asp?sort=北京&id=605">
            //先将 "" 替换为 空
            strContent = Regex.Replace(strContent, "\"", "");
            

            string strRegex = @"demo_class.asp?sort=([^a-zA-Z0-9])+";
            strRegex = @"demo_class.asp\?sort=([^x00-xff]{4})&id=606";
            strRegex = @"demo_class.asp\?sort=([^x00-xff]{4})&id\=([a-z0-9]+)";
           
            Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
            MatchCollection m = r.Matches(strContent);

            for (int i = 0; i <= m.Count - 1; i++)
            {
                bool rep = false;
                string strNew = m[i].ToString();

                // 过滤重复的URL 
                foreach (string str in al)
                {
                    if (strNew == str)
                    {
                        rep = true;
                        break;
                    }
                }

                if (!rep) al.Add(strNew);
            }

            al.Sort();
        }

    }
}

最终得到的在线试听地址如下

witch 2013-1-9