提取网页中的链接并生成xml代码,C#代码库,德仔网

代码语言
代码分类
【C#】提取网页中的链接并生成xml
作者:欣雨 / 发布于2016/7/27/ 660
using System;  
using System.Xml;  
using System.Text;  
using System.Net;  
using System.IO;  
using System.Collections;  
using System.Text.RegularExpressions;  
  
public class App  
{  
    public static void Main()  
    {  
        string strCode;  
        ArrayList alLinks;   
        Console.Write("请输入一个网页地址：");  
        string strURL = Console.ReadLine();  
        if (strURL.Substring(0, 7) != @"http://")  
        {  
            strURL = @"http://" + strURL;  
        }   
        Console.WriteLine("正在获取页面代码，请稍侯...");  
        strCode = GetPageSource(strURL);  
  
        Console.WriteLine("正在提取超链接，请稍侯...");  
        alLinks = GetHyperLinks(strCode);  
  
        Console.WriteLine("正在写入文件，请稍侯...");  
        WriteToXml(strURL, alLinks);  
    }  
  
    // 获取指定网页的HTML代码   
    static string GetPageSource(string URL)  
    {  
        Uri uri = new Uri(URL);   
        HttpWebRequest hwReq = (HttpWebRequest)WebRequest.Create(uri);  
        HttpWebResponse hwRes = (HttpWebResponse)hwReq.GetResponse();   
        hwReq.Method = "Get";   
        hwReq.KeepAlive = false;   
        StreamReader reader = new StreamReader(hwRes.GetResponseStream(), System.Text.Encoding.GetEncoding("GB2312"));   
        return reader.ReadToEnd();  
    }  
  
    // 提取HTML代码中的网址   
    static ArrayList GetHyperLinks(string htmlCode)  
    {  
        ArrayList al = new ArrayList();   
        string strRegex = @"http://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?";   
        Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);  
        MatchCollection m = r.Matches(htmlCode);   
        for (int i = 0; i <= m.Count - 1; i++)  
        {  
            bool rep = false;  
            string strNew = m[i].ToString();  
  
            // 过滤重复的URL   
            foreach (string str in al)  
            {  
                if (strNew == str)  
                {  
                    rep = true;  
                    break;  
                }  
            }   
            if (!rep) al.Add(strNew);  
        }   
        al.Sort();   
        return al;  
    }  
  
    // 把网址写入xml文件   
    static void WriteToXml(string strURL, ArrayList alHyperLinks)  
    {  
        XmlTextWriter writer = new XmlTextWriter("HyperLinks.xml", Encoding.UTF8);   
        writer.Formatting = Formatting.Indented;  
        writer.WriteStartDocument(false);  
        writer.WriteDocType("HyperLinks", null, "urls.dtd", null);  
        writer.WriteComment("提取自" + strURL + "的超链接");  
        writer.WriteStartElement("HyperLinks");  
        writer.WriteStartElement("HyperLinks", null);  
        writer.WriteAttributeString("DateTime", DateTime.Now.ToString());   
        foreach (string str in alHyperLinks)  
        {  
            string title = GetDomain(str);  
            string body = str;  
            writer.WriteElementString(title, null, body);  
        }   
        writer.WriteEndElement();  
        writer.WriteEndElement();   
        writer.Flush();  
        writer.Close();  
    }  
  
    // 获取网址的域名后缀   
    static string GetDomain(string strURL)  
    {  
        string retVal;   
        string strRegex = @"(\.com/|\.net/|\.cn/|\.org/|\.gov/)";   
        Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);  
        Match m = r.Match(strURL);  
        retVal = m.ToString();   
        strRegex = @"\.|/{1}quot;;  
        retVal = Regex.Replace(retVal, strRegex, "").ToString();   
        if (retVal == "")  
            retVal = "other";   
        return retVal;  
    }  
}
试试其它关键字
　生成xml　　链接　
同语言下
可能有用的
欣雨贡献的其它代码(10)