File size: 1,722 Bytes
e26fba6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using HtmlAgilityPack;
using Fizzler.Systems.HtmlAgilityPack;
using System.Linq;
namespace Client.Scraper
{
public abstract class ScraperBase
{
#region Fields
#endregion
protected HtmlNode GotoUrl(string url)
{
//HtmlDocument doc = new HtmlDocument();
//using (var client = new WebClient())
//{
// //doc.Load(client.OpenRead(url));
// byte[] myDataBuffer = client.DownloadData(url);
// string download = Encoding.ASCII.GetString(myDataBuffer);
// doc.LoadHtml2(download);
// return doc.DocumentNode;
//}
return new HtmlWeb().Load(url).DocumentNode;
}
protected IEnumerable<HtmlNode> SelectItems(HtmlNode cq, string css)
{
return cq.QuerySelectorAll(css);
}
protected HtmlNode SelectItem(HtmlNode cq, string css)
{
return cq.QuerySelector(css);
}
protected string ReadAttribute(HtmlNode element, string attribute)
{
return element.Attributes[attribute] != null ? element.Attributes[attribute].Value : String.Empty;
}
protected string ReadText(HtmlNode element)
{
return element.InnerText;// WebUtility.HtmlDecode(StepReadText.Process(element).ToString());
}
public abstract string RootUrl { get;}
public abstract string ID { get; }
public abstract string Title { get; }
}
}
|