CrawlerHandler.cs
3.11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Web;
/// <summary>
/// Crawler 的摘要说明
/// </summary>
public class CrawlerHandler : Handler
{
private string[] Sources;
private Crawler[] Crawlers;
public CrawlerHandler(HttpContext context) : base(context) { }
public override void Process()
{
Sources = Request.Form.GetValues("source[]");
if (Sources == null || Sources.Length == 0)
{
WriteJson(new
{
state = "参数错误:没有指定抓取源"
});
return;
}
Crawlers = Sources.Select(x => new Crawler(x, Server).Fetch()).ToArray();
WriteJson(new
{
state = "SUCCESS",
list = Crawlers.Select(x => new
{
state = x.State,
source = x.SourceUrl,
url = x.ServerUrl
})
});
}
}
public class Crawler
{
public string SourceUrl { get; set; }
public string ServerUrl { get; set; }
public string State { get; set; }
private HttpServerUtility Server { get; set; }
public Crawler(string sourceUrl, HttpServerUtility server)
{
this.SourceUrl = sourceUrl;
this.Server = server;
}
public Crawler Fetch()
{
var request = HttpWebRequest.Create(this.SourceUrl) as HttpWebRequest;
using (var response = request.GetResponse() as HttpWebResponse)
{
if (response.StatusCode != HttpStatusCode.OK)
{
State = "Url returns " + response.StatusCode + ", " + response.StatusDescription;
return this;
}
if (response.ContentType.IndexOf("image") == -1)
{
State = "Url is not an image";
return this;
}
ServerUrl = PathFormatter.Format(Path.GetFileName(this.SourceUrl), Config.GetString("catcherPathFormat"));
var savePath = Server.MapPath(ServerUrl);
if (!Directory.Exists(Path.GetDirectoryName(savePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(savePath));
}
try
{
var stream = response.GetResponseStream();
var reader = new BinaryReader(stream);
byte[] bytes;
using (var ms = new MemoryStream())
{
byte[] buffer = new byte[4096];
int count;
while ((count = reader.Read(buffer, 0, buffer.Length)) != 0)
{
ms.Write(buffer, 0, count);
}
bytes = ms.ToArray();
}
File.WriteAllBytes(savePath, bytes);
State = "SUCCESS";
}
catch (Exception e)
{
State = "抓取错误:" + e.Message;
}
return this;
}
}
}