紧跟着上一节说的文章，虽然已经放出了所写的全代码，但还是再解释一下另外一个页面的请求和分析过程吧。
PS：又可以愉快的水一章了，咕嘿嘿。
页面分析
上回说到下载按钮的href属性是javascript:;伪协议，导致了新打开的页面链接携带#符号，但是我们通过了phantomjs已经解决了第一次跳转的问题。
下载页面
事实证明，这里更加狠，连个伪协议都没有。不过没关系，我们还是沿用上回的那个方法，使用phantomjs来渲染页面并且将跳转的页面链接以响应返回给我们的客户端请求。
实现
采用上一节所说的让Phantomjs作为服务端，然后去请求它，让它把要爬取的结果反馈给.net。注意，这里的返回给客户端的响应结果可以是网页页面，也可以是Phantomjs进行HTML解析完的真实数据。
.Net Core代码
public async Task<string> GetDownloadPageAsync(string url)
{
string result = string.Empty;
//请求phantomjs 获取下载页面
string dom = "Tappable-inactive animated fadeIn";
KeyValuePair<string, string> url2dom = new KeyValuePair<string, string>(url, dom);
var postData = JsonConvert.SerializeObject(url2dom);
CookieContainer cc = new CookieContainer();
HttpHelpers helper = new HttpHelpers();
HttpItems items = new HttpItems();
HttpResults hr = new HttpResults();
items.Url = this.PostUrl1;
items.Method = "POST";
items.Container = cc;
items.Postdata = postData;
items.Timeout = 100000;
hr = await helper.GetHtmlAsync(items);
var downloadPageUrl = hr.Html;
Console.WriteLine($"first => { downloadPageUrl }");
if(downloadPageUrl.Contains("http"))
{
//获取百度云下载地址和分享密码
//string code1 = "1";
dom = "Tappable-inactive btn btn-success btn-block"; // 下载链接
url2dom = new KeyValuePair<string, string>(downloadPageUrl, dom);
postData = JsonConvert.SerializeObject(url2dom);
items = new HttpItems
{
Url = this.PostUrl2
};
items.Method = "POST";
items.Container = cc;
items.Postdata = postData;
items.Timeout = 1000000;
hr = await helper.GetHtmlAsync(items);
result = hr.Html; //返回json数据
Console.WriteLine($"second => { result }");
}
else
{
result = downloadPageUrl; //输出错误信息
}
return result;
}
这里包含了第一次在详情页获取下载页的那个请求，以及下载页面获取百度云链接和分享密码的请求。
JavaScript代码
"use strict";
var port = 8089;
var server = require('webserver').create();
server.listen(8089, function (request, response) {
//传入的参数有待更改，目前为
//{"Key":"https://acg12.com/download/#60e21d8417ab60fbfJfcqnT1BC8Qd20PehAIKv3J4ZO%2FJCo0htE9hP5IFZU",
//"Value":"Tappable-inactive btn btn-success btn-block"}的json字符窜
//第一个参数为经过第一次请求所返回的下载页面，第二个为下载按钮的Dom
var data = JSON.parse(request.postRaw);
var url = data.Key.toString();
console.log(url);
var dom = data.Value.toString();
console.log(dom);
var code = 200;
var pwdArray = new Array();
var result = new Array();
var page = require('webpage').create();
page.onInitialized = function() {
page.customHeaders = {};
};
page.settings.loadImages = false;
page.customHeaders = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36",
"Referer": url
};
response.headers = {
'Cache': 'no-cache',
'Content-Type': 'text/plain',
'Connection': 'Keep-Alive',
'Keep-Alive': 'timeout=40, max=100'
};
//根据Phantomjs的官网，这个回调在打开新标签页会触发
page.onPageCreated = function(newPage) {
//console.log('A new child page was created! Its requested URL is not yet available, though.');
page.onInitialized = function() {
newPage.customHeaders = {};
};
newPage.settings.loadImages = false;
newPage.customHeaders = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36"
};
//newPage.viewportSize = { width: 1920, height: 1080 };
//当百度云页面打开并渲染完成时触发
newPage.onLoadFinished = function(status) {
//console.log('A child page is Loaded: ' + newPage.url);
//newPage.render('newPage.png', {format: 'png', quality: '100'});
//console.log(pwdArray.length);
if(pwdArray.length > 0){
//console.log("enter");
//从数组中pop出密码，当无密码时pop的数据为null字符窜
var temp = {"url": newPage.url.toString(), "password": pwdArray.pop().toString()};
console.log(JSON.stringify(temp));
result.push(temp); // 将json数据push进返回结果
}
};
};
page.open(url, function (status) {
console.log("----" + status);
if (status !== 'success') {
code = 400;
response.write('4XX');
response.statusCode = code;
response.close();
} else {
code = 200;
window.setTimeout(function (){
//var dom = dom;
pwdArray = page.evaluate(function(dom) {
console.log(dom);
var pwdArray = new Array();
var btnList = document.getElementsByClassName(dom); // 百度云链接
for(var i = 0; i < btnList.length;i ++ ){
//猜测所有下载节点都有密码
var temp = document.getElementById("downloadPwd-" + i);
if(temp != undefined){
//console.log("****" + temp.value);
pwdArray.push(temp.value); // 有密码push进数组
}else{
//console.log("****null");
pwdArray.push("null"); // 无密码则push进null字符窜，这样做到和url的一一对应
}
}
for(var i = 0; i < btnList.length;i ++ ){
//console.log("click");
btnList[i].click(); // 点击下载，打开新标签页
}
return pwdArray;
}, dom);
}, 6000);
}
});
//设置等待20秒后才发送客户端的响应结果，这样保证上述方法都能成功运行结束
window.setTimeout(function(){
var rs = JSON.stringify(result)
console.log(rs);
response.write(rs);
response.statusCode = code;
response.close();
}, 20000);
page.onConsoleMessage = function(msg, lineNum, sourceId) {
console.log("$$$$$" + msg);
};
page.onError = function(msg, trace) {
var msgStack = ['PHANTOM ERROR: ' + msg];
if (trace && trace.length) {
msgStack.push('TRACE:');
trace.forEach(function(t) {
msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line + (t.function ? ' (in function ' + t.function +')' : ''));
});
}
console.log(msgStack.join('\n'));
phantom.exit(1);
};
});
phantom.onError = function(msg, trace) {
var msgStack = ['PHANTOM ERROR: ' + msg];
if (trace && trace.length) {
msgStack.push('TRACE:');
trace.forEach(function(t) {
msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line + (t.function ? ' (in function ' + t.function +')' : ''));
});
}
console.log(msgStack.join('\n'));
phantom.exit(1);
};
完整的源代码已经放在Github上了，里面有写好的bat文件，直接运行run.bat就行。当然前提，第一节的那些环境都配置完成了。大家下周见，下周可能可以尝试用用DotnetSpider，这是借鉴了WebMagic写的.net core地爬虫框架，有兴趣的可以先去尝试一下玩玩。
谢谢~~
网友评论