网络爬虫主要功能就是对网页内容进行爬取，然后根据特定需求对内容进行过滤分析。
针对网页内容爬取，假设需求为要对一个网站进行全站爬取，将爬取的文件按类型保存在本地磁盘，并提供配置网站爬取的最大层次、最大链接数、爬取类型范围等。
这里使用kafka主题做爬虫队列，使用springboot做了一个简单的实现。
任务创建接口
这里提供了两个接口，一个是输入网站爬取配置创建爬取任务，一个是根据任务id查询任务状态，没有提供详细结果查询，结果可直接数据库查看。
@RestController
public class CrawlerTaskController {
@Autowired
private WebsiteTaskService websiteTaskService;
@Autowired
private WebsiteTaskDao websiteTaskDao;
@Autowired
private TaskProducer taskProducer;
@PostMapping("task/add")
@ResponseBody
public Map<String, Object> addWebsiteTask(WebsiteTask item) {
item.setTaskCount(1);
websiteTaskService.put(item);
UrlTask task = new UrlTask();
task.setUrl(item.getUrl());
task.setParentId(-1);
task.setRootId(task.getRootId());
task.setLevel(0);
taskProducer.sendUrlTask(task);
Map<String, Object> map = new HashMap<>();
map.put("id", item.getId());
map.put("message", "爬虫任务添加成功！");
return map;
}
@PostMapping("task/get")
@ResponseBody
public WebsiteTask getWebsiteTask(int id) {
return websiteTaskDao.findById(id).get();
}
}
网站任务实体
@Entity
@EntityListeners(AuditingEntityListener.class)
public class WebsiteTask {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private int id;
@Column(length = 1024)
private String url;//网站url，一般为首页链接
private int maxLevel;//最大爬取层次
private int maxCount;//最大爬取链接数
private int outerLevel;//最大爬取外链层次
private String range;//爬取类型范围
private int taskCount;//任务数
private int finishCount;//爬取完成任务数
private int state = 1;//状态：1=执行中;2=已完成
@CreatedDate
private Date createTime;//创建时间
private Date finishTime;//完成时间
@Transient
private List<String> ranges;
......
}
url任务实体
@Entity
@EntityListeners(AuditingEntityListener.class)
public class UrlTask {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private int id;
private int parentId;//父页面任务id
private int rootId;//网站任务id
@Column(length = 1024)
private String url;
private String contentType;//网页类型
private long contentLength;//内容长度
private int level;//当前层次
private long useMillis;//爬取用时
private int respCode;//响应状态码
private String remark;//备注
private String filePath;//保存磁盘文件路径
@CreatedDate
private Date createTime;//创建时间
......
}
爬取的网页模型定义
public class WebPageModel {
public int respCode = 200;//响应状态码
public String message;//异常信息
public Document document;//html页面Document对象
public String encoding;//页面编码
public String contentType;//网页类型
public long contentLength;//内容长度
public String filePath;//文件路径
public String fileExt;//文件后缀
public PageFormat format = PageFormat.OTHER;
public enum PageFormat {
HTML, IMAGE, AUDIO, VIDEO, TXT, WORD, EXCEL, PPT, PDF, COMPRESS, APK, IPA, OTHER
}
public void updateFormat() {
String type = contentType;
if (ContentTypeUtil.OCTET_STREAM_TYPE.equalsIgnoreCase(contentType)) {
type = ContentTypeUtil.getContentType(fileExt);
}
if (ContentTypeUtil.isHtml(type)) {
format = PageFormat.HTML;
} else if (ContentTypeUtil.isImage(type)) {
format = PageFormat.IMAGE;
} else if (ContentTypeUtil.isAudio(type)) {
format = PageFormat.AUDIO;
} else if (ContentTypeUtil.isVideo(type)) {
format = PageFormat.VIDEO;
} else if (ContentTypeUtil.isTxt(type)) {
format = PageFormat.TXT;
} else if (ContentTypeUtil.isWord(type)) {
format = PageFormat.WORD;
} else if (ContentTypeUtil.isExcel(type)) {
format = PageFormat.EXCEL;
} else if (ContentTypeUtil.isPpt(type)) {
format = PageFormat.PPT;
} else if (ContentTypeUtil.isPdf(type)) {
format = PageFormat.PDF;
} else if (ContentTypeUtil.isCompress(type)) {
format = PageFormat.COMPRESS;
} else if (ContentTypeUtil.isApk(type)) {
format = PageFormat.APK;
} else if (ContentTypeUtil.isIpa(type)) {
format = PageFormat.IPA;
}
}
}
url去重
public class UrlDuplicateFilter {
private final Object lock = new Object();
private final Set<String> set = new HashSet<>();
private final int maxCount; // 最大不相同数量
public UrlDuplicateFilter(int maxCount) {
this.maxCount = maxCount;
}
/**
* 过滤重复url
*/
public boolean filter(String url) {
if (StringUtils.isBlank(url)) {
return false;
}
synchronized (lock) {
if (reachMaxCount() || set.contains(url)) {
return false;
}
set.add(url);
}
return true;
}
/**
* 判断数量是否达到上限
*/
private boolean reachMaxCount() {
return set.size() >= maxCount;
}
}
一条url的爬取执行过程
public class CrawlerTask implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(CrawlerTask.class);
private UrlTask task;
private WebsiteTaskService websiteTaskService;
private UrlTaskDao urlTaskDao;
private TaskProducer taskProducer;
public CrawlerTask(UrlTask task, WebsiteTaskService websiteTaskService, UrlTaskDao urlTaskDao,
TaskProducer taskProducer) {
this.task = task;
this.websiteTaskService = websiteTaskService;
this.urlTaskDao = urlTaskDao;
this.taskProducer = taskProducer;
}
@Override
public void run() {
long millis = System.currentTimeMillis();
WebsiteTask website = websiteTaskService.getWebsiteTask(task.getRootId());
//爬取链接内容
WebPageModel page = PageDownloadUtil.executeGet(task.getUrl(), website.getRanges());
task.setContentLength(page.contentLength);
task.setContentType(page.contentType);
task.setRespCode(page.respCode);
task.setRemark(page.message);
task.setFilePath(page.filePath);
task.setUseMillis(System.currentTimeMillis() - millis);
urlTaskDao.saveAndFlush(task);
if (task.getLevel() < website.getMaxLevel()) {
Set<String> childUrls = new UrlExtract(page.document, task.getUrl()).extractFromA().extractFromFrame()
.extractFromIframe().extractFromImg().getUrls();
if (!childUrls.isEmpty()) {
UrlDuplicateFilter dupFilter = websiteTaskService.getUrlDuplicateFilter(task.getRootId());
int addCount = 0;
for (String childUrl : childUrls) {
if (CrawlerUtil.isOuterUrl(task.getUrl(), childUrl) && task.getLevel() >= website.getOuterLevel()) {
continue;
}
//提取出的子链接去重
if (dupFilter.filter(childUrl)) {
UrlTask childTask = new UrlTask();
childTask.setUrl(childUrl);
childTask.setParentId(task.getId());
childTask.setRootId(task.getRootId());
childTask.setLevel(task.getLevel() + 1);
taskProducer.sendUrlTask(childTask);
addCount++;
}
}
//任务数更新
websiteTaskService.addTaskCount(task.getRootId(), addCount);
}
}
//完成任务数更新
websiteTaskService.addFinishCount(task.getRootId());
LOG.info(String.format("爬取用时=%s，url=%s", System.currentTimeMillis() - millis, task.getUrl()));
}
}
网页爬取工具
public class PageDownloadUtil {
private static final Logger LOG = LoggerFactory.getLogger(PageDownloadUtil.class);
private static final int MAX_HTML_LENGTH = 20 * 1024 * 1024;//html页面限制20M
private static final int MAX_FILE_LENGTH = 500 * 1024 * 1024;//其它附件类型限制500M
private static final String FOLDER_NAME = "d:/temp/" + UUID.randomUUID().toString().replace("-", "") + "/";
private static final AtomicInteger INDEX = new AtomicInteger();
private static final CloseableHttpClient client = HttpClientUtil.createHttpClient();
//自定义错误返回值
private static final Map<Integer,String> CODE_MAP = new HashMap<>();
static{
CODE_MAP.put(-501,"uri解析异常");
CODE_MAP.put(-502,"网络协议异常");
CODE_MAP.put(-503,"域名解析异常");
CODE_MAP.put(-504,"http连接异常");
CODE_MAP.put(-505,"网络IO异常");
CODE_MAP.put(-506,"页面解析异常");
CODE_MAP.put(-507,"编码格式异常");
CODE_MAP.put(-508,"内容长度超出限制");
CODE_MAP.put(-509,"网页类型超出可爬取范围");
}
public static WebPageModel executeGet(String url, List<String> ranges) {
WebPageModel page = new WebPageModel();
int redirectTimes = 0;
boolean redirect;
URI uri = CrawlerUtil.urlConvertToUri(url);
if (uri == null) {
page.respCode = -501;
page.message = CODE_MAP.get(page.respCode);
return page;
}
do {
redirectTimes++;
redirect = false;
HttpGet method = new HttpGet(uri);
HttpClientUtil.setHeader(method, url);
CloseableHttpResponse response = null;
long millis = System.currentTimeMillis();
try {
response = client.execute(method);
page.respCode = response.getStatusLine().getStatusCode();
if (page.respCode == HttpStatus.SC_OK) {
download(page, url, response, ranges);
} else if (page.respCode >= 300 && page.respCode < 400) {// 页面跳转
Header[] locationHeader = response.getHeaders("location");
if (locationHeader != null && locationHeader.length > 0) {
String redirectUrl = locationHeader[0].getValue();
if (StringUtils.isNotBlank(redirectUrl) && !url.equals(redirectUrl)) {
uri = CrawlerUtil.urlConvertToUri(redirectUrl);
redirect = true;
}
}
}
} catch (ClientProtocolException e) {
LOG.error("", e);
page.respCode = -502;
page.message = CODE_MAP.get(page.respCode);
} catch (UnknownHostException e) {
LOG.error("", e);
page.respCode = -503;
page.message = CODE_MAP.get(page.respCode);
} catch (HttpHostConnectException e) {
LOG.error("", e);
page.respCode = -504;
page.message = CODE_MAP.get(page.respCode);
} catch (IOException e) {//连接超时尝试重连3次
redirectTimes++;
redirect = true;
LOG.error(String.format("第%s次链接失败，executeusetime=%s", redirectTimes / 2,
System.currentTimeMillis() - millis), e);
page.respCode = -505;
page.message = CODE_MAP.get(page.respCode);
} finally {
if (response != null) {
EntityUtils.consumeQuietly(response.getEntity());
try {
response.close();
} catch (IOException e) {
LOG.error("responseclose", e);
}
}
method.releaseConnection();
}
} while (redirect && redirectTimes <= 5);
return page;
}
private static void download(WebPageModel page, String url, CloseableHttpResponse response,
List<String> ranges) {
HttpEntity entity = response.getEntity();
page.contentLength = entity.getContentLength();// 此方法不准确，经常返回-1，后面重新赋值
// ContentType.getOrDefault(entity).getMimeType()提取可能会因为非支持的charset类型而报错，所以这里改为手工提取mimeType
Header header = entity.getContentType();
if (header != null) {
HeaderElement[] headerElements = header.getElements();
if (headerElements != null && headerElements.length > 0) {
page.contentType = headerElements[0].getName();
}
}
if (ContentTypeUtil.OCTET_STREAM_TYPE.equalsIgnoreCase(page.contentType)) {
page.fileExt = HttpClientUtil.getOctetStreamFileExt(url, response);
} else if (page.contentType == null) {
//若未从header中取到contentType，根据url后缀判断
if (url.lastIndexOf("/") > 8) {
String name = url.substring(url.lastIndexOf("/"));
if (name.contains(".")) {
page.contentType = ContentTypeUtil.getContentType(name.substring(name.lastIndexOf(".")));
}
}
}
page.updateFormat();
if (ranges.contains(page.format.toString())) {
if (page.format == WebPageModel.PageFormat.HTML) {
if (page.contentLength == 0 || page.contentLength > MAX_HTML_LENGTH) {
page.respCode = -508;
page.message = CODE_MAP.get(page.respCode);
return;
}
try {
String html = null;
Document document = null;
String charset = null;
if (header != null) {
charset = CrawlerUtil.judgeCharset(header.toString());
}
if (charset != null) {
html = EntityUtils.toString(entity, charset);
document = Jsoup.parse(html);
} else {
byte[] data = EntityUtils.toByteArray(entity);
html = new String(data, CrawlerUtil.UTF_8);
document = Jsoup.parse(html);
charset = CrawlerUtil.getCharsetFromMeta(document);
if (charset != null && !CrawlerUtil.UTF_8.equals(charset)) {
html = new String(data, charset);
document = Jsoup.parse(html);
}
}
byte[] data = html.getBytes(CrawlerUtil.UTF_8);
page.contentLength = data.length;
if (page.contentLength <= 0 || page.contentLength > MAX_HTML_LENGTH) {
page.respCode = -508;
page.message = CODE_MAP.get(page.respCode);
return;
}
page.encoding = CrawlerUtil.UTF_8;
page.document = document;
createFilePath(page);
HttpClientUtil.exportDataAsFile(data, page.filePath);
} catch (ParseException e) {
LOG.error("", e);
page.respCode = -506;
page.message = CODE_MAP.get(page.respCode);
} catch (UnsupportedEncodingException e) {
LOG.error("", e);
page.respCode = -507;
page.message = CODE_MAP.get(page.respCode);
} catch (IOException e) {
LOG.error("", e);
page.respCode = -505;
page.message = CODE_MAP.get(page.respCode);
}
} else {// 如果是非html页面直接下载
if (page.contentLength == 0 || page.contentLength > MAX_FILE_LENGTH) {
page.respCode = -508;
page.message = CODE_MAP.get(page.respCode);
return;
}
if (page.fileExt == null) {
page.fileExt = ContentTypeUtil.getExtendFileName(page.contentType);
}
createFilePath(page);
HttpClientUtil.exportEntityAsFile(entity, page.filePath);
}
} else {
page.respCode = -509;
page.message = CODE_MAP.get(page.respCode);
}
}
private static void createFilePath(WebPageModel page) {
String filePath = FOLDER_NAME + page.format.toString() + "/";
File file = new File(filePath);
file.mkdirs();
page.filePath = filePath + INDEX.getAndIncrement() + page.fileExt;
}
}
项目地址
详细项目代码可到我的Github上查看下载：https://github.com/DexterQY/website-crawler
网友评论