node 爬虫案例

https://www.cnblogs.com/xiaxuexiaoab/p/7124956.html

存数据库

save.js

var mysql = require('mysql');
var async = require('async');

var pool = mysql.createPool({
   host : 'localhost',
   user : 'root',
   password : 'root',
   database : 'lovelydong'
});

exports.videoSave = function(list,callback){
   
   pool.getConnection(function(err,connection){
       if(err){
           return callback(err);
       }
     var findsql = 'select * from sx_list where id=?';
     var updatesql = 'update sx_list set title=?,url=?,time=?,where id=?';
       var insertsql = 'insert into sx_list(title,url,time) values(?,?,?)';
       async.eachSeries(list,function(item,next){
       connection.query(findsql,[item.id],function(err,result){
           if(err){
               return next(err);
           }
           if(result.length>=1){
               connection.query(updatesql,[item.title,item.url,item.time],next);
           }else{
               connection.query(insertsql,[item.title,item.url,item.time],next);
           }
       });         
       },callback);
       connection.release();
   });
};

爬取数据

router.js

router.get("/sxjy",function(req,res){
    //var aId = req.query.aId;
    var detail = {};
    var _html;
    detail.lists = [];
    console.log('http://www.shanxiangjiaoyu.com/huodong');
    http.get("http://www.shanxiangjiaoyu.com/huodong",function(response){
        response.on("data",function(chunk){
            _html += chunk;
        });
        response.on("end",function(){
            console.log("爬取结束");
            $ = cheerio.load(_html,{decodeEntities: false});
            detail.title = $(".normal_top").text();
            /*detail.author = $(".aticle_bot>span:nth-last-child(1) i").text();
            detail.article = $(".aticle_mas").html();*/
            
            $(".hdt_r").each(function(){
                detail.lists.push({"title":$(this).find(".hdt_title").text(),"url":$(this).find("a").attr("href"),"time":$(this).find(".hdt_time").text()});
            })
            
            res.charset = 'utf-8';
            save.videoSave(detail.lists);
            res.send({"detail":detail})
        });
    }).on("error",function(err){
        console.log(err)
    })
});