爬虫

爬虫

方法一

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Iterator;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Test {

    public static void main(String[] args) throws IOException {
        Document document = Jsoup.connect("https://www.doutula.com/article/list/?page=8").get();
        //获取图片链接 逗号是同一级元素
        Elements els = document.select(".lazy,.image_dtb,.img-responsive");//css选择器 id class tagName
        
        Iterator<Element> iter = els.iterator();
        while (iter.hasNext()) {
            Element object = (Element) iter.next();
            String imgUrl = object.attr("data-original");
            //获取图片
            if (!imgUrl.contains("http")) {
                continue;
            }
            URL url = new URL(imgUrl);
            URLConnection con = url.openConnection();
            con.connect();
            InputStream stream = con.getInputStream();
            byte[] by = new byte[1024];
            int length = -1;
            //out的流
            String[] imageUrlArr = imgUrl.split("/");
            String fileName = "img/" + imageUrlArr[imageUrlArr.length - 1];
            FileOutputStream fileOutputStream = new FileOutputStream(fileName);
            while ((length = stream.read(by)) != -1) {
                //把流写入到本地文件夹
                fileOutputStream.write(by,0,length);
            }
            fileOutputStream.close();
            stream.close();
        }
    }
}

方法二

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Iterator;
import java.util.LinkedList;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Test02 {

    public static void main(String[] args) {
        // 创建图片下载对象
        final ImgDownload imgDownload = new ImgDownload();
        // 生产图片链接的
        new Thread(new Runnable() {
            @Override
            public void run() {
                for (int i = 1; i < 500; i++) {
                    try {
                        imgDownload.parseImageUrl(i);
                    } catch (Exception e) {
                        // TODO: handle exception
                    }
                }
            }
        }).start();
        new Thread(new Runnable() {
            @Override
            public void run() {
                for (int i = 500; i < 1000; i++) {
                    try {
                        imgDownload.parseImageUrl(i);
                    } catch (Exception e) {
                        // TODO: handle exception
                    }
                }
            }
        }).start();
        new Thread(new Runnable() {
            @Override
            public void run() {
                for (int i = 1000; i < 1500; i++) {
                    try {
                        imgDownload.parseImageUrl(i);
                    } catch (Exception e) {
                        // TODO: handle exception
                    }
                }
            }
        }).start();

        // 下载
        new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    while (true) {
                        imgDownload.downloadImage();
                    }
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
        }).start();
        new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    while (true) {
                        imgDownload.downloadImage();
                    }
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
        }).start();
        new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    while (true) {
                        imgDownload.downloadImage();
                    }
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
        }).start();
        new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    while (true) {
                        imgDownload.downloadImage();
                    }
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
        }).start();
        new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    while (true) {
                        imgDownload.downloadImage();
                    }
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
        }).start();
    }
}

class ImgDownload {
    // imgUrl的数组
    LinkedList<String> list = new LinkedList<>();

    // 解析链接
    void parseImageUrl(int page) throws Exception {
        synchronized (this) {
            if (list.size() > 100) {
                wait();
            }
            String htmlUrl = "https://www.doutula.com/article/list/?page=" + page;
            Document document = Jsoup.connect(htmlUrl).get();
            // 获取图片链接 逗号是同一级元素
            Elements els = document.select(".lazy,.image_dtb,.img-responsive");// css选择器
                                                                                // id
                                                                                // class
                                                                                // tagName

            Iterator<Element> iter = els.iterator();
            while (iter.hasNext()) {
                Element object = (Element) iter.next();
                String imgUrl = object.attr("data-original");
                if (!imgUrl.contains("http")) {//如果不包含,则跳过
                    continue;
                }
                this.list.add(imgUrl);
            }
            // 通知可以下载图片
            notifyAll();
        }
    }

    // 下载图片
    void downloadImage() throws Exception {
        synchronized (this) {
            if (this.list.size() <= 1) {
                wait();
            }
//          String imgUrl = null;
//          if (this.list.size() > 0) {
//              imgUrl = this.list.removeFirst();
//          } else {
//              return;
//          }
            String imgUrl = this.list.removeFirst();
            URL url = new URL(imgUrl);
            URLConnection con = url.openConnection();
            con.connect();
            InputStream stream = con.getInputStream();
            byte[] by = new byte[1024];
            int length = -1;
            // out的流
            String[] imageUrlArr = imgUrl.split("/");
            String fileName = "img/" + imageUrlArr[imageUrlArr.length - 1];
            FileOutputStream fileOutputStream = new FileOutputStream(fileName);
            while ((length = stream.read(by)) != -1) {
                // 把流写入到本地文件夹
                fileOutputStream.write(by, 0, length);
            }
            fileOutputStream.close();
            stream.close();
            
            if (this.list.size() < 100) {
                notify();
                //notifyAll();
            }
        }
    }
}