java爬取网易云音乐评论

网易云神评论，永远不会让人失望，下面，我们就一起来爬一爬

导包

<dependency>
    <groupId>org.apache.cxf</groupId>
    <artifactId>cxf-rt-transports-http</artifactId>
    <version>3.3.2</version>
</dependency>
<dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
</dependency>
<dependency>
    <groupId>com.alibaba</groupId>
    <artifactId>fastjson</artifactId>
    <version>1.2.47</version>
</dependency>
<dependency>
    <groupId>org.projectlombok</groupId>
    <artifactId>lombok</artifactId>
    <optional>true</optional>
</dependency>

实现代码Controller及实体类

package com.example.api.controller;


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.example.api.util.BaseResult;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;


/**
 * @Auyher: Ktry
 * @Date: 2020/3/22 22:25
 */
@RestController
public class wyy {

    @RequestMapping(value = {"buzz"},method = {RequestMethod.GET})
    public BaseResult buzz(@RequestParam(required = false,defaultValue = "") String id)  throws Exception {


        CloseableHttpClient closeableHttpClient = HttpClients.createDefault() ;
        HttpPost httpPost = new HttpPost("http://music.163.com/weapi/v1/resource/comments/R_SO_4_"+id+"?csrf_token=") ;
        httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36");

        List<NameValuePair> list=new ArrayList<NameValuePair>();
        list.add(new BasicNameValuePair("params","RlBC7U1bfy/boPwg9ag7/a7AjkQOgsIfd+vsUjoMY2tyQCPFgnNoxHeCY+ZuHYqtM1zF8DWIBwJWbsCOQ6ZYxBiPE3bk+CI1U6Htoc4P9REBePlaiuzU4M3rDAxtMfNN3y0eimeq3LVo28UoarXs2VMWkCqoTXSi5zgKEKbxB7CmlBJAP9pn1aC+e3+VOTr0"));
        list.add(new BasicNameValuePair("encSecKey","76a0d8ff9f6914d4f59be6b3e1f5d1fc3998317195464f00ee704149bc6672c587cd4a37471e3a777cb283a971d6b9205ce4a7187e682bdaefc0f225fb9ed1319f612243096823ddec88b6d6ea18f3fec883d2489d5a1d81cb5dbd0602981e7b49db5543b3d9edb48950e113f3627db3ac61cbc71d811889d68ff95d0eba04e9"));

        httpPost.setEntity(new UrlEncodedFormEntity(list));
        CloseableHttpResponse response=closeableHttpClient.execute(httpPost);

        HttpEntity entity=response.getEntity();
        String ux = EntityUtils.toString(entity,"utf-8") ;
        JSONObject jsonObject = JSON.parseObject(ux);

        LinkedHashMap<String, ArrayList> map = new LinkedHashMap<>();
        map.put("hot",Text1(jsonObject.getJSONArray("hotComments")));
        map.put("newest",Text1(jsonObject.getJSONArray("comments")));

        return new BaseResult("200",map);
    }

    public ArrayList Text1(JSONArray objects){
        ArrayList List = new ArrayList();
        for (Object v : objects) {
            poj1 poj1 = new poj1();
            JSONObject object = (JSONObject) v;
            poj1.setContent(object.getString("content"));
            poj1.setUserimg(object.getJSONObject("user").getString("avatarUrl"));
            poj1.setUsername(object.getJSONObject("user").getString("nickname"));
            poj1.setLikedCount(object.getString("likedCount"));
            poj1.setTime(object.getString("time"));

            ArrayList arrayList = new ArrayList();
            JSONArray beReplied = object.getJSONArray("beReplied");


            for (Object i : beReplied) {
                poj2 poj2 = new poj2();
                JSONObject objecti = (JSONObject) i;
                poj2.setContent(objecti.getString("content"));
                objecti = objecti.getJSONObject("user");
                poj2.setUserimg(objecti.getString("avatarUrl"));
                poj2.setUsername(objecti.getString("nickname"));
                arrayList.add(poj2);
            }
            poj1.setBeReplied(arrayList);
            List.add(poj1);
        };
        return List;
    }

    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    @ToString
    class poj1{
        private String content;
        private String likedCount;
        private String time;
        private String userimg;
        private String username;
        private List beReplied;
    }
    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    @ToString
    class poj2{
        private String content;
        private String userimg;
        private String username;
    }
    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    @ToString
    public class BaseResult{
        private String code;
        private Object data;
    }
}

浏览器测试

以《可乐》这首歌为例

http://127.0.0.1:8080/buzz?id=1340200924

{
"code": "200",
"data": {
"hot": [
{
"content": "希望能在某一刻安放你们的些许悲伤情绪，就很好。",
"likedCount": "38185",
"time": "1547773069906",
"userimg": "http://p2.music.126.net/64m6LNpiFSMd029k6Ey_pQ==/109951164600709805.jpg",
"username": "赵紫骅",
"beReplied": []
},
{
"content": "99546949399 593693948 等我什么时候破解了你给我这一串数字 就快乐了吧",
"likedCount": "30348",
"time": "1559978151502",
"userimg": "http://p2.music.126.net/28gmImck5513o9Wn7OU2xw==/109951164589100157.jpg",
"username": "秦留香",
"beReplied": []
},
{
"content": "以前的《可乐》是遗憾\n现在的《可乐》是释怀.",
"likedCount": "29850",
"time": "1547805383581",
"userimg": "http://p1.music.126.net/b9ybvjeGkTEXz2TDlSLHDw==/109951164834372225.jpg",
"username": "不是梨洲",
"beReplied": []
},
{
"content": "人和人的出场顺序其实太重要了，陪你伶仃大醉的人是没办法送你回家的",
"likedCount": "21014",
"time": "1547683044256",
"userimg": "http://p1.music.126.net/187tCfgzd1oxoNgPsgHteQ==/109951164809444338.jpg",
"username": "迢往",
"beReplied": []
},
{
"content": "“其实你过得好的话我比谁都开心，可是我比谁都不愿意做那个人”",
"likedCount": "12365",
"time": "1547659146154",
"userimg": "http://p1.music.126.net/M-ZX4rPWIyguJ_PnekJVPw==/109951163271184431.jpg",
"username": "后原",
"beReplied": []
},
{
"content": "我想你了 我没喝酒",
"likedCount": "11714",
"time": "1547701336701",
"userimg": "http://p1.music.126.net/-YxgiQ65tet2IFR3AcbbmA==/109951164813822025.jpg",
"username": "Depravlty",
"beReplied": []
},
{
"content": "不知道少了什么 总觉得没有以前快乐.",
"likedCount": "9352",
"time": "1547682309807",
"userimg": "http://p1.music.126.net/kGeghWvELmzZsu_R4ENiDg==/109951164820805514.jpg",
"username": "岸域",
"beReplied": []
},
{
"content": "其实我们最遗憾的就是 从来没有感受过被别人坚定选择的感觉 就好像他刚好需要 而你刚好在",
"likedCount": "8519",
"time": "1547655100890",
"userimg": "http://p1.music.126.net/9II5W61HR719sg2IWokJzA==/109951164231479140.jpg",
"username": "黑夜难逃",
"beReplied": []
},
{
"content": "布衣饭菜，可乐终身，不必作远游计矣。",
"likedCount": "6925",
"time": "1548376801076",
"userimg": "http://p1.music.126.net/hRr7oaN_TXc_RWD2MVw49A==/109951163997689965.jpg",
"username": "假如我年少有为不自卑",
"beReplied": []
},
{
"content": "可怜情人节咱没约会对象，愚人节咱没表白的对象，还是清明节好，有祭拜的对象。",
"likedCount": "6573",
"time": "1550151892504",
"userimg": "http://p1.music.126.net/SLiXOtJf4jnAw7Tzd2mB3w==/109951163859899074.jpg",
"username": "潮货snaker",
"beReplied": []
},
{
"content": "曲子都变了，你怎么还没回来",
"likedCount": "5212",
"time": "1547659100942",
"userimg": "http://p1.music.126.net/x1bEPmHXWUzqGsA8-Igzuw==/109951164307928818.jpg",
"username": "超凶的小林",
"beReplied": []
},
{
"content": "我过的很好，希望你也是",
"likedCount": "4698",
"time": "1560079597985",
"userimg": "http://p1.music.126.net/aDTajkqhnkrvwGmMvDE67A==/109951164375302088.jpg",
"username": "忘记了寻找",
"beReplied": [
{
"content": "99546949399 593693948 等我什么时候破解了你给我这一串数字 就快乐了吧",
"userimg": "http://p1.music.126.net/28gmImck5513o9Wn7OU2xw==/109951164589100157.jpg",
"username": "秦留香"
}
]
},
{
"content": "我还在等你,可我无能为力",
"likedCount": "3733",
"time": "1547687232224",
"userimg": "http://p1.music.126.net/RLeBJe4D1ZzUtltxfoKDMg==/109951163250239066.jpg",
"username": "帐号已注销",
"beReplied": []
},
{
"content": "这是同一句话，You will always have a special place in my heart!你在我心中一直有个特别的位置 我是查百度的只有一个追问追答那是你吗？",
"likedCount": "3512",
"time": "1560102559659",
"userimg": "http://p1.music.126.net/vEvhKYhcIA2iB9L708dudw==/109951164356403668.jpg",
"username": "Goto丶sleep丶Archie",
"beReplied": [
{
"content": "99546949399 593693948 等我什么时候破解了你给我这一串数字 就快乐了吧",
"userimg": "http://p1.music.126.net/28gmImck5513o9Wn7OU2xw==/109951164589100157.jpg",
"username": "秦留香"
}
]
},
{
"content": "乐源自心 如果真的可以快乐 放手也是一种成全吧。",
"likedCount": "3027",
"time": "1547659834081",
"userimg": "http://p1.music.126.net/jp6GwSYYrTLg0YK_tpD56Q==/109951163878593989.jpg",
"username": "L陆嘉诚",
"beReplied": []
}
],
"newest": [
{
"content": "怦然心动最后也沦为泛泛之交 .",
"likedCount": "0",
"time": "1585061627118",
"userimg": "http://p1.music.126.net/2Ysi_kw1pMMBPkNRv5CMCg==/109951164642583015.jpg",
"username": "小彤ovo",
"beReplied": []
},
{
"content": "听哭了",
"likedCount": "1",
"time": "1585059337196",
"userimg": "http://p1.music.126.net/KLzEuf8kOh3jc5aAxofr0A==/109951164781356925.jpg",
"username": "文铭兰",
"beReplied": []
},
{
"content": "一个账号资料你还要看多少遍.",
"likedCount": "3",
"time": "1585058484202",
"userimg": "http://p1.music.126.net/EpuJMPVTTS4efxq9LKqECQ==/109951164833462861.jpg",
"username": "Eliaukub-a",
"beReplied": []
},
{
"content": "人这一生会遇到两个人，：一个教会你如何去爱，一个就是被你爱的人。",
"likedCount": "5",
"time": "1585058442322",
"userimg": "http://p1.music.126.net/M25treYpkHXqoLbxNyzX7w==/109951164828417863.jpg",
"username": "爱吃鱼的Climbing",
"beReplied": []
},
{
"content": "✓！",
"likedCount": "0",
"time": "1585057922082",
"userimg": "http://p1.music.126.net/3A5ocL7pRHguwq7BPcnbiA==/109951164832698314.jpg",
"username": "迪士尼在逃公主200317",
"beReplied": [
{
"content": "加油啊，感觉自己一个人真的挺开心的，什么都不用去想就只做自己想做的吃自己想吃的玩自己想玩的，真的挺好",
"userimg": "http://p1.music.126.net/w9XvkO7Dc4bveR0szLVOng==/109951163849019827.jpg",
"username": "半晩"
}
]
},
{
"content": "没有很长的故事没有过激的行为 只是爱而不得",
"likedCount": "0",
"time": "1585057169572",
"userimg": "http://p1.music.126.net/pY-lbpad70252OOvwbDBUA==/109951164442499198.jpg",
"username": "p_5t",
"beReplied": [
{
"content": "布衣饭菜，可乐终身，不必作远游计矣。",
"userimg": "http://p1.music.126.net/hRr7oaN_TXc_RWD2MVw49A==/109951163997689965.jpg",
"username": "假如我年少有为不自卑"
}
]
},
{
"content": "你怕不怕这辈子再也心动不起来了.",
"likedCount": "3",
"time": "1585057020303",
"userimg": "http://p1.music.126.net/EaA5fcn4uJ9Dku_vnvRRaA==/109951164367631612.jpg",
"username": "想看月亮_qian",
"beReplied": []
},
{
"content": "如果分手让你快乐，让你可以不用长大，没有负担，那就分开吧",
"likedCount": "0",
"time": "1585056570643",
"userimg": "http://p1.music.126.net/Y21h_7EVqUDfEu4kL0JJMw==/109951164259118456.jpg",
"username": "叫我小仙妹",
"beReplied": []
},
{
"content": "重不重要 都不要了",
"likedCount": "2",
"time": "1585056497555",
"userimg": "http://p1.music.126.net/_CJ_WQRAtPTmswqzSvMz6w==/109951164186912232.jpg",
"username": "玖-North",
"beReplied": []
},
{
"content": "冬天的事就让它在夏天来临前过去吧",
"likedCount": "1",
"time": "1585056393300",
"userimg": "http://p1.music.126.net/5AE-Rvbzw6HduyiZcALajg==/109951163530184022.jpg",
"username": "Verney倩",
"beReplied": []
},
{
"content": "经常会想下次见你穿什么款式的衣服，涂哪支口红\n以后不想了",
"likedCount": "2",
"time": "1585056349247",
"userimg": "http://p1.music.126.net/5AE-Rvbzw6HduyiZcALajg==/109951163530184022.jpg",
"username": "Verney倩",
"beReplied": []
},
{
"content": "我再也不要喜欢警察哥哥了\n即使你的眼睛再好看",
"likedCount": "0",
"time": "1585056277844",
"userimg": "http://p1.music.126.net/5AE-Rvbzw6HduyiZcALajg==/109951163530184022.jpg",
"username": "Verney倩",
"beReplied": []
},
{
"content": "嗯，一样。",
"likedCount": "0",
"time": "1585055752205",
"userimg": "http://p1.music.126.net/us1ZxAa8D6FmvrykoqR4mw==/109951164829671763.jpg",
"username": "老歌诉情",
"beReplied": [
{
"content": "这首歌很好听，当时刚分手的时候一直听，单曲循环，觉得无论是歌词还是评论，都很有感觉。现在再回来听这首歌让觉得已经放下的我又湿了眼角，前段时间她给我发了一个在么，我的第一反应竟然是她的号被盗了，来问我微信绑没绑定银行卡，那天我俩唠了一个多小时，她跟我说她已经放下了。",
"userimg": "http://p1.music.126.net/ibITxPWCkwmKAk8Xf72-2A==/109951164363816644.jpg",
"username": "gxzcjs"
}
]
},
{
"content": "我把他所有联系方式都删了 唯独之前跟他的聊天记录没删 我也想放过我自己",
"likedCount": "0",
"time": "1585055506200",
"userimg": "http://p1.music.126.net/dOui50pBTXhuJbSsPSz3GA==/109951164755756212.jpg",
"username": "H-九娘",
"beReplied": [
{
"content": "我把跟他有关的东西全都删了，唯独跟他的联系方式没删。我啊，只是想放过自己",
"userimg": "http://p1.music.126.net/_Iqg9kpXSohtxWN_5dYu8g==/109951164753698871.jpg",
"username": "颜海吧"
}
]
},
{
"content": "黄钻我就不续费了 你空间我就不去了",
"likedCount": "0",
"time": "1585055405835",
"userimg": "http://p1.music.126.net/dOui50pBTXhuJbSsPSz3GA==/109951164755756212.jpg",
"username": "H-九娘",
"beReplied": []
},
{
"content": "不打扰是我爱你的最后方式",
"likedCount": "1",
"time": "1585055339204",
"userimg": "http://p1.music.126.net/dOui50pBTXhuJbSsPSz3GA==/109951164755756212.jpg",
"username": "H-九娘",
"beReplied": []
},
{
"content": "放过你了",
"likedCount": "0",
"time": "1585055266915",
"userimg": "http://p1.music.126.net/lQXuAvoy6IbquHG9cpLRow==/109951164301256152.jpg",
"username": "632喜歡",
"beReplied": []
},
{
"content": "有些事只适合烂在心里.",
"likedCount": "1",
"time": "1585054967299",
"userimg": "http://p1.music.126.net/JJOubsAPvWbUBJVZdKio-A==/109951164641943714.jpg",
"username": "椿湫1118",
"beReplied": []
},
{
"content": "我应该觉得高兴的，可是，我觉得孤单了[可爱]",
"likedCount": "1",
"time": "1585054800171",
"userimg": "http://p1.music.126.net/JDfzSSnq7RTGVXbUlABE1w==/109951164664639297.jpg",
"username": "小宁姐姐丫",
"beReplied": []
},
{
"content": "我找不到可以偷偷发泄的地方，所以只能在网易云评论里悄悄崩溃",
"likedCount": "1",
"time": "1585054786848",
"userimg": "http://p1.music.126.net/qkRARfOrSyIaLJIuERKn-g==/109951163075775692.jpg",
"username": "BNKNM",
"beReplied": []
}
]
}
}