springboot批量爬取微信公众号信息及视频下载

springboot批量爬取微信公众号信息及视频下载

1. 准备需要爬取的公众号链接(例如:https://mp.weixin.qq.com/s/GPz-w3_gS8jsgINJH9t6vw).下面的是整合了160多个公众号文章的地址.

2. 你们应该知道需要获取160个视频,先需要知道这160视频文章的地址.

• a. 搭建springboot框架.demo直通车https://chenqiwei.lanzoui.com/isaWAschwji

• b.导入爬取网页的依赖在pom文件下.

<!-- 这个是爬取网页地址 -->
　　　　<dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.2</version>
        </dependency>
<!-- 这个主要数据解析存储用到-->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.47</version>
        </dependency>

3.获取每个视频文章的地址

String url="https://mp.weixin.qq.com/s/GPz-w3_gS8jsgINJH9t6vw";
        Document document = Jsoup.parse(new URL(url), 30000);
        Elements section = document.getElementsByTag("a");
        StringBuffer stringBuffer = new StringBuffer();
        for (Element s:section){ //遍历每个视频文章集数
            VideoInfo videoInfo = new VideoInfo();
            String href = s.attr("href");//获取每个文章的地址
            String name = s.text();//获取每个文章的文本
}

4.每个的文章的视频,是需要再次请求微信服务器才会给你的 (获取的每个文章地址的mpvip的值替换下面这个地址的vid的地址才会返回真正的视频地址给你,就是文章mpcid替换途中红色部分再次请求,返回的才是视频地址)

https://mp.weixin.qq.com/mp/videoplayer?action=get_mp_video_play_url&preview=0&__biz=&mid=&idx=&vid=1070106698888740864&uin=&key=&pass_ticket=&wxtoken=&appmsg_token=&x5=0&f=json

if (href.contains("http")){
                Document parse = Jsoup.parse(new URL(href), 30000);
                String mpvid=parse.getElementsByTag("iframe").attr("data-mpvid");
                String sun="https://mp.weixin.qq.com/mp/videoplayer?action=get_mp_video_play_url&preview=0&__biz=&mid=&idx=&vid=1070106698888740864&uin=&key=&pass_ticket=&wxtoken=&appmsg_token=&x5=0&f=json";
                sun=sun.replace("1070106698888740864",mpvid);
                String s1 = sendGet(sun,"");
                String videourl=JSONObject.parseObject(s1).getJSONArray("url_info").getJSONObject(0).get("url").toString();
                String title=JSONObject.parseObject(s1).get("title").toString();
                String sss="<tr>\n" +
                        "        <td>"+name+"</td>\n" +
                        "        <td>"+title+"</td>\n" +
                        "        <td><a href='"+videourl.replace("http://","https://")+"'  target=\"_blank\">"+title+"视频地址点击下载</a></td>\n" +
                        "    </tr>";
                stringBuffer.append(sss);
            }

5.出来的效果.(每个视频地址就整理出来了)

6.完整的源码

public static void main(String[] args) throws Exception {
       String url="https://mp.weixin.qq.com/s/GPz-w3_gS8jsgINJH9t6vw";
       Document document = Jsoup.parse(new URL(url), 30000);
       Elements section = document.getElementsByTag("a");
       StringBuffer stringBuffer = new StringBuffer();
       for (Element s:section){
           VideoInfo videoInfo = new VideoInfo();
           String href = s.attr("href");
           String name = s.text();
           if (href.contains("http")){
               Document parse = Jsoup.parse(new URL(href), 30000);
               String mpvid=parse.getElementsByTag("iframe").attr("data-mpvid");
               String sun="https://mp.weixin.qq.com/mp/videoplayer?action=get_mp_video_play_url&preview=0&__biz=&mid=&idx=&vid=1070106698888740864&uin=&key=&pass_ticket=&wxtoken=&appmsg_token=&x5=0&f=json";
               sun=sun.replace("1070106698888740864",mpvid);
               String s1 = sendGet(sun,"");
               String videourl=JSONObject.parseObject(s1).getJSONArray("url_info").getJSONObject(0).get("url").toString();
               String title=JSONObject.parseObject(s1).get("title").toString();
               String sss="<tr>\n" +
                       "        <td>"+name+"</td>\n" +
                       "        <td>"+title+"</td>\n" +
                       "        <td><a href='"+videourl.replace("http://","https://")+"'  target=\"_blank\">"+title+"视频地址点击下载</a></td>\n" +
                       "    </tr>";
               stringBuffer.append(sss);
           }
       }
       System.out.println(stringBuffer.toString());
   }

   public static String sendGet(String url, String param) {
       String result = "";
       BufferedReader in = null;
       try {
           String urlNameString = url + "?" + param;
           URL realUrl = new URL(urlNameString);
           // 打开和URL之间的连接
           URLConnection connection = realUrl.openConnection();
           // 设置通用的请求属性
           connection.setRequestProperty("accept", "*/*");
           connection.setRequestProperty("connection", "Keep-Alive");
           connection.setRequestProperty("user-agent",
                   "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
           // 建立实际的连接
           connection.connect();
           // 获取所有响应头字段
           Map<String, List<String>> map = connection.getHeaderFields();
           // 遍历所有的响应头字段
           for (String key : map.keySet()) {
               System.out.println(key + "--->" + map.get(key));
           }
           // 定义 BufferedReader输入流来读取URL的响应
           in = new BufferedReader(new InputStreamReader(
                   connection.getInputStream()));
           String line;
           while ((line = in.readLine()) != null) {
               result += line;
           }
       } catch (Exception e) {
           System.out.println("发送GET请求出现异常！" + e);
           e.printStackTrace();
       }
       // 使用finally块来关闭输入流
       finally {
           try {
               if (in != null) {
                   in.close();
               }
           } catch (Exception e2) {
               e2.printStackTrace();
           }
       }
       return result;
   }