bidi_Internship/src/main/java/space/anyi/Application.java

package space.anyi;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Application {
    static String indexUrl = "https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/index%s.html";
    static String baseUrl = "https://www.tjdl.gov.cn";
    public static void main(String[] args) throws Exception {
//        String res = doGet(listUrl);
//        System.out.println(res);
        getList();
    }

    static int total = 1;
    static int index = 0;
    static int count = 0;
    public static void getList() throws Exception {
        while (index< total){
            String listUrl = index == 0 ? String.format(indexUrl, "") : String.format(indexUrl, "_" + index);
            System.out.println(listUrl);
            String res = doGet(listUrl);
            //var countPage = 5//共多少页
            Pattern pattern = Pattern.compile("var countPage = (\\d+)//共多少页");
            Matcher matcher = pattern.matcher(res);
            if (matcher.find()) {
                String totalStr = matcher.group(1);
                total =  Integer.parseInt(totalStr);
            }
        Document doc = Jsoup.parse(res);

//        Elements eachTags = doc.select("div[class=news-items] ul li:has(a)");
        //#xxgk_right > div.zfxxgk_zd2.borderTop.zfxxgkzd.border0
        Elements eachTags = doc.select("div.zfxxgk_zd2.borderTop.zfxxgkzd.border0 ul li:has(a)");
        if (eachTags != null) {
            if (eachTags.size() > 0) {
                for (Element eachTag : eachTags) {
                    count++;
                    String id = eachTag.select("a").first().attr("href");
                    id = id.substring(id.indexOf("./") + 1);
                    String link = listUrl.substring(0, listUrl.lastIndexOf("/")) + id;
                    String title = eachTag.select("a").first().text().trim();
                    System.out.println(title);
                    System.out.println(link);
                    String detail = doGet(link);
                    detail = detail.replace("display:none;", "");
                    Document document = Jsoup.parse(detail);
                    //body > div.bgf7f7f7 > div > div.listCon > div > div.contentCons > div.pages_content
                    //body > div.bgf7f7f7 > div
                    Element content = document.select("body > div.bgf7f7f7 > div").first();
//                    Element pageContent = content.select("div.pages_content").first();
                    //文件
                    Elements as = content.select("a");
                    for (Element a : as) {
                        String href = a.attr("href");
                        if (href.startsWith("./")) {
                            href = link.substring(0, link.lastIndexOf('/')) + href.substring(href.indexOf("./") + 1);
                            a.attr("href", href);
                        } else if (href.startsWith("../")) {
                            href = baseUrl + href.substring(href.lastIndexOf("../") + 2);
                            a.attr("href", href);
                        } else if (href.startsWith("/")) {
                            href = baseUrl + href;
                            a.attr("href", href);
                        }
//                        String fileTitle = a.text();
//                        System.out.println(fileTitle);
                    }
                    //图片
                    Elements imgs = content.select("img");
                    for (Element img : imgs) {
                        String src = img.attr("src");
                        if (src.startsWith("./")) {
                            //https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/202507/W020250723358692283424_ORIGIN.jpg
                            src = link.substring(0, link.lastIndexOf('/')) + src.substring(src.indexOf("./") + 1);
                            img.attr("src", src);
                        } else if (src.startsWith("../")) {
                            src = baseUrl + src.substring(src.lastIndexOf("../") + 2);
                            img.attr("src", src);
                        } else if (src.startsWith("/")) {
                            src = baseUrl + src;
                            img.attr("src", src);
                        }
                    }
                    //头部
                    content.select("div.dqwz").remove();
                    content.select("div.contentMemo").remove();
                    content.select("#articlePlayer").remove();
                    content.select("#share").remove();
                    content.select("dl.QrcodeBot").remove();
                    //底部
                    content.select("#file_document").remove();
                    content.select("div.inRight").remove();
                    content.select("script").remove();
                    content.select("style").remove();
                    String text = content.text();

                    //save
                    try (BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(Files.newOutputStream(Paths.get(System.getProperty("user.dir") + File.separator + "data" + File.separator + title.substring(2) + ".html"))))) {
                        bufferedWriter.write(content.toString());
                    }
                }
                }
            }
        index++;
        }
        System.out.println(count);
    }

    private static int getTotal(String res) {
        //var countPage = 5//共多少页
        Pattern pattern = Pattern.compile("var countPage = (\\d+)//共多少页");
        Matcher matcher = pattern.matcher(res);
        if (matcher.find()) {
            String totalStr = matcher.group(1);
            return Integer.parseInt(totalStr);
        }
        return total;
    }

    public static String doGet(String url) throws Exception {
        String result = "";
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(url);
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
                .build();
        httpGet.setConfig(requestConfig);
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
                result = content;
            }
        } finally {
            if (response != null) {
                response.close();
            }
            httpClient.hashCode();
        }
        return result;
    }
}