bidi_Internship/src/main/java/space/anyi/Application2.java

package space.anyi;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.sql.*;
import java.time.LocalDate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Application2 implements PageProcessor {

    //初始url
    static String indexUrl = "https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/index%s.html";
    static int total = 1;
    static int index = 0;
    public static final String baseUrl = "https://www.tjdl.gov.cn";
    public static Pattern p = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");

    //配置
    private Site site = new Site()
            .setTimeOut(100 * 1000)     //超时时间
            .setRetryTimes(3)         //重试请求次数
            .setRetrySleepTime(3000)  //重试间隔时间                  请求头
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");

    @Override
    public void process(Page page) {
        String currentUrl = page.getUrl().toString();
        if (currentUrl.contains("index")) {
            String rawText = page.getRawText();
            Pattern pattern = Pattern.compile("var countPage = (\\d+)//共多少页");
            Matcher matcher = pattern.matcher(rawText);
            if (matcher.find()) {
                String totalStr = matcher.group(1);
                total =  Integer.parseInt(totalStr);
            }
            Document document =  Jsoup.parse(rawText);
            Elements eachTags = document.select("div.zfxxgk_zd2.borderTop.zfxxgkzd.border0 ul li:has(a)");
            if (eachTags != null && eachTags.size() > 0) {
                for (Element eachTag : eachTags) {
                    //解析详情页url
                    String id = eachTag.select("a").first().attr("href");
                    id = id.substring(id.indexOf("./") + 1);
                    String link = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + id;
                    String title = eachTag.select("a").first().text().trim();
                    String dateStr = eachTag.select("b").first().text().trim();
                    System.out.println(title);
                    System.out.println(link);
                    //数据库是否已存在
                    String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
                    try(Connection connection = MyDao.getConnection();
                        PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
                        prepareStatement.setString(1,link);
                        ResultSet resultSet = prepareStatement.executeQuery();
                        if (resultSet.next()) {
                            System.out.println("跳过" + link);
                            resultSet.close();
                            continue;
                        }
                        resultSet.close();
                    } catch (SQLException e) {
                        throw new RuntimeException(e);
                    }
                    //todo
                    Request request = new Request(link);
                    request.putExtra("title",title.substring(2));
                    request.putExtra("listDate",LocalDate.parse(dateStr));
                    page.addTargetRequest(request);
                }
            }
            page.addTargetRequest(String.format(indexUrl, "_" + (++index)));
        } else {
            //处理详情网址
            String rawText = page.getRawText();
            rawText = rawText.replace("display:none;", "");
            Document document = Jsoup.parse(rawText);
            Element content = document.select("body > div.bgf7f7f7 > div").first();
            //文件url清洗
            Elements as = content.select("a");
            for (Element a : as) {
                String href = a.attr("href");
                if (href.startsWith("./")) {
                    href = currentUrl.substring(0, currentUrl.lastIndexOf('/')) + href.substring(href.indexOf("./") + 1);
                    a.attr("href", href);
                } else if (href.startsWith("../")) {
                    href = baseUrl + href.substring(href.lastIndexOf("../") + 2);
                    a.attr("href", href);
                } else if (href.startsWith("/")) {
                    href = baseUrl + href;
                    a.attr("href", href);
                }
            }
            //图片url清洗
            Elements imgs = content.select("img");
            for (Element img : imgs) {
                String src = img.attr("src");
                if (src.startsWith("./")) {
                    //https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/202507/W020250723358692283424_ORIGIN.jpg
                    src = currentUrl.substring(0, currentUrl.lastIndexOf('/')) + src.substring(src.indexOf("./") + 1);
                    img.attr("src", src);
                } else if (src.startsWith("../")) {
                    src = baseUrl + src.substring(src.lastIndexOf("../") + 2);
                    img.attr("src", src);
                } else if (src.startsWith("/")) {
                    src = baseUrl + src;
                    img.attr("src", src);
                }
            }
            //头部清洗
            content.select("div.dqwz").remove();
            content.select("div.contentMemo").remove();
            content.select("#articlePlayer").remove();
            content.select("#share").remove();
            content.select("dl.QrcodeBot").remove();
            //底部清洗
            content.select("#file_document").remove();
            content.select("div.inRight").remove();
            content.select("script").remove();
            content.select("style").remove();
            //save
            String SOURCE_NAME = "天津市东丽区民政局";
            String DETAIL_LINK = currentUrl;
            String DETAIL_TITLE = content.select("h1").first().text().trim();
            String DETAIL_CONTENT = content.toString();
            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
            LocalDate CREATE_TIME = LocalDate.now();
            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
            String CREATE_BY = "黄略";

            String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
            try(Connection connection = MyDao.getConnection();
                PreparedStatement preparedStatement = connection.prepareStatement(sql);){
                preparedStatement.setString(1, SOURCE_NAME);
                preparedStatement.setString(2, DETAIL_LINK);
                preparedStatement.setString(3, DETAIL_TITLE);
                preparedStatement.setString(4, DETAIL_CONTENT);
                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
                preparedStatement.setString(7, LIST_TITLE);
                preparedStatement.setString(8, CREATE_BY);
                preparedStatement.executeUpdate();
            } catch (SQLException e) {
                throw new RuntimeException(e);
            }
        }
    }


    @Override
    public Site getSite() {
        return site;
    }

    //启动
    public static void main(String[] args) {
        Spider spider = Spider.create(new Application2())
                .addUrl("https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/index.html");
        System.out.println("start");
        spider.run();
        System.out.println("end");
    }
}