bidi_Internship/src/main/java/space/anyi/Application4.java

package space.anyi;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.model.HttpRequestBody;
import us.codecraft.webmagic.processor.PageProcessor;

import java.nio.charset.StandardCharsets;
import java.sql.*;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Application4 implements PageProcessor {

    //初始url
    static String indexUrl = "http://ztb.chinays.gov.cn/ceinwz/WebInfo_List.aspx?newsid=81&jsgc=01100000&zfcg=&tdjy=&cqjy=&qtjy=&PubDateSort=1&ShowPre=1&CbsZgys=0&zbfs=&qxxx=0&showqxname=0&NewsShowPre=1&wsjj=0&showCgr=0&ShowOverDate=0&showdate=1&FromUrl=zbgg";
    static int total = 1;
    static int index = 1;
    public static final String baseUrl = "http://ztb.chinays.gov.cn";
    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");


    private static final Map<String, Map<String,Object>> context = new HashMap<>();

    //配置
    private Site site = new Site()
            .setTimeOut(100 * 1000)     //超时时间
            .setRetryTimes(3)         //重试请求次数
            .setRetrySleepTime(3000)  //重试间隔时间                  请求头
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");

    public static String doGet(String url) throws Exception {
        String result = "";
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(url);
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
                .build();
        httpGet.setConfig(requestConfig);
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
                result = content;
            }
        } finally {
            if (response != null) {
                response.close();
            }
            httpClient.hashCode();
        }
        return result;
    }

    public static String urlClean(String url,String baseUrl,String currentUrl){
        String result = "";
        if (url.startsWith("/")) {
            //根路径开始
            result = baseUrl + url;
        }else if (url.startsWith("./")){
            //当前同级目录
            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
        }else if (url.startsWith("../")){
            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
            result = baseUrl + result.substring(result.lastIndexOf("../") + 2);
        }else {
            //等价于当前目录
            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
        }
        return result;
    }

    @Override
    public void process(Page page) {
        String currentUrl = page.getUrl().toString();
        if (currentUrl.contains("WebInfo_List")) {
            String rawText = page.getRawText();
            Document document =  Jsoup.parse(rawText);
            //总页数
            Elements eachTags = document.select("table.myGVClass tr:has(a)");
            if (eachTags != null && eachTags.size() > 0) {
                for (Element eachTag : eachTags) {
                    //解析详情页url
                    String href = eachTag.select("a").first().attr("href");
                    String link = "";
                    link = urlClean(href,baseUrl,currentUrl);
                    String title = eachTag.select("a").first().text().trim();
                    //有可能出現 ”2026-10-14 [正在进行]“
                    String dateStr = eachTag.select("td.fFbDate").first().text().trim();
                    Matcher matcher = pattern.matcher(dateStr);
                    if (matcher.find()) {
                        String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
                        String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
                        dateStr = matcher.group(1) + "-" + month + "-" + day;
                    }
                    System.out.println(title);
                    System.out.println(link);
                    //数据库是否已存在
                    String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
                    try(Connection connection = MyDao.getConnection();
                        PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
                        prepareStatement.setString(1,link);
                        ResultSet resultSet = prepareStatement.executeQuery();
                        if (resultSet.next()) {
                            System.out.println("跳过" + link);
                            resultSet.close();
                            continue;
                        }
                        resultSet.close();
                    } catch (SQLException e) {
                        throw new RuntimeException(e);
                    }
                    Request request = new Request(link);
//                    request.putExtra("title",title.substring(2));
//                    request.putExtra("listDate",LocalDate.parse(dateStr));
                    Map<String, Object> map = new HashMap<>();
                    map.put("title",title);
                    map.put("listDate",LocalDate.parse(dateStr));
                    context.put(link,map);
                    page.addTargetRequest(request);
                }
            }
            //翻頁
            String __EVENTVALIDATION = document.select("#__EVENTVALIDATION").first().val();
            String __VIEWSTATEGENERATOR = document.select("#__VIEWSTATEGENERATOR").first().val();
            String ctl00_myTreeView_ExpandState = document.select("#ctl00_myTreeView_ExpandState").first().val();
            String __VIEWSTATE = document.select("#__VIEWSTATE").first().val();
            Element nextPage = document.getElementById("ctl00_ContentPlaceHolder1_LinkButtonNextPage");
            if (Objects.nonNull(nextPage)) {
                Map<String, Object> params = new HashMap<>();
                params.put("__EVENTVALIDATION",__EVENTVALIDATION);
                params.put("__VIEWSTATEGENERATOR",__VIEWSTATEGENERATOR);
                params.put("ctl00_myTreeView_ExpandState",ctl00_myTreeView_ExpandState);
                params.put("__VIEWSTATE",__VIEWSTATE);
                params.put("__EVENTTARGET","ctl00$ContentPlaceHolder1$LinkButtonNextPage");
                Request request = new Request(indexUrl);
                request.setMethod("post");
                HttpRequestBody requestBody = HttpRequestBody.form(params, StandardCharsets.UTF_8.name());
                request.setRequestBody(requestBody);
                page.addTargetRequest(request);
            }
            page.setSkip(true);
        } else {
            //处理详情网址
            String rawText = page.getRawText();
            rawText = rawText.replace("display:none;", "");
            Document document = Jsoup.parse(rawText);
            Element content = document.getElementById("form1");
            if (Objects.isNull(content)) {
                content = document.select("#aspnetForm > div:nth-child(3) > table:nth-child(4) > tbody:nth-child(2) > tr > td:nth-child(3) > div > table").first();
            }
            //文件url清洗
            Elements elements = content.select("a.lan");
            if (Objects.nonNull(elements) && elements.size() > 0) {
                elements.remove();
            }
            Elements as = content.select("a");
            for (Element a : as) {
                String href = a.attr("href");
                href = urlClean(href,baseUrl,currentUrl);
                a.attr("href", href);
            }
            //图片url清洗
            Elements imgs = content.select("img");
            for (Element img : imgs) {
                String src = img.attr("src");
                src = urlClean(src,baseUrl,currentUrl);
                img.attr("src", src);
            }
            //正文清洗
//            Element frmBestwordHtml = content.getElementById("frmBestwordHtml");
//            if (Objects.nonNull(frmBestwordHtml)) {
//                String src = frmBestwordHtml.attr("src");
//                src = urlClean(src,baseUrl,currentUrl);
//                frmBestwordHtml.attr("src", src);
//                try {
//                    String res = doGet(src);
//                    Element element = Jsoup.parse(res);
//                    frmBestwordHtml.replaceWith(element);
//                } catch (Exception e) {
//                    throw new RuntimeException(e);
//                }
//            }
            //清洗
//            content.select("td.info_text").remove();
            //save
            Map<String, Object> map = context.get(currentUrl);
            String SOURCE_NAME = "英山县公共资源交易中心-英山县政府采购中心";
            String DETAIL_LINK = currentUrl;
            Element titleElement = content.getElementById("lblXmmc");
            if (Objects.isNull(titleElement)) {
                titleElement = content.getElementById("ctl00_ContentPlaceHolder1_TitleLabel");
            }
            String DETAIL_TITLE = titleElement.text().trim();
            String DETAIL_CONTENT = content.toString();
            LocalDate PAGE_TIME = ((LocalDate) map.get("listDate"));
//            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
            LocalDate CREATE_TIME = LocalDate.now();
            String LIST_TITLE = ((String) map.get("title"));
//            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
            String CREATE_BY = "黄略";

            String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
            try(Connection connection = MyDao.getConnection();
                PreparedStatement preparedStatement = connection.prepareStatement(sql);){
                preparedStatement.setString(1, SOURCE_NAME);
                preparedStatement.setString(2, DETAIL_LINK);
                preparedStatement.setString(3, DETAIL_TITLE);
                preparedStatement.setString(4, DETAIL_CONTENT);
                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
                preparedStatement.setString(7, LIST_TITLE);
                preparedStatement.setString(8, CREATE_BY);
                preparedStatement.executeUpdate();
            } catch (SQLException e) {
                throw new RuntimeException(e);
            }
        }
    }


    @Override
    public Site getSite() {
        return site;
    }

    //启动
    public static void main(String[] args) {
        Spider spider = Spider.create(new Application4())
                .addUrl("http://ztb.chinays.gov.cn/ceinwz/WebInfo_List.aspx?newsid=81&jsgc=01100000&zfcg=&tdjy=&cqjy=&qtjy=&PubDateSort=1&ShowPre=1&CbsZgys=0&zbfs=&qxxx=0&showqxname=0&NewsShowPre=1&wsjj=0&showCgr=0&ShowOverDate=0&showdate=1&FromUrl=zbgg");
        System.out.println("start");
        spider.run();
        System.out.println("end");
    }
}