bidi_Internship/src/main/java/space/anyi/Application5.java

package space.anyi;

import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Application5 {
    static String indexUrl = "http://ztb.chinays.gov.cn/ceinwz/WebInfo_List.aspx?newsid=81&jsgc=01100000&zfcg=&tdjy=&cqjy=&qtjy=&PubDateSort=1&ShowPre=1&CbsZgys=0&zbfs=&qxxx=0&showqxname=0&NewsShowPre=1&wsjj=0&showCgr=0&ShowOverDate=0&showdate=1&FromUrl=zbgg";
    static String baseUrl = "https://www.tjdl.gov.cn";
    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");

    static MyDao myDao = new MyDao();
    public static void main(String[] args) throws Exception {
        getList();
    }

//    static int total = 1;
//    static int index = 0;
    static int count = 0;
    static String __EVENTVALIDATION = "";
    static String __VIEWSTATEGENERATOR = "";
    static String ctl00_myTreeView_ExpandState = "ennnnn";
    static String __VIEWSTATE = "";
    public static void getList() throws Exception {
        while (true){
            //列表url构建
            String listUrl = indexUrl;
            System.out.println(listUrl);
            //采集列表页
            String res;
            if (count == 0) {
                res = doGet(listUrl);
            }else {
                res = doPost(listUrl);
            }
            //采集总页数
            Document doc = Jsoup.parse(res);
            //有没有下一页
            __EVENTVALIDATION = doc.select("#__EVENTVALIDATION").first().val();
            __VIEWSTATEGENERATOR = doc.select("#__VIEWSTATEGENERATOR").first().val();
//            ctl00_myTreeView_ExpandState = doc.select("#ctl00_myTreeView_ExpandState").first().val();
            __VIEWSTATE = doc.select("#__VIEWSTATE").first().val();
            Element nextPage = doc.getElementById("ctl00_ContentPlaceHolder1_LinkButtonNextPage");
            //列表
            Elements eachTags = doc.select("table.myGVClass tr:has(a)");
            if (eachTags != null) {
                if (eachTags.size() > 0) {
                    for (Element eachTag : eachTags) {
                        count++;
                        //解析详情页url
                        String href = eachTag.select("a").first().attr("href");
                        String link = "";
                        String currentUrl = listUrl;
                        link = urlClean(href,baseUrl,currentUrl);
                        String title = eachTag.select("a").first().text().trim();
                        System.out.println(title);
                        System.out.println(link);
                        //数据库是否已存在
                        String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
                        try(Connection connection = MyDao.getConnection();
                            PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
                            prepareStatement.setString(1,link);
                            ResultSet resultSet = prepareStatement.executeQuery();
                            if (resultSet.next()) {
                                System.out.println("跳过" + link);
                                resultSet.close();
                                continue;
                            }
                            resultSet.close();
                        }
                        //采集详情页
                        String detail = doGet(link);
                        detail = detail.replace("display:none;", "");
                        Document document = Jsoup.parse(detail);
                        //#form1 > div:nth-child(3) > table > tbody > tr:nth-child(1) > td > table
                        Element content = document.getElementById("form1");
                        if (Objects.isNull(content)) {
                            content = document.select("#aspnetForm > div:nth-child(3) > table:nth-child(4) > tbody:nth-child(2) > tr > td:nth-child(3) > div > table").first();
                        }
                        //文件url清洗
                        Elements elements = content.select("a.lan");
                        if (Objects.nonNull(elements) && elements.size() > 0) {
                            elements.remove();
                        }
                        Elements as = content.select("a");
                        for (Element a : as) {
                            href = a.attr("href");
                            href = urlClean(href,baseUrl,link);
                            a.attr("href", href);
                        }
                        //图片url清洗
                        Elements imgs = content.select("img");
                        for (Element img : imgs) {
                            String src = img.attr("src");
                            src = urlClean(src,baseUrl,link);
                            img.attr("src", src);
                        }
                        //正文清洗
//                        Element frmBestwordHtml = content.getElementById("frmBestwordHtml");
//                        if (Objects.nonNull(frmBestwordHtml)) {
//                            String src = frmBestwordHtml.attr("src");
//                            src = urlClean(src, baseUrl, link);
//                            frmBestwordHtml.attr("src", src);
//                            try {
//                                res = doGet(src);
//                                Element element = Jsoup.parse(res);
//                                frmBestwordHtml.replaceWith(element);
//                            } catch (Exception e) {
//                                throw new RuntimeException(e);
//                            }
//                        }

                        //save
                        //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
                        String SOURCE_NAME = "英山县公共资源交易中心-英山县政府采购中心";
                        String DETAIL_LINK = link;
                        Element titleElement = content.getElementById("lblXmmc");
                        if (Objects.isNull(titleElement)) {
                            titleElement = content.getElementById("ctl00_ContentPlaceHolder1_TitleLabel");
                        }
                        String DETAIL_TITLE = titleElement.text().trim();
                        String DETAIL_CONTENT = content.toString();
                        //有可能出現 ”2026-10-14 [正在进行]“
                        String dateStr = eachTag.select("td.fFbDate").first().text().trim();
                        Matcher matcher = pattern.matcher(dateStr);
                        if (matcher.find()) {
                            String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
                            String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
                            dateStr = matcher.group(1) + "-" + month + "-" + day;
                        }
                        LocalDate PAGE_TIME = LocalDate.parse(dateStr);
                        LocalDate CREATE_TIME = LocalDate.now();
                        String LIST_TITLE = title;
                        String CREATE_BY = "黄略";

                        String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
                        try(Connection connection = MyDao.getConnection();
                            PreparedStatement preparedStatement = connection.prepareStatement(sql);){
                            preparedStatement.setString(1, SOURCE_NAME);
                            preparedStatement.setString(2, DETAIL_LINK);
                            preparedStatement.setString(3, DETAIL_TITLE);
                            preparedStatement.setString(4, DETAIL_CONTENT);
                            preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
                            preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
                            preparedStatement.setString(7, LIST_TITLE);
                            preparedStatement.setString(8, CREATE_BY);
                            preparedStatement.executeUpdate();
                        }
                    }
                    }
                }
            if (Objects.isNull(nextPage)) {
                break;
            }
        }
        System.out.println(count);
    }


    public static String doGet(String url) throws Exception {
        String result = "";
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(url);
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
                .build();
        httpGet.setConfig(requestConfig);
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
                result = content;
            }
        } finally {
            if (response != null) {
                response.close();
            }
            httpClient.hashCode();
        }
        return result;
    }

    public static String doPost(String path) throws Exception {
        String result = "";

        CloseableHttpClient httpclient = HttpClients.createDefault();
        // 创建http POST请求
        HttpPost httpPost = new HttpPost(path);
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
                .build();
        httpPost.setConfig(requestConfig);
        List<NameValuePair> parameters = new ArrayList<NameValuePair>(0);
        parameters.add(new BasicNameValuePair("__EVENTVALIDATION",__EVENTVALIDATION));
        parameters.add(new BasicNameValuePair("__VIEWSTATEGENERATOR",__VIEWSTATEGENERATOR));
        parameters.add(new BasicNameValuePair("ctl00_myTreeView_ExpandState",ctl00_myTreeView_ExpandState));
        parameters.add(new BasicNameValuePair("__VIEWSTATE",__VIEWSTATE));
        parameters.add(new BasicNameValuePair("__EVENTTARGET","ctl00$ContentPlaceHolder1$LinkButtonNextPage"));

        // 构造一个form表单式的实体
        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(parameters);
        // 将请求实体设置到httpPost对象中
        httpPost.setEntity(formEntity);
        httpPost.setHeader("User-Agent",
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");
        CloseableHttpResponse response = null;
        try {
            response = httpclient.execute(httpPost);
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
                return content;
            }
        } finally {
            if (response != null) {
                response.close();
            }
            httpclient.close();
        }
        return result;
    }

    public static String urlClean(String url,String baseUrl,String currentUrl){
        String result = "";
        if (url.startsWith("/")) {
            //根路径开始
            result = baseUrl + url;
        }else if (url.startsWith("./")){
            //当前同级目录
            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
        }else if (url.startsWith("../")){
            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
            result = baseUrl + url.substring(url.lastIndexOf("../") + 2);
        }else {
            //等价于当前目录
            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
        }
        return result;
    }
}