feat:爬虫实习第二天

2026.10.20

@@ -1,3 +1,2 @@
 今日学习日报:
-学习通正则表达式处理Jsoup无法处理的特殊情况，例如总页数不在html元素显示，而是通过javascript动态渲染的方式显示，所以需要使用正则表达式提取混在js代码中的总页数数据;
-学习通过ojdbc6操作数据，预处理sql编写、sql参数动态填充、执行预编译sql,将采集、清洗完成的数据入库;
+学习采集动态参数校验的复杂网页,学习使用post请求携带校验参数采集数据;

2026.10.21

@@ -0,0 +1,2 @@
+今日学习日报:
+学习采集动态参数校验的复杂网页,学习使用post请求携带校验参数采集数据;

pom.xml

@@ -26,10 +26,10 @@
             <artifactId>webmagic-core</artifactId>
             <version>0.7.3</version>
             <exclusions>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-api</artifactId>
-                </exclusion>
+<!--                <exclusion>-->
+<!--                    <groupId>org.slf4j</groupId>-->
+<!--                    <artifactId>slf4j-api</artifactId>-->
+<!--                </exclusion>-->
                 <exclusion>
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-log4j12</artifactId>
@@ -56,5 +56,11 @@
             <artifactId>json</artifactId>
             <version>20140107</version>
         </dependency>
+
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.2.3</version>
+        </dependency>
     </dependencies>
 </project>

src/main/java/space/anyi/Application2.java

@@ -0,0 +1,170 @@
+package space.anyi;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.sql.*;
+import java.time.LocalDate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application2 implements PageProcessor {
+
+    //初始url
+    static String indexUrl = "https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/index%s.html";
+    static int total = 1;
+    static int index = 0;
+    public static final String baseUrl = "https://www.tjdl.gov.cn";
+    public static Pattern p = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+    //配置
+    private Site site = new Site()
+            .setTimeOut(100 * 1000)     //超时时间
+            .setRetryTimes(3)         //重试请求次数
+            .setRetrySleepTime(3000)  //重试间隔时间                  请求头
+            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");
+
+    @Override
+    public void process(Page page) {
+        String currentUrl = page.getUrl().toString();
+        if (currentUrl.contains("index")) {
+            String rawText = page.getRawText();
+            Pattern pattern = Pattern.compile("var countPage = (\\d+)//共多少页");
+            Matcher matcher = pattern.matcher(rawText);
+            if (matcher.find()) {
+                String totalStr = matcher.group(1);
+                total =  Integer.parseInt(totalStr);
+            }
+            Document document =  Jsoup.parse(rawText);
+            Elements eachTags = document.select("div.zfxxgk_zd2.borderTop.zfxxgkzd.border0 ul li:has(a)");
+            if (eachTags != null && eachTags.size() > 0) {
+                for (Element eachTag : eachTags) {
+                    //解析详情页url
+                    String id = eachTag.select("a").first().attr("href");
+                    id = id.substring(id.indexOf("./") + 1);
+                    String link = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + id;
+                    String title = eachTag.select("a").first().text().trim();
+                    String dateStr = eachTag.select("b").first().text().trim();
+                    System.out.println(title);
+                    System.out.println(link);
+                    //数据库是否已存在
+                    String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                    try(Connection connection = MyDao.getConnection();
+                        PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                        prepareStatement.setString(1,link);
+                        ResultSet resultSet = prepareStatement.executeQuery();
+                        if (resultSet.next()) {
+                            System.out.println("跳过" + link);
+                            resultSet.close();
+                            continue;
+                        }
+                        resultSet.close();
+                    } catch (SQLException e) {
+                        throw new RuntimeException(e);
+                    }
+                    //todo
+                    Request request = new Request(link);
+                    request.putExtra("title",title.substring(2));
+                    request.putExtra("listDate",LocalDate.parse(dateStr));
+                    page.addTargetRequest(request);
+                }
+            }
+            page.addTargetRequest(String.format(indexUrl, "_" + (++index)));
+        } else {
+            //处理详情网址
+            String rawText = page.getRawText();
+            rawText = rawText.replace("display:none;", "");
+            Document document = Jsoup.parse(rawText);
+            Element content = document.select("body > div.bgf7f7f7 > div").first();
+            //文件url清洗
+            Elements as = content.select("a");
+            for (Element a : as) {
+                String href = a.attr("href");
+                if (href.startsWith("./")) {
+                    href = currentUrl.substring(0, currentUrl.lastIndexOf('/')) + href.substring(href.indexOf("./") + 1);
+                    a.attr("href", href);
+                } else if (href.startsWith("../")) {
+                    href = baseUrl + href.substring(href.lastIndexOf("../") + 2);
+                    a.attr("href", href);
+                } else if (href.startsWith("/")) {
+                    href = baseUrl + href;
+                    a.attr("href", href);
+                }
+            }
+            //图片url清洗
+            Elements imgs = content.select("img");
+            for (Element img : imgs) {
+                String src = img.attr("src");
+                if (src.startsWith("./")) {
+                    //https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/202507/W020250723358692283424_ORIGIN.jpg
+                    src = currentUrl.substring(0, currentUrl.lastIndexOf('/')) + src.substring(src.indexOf("./") + 1);
+                    img.attr("src", src);
+                } else if (src.startsWith("../")) {
+                    src = baseUrl + src.substring(src.lastIndexOf("../") + 2);
+                    img.attr("src", src);
+                } else if (src.startsWith("/")) {
+                    src = baseUrl + src;
+                    img.attr("src", src);
+                }
+            }
+            //头部清洗
+            content.select("div.dqwz").remove();
+            content.select("div.contentMemo").remove();
+            content.select("#articlePlayer").remove();
+            content.select("#share").remove();
+            content.select("dl.QrcodeBot").remove();
+            //底部清洗
+            content.select("#file_document").remove();
+            content.select("div.inRight").remove();
+            content.select("script").remove();
+            content.select("style").remove();
+            //save
+            String SOURCE_NAME = "天津市东丽区民政局";
+            String DETAIL_LINK = currentUrl;
+            String DETAIL_TITLE = content.select("h1").first().text().trim();
+            String DETAIL_CONTENT = content.toString();
+            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
+            LocalDate CREATE_TIME = LocalDate.now();
+            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
+            String CREATE_BY = "黄略";
+
+            String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+            try(Connection connection = MyDao.getConnection();
+                PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                preparedStatement.setString(1, SOURCE_NAME);
+                preparedStatement.setString(2, DETAIL_LINK);
+                preparedStatement.setString(3, DETAIL_TITLE);
+                preparedStatement.setString(4, DETAIL_CONTENT);
+                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                preparedStatement.setString(7, LIST_TITLE);
+                preparedStatement.setString(8, CREATE_BY);
+                preparedStatement.executeUpdate();
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    //启动
+    public static void main(String[] args) {
+        Spider spider = Spider.create(new Application2())
+                .addUrl("https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/index.html");
+        System.out.println("start");
+        spider.run();
+        System.out.println("end");
+    }
+}

src/main/java/space/anyi/Application3.java

@@ -0,0 +1,185 @@
+package space.anyi;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.sql.*;
+import java.time.LocalDate;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application3 implements PageProcessor {
+
+    //初始url
+    static String indexUrl = "http://fykye.com/xinwenzixun/zhaotoubiaoxinxi/index%s.html";
+    static int total = 1;
+    static int index = 1;
+    public static final String baseUrl = "http://fykye.com";
+
+    private static final Map<String, Map<String,Object>> context = new HashMap<>();
+
+    //配置
+    private Site site = new Site()
+            .setTimeOut(100 * 1000)     //超时时间
+            .setRetryTimes(3)         //重试请求次数
+            .setRetrySleepTime(3000)  //重试间隔时间                  请求头
+            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");
+
+    @Override
+    public void process(Page page) {
+        String currentUrl = page.getUrl().toString();
+        if (currentUrl.contains("index")) {
+            String rawText = page.getRawText();
+            //body > div:nth-child(6) > table > tbody > tr > td:nth-child(2) > table:nth-child(4) > tbody > tr > td > div > a:nth-child(1) > b
+            Document document =  Jsoup.parse(rawText);
+            //总页数
+            String totalStr = document.select("body > div:nth-child(6) > table > tbody > tr > td:nth-child(2) > table:nth-child(4) > tbody > tr > td > div > a:nth-child(1) > b").first().text();
+
+            int totalRecord = Integer.parseInt(totalStr);
+            total = totalRecord % 5 ==0 ? totalRecord / 5 : (totalRecord / 5) + 1;
+            //body > div:nth-child(6) > table > tbody > tr > td:nth-child(2) > table:nth-child(3) > tbody > tr:nth-child(6) > td:nth-child(2) > a
+            //body > div:nth-child(6) > table > tbody > tr > td:nth-child(2) > table:nth-child(3) tr
+            Elements eachTags = document.select("body > div:nth-child(6) > table > tbody > tr > td:nth-child(2) > table:nth-child(3) tr:has(a)");
+            if (eachTags != null && eachTags.size() > 0) {
+                for (Element eachTag : eachTags) {
+                    //解析详情页url
+                    String href = eachTag.select("a").first().attr("href");
+                    String link = "";
+                    if (href.startsWith("/")) {
+                        link = baseUrl + href;
+                    }else if (href.startsWith("./")){
+                        link = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + href.substring(href.indexOf("./") + 1);
+                    }else if (href.startsWith("../")){
+                        link = baseUrl + link.substring(link.lastIndexOf("../") + 2);
+                    }else {
+                        throw new RuntimeException("列表中详情url非法,请检查");
+                    }
+                    String title = eachTag.select("a").first().text().trim();
+                    String dateStr = eachTag.select("td").last().text().trim();
+                    System.out.println(title);
+                    System.out.println(link);
+                    //数据库是否已存在
+                    String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                    try(Connection connection = MyDao.getConnection();
+                        PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                        prepareStatement.setString(1,link);
+                        ResultSet resultSet = prepareStatement.executeQuery();
+                        if (resultSet.next()) {
+                            System.out.println("跳过" + link);
+                            resultSet.close();
+                            continue;
+                        }
+                        resultSet.close();
+                    } catch (SQLException e) {
+                        throw new RuntimeException(e);
+                    }
+                    //todo
+                    Request request = new Request(link);
+//                    request.putExtra("title",title.substring(2));
+//                    request.putExtra("listDate",LocalDate.parse(dateStr));
+                    Map<String, Object> map = new HashMap<>();
+                    map.put("title",title);
+                    map.put("listDate",LocalDate.parse(dateStr));
+                    context.put(link,map);
+                    page.addTargetRequest(request);
+                }
+            }
+            if (index <= total) {
+                page.addTargetRequest(String.format(indexUrl, "_" + (++index)));
+            }
+            page.setSkip(true);
+        } else {
+            //处理详情网址
+            String rawText = page.getRawText();
+            rawText = rawText.replace("display:none;", "");
+            Document document = Jsoup.parse(rawText);
+            //body > div:nth-child(6) > table > tbody > tr > td:nth-child(2) > table:nth-child(2)
+            Element content = document.select("body > div:nth-child(6) > table > tbody > tr > td:nth-child(2) > table:nth-child(2)").first();
+            //文件url清洗
+            Elements as = content.select("a");
+            for (Element a : as) {
+                String href = a.attr("href");
+                if (href.startsWith("./")) {
+                    href = currentUrl.substring(0, currentUrl.lastIndexOf('/')) + href.substring(href.indexOf("./") + 1);
+                    a.attr("href", href);
+                } else if (href.startsWith("../")) {
+                    href = baseUrl + href.substring(href.lastIndexOf("../") + 2);
+                    a.attr("href", href);
+                } else if (href.startsWith("/")) {
+                    href = baseUrl + href;
+                    a.attr("href", href);
+                }
+            }
+            //图片url清洗
+            Elements imgs = content.select("img");
+            for (Element img : imgs) {
+                String src = img.attr("src");
+                if (src.startsWith("./")) {
+                    src = currentUrl.substring(0, currentUrl.lastIndexOf('/')) + src.substring(src.indexOf("./") + 1);
+                    img.attr("src", src);
+                } else if (src.startsWith("../")) {
+                    src = baseUrl + src.substring(src.lastIndexOf("../") + 2);
+                    img.attr("src", src);
+                } else if (src.startsWith("/")) {
+                    src = baseUrl + src;
+                    img.attr("src", src);
+                }
+            }
+            //清洗
+            content.select("td.info_text").remove();
+            //save
+            Map<String, Object> map = context.get(currentUrl);
+            String SOURCE_NAME = "凤阳琅琊山矿业股份有限公司";
+            String DETAIL_LINK = currentUrl;
+            String DETAIL_TITLE = content.select("h1").first().text().trim();
+            String DETAIL_CONTENT = content.toString();
+            LocalDate PAGE_TIME = ((LocalDate) map.get("listDate"));
+//            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
+            LocalDate CREATE_TIME = LocalDate.now();
+            String LIST_TITLE = ((String) map.get("title"));
+//            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
+            String CREATE_BY = "黄略";
+
+            String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+            try(Connection connection = MyDao.getConnection();
+                PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                preparedStatement.setString(1, SOURCE_NAME);
+                preparedStatement.setString(2, DETAIL_LINK);
+                preparedStatement.setString(3, DETAIL_TITLE);
+                preparedStatement.setString(4, DETAIL_CONTENT);
+                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                preparedStatement.setString(7, LIST_TITLE);
+                preparedStatement.setString(8, CREATE_BY);
+                preparedStatement.executeUpdate();
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    //启动
+    public static void main(String[] args) {
+        Spider spider = Spider.create(new Application3())
+                .addUrl("http://fykye.com/xinwenzixun/zhaotoubiaoxinxi/index.html");
+        System.out.println("start");
+        spider.run();
+        System.out.println("end");
+    }
+}

src/main/java/space/anyi/Application4.java

@@ -0,0 +1,252 @@
+package space.anyi;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.model.HttpRequestBody;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.time.LocalDate;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application4 implements PageProcessor {
+
+    //初始url
+    static String indexUrl = "http://ztb.chinays.gov.cn/ceinwz/WebInfo_List.aspx?newsid=81&jsgc=01100000&zfcg=&tdjy=&cqjy=&qtjy=&PubDateSort=1&ShowPre=1&CbsZgys=0&zbfs=&qxxx=0&showqxname=0&NewsShowPre=1&wsjj=0&showCgr=0&ShowOverDate=0&showdate=1&FromUrl=zbgg";
+    static int total = 1;
+    static int index = 1;
+    public static final String baseUrl = "http://ztb.chinays.gov.cn";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+
+    private static final Map<String, Map<String,Object>> context = new HashMap<>();
+
+    //配置
+    private Site site = new Site()
+            .setTimeOut(100 * 1000)     //超时时间
+            .setRetryTimes(3)         //重试请求次数
+            .setRetrySleepTime(3000)  //重试间隔时间                  请求头
+            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + result.substring(result.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+
+    @Override
+    public void process(Page page) {
+        String currentUrl = page.getUrl().toString();
+        if (currentUrl.contains("WebInfo_List")) {
+            String rawText = page.getRawText();
+            Document document =  Jsoup.parse(rawText);
+            //总页数
+            Elements eachTags = document.select("table.myGVClass tr:has(a)");
+            if (eachTags != null && eachTags.size() > 0) {
+                for (Element eachTag : eachTags) {
+                    //解析详情页url
+                    String href = eachTag.select("a").first().attr("href");
+                    String link = "";
+                    link = urlClean(href,baseUrl,currentUrl);
+                    String title = eachTag.select("a").first().text().trim();
+                    //有可能出現 ”2026-10-14 [正在进行]“
+                    String dateStr = eachTag.select("td.fFbDate").first().text().trim();
+                    Matcher matcher = pattern.matcher(dateStr);
+                    if (matcher.find()) {
+                        String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                        String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                        dateStr = matcher.group(1) + "-" + month + "-" + day;
+                    }
+                    System.out.println(title);
+                    System.out.println(link);
+                    //数据库是否已存在
+                    String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                    try(Connection connection = MyDao.getConnection();
+                        PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                        prepareStatement.setString(1,link);
+                        ResultSet resultSet = prepareStatement.executeQuery();
+                        if (resultSet.next()) {
+                            System.out.println("跳过" + link);
+                            resultSet.close();
+                            continue;
+                        }
+                        resultSet.close();
+                    } catch (SQLException e) {
+                        throw new RuntimeException(e);
+                    }
+                    Request request = new Request(link);
+//                    request.putExtra("title",title.substring(2));
+//                    request.putExtra("listDate",LocalDate.parse(dateStr));
+                    Map<String, Object> map = new HashMap<>();
+                    map.put("title",title);
+                    map.put("listDate",LocalDate.parse(dateStr));
+                    context.put(link,map);
+                    page.addTargetRequest(request);
+                }
+            }
+            //翻頁
+            String __EVENTVALIDATION = document.select("#__EVENTVALIDATION").first().val();
+            String __VIEWSTATEGENERATOR = document.select("#__VIEWSTATEGENERATOR").first().val();
+            String ctl00_myTreeView_ExpandState = document.select("#ctl00_myTreeView_ExpandState").first().val();
+            String __VIEWSTATE = document.select("#__VIEWSTATE").first().val();
+            Element nextPage = document.getElementById("ctl00_ContentPlaceHolder1_LinkButtonNextPage");
+            if (Objects.nonNull(nextPage)) {
+                Map<String, Object> params = new HashMap<>();
+                params.put("__EVENTVALIDATION",__EVENTVALIDATION);
+                params.put("__VIEWSTATEGENERATOR",__VIEWSTATEGENERATOR);
+                params.put("ctl00_myTreeView_ExpandState",ctl00_myTreeView_ExpandState);
+                params.put("__VIEWSTATE",__VIEWSTATE);
+                params.put("__EVENTTARGET","ctl00$ContentPlaceHolder1$LinkButtonNextPage");
+                Request request = new Request(indexUrl);
+                request.setMethod("post");
+                HttpRequestBody requestBody = HttpRequestBody.form(params, StandardCharsets.UTF_8.name());
+                request.setRequestBody(requestBody);
+                page.addTargetRequest(request);
+            }
+            page.setSkip(true);
+        } else {
+            //处理详情网址
+            String rawText = page.getRawText();
+            rawText = rawText.replace("display:none;", "");
+            Document document = Jsoup.parse(rawText);
+            Element content = document.getElementById("form1");
+            if (Objects.isNull(content)) {
+                content = document.select("#aspnetForm > div:nth-child(3) > table:nth-child(4) > tbody:nth-child(2) > tr > td:nth-child(3) > div > table").first();
+            }
+            //文件url清洗
+            Elements elements = content.select("a.lan");
+            if (Objects.nonNull(elements) && elements.size() > 0) {
+                elements.remove();
+            }
+            Elements as = content.select("a");
+            for (Element a : as) {
+                String href = a.attr("href");
+                href = urlClean(href,baseUrl,currentUrl);
+                a.attr("href", href);
+            }
+            //图片url清洗
+            Elements imgs = content.select("img");
+            for (Element img : imgs) {
+                String src = img.attr("src");
+                src = urlClean(src,baseUrl,currentUrl);
+                img.attr("src", src);
+            }
+            //正文清洗
+//            Element frmBestwordHtml = content.getElementById("frmBestwordHtml");
+//            if (Objects.nonNull(frmBestwordHtml)) {
+//                String src = frmBestwordHtml.attr("src");
+//                src = urlClean(src,baseUrl,currentUrl);
+//                frmBestwordHtml.attr("src", src);
+//                try {
+//                    String res = doGet(src);
+//                    Element element = Jsoup.parse(res);
+//                    frmBestwordHtml.replaceWith(element);
+//                } catch (Exception e) {
+//                    throw new RuntimeException(e);
+//                }
+//            }
+            //清洗
+//            content.select("td.info_text").remove();
+            //save
+            Map<String, Object> map = context.get(currentUrl);
+            String SOURCE_NAME = "英山县公共资源交易中心-英山县政府采购中心";
+            String DETAIL_LINK = currentUrl;
+            Element titleElement = content.getElementById("lblXmmc");
+            if (Objects.isNull(titleElement)) {
+                titleElement = content.getElementById("ctl00_ContentPlaceHolder1_TitleLabel");
+            }
+            String DETAIL_TITLE = titleElement.text().trim();
+            String DETAIL_CONTENT = content.toString();
+            LocalDate PAGE_TIME = ((LocalDate) map.get("listDate"));
+//            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
+            LocalDate CREATE_TIME = LocalDate.now();
+            String LIST_TITLE = ((String) map.get("title"));
+//            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
+            String CREATE_BY = "黄略";
+
+            String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+            try(Connection connection = MyDao.getConnection();
+                PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                preparedStatement.setString(1, SOURCE_NAME);
+                preparedStatement.setString(2, DETAIL_LINK);
+                preparedStatement.setString(3, DETAIL_TITLE);
+                preparedStatement.setString(4, DETAIL_CONTENT);
+                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                preparedStatement.setString(7, LIST_TITLE);
+                preparedStatement.setString(8, CREATE_BY);
+                preparedStatement.executeUpdate();
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    //启动
+    public static void main(String[] args) {
+        Spider spider = Spider.create(new Application4())
+                .addUrl("http://ztb.chinays.gov.cn/ceinwz/WebInfo_List.aspx?newsid=81&jsgc=01100000&zfcg=&tdjy=&cqjy=&qtjy=&PubDateSort=1&ShowPre=1&CbsZgys=0&zbfs=&qxxx=0&showqxname=0&NewsShowPre=1&wsjj=0&showCgr=0&ShowOverDate=0&showdate=1&FromUrl=zbgg");
+        System.out.println("start");
+        spider.run();
+        System.out.println("end");
+    }
+}

src/main/java/space/anyi/Application5.java

@@ -0,0 +1,262 @@
+package space.anyi;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application5 {
+    static String indexUrl = "http://ztb.chinays.gov.cn/ceinwz/WebInfo_List.aspx?newsid=81&jsgc=01100000&zfcg=&tdjy=&cqjy=&qtjy=&PubDateSort=1&ShowPre=1&CbsZgys=0&zbfs=&qxxx=0&showqxname=0&NewsShowPre=1&wsjj=0&showCgr=0&ShowOverDate=0&showdate=1&FromUrl=zbgg";
+    static String baseUrl = "https://www.tjdl.gov.cn";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+    static MyDao myDao = new MyDao();
+    public static void main(String[] args) throws Exception {
+        getList();
+    }
+
+//    static int total = 1;
+//    static int index = 0;
+    static int count = 0;
+    static String __EVENTVALIDATION = "";
+    static String __VIEWSTATEGENERATOR = "";
+    static String ctl00_myTreeView_ExpandState = "ennnnn";
+    static String __VIEWSTATE = "";
+    public static void getList() throws Exception {
+        while (true){
+            //列表url构建
+            String listUrl = indexUrl;
+            System.out.println(listUrl);
+            //采集列表页
+            String res;
+            if (count == 0) {
+                res = doGet(listUrl);
+            }else {
+                res = doPost(listUrl);
+            }
+            //采集总页数
+            Document doc = Jsoup.parse(res);
+            //有没有下一页
+            __EVENTVALIDATION = doc.select("#__EVENTVALIDATION").first().val();
+            __VIEWSTATEGENERATOR = doc.select("#__VIEWSTATEGENERATOR").first().val();
+//            ctl00_myTreeView_ExpandState = doc.select("#ctl00_myTreeView_ExpandState").first().val();
+            __VIEWSTATE = doc.select("#__VIEWSTATE").first().val();
+            Element nextPage = doc.getElementById("ctl00_ContentPlaceHolder1_LinkButtonNextPage");
+            //列表
+            Elements eachTags = doc.select("table.myGVClass tr:has(a)");
+            if (eachTags != null) {
+                if (eachTags.size() > 0) {
+                    for (Element eachTag : eachTags) {
+                        count++;
+                        //解析详情页url
+                        String href = eachTag.select("a").first().attr("href");
+                        String link = "";
+                        String currentUrl = listUrl;
+                        link = urlClean(href,baseUrl,currentUrl);
+                        String title = eachTag.select("a").first().text().trim();
+                        System.out.println(title);
+                        System.out.println(link);
+                        //数据库是否已存在
+                        String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                        try(Connection connection = MyDao.getConnection();
+                            PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                            prepareStatement.setString(1,link);
+                            ResultSet resultSet = prepareStatement.executeQuery();
+                            if (resultSet.next()) {
+                                System.out.println("跳过" + link);
+                                resultSet.close();
+                                continue;
+                            }
+                            resultSet.close();
+                        }
+                        //采集详情页
+                        String detail = doGet(link);
+                        detail = detail.replace("display:none;", "");
+                        Document document = Jsoup.parse(detail);
+                        //#form1 > div:nth-child(3) > table > tbody > tr:nth-child(1) > td > table
+                        Element content = document.getElementById("form1");
+                        if (Objects.isNull(content)) {
+                            content = document.select("#aspnetForm > div:nth-child(3) > table:nth-child(4) > tbody:nth-child(2) > tr > td:nth-child(3) > div > table").first();
+                        }
+                        //文件url清洗
+                        Elements elements = content.select("a.lan");
+                        if (Objects.nonNull(elements) && elements.size() > 0) {
+                            elements.remove();
+                        }
+                        Elements as = content.select("a");
+                        for (Element a : as) {
+                            href = a.attr("href");
+                            href = urlClean(href,baseUrl,link);
+                            a.attr("href", href);
+                        }
+                        //图片url清洗
+                        Elements imgs = content.select("img");
+                        for (Element img : imgs) {
+                            String src = img.attr("src");
+                            src = urlClean(src,baseUrl,link);
+                            img.attr("src", src);
+                        }
+                        //正文清洗
+//                        Element frmBestwordHtml = content.getElementById("frmBestwordHtml");
+//                        if (Objects.nonNull(frmBestwordHtml)) {
+//                            String src = frmBestwordHtml.attr("src");
+//                            src = urlClean(src, baseUrl, link);
+//                            frmBestwordHtml.attr("src", src);
+//                            try {
+//                                res = doGet(src);
+//                                Element element = Jsoup.parse(res);
+//                                frmBestwordHtml.replaceWith(element);
+//                            } catch (Exception e) {
+//                                throw new RuntimeException(e);
+//                            }
+//                        }
+
+                        //save
+                        //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+                        String SOURCE_NAME = "英山县公共资源交易中心-英山县政府采购中心";
+                        String DETAIL_LINK = link;
+                        Element titleElement = content.getElementById("lblXmmc");
+                        if (Objects.isNull(titleElement)) {
+                            titleElement = content.getElementById("ctl00_ContentPlaceHolder1_TitleLabel");
+                        }
+                        String DETAIL_TITLE = titleElement.text().trim();
+                        String DETAIL_CONTENT = content.toString();
+                        //有可能出現 ”2026-10-14 [正在进行]“
+                        String dateStr = eachTag.select("td.fFbDate").first().text().trim();
+                        Matcher matcher = pattern.matcher(dateStr);
+                        if (matcher.find()) {
+                            String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                            String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                            dateStr = matcher.group(1) + "-" + month + "-" + day;
+                        }
+                        LocalDate PAGE_TIME = LocalDate.parse(dateStr);
+                        LocalDate CREATE_TIME = LocalDate.now();
+                        String LIST_TITLE = title;
+                        String CREATE_BY = "黄略";
+
+                        String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+                        try(Connection connection = MyDao.getConnection();
+                            PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                            preparedStatement.setString(1, SOURCE_NAME);
+                            preparedStatement.setString(2, DETAIL_LINK);
+                            preparedStatement.setString(3, DETAIL_TITLE);
+                            preparedStatement.setString(4, DETAIL_CONTENT);
+                            preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                            preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                            preparedStatement.setString(7, LIST_TITLE);
+                            preparedStatement.setString(8, CREATE_BY);
+                            preparedStatement.executeUpdate();
+                        }
+                    }
+                    }
+                }
+            if (Objects.isNull(nextPage)) {
+                break;
+            }
+        }
+        System.out.println(count);
+    }
+
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    public static String doPost(String path) throws Exception {
+        String result = "";
+
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        // 创建http POST请求
+        HttpPost httpPost = new HttpPost(path);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpPost.setConfig(requestConfig);
+        List<NameValuePair> parameters = new ArrayList<NameValuePair>(0);
+        parameters.add(new BasicNameValuePair("__EVENTVALIDATION",__EVENTVALIDATION));
+        parameters.add(new BasicNameValuePair("__VIEWSTATEGENERATOR",__VIEWSTATEGENERATOR));
+        parameters.add(new BasicNameValuePair("ctl00_myTreeView_ExpandState",ctl00_myTreeView_ExpandState));
+        parameters.add(new BasicNameValuePair("__VIEWSTATE",__VIEWSTATE));
+        parameters.add(new BasicNameValuePair("__EVENTTARGET","ctl00$ContentPlaceHolder1$LinkButtonNextPage"));
+
+        // 构造一个form表单式的实体
+        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(parameters);
+        // 将请求实体设置到httpPost对象中
+        httpPost.setEntity(formEntity);
+        httpPost.setHeader("User-Agent",
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");
+        CloseableHttpResponse response = null;
+        try {
+            response = httpclient.execute(httpPost);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                return content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpclient.close();
+        }
+        return result;
+    }
+
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + url.substring(url.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+}