feat:爬虫实习第五天

2026.10.23

@@ -1,7 +1,2 @@
 今日学习日报:
-学习列表接口的分页机制，根据总数与每页条数计算总页数并循环翻页至末页；
-学习同一POST接口的两种请求体构造方式，表单参数用NameValuePair+UrlEncodedFormEntity，JSON参数用StringEntity，并补充Content-Type、User-Agent请求头；
-学习HTML中内嵌JSON的提取，用正则截取subjectInfoResult、notice片段后再转为JSONObject解析；
-学习按noticeType多分支处理不同公告类型，分别取字段填充到对应的HTML模板中生成详情；
-学习时间与链接清洗，毫秒时间戳转LocalDateTime、文本日期统一为yyyy-MM-dd，并将iframe、附件、相对a/img地址重写为可访问链接；
-学习断点续采与WebMagic进阶，按DETAIL_LINK查重跳过已采数据，用HttpRequestBody.form发POST、addTargetRequest下钻详情、setSkip跳过列表页、context在列表页与详情页间传参。
+练习webmagic和httpclient分别采集json格式和html格式的数据;

record.md

@@ -0,0 +1,23 @@
+## 根据title包含的关键词决定入哪个库
+```java
+//招标公告
+//分解
+String key = "招标、采购、询价、询比、竞标、竞价、竞谈、竞拍、竞卖、竞买、竞投、竞租、比选、比价、竞争性、谈判、磋商、投标、邀标、议标、议价、单一来源、遴选、标段、明标、明投、出让、转让、拍卖、招租、预审、发包、开标、答疑、补遗、澄清、挂牌";
+String[] keys = key.split("、");
+if (!CheckProclamationUtil.isProclamationValuable(title, keys)) {
+continue;
+}
+//中标信息
+//分解
+if (!CheckProclamationUtil.isProclamationValuable(title)) {
+continue;
+}
+String key = "决算、预算";
+String[] keys = key.split("、");
+if (CheckProclamationUtil.isValuableByExceptTitleKeyWords(title, keys)) {
+continue;
+}
+if (!CheckProclamationUtil.isProclamationValuable(title, null)) {
+continue;
+}
+```

src/main/java/space/anyi/Application10.java

@@ -0,0 +1,224 @@
+package space.anyi;
+
+import org.apache.http.Header;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application10 {
+    static String indexUrl = "http://www.njsgczyy.cn/lnys/lnys6.html?mtt=6";
+    static String baseUrl = "http://www.njsgczyy.cn";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+
+    public static void main(String[] args) throws Exception {
+        getList();
+    }
+
+    static int total = 1;
+    static int index = 1;
+    static int pageSize = 10;
+    static int count = 0;
+    static String __EVENTTARGET = "AspNetPager1";
+    static int __EVENTARGUMENT = index;
+    static String __EVENTVALIDATION = "/wEdAAKvVXD1oYELeveMr0vHCmYPNOl94asWmGCaF2LdqKNWh2llJ1q5QHivvzgg/3iFHWM214ccCG8N+P8o1exiW9Fq";
+
+    static String cookie = "ASP.NET_SessionId=r32rndzwmah1lem5rt5xsecm";
+    public static void getList() throws Exception {
+        while (true){
+            //列表url构建
+            String listUrl = indexUrl;
+            System.out.println(listUrl);
+            //采集列表页
+            String res = doPost(listUrl);
+            Document document = Jsoup.parse(res);
+            __EVENTVALIDATION = document.select("#__EVENTVALIDATION").first().val();
+            Elements items = document.select("div.in-news2 li");
+            for (int i = 0; i < items.size(); i++) {
+                Element item = items.get(i);
+                Element a = item.select("a").first();
+                String title = a.text().trim();
+                String href = a.attr("href");
+                String dateStr = item.select("span").first().text().trim();
+                String link  = urlClean(href,baseUrl,listUrl);
+                System.out.println(link);
+                System.out.println(title);
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                    prepareStatement.setString(1,link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                }
+                //详情
+                String detailRes = doGet(link);
+                Element content = Jsoup.parse(detailRes).select("div.ri-neir").first();
+                String detailTitle = content.select("div.detail.wow.dh3").first().text().trim();
+                //清洗
+                Elements imgs = content.select("img");
+                for (int j = 0; j < imgs.size(); j++) {
+                    Element img = imgs.get(j);
+                    img.attr("src",urlClean(img.attr("src"),baseUrl,link));
+                }
+                Elements as = content.select("a");
+                for (int j = 0; j < as.size(); j++) {
+                    a = as.get(j);
+                    a.attr("href",urlClean(a.attr("href"),baseUrl,link));
+                }
+                content.select("div.xx.wow.dh3").remove();
+                //save
+                //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+                String SOURCE_NAME = "南京市高淳中医院";
+                String DETAIL_LINK = link;
+                String DETAIL_TITLE = detailTitle;
+                String DETAIL_CONTENT = content.toString();
+                Matcher matcher = pattern.matcher(dateStr);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    dateStr = matcher.group(1) + "-" + month + "-" + day;
+                }
+                LocalDate PAGE_TIME = LocalDate.parse(dateStr);
+                LocalDate CREATE_TIME = LocalDate.now();
+                String LIST_TITLE = title;
+                String CREATE_BY = "黄略";
+
+                String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                    preparedStatement.setString(1, SOURCE_NAME);
+                    preparedStatement.setString(2, DETAIL_LINK);
+                    preparedStatement.setString(3, DETAIL_TITLE);
+                    preparedStatement.setString(4, DETAIL_CONTENT);
+                    preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                    preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                    preparedStatement.setString(7, LIST_TITLE);
+                    preparedStatement.setString(8, CREATE_BY);
+                    preparedStatement.executeUpdate();
+                }
+                count++;
+            }
+            //下一页
+            Element nextPage = document.select("#AspNetPager1 a[disabled=disabled]:contains(下一页)").first();
+            if (Objects.isNull(nextPage)) {
+                index++;
+            }else {
+                break;
+            }
+        }
+        System.out.println(count);
+    }
+
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    public static String doPost(String path) throws Exception {
+        String result = "";
+
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        // 创建http POST请求
+        HttpPost httpPost = new HttpPost(path);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpPost.setConfig(requestConfig);
+        List<NameValuePair> parameters = new ArrayList<NameValuePair>(0);
+        parameters.add(new BasicNameValuePair("__EVENTTARGET",__EVENTTARGET));
+        __EVENTARGUMENT = index;
+        parameters.add(new BasicNameValuePair("__EVENTARGUMENT",__EVENTARGUMENT+""));
+        parameters.add(new BasicNameValuePair("__EVENTVALIDATION",__EVENTVALIDATION));
+        // 构造一个form表单式的实体
+        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(parameters);
+        // 将请求实体设置到httpPost对象中
+        httpPost.setEntity(formEntity);
+        httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded");
+        httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");
+        httpPost.setHeader("cookie", cookie);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpclient.execute(httpPost);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                return content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpclient.close();
+        }
+        return result;
+    }
+
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + url.substring(url.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+}

src/main/java/space/anyi/Application11.java

@@ -0,0 +1,199 @@
+package space.anyi;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.model.HttpRequestBody;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.time.LocalDate;
+import java.util.HashMap;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application11 implements PageProcessor {
+
+    // 初始url
+    static String indexUrl = "http://www.njsgczyy.cn/lnys/lnys6.html?mtt=6";
+    static String baseUrl = "http://www.njsgczyy.cn";
+    static int total = 0;
+    static int index = 1;
+    static String __EVENTTARGET = "AspNetPager1";
+    static int __EVENTARGUMENT = index;
+    static String __EVENTVALIDATION =
+            "/wEdAAKvVXD1oYELeveMr0vHCmYPNOl94asWmGCaF2LdqKNWh2llJ1q5QHivvzgg/3iFHWM214ccCG8N+P8o1exiW9Fq";
+    public static Pattern pattern =
+            Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+    // 配置
+    private Site site =
+            new Site()
+                    .setCharset(StandardCharsets.UTF_8.name())
+                    .setTimeOut(100 * 1000) // 超时时间
+                    .setRetryTimes(3) // 重试请求次数
+                    .setRetrySleepTime(3000) // 重试间隔时间                  请求头
+                    .setUserAgent(
+                            "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");
+
+    public static String urlClean(String url, String baseUrl, String currentUrl) {
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            // 根路径开始
+            result = baseUrl + url;
+        } else if (url.startsWith("./")) {
+            // 当前同级目录
+            result =
+                    currentUrl.substring(0, currentUrl.lastIndexOf("/"))
+                            + url.substring(url.indexOf("./") + 1);
+        } else if (url.startsWith("../")) {
+            // 多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + result.substring(result.lastIndexOf("../") + 2);
+        } else {
+            // 等价于当前目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+
+    @Override
+    public void process(Page page) {
+        String currentUrl = page.getUrl().toString();
+        if (currentUrl.equals(indexUrl)) {
+            String res = page.getRawText();
+            Document document = Jsoup.parse(res);
+            __EVENTVALIDATION = document.select("#__EVENTVALIDATION").first().val();
+            Elements items = document.select("div.in-news2 li");
+            for (int i = 0; i < items.size(); i++) {
+                Element item = items.get(i);
+                Element a = item.select("a").first();
+                String title = a.text().trim();
+                String href = a.attr("href");
+                String dateStr = item.select("span").first().text().trim();
+                Matcher matcher = pattern.matcher(dateStr);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    dateStr = matcher.group(1) + "-" + month + "-" + day;
+                }
+                String link = urlClean(href, baseUrl, currentUrl);
+                System.out.println(link);
+                System.out.println(title);
+                // 数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try (Connection connection = MyDao.getConnection();
+                     PreparedStatement prepareStatement = connection.prepareStatement(querySql);) {
+                    prepareStatement.setString(1, link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                } catch (SQLException e) {
+                    throw new RuntimeException(e);
+                }
+                // 详情请求
+                Request detailRequest = new Request(link);
+                detailRequest.putExtra("title", title);
+                detailRequest.putExtra("listDate", LocalDate.parse(dateStr));
+                page.addTargetRequest(detailRequest);
+            }
+            // 下一页
+            Element nextPage = document.select("#AspNetPager1 a[disabled=disabled]:contains(下一页)").first();
+            if (Objects.isNull(nextPage)) {
+                index++;
+                HashMap<String, Object> params = new HashMap<>();
+                params.put("__EVENTTARGET", __EVENTTARGET);
+                __EVENTARGUMENT = index;
+                params.put("__EVENTARGUMENT", __EVENTARGUMENT);
+                params.put("__EVENTVALIDATION", __EVENTVALIDATION);
+                HttpRequestBody requestBody = HttpRequestBody.form(params, StandardCharsets.UTF_8.name());
+                Request request = new Request(indexUrl).setMethod("post");
+                request.addHeader("content-type", "application/x-www-form-urlencoded; charset=UTF-8");
+                request.setRequestBody(requestBody);
+                page.addTargetRequest(request);
+            }
+            page.setSkip(true);
+        } else {
+            // 处理详情网址
+            String detailRes = page.getRawText();
+            Element content = Jsoup.parse(detailRes).select("div.ri-neir").first();
+            String detailTitle = content.select("div.detail.wow.dh3").first().text().trim();
+            // 清洗
+            Elements imgs = content.select("img");
+            for (int j = 0; j < imgs.size(); j++) {
+                Element img = imgs.get(j);
+                img.attr("src", urlClean(img.attr("src"), baseUrl, currentUrl));
+            }
+            Elements as = content.select("a");
+            for (int j = 0; j < as.size(); j++) {
+                Element a = as.get(j);
+                a.attr("href", urlClean(a.attr("href"), baseUrl, currentUrl));
+            }
+            content.select("div.xx.wow.dh3").remove();
+
+            // save
+            // 字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+            String SOURCE_NAME = "南京市高淳中医院";
+            String DETAIL_LINK = currentUrl;
+            String DETAIL_TITLE = detailTitle;
+            String DETAIL_CONTENT = content.toString();
+            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
+            LocalDate CREATE_TIME = LocalDate.now();
+            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
+            String CREATE_BY = "黄略";
+
+            String sql =
+                    "insert into "
+                            + MyDao.TABLE_NAME
+                            + "(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+            try (Connection connection = MyDao.getConnection();
+                 PreparedStatement preparedStatement = connection.prepareStatement(sql);) {
+                preparedStatement.setString(1, SOURCE_NAME);
+                preparedStatement.setString(2, DETAIL_LINK);
+                preparedStatement.setString(3, DETAIL_TITLE);
+                preparedStatement.setString(4, DETAIL_CONTENT);
+                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME)); // LocalDate -> java.sql.Date
+                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                preparedStatement.setString(7, LIST_TITLE);
+                preparedStatement.setString(8, CREATE_BY);
+                preparedStatement.executeUpdate();
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    // 启动
+    public static void main(String[] args) {
+        HashMap<String, Object> params = new HashMap<>();
+        params.put("__EVENTTARGET", __EVENTTARGET);
+        params.put("__EVENTARGUMENT", __EVENTARGUMENT);
+        params.put("__EVENTVALIDATION", __EVENTVALIDATION);
+        HttpRequestBody requestBody = HttpRequestBody.form(params, StandardCharsets.UTF_8.name());
+        Request request = new Request(indexUrl).setMethod("post");
+        request.addHeader("content-type", "application/x-www-form-urlencoded; charset=UTF-8");
+        request.setRequestBody(requestBody);
+        Spider spider = Spider.create(new Application11()).addRequest(request);
+        System.out.println("start");
+        spider.run();
+        System.out.println("end");
+    }
+}

src/main/java/space/anyi/Application12.java

@@ -0,0 +1,267 @@
+package space.anyi;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.model.HttpRequestBody;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.utils.HttpConstant;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application12 implements PageProcessor {
+
+    //初始url
+    static String indexUrl = "https://ggzy.hainan.gov.cn/inteligentsearch/rest/esinteligentsearch/getFullTextDataNew";
+    static String baseUrl = "https://ggzy.hainan.gov.cn";
+    static int total = 0;
+    static int index = 0;
+    static int pageSize = 10;
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+    //配置
+    private Site site = new Site().setCharset(StandardCharsets.UTF_8.name())
+            .setTimeOut(100 * 1000)     //超时时间
+            .setRetryTimes(3)         //重试请求次数
+            .setRetrySleepTime(3000)  //重试间隔时间                  请求头
+            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + result.substring(result.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+
+    @Override
+    public void process(Page page) {
+        String currentUrl = page.getUrl().toString();
+        if (currentUrl.equals(indexUrl)) {
+            String res = page.getRawText();
+            JSONObject result = new JSONObject(res).getJSONObject("result");
+            total = result.getInt("totalcount");
+            JSONArray items = result.getJSONArray("records");
+            for (int i = 0; i < items.length(); i++) {
+                JSONObject item = items.getJSONObject(i);
+                String title = item.get("title").toString();
+                String href = item.get("linkurl").toString();
+                String dateStr = item.get("infodate").toString();
+                String link  = urlClean(href,baseUrl,currentUrl);
+                System.out.println(link);
+                System.out.println(title);
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                    prepareStatement.setString(1,link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                } catch (SQLException e) {
+                    throw new RuntimeException(e);
+                }
+                //详情请求
+                Request detailRequest = new Request(link);
+                detailRequest.putExtra("title",title);
+                detailRequest.putExtra("listDate",LocalDate.parse(dateStr));
+                page.addTargetRequest(detailRequest);
+            }
+            //下一页
+            if (index<=total) {
+                index+=pageSize;
+                String endTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
+                String jsonStr = "{\n" +
+                        "  \"token\": \"\",\n" +
+                        "  \"pn\": "+index+",\n" +
+                        "  \"rn\": "+pageSize+",\n" +
+                        "  \"sdt\": \"\",\n" +
+                        "  \"edt\": \"\",\n" +
+                        "  \"wd\": \" \",\n" +
+                        "  \"inc_wd\": \"\",\n" +
+                        "  \"exc_wd\": \"\",\n" +
+                        "  \"fields\": \"title\",\n" +
+                        "  \"cnum\": \"001\",\n" +
+                        "  \"sort\": \"{\\\"istop\\\":\\\"0\\\",\\\"ordernum\\\":\\\"0\\\",\\\"webdate\\\":\\\"0\\\"}\",\n" +
+                        "  \"ssort\": \"title\",\n" +
+                        "  \"cl\": 200,\n" +
+                        "  \"terminal\": \"\",\n" +
+                        "  \"condition\": [\n" +
+                        "    {\n" +
+                        "      \"fieldName\": \"categorynum\",\n" +
+                        "      \"equal\": \"003001001\",\n" +
+                        "      \"notEqual\": null,\n" +
+                        "      \"equalList\": null,\n" +
+                        "      \"notEqualList\": null,\n" +
+                        "      \"isLike\": true,\n" +
+                        "      \"likeType\": 2\n" +
+                        "    }\n" +
+                        "  ],\n" +
+                        "  \"time\": [\n" +
+                        "    {\n" +
+                        "      \"fieldName\": \"webdate\",\n" +
+                        "      \"startTime\": \"1970-01-01 00:00:00\",\n" +
+                        "      \"endTime\": \""+endTime+"\"\n" +
+                        "    }\n" +
+                        "  ],\n" +
+                        "  \"highlights\": \"title\",\n" +
+                        "  \"statistics\": null,\n" +
+                        "  \"unionCondition\": null,\n" +
+                        "  \"accuracy\": \"\",\n" +
+                        "  \"noParticiple\": \"1\",\n" +
+                        "  \"searchRange\": [],\n" +
+                        "  \"isBusiness\": \"1\"\n" +
+                        "}";
+                HttpRequestBody requestBody =  HttpRequestBody.json(jsonStr,StandardCharsets.UTF_8.name());
+                Request request = new Request(indexUrl).setMethod(HttpConstant.Method.POST);
+                request.addHeader("content-type","application/json;");
+                request.setRequestBody(requestBody);
+                page.addTargetRequest(request);
+            }
+            page.setSkip(true);
+        } else {
+            //处理详情网址
+            String detailRes = page.getRawText();
+            Document detailDocument = Jsoup.parse(detailRes);
+            Element content = detailDocument.select("div.article-info.jyxx-info").first();
+            String detailTitle = detailDocument.select("h3.infotitle").first().text().trim();
+            //清洗
+            Elements imgs = content.select("img");
+            for (int j = 0; j < imgs.size(); j++) {
+                Element img = imgs.get(j);
+                img.attr("src",urlClean(img.attr("src"),baseUrl,currentUrl));
+            }
+            Elements as = content.select("a");
+            for (int j = 0; j < as.size(); j++) {
+                Element a = as.get(j);
+                String onclick = a.attr("onclick");
+                Pattern p = Pattern.compile("ztbfjyz\\('([^']+)'");
+                Matcher m = p.matcher(onclick);
+                if (m.find()) {
+                    String url = m.group(1);
+                    a.attr("href",urlClean(url,baseUrl,currentUrl));
+                }
+            }
+            content.select("div.hidden").remove();
+
+            //save
+            //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+            String SOURCE_NAME = "海南省公共资源交易服务平台";
+            String DETAIL_LINK = currentUrl;
+            String DETAIL_TITLE = detailTitle;
+            String DETAIL_CONTENT = content.toString();
+            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
+            LocalDate CREATE_TIME = LocalDate.now();
+            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
+            String CREATE_BY = "黄略";
+
+            String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+            try(Connection connection = MyDao.getConnection();
+                PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                preparedStatement.setString(1, SOURCE_NAME);
+                preparedStatement.setString(2, DETAIL_LINK);
+                preparedStatement.setString(3, DETAIL_TITLE);
+                preparedStatement.setString(4, DETAIL_CONTENT);
+                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                preparedStatement.setString(7, LIST_TITLE);
+                preparedStatement.setString(8, CREATE_BY);
+                preparedStatement.executeUpdate();
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    //启动
+    public static void main(String[] args) {
+        String endTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
+        String jsonStr = "{\n" +
+                "  \"token\": \"\",\n" +
+                "  \"pn\": "+index+",\n" +
+                "  \"rn\": "+pageSize+",\n" +
+                "  \"sdt\": \"\",\n" +
+                "  \"edt\": \"\",\n" +
+                "  \"wd\": \" \",\n" +
+                "  \"inc_wd\": \"\",\n" +
+                "  \"exc_wd\": \"\",\n" +
+                "  \"fields\": \"title\",\n" +
+                "  \"cnum\": \"001\",\n" +
+                "  \"sort\": \"{\\\"istop\\\":\\\"0\\\",\\\"ordernum\\\":\\\"0\\\",\\\"webdate\\\":\\\"0\\\"}\",\n" +
+                "  \"ssort\": \"title\",\n" +
+                "  \"cl\": 200,\n" +
+                "  \"terminal\": \"\",\n" +
+                "  \"condition\": [\n" +
+                "    {\n" +
+                "      \"fieldName\": \"categorynum\",\n" +
+                "      \"equal\": \"003001001\",\n" +
+                "      \"notEqual\": null,\n" +
+                "      \"equalList\": null,\n" +
+                "      \"notEqualList\": null,\n" +
+                "      \"isLike\": true,\n" +
+                "      \"likeType\": 2\n" +
+                "    }\n" +
+                "  ],\n" +
+                "  \"time\": [\n" +
+                "    {\n" +
+                "      \"fieldName\": \"webdate\",\n" +
+                "      \"startTime\": \"1970-01-01 00:00:00\",\n" +
+                "      \"endTime\": \""+endTime+"\"\n" +
+                "    }\n" +
+                "  ],\n" +
+                "  \"highlights\": \"title\",\n" +
+                "  \"statistics\": null,\n" +
+                "  \"unionCondition\": null,\n" +
+                "  \"accuracy\": \"\",\n" +
+                "  \"noParticiple\": \"1\",\n" +
+                "  \"searchRange\": [],\n" +
+                "  \"isBusiness\": \"1\"\n" +
+                "}";
+        HttpRequestBody requestBody =  HttpRequestBody.json(jsonStr,StandardCharsets.UTF_8.name());
+        Request request = new Request(indexUrl).setMethod(HttpConstant.Method.POST);
+        request.addHeader("content-type","application/json;");
+        request.setRequestBody(requestBody);
+        Spider spider = Spider.create(new Application12())
+                .addRequest(request);
+        System.out.println("start");
+        spider.run();
+        System.out.println("end");
+    }
+}

src/main/java/space/anyi/Application13.java

@@ -0,0 +1,225 @@
+package space.anyi;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import space.anyi.MyDao;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application13 {
+    static String indexUrl = "https://ggzy.hainan.gov.cn/inteligentsearch/rest/esinteligentsearch/getFullTextDataNew";
+    static String baseUrl = "https://ggzy.hainan.gov.cn";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+
+    public static void main(String[] args) throws Exception {
+        getList();
+    }
+
+    static int total = 0;
+    static int index = 0;
+    static int pageSize = 10;
+    static int count = 0;
+    public static void getList() throws Exception {
+        while (true){
+            //列表url构建
+            String listUrl = indexUrl;
+            System.out.println(listUrl);
+            //采集列表页
+            String res = doPost(listUrl);
+            JSONObject result = new JSONObject(res).getJSONObject("result");
+            total = result.getInt("totalcount");
+            JSONArray items = result.getJSONArray("records");
+            for (int i = 0; i < items.length(); i++) {
+                JSONObject item = items.getJSONObject(i);
+                String title = item.get("title").toString();
+                String href = item.get("linkurl").toString();
+                String dateStr = item.get("infodate").toString();
+                String link  = urlClean(href,baseUrl,listUrl);
+                System.out.println(link);
+                System.out.println(title);
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                    prepareStatement.setString(1,link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                }
+                //详情
+                String detailRes = doGet(link);
+                Document detailDocument = Jsoup.parse(detailRes);
+                Element content = detailDocument.select("div.article-info.jyxx-info").first();
+                String detailTitle = detailDocument.select("h3.infotitle").first().text().trim();
+                //清洗
+                Elements imgs = content.select("img");
+                for (int j = 0; j < imgs.size(); j++) {
+                    Element img = imgs.get(j);
+                    img.attr("src",urlClean(img.attr("src"),baseUrl,link));
+                }
+                Elements as = content.select("a");
+                for (int j = 0; j < as.size(); j++) {
+                    Element a = as.get(j);
+                    String onclick = a.attr("onclick");
+                    Pattern p = Pattern.compile("ztbfjyz\\('([^']+)'");
+                    Matcher m = p.matcher(onclick);
+                    if (m.find()) {
+                        String url = m.group(1);
+                        a.attr("href",urlClean(url,baseUrl,link));
+                    }
+                }
+                content.select("div.hidden").remove();
+                //save
+                //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+                String SOURCE_NAME = "海南省公共资源交易服务平台";
+                String DETAIL_LINK = link;
+                String DETAIL_TITLE = detailTitle;
+                String DETAIL_CONTENT = content.toString();
+                Matcher matcher = pattern.matcher(dateStr);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    dateStr = matcher.group(1) + "-" + month + "-" + day;
+                }
+                LocalDate PAGE_TIME = LocalDate.parse(dateStr);
+                LocalDate CREATE_TIME = LocalDate.now();
+                String LIST_TITLE = title;
+                String CREATE_BY = "黄略";
+
+                String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                    preparedStatement.setString(1, SOURCE_NAME);
+                    preparedStatement.setString(2, DETAIL_LINK);
+                    preparedStatement.setString(3, DETAIL_TITLE);
+                    preparedStatement.setString(4, DETAIL_CONTENT);
+                    preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                    preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                    preparedStatement.setString(7, LIST_TITLE);
+                    preparedStatement.setString(8, CREATE_BY);
+                    preparedStatement.executeUpdate();
+                }
+                count++;
+            }
+            //下一页
+            if (index<=total) {
+                index+=pageSize;
+            }else {
+                break;
+            }
+        }
+        System.out.println(count);
+    }
+
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    public static String doPost(String path) throws Exception {
+        String result = "";
+
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        // 创建http POST请求
+        HttpPost httpPost = new HttpPost(path);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpPost.setConfig(requestConfig);
+        String endTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
+        String jsonStr  = String.format("{\"token\":\"\",\"pn\":%d,\"rn\":%d,\"sdt\":\"\",\"edt\":\"\",\"wd\":\"\",\"inc_wd\":\"\",\"exc_wd\":\"\",\"fields\":\"title\",\"cnum\":\"001\",\"sort\":\"{\\\"istop\\\":\\\"0\\\",\\\"ordernum\\\":\\\"0\\\",\\\"webdate\\\":\\\"0\\\"}\",\"ssort\":\"title\",\"cl\":200,\"terminal\":\"\",\"condition\":[{\"fieldName\":\"categorynum\",\"equal\":\"003001001\",\"notEqual\":null,\"equalList\":null,\"notEqualList\":null,\"isLike\":true,\"likeType\":2}],\"time\":[{\"fieldName\":\"webdate\",\"startTime\":\"1970-01-01 00:00:00\",\"endTime\":\"%s\"}],\"highlights\":\"title\",\"statistics\":null,\"unionCondition\":null,\"accuracy\":\"\",\"noParticiple\":\"1\",\"searchRange\":[],\"isBusiness\":\"1\"}", index,pageSize,endTime);
+        // 构造一个json实体
+        StringEntity jsonEntity = new StringEntity(jsonStr, StandardCharsets.UTF_8);
+        // 将请求实体设置到httpPost对象中
+        httpPost.setEntity(jsonEntity);
+        httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");
+        httpPost.setHeader("content-type", "application/json;");
+        CloseableHttpResponse response = null;
+        try {
+            response = httpclient.execute(httpPost);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                return content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpclient.close();
+        }
+        return result;
+    }
+
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + url.substring(url.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+}

src/main/java/space/anyi/Application14.java

@@ -0,0 +1,307 @@
+package space.anyi;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import space.anyi.MyDao;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application14 {
+    static String indexUrl = "https://zfcj.gz.gov.cn/ysqgk/Api/WebApi/xfsygsxxlb.ashx?gcmc&page=%d&pageSize=%d";
+    static String baseUrl = "https://zfcj.gz.gov.cn";
+    static String linkTemplate = "https://zfcj.gz.gov.cn/zfcj/gczlaq/fireInspectionInformation/?md5=%s";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+
+    public static void main(String[] args) throws Exception {
+        getList();
+    }
+
+    static int total = 1;
+    static int index = 1;
+    static int pageSize = 10;
+    static int count = 0;
+
+    public static void getList() throws Exception {
+        while (true) {
+            //列表url构建
+            String listUrl = String.format(indexUrl, index, pageSize);
+            System.out.println(listUrl);
+            //采集列表页
+            String res = doPost(listUrl);
+            JSONObject result = new JSONObject(res.trim());
+            JSONArray items = result.getJSONArray("data");
+            total = result.getInt("totalPage");
+            for (int i = 0; i < items.length(); i++) {
+                JSONObject item = items.getJSONObject(i);
+                //工程名称
+                String gcmc = item.get("gcmc").toString();
+                //单位名称
+                String jsdw = item.get("jsdw").toString();
+                //工程地点
+                String gcdz = item.get("gcdz").toString();
+                //文书编号
+                String wsbh = item.get("wsbh").toString();
+                //办结时间
+                String jasj = item.get("jasj").toString();
+                //结果
+                String jg = item.get("jg").toString();
+                //办理部门
+                String spbm = item.get("spbm").toString();
+                //事项名称
+                String splx = item.get("splx").toString();
+                String md5 = md5(new StringBuilder(gcmc)
+                        .append(jsdw)
+                        .append(gcdz)
+                        .append(wsbh)
+                        .append(jasj)
+                        .append(jg)
+                        .append(spbm)
+                        .append(splx)
+                        .toString());
+                String title = gcmc;
+                String dateStr = jasj;
+                String link = String.format(linkTemplate, md5);
+                System.out.println(link);
+                System.out.println(title);
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try (Connection connection = MyDao.getConnection();
+                     PreparedStatement prepareStatement = connection.prepareStatement(querySql);) {
+                    prepareStatement.setString(1, link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                }
+                //详情
+//                String detailRes = doGet(link);
+                String content = "<table class=\"table-list\">\n" +
+                        "\t<thead>\n" +
+                        "\t\t<tr>\n" +
+                        "\t\t\t<th width=\"18%\">\n" +
+                        "\t\t\t\t工程名称\n" +
+                        "\t\t\t</th>\n" +
+                        "\t\t\t<th width=\"12%\">\n" +
+                        "\t\t\t\t单位名称\n" +
+                        "\t\t\t</th>\n" +
+                        "\t\t\t<th width=\"20%\">\n" +
+                        "\t\t\t\t工程地点\n" +
+                        "\t\t\t</th>\n" +
+                        "\t\t\t<th width=\"10%\">\n" +
+                        "\t\t\t\t文书编号\n" +
+                        "\t\t\t</th>\n" +
+                        "\t\t\t<th width=\"8%\">\n" +
+                        "\t\t\t\t办结时间\n" +
+                        "\t\t\t</th>\n" +
+                        "\t\t\t<th width=\"7%\">\n" +
+                        "\t\t\t\t结果\n" +
+                        "\t\t\t</th>\n" +
+                        "\t\t\t<th width=\"13%\">\n" +
+                        "\t\t\t\t办理部门\n" +
+                        "\t\t\t</th>\n" +
+                        "\t\t\t<th width=\"15%\">\n" +
+                        "\t\t\t\t事项名称\n" +
+                        "\t\t\t</th>\n" +
+                        "\t\t</tr>\n" +
+                        "\t</thead>\n" +
+                        "\t<tbody>\n" +
+                        "\t\t<tr>\n" +
+                        "\t\t\t<td class=\"text-left\">\n" +
+                        "\t\t\t\t" + gcmc + "\n" +
+                        "\t\t\t</td>\n" +
+                        "\t\t\t<td class=\"text-left\">\n" +
+                        "\t\t\t\t" + jsdw + "\n" +
+                        "\t\t\t</td>\n" +
+                        "\t\t\t<td class=\"text-left\">\n" +
+                        "\t\t\t\t" + gcdz + "\n" +
+                        "\t\t\t</td>\n" +
+                        "\t\t\t<td class=\"text-left\">\n" +
+                        "\t\t\t\t" + wsbh + "\n" +
+                        "\t\t\t</td>\n" +
+                        "\t\t\t<td>\n" +
+                        "\t\t\t\t" + jasj + "\n" +
+                        "\t\t\t</td>\n" +
+                        "\t\t\t<td>\n" +
+                        "\t\t\t\t" + jg + "\n" +
+                        "\t\t\t</td>\n" +
+                        "\t\t\t<td class=\"text-left\">\n" +
+                        "\t\t\t\t" + spbm + "\n" +
+                        "\t\t\t</td>\n" +
+                        "\t\t\t<td class=\"text-left\">\n" +
+                        "\t\t\t\t" + splx + "\n" +
+                        "\t\t\t</td>\n" +
+                        "\t\t</tr>\n" +
+                        "\t</tbody>\n" +
+                        "</table>";
+
+                //save
+                //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+                String SOURCE_NAME = "广州市住房和城乡建设局";
+                String DETAIL_LINK = link;
+                String DETAIL_TITLE = gcmc;
+                String DETAIL_CONTENT = content.toString();
+                Matcher matcher = pattern.matcher(dateStr);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    dateStr = matcher.group(1) + "-" + month + "-" + day;
+                }
+                LocalDate PAGE_TIME = LocalDate.parse(dateStr);
+                LocalDate CREATE_TIME = LocalDate.now();
+                String LIST_TITLE = title;
+                String CREATE_BY = "黄略";
+
+                String sql = "insert into " + MyDao.TABLE_NAME + "(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+                try (Connection connection = MyDao.getConnection();
+                     PreparedStatement preparedStatement = connection.prepareStatement(sql);) {
+                    preparedStatement.setString(1, SOURCE_NAME);
+                    preparedStatement.setString(2, DETAIL_LINK);
+                    preparedStatement.setString(3, DETAIL_TITLE);
+                    preparedStatement.setString(4, DETAIL_CONTENT);
+                    preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                    preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                    preparedStatement.setString(7, LIST_TITLE);
+                    preparedStatement.setString(8, CREATE_BY);
+                    preparedStatement.executeUpdate();
+                }
+                count++;
+            }
+            //下一页
+            if (index < total) {
+                index++;
+            } else {
+                break;
+            }
+        }
+        System.out.println(count);
+    }
+
+    private static String md5(String message) {
+        if (message == null) {
+            return null;
+        }
+        try {
+            MessageDigest md = MessageDigest.getInstance("MD5");
+            byte[] digest = md.digest(message.getBytes(StandardCharsets.UTF_8));
+
+            // 字节数组转 32 位小写十六进制
+            StringBuilder sb = new StringBuilder(digest.length * 2);
+            for (byte b : digest) {
+                sb.append(Character.forDigit((b >> 4) & 0xF, 16));
+                sb.append(Character.forDigit(b & 0xF, 16));
+            }
+            return sb.toString();
+        } catch (NoSuchAlgorithmException e) {
+            // MD5 是 JDK 必备算法，正常不会走到这里
+            throw new IllegalStateException("MD5 algorithm not available", e);
+        }
+    }
+
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    public static String doPost(String path) throws Exception {
+        String result = "";
+
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        // 创建http POST请求
+        HttpPost httpPost = new HttpPost(path);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpPost.setConfig(requestConfig);
+        List<NameValuePair> parameters = new ArrayList<NameValuePair>(0);
+//        parameters.add(new BasicNameValuePair("__EVENTTARGET",__EVENTTARGET));
+//        __EVENTARGUMENT = index;
+//        parameters.add(new BasicNameValuePair("__EVENTARGUMENT",__EVENTARGUMENT+""));
+//        parameters.add(new BasicNameValuePair("__EVENTVALIDATION",__EVENTVALIDATION));
+
+        // 构造一个form表单式的实体
+        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(parameters);
+        // 将请求实体设置到httpPost对象中
+        httpPost.setEntity(formEntity);
+        httpPost.setHeader("User-Agent",
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");
+        CloseableHttpResponse response = null;
+        try {
+            response = httpclient.execute(httpPost);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                return content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpclient.close();
+        }
+        return result;
+    }
+
+    public static String urlClean(String url, String baseUrl, String currentUrl) {
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        } else if (url.startsWith("./")) {
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        } else if (url.startsWith("../")) {
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + url.substring(url.lastIndexOf("../") + 2);
+        } else {
+            //等价于当前目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+}

src/main/java/space/anyi/Application15.java

@@ -0,0 +1,243 @@
+package space.anyi;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application15 {
+    static String indexUrl = "https://hjpt.580168.net:32443/cbmsApi/api/WebNews/list";
+    static String baseUrl = "https://hjpt.580168.net:32443";
+    static String detailUrlTemplate = "https://hjpt.580168.net:32443/web/#/Sub/NewsDetail?id=%s&webKind=%s&bydyUrl=";
+    static String detailUrl = "https://hjpt.580168.net:32443/cbmsApi/api/WebNews/website-detail";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+
+    public static void main(String[] args) throws Exception {
+        getList();
+    }
+
+    static int total = 1;
+    static int index = 1;
+    static int pageSize = 10;
+    static int count = 0;
+
+    public static void getList() throws Exception {
+        while (true) {
+            //列表url构建
+            String listUrl = indexUrl;
+            System.out.println(listUrl);
+            //采集列表页
+            String res = doPost(listUrl, String.format("{\"KW\": \"\", \"pageNumber\": %d, \"pageSize\": %d, \"searchContent\": false, \"subIndex\": 0, \"u\": \"zfcg=01000000&newsid=11,17,20,111\", \"xmlx\": \"\"}", index, pageSize));
+            JSONObject result = new JSONObject(res).getJSONArray("outDatas").getJSONObject(0);
+            total = result.getInt("total");
+            JSONArray items = result.getJSONArray("rows");
+            for (int i = 0; i < items.length(); i++) {
+                JSONObject item = items.getJSONObject(i);
+                String title = item.get("title").toString();
+                String dateStr = item.get("pubdate").toString();
+                String nid = item.get("nid").toString();
+                String webKind = item.get("webKind").toString();
+                String link = String.format(detailUrlTemplate, nid, webKind);
+                System.out.println(link);
+                System.out.println(title);
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try (Connection connection = MyDao.getConnection();
+                     PreparedStatement prepareStatement = connection.prepareStatement(querySql);) {
+                    prepareStatement.setString(1, link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                }
+                //详情
+                String gcmc = "";
+                String contentStr = "";
+                switch (webKind){
+                    case "zbgg":
+                        String detailRes = doPost(detailUrl, String.format("{\"id\":\"%s\",\"webKind\":\"%s\"}", nid, webKind));
+                        JSONObject detailResult = new JSONObject(detailRes).getJSONArray("outDatas").getJSONObject(0);
+                        //项目编号
+                        String sgzbbm = detailResult.get("sgzbbm").toString();
+                        //项目名称
+                        gcmc = detailResult.get("gcmc").toString();
+                        //需求公示
+                        String yggContent = detailResult.optString("yggContent");
+                        //采购公告
+                        String zbggContent = detailResult.optString("zbggContent","");
+                        //结果公告
+                        String jggsContent = detailResult.optString("jggsContent","");
+                        contentStr = String.format("<h1>项目编号:%s</h1><h1>项目名称:%s</h1>", sgzbbm, gcmc)
+                                .concat(yggContent)
+                                .concat(zbggContent)
+                                .concat(jggsContent);
+                        break;
+                    case "news":
+                        detailRes = doPost(detailUrl, String.format("{\"id\":\"%s\",\"webKind\":\"%s\"}", nid, webKind));
+                        detailResult = new JSONObject(detailRes).getJSONArray("outDatas").getJSONObject(0);
+                        //项目名称
+                        gcmc = detailResult.get("title").toString();
+                        String contentRes = doPost("https://hjpt.580168.net:32443/cbmsApi/api/WebNews/website-content", String.format("{\"id\":\"%s\"}", nid));
+                        contentStr = new JSONObject(contentRes).getJSONArray("outDatas").getJSONObject(0).optString("content", "");
+                        break;
+                    default:
+                        System.err.println(String.format("未处理的类型:%s", webKind));
+                }
+
+                String detailTitle = gcmc;
+                Document content = Jsoup.parse(contentStr);
+                //清洗
+                Elements imgs = content.select("img");
+                for (int j = 0; j < imgs.size(); j++) {
+                    Element img = imgs.get(j);
+                    img.attr("src", urlClean(img.attr("src"), baseUrl, link));
+                }
+                Elements as = content.select("a");
+                for (int j = 0; j < as.size(); j++) {
+                    Element a = as.get(j);
+                    a.attr("href", urlClean(a.attr("href"), baseUrl, link));
+                }
+                //save
+                //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+                String SOURCE_NAME = "百思特CBMS电子交易平台";
+                String DETAIL_LINK = link;
+                String DETAIL_TITLE = detailTitle;
+                String DETAIL_CONTENT = content.toString();
+                Matcher matcher = pattern.matcher(dateStr);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    dateStr = matcher.group(1) + "-" + month + "-" + day;
+                }
+                LocalDate PAGE_TIME = LocalDate.parse(dateStr);
+                LocalDate CREATE_TIME = LocalDate.now();
+                String LIST_TITLE = title;
+                String CREATE_BY = "黄略";
+
+                String sql = "insert into " + MyDao.TABLE_NAME + "(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+                try (Connection connection = MyDao.getConnection();
+                     PreparedStatement preparedStatement = connection.prepareStatement(sql);) {
+                    preparedStatement.setString(1, SOURCE_NAME);
+                    preparedStatement.setString(2, DETAIL_LINK);
+                    preparedStatement.setString(3, DETAIL_TITLE);
+                    preparedStatement.setString(4, DETAIL_CONTENT);
+                    preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                    preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                    preparedStatement.setString(7, LIST_TITLE);
+                    preparedStatement.setString(8, CREATE_BY);
+                    preparedStatement.executeUpdate();
+                }
+                count++;
+            }
+            //下一页
+            if (index < (total % pageSize == 0 ? total / pageSize : total / pageSize + 1)) {
+                index++;
+            } else {
+                break;
+            }
+        }
+        System.out.println(count);
+    }
+
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    public static String doPost(String path, String json) throws Exception {
+        String result = "";
+
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        // 创建http POST请求
+        HttpPost httpPost = new HttpPost(path);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpPost.setConfig(requestConfig);
+
+        // 构造一个form表单式的实体
+        StringEntity stringEntity = new StringEntity(json, StandardCharsets.UTF_8);
+        // 将请求实体设置到httpPost对象中
+        httpPost.setEntity(stringEntity);
+        httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");
+        httpPost.setHeader("Content-Type", "application/json");
+        CloseableHttpResponse response = null;
+        try {
+            response = httpclient.execute(httpPost);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                return content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpclient.close();
+        }
+        return result;
+    }
+
+    public static String urlClean(String url, String baseUrl, String currentUrl) {
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        } else if (url.startsWith("./")) {
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        } else if (url.startsWith("../")) {
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + url.substring(url.lastIndexOf("../") + 2);
+        } else {
+            //等价于当前目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+}

src/main/java/space/anyi/Application9.java

@@ -98,9 +98,8 @@ public class Application9 {
                         iframe.remove();
                         content.append(a);
                     }
-                    content.select("div.media-notice-link").remove();
                     Elements attachmentLinks = content.select("div.attachment-area a");
-                    content.select("div.attachment-area div.attachment-icon").remove();
+
                     for (int j = 0; j < attachmentLinks.size(); j++) {
                         Element attachmentLink = attachmentLinks.get(j);
                         String onclick = attachmentLink.attr("onclick");
@@ -114,6 +113,8 @@ public class Application9 {
                         }
                     }
 
+                    content.select("div.media-notice-link").remove();
+                    content.select("div.attachment-area div.attachment-icon").remove();
                     //save
                     //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
                     String SOURCE_NAME = "四川国锃招标代理有限公司";

src/main/java/space/anyi/template/HTTPTemplate.java

@@ -0,0 +1,221 @@
+package space.anyi.template;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import space.anyi.MyDao;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HTTPTemplate {
+    static String indexUrl = "http://www.njsgczyy.cn/lnys/lnys6.html?mtt=6";
+    static String baseUrl = "http://www.njsgczyy.cn";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+
+    public static void main(String[] args) throws Exception {
+        getList();
+    }
+
+    static int total = 1;
+    static int index = 1;
+    static int pageSize = 10;
+    static int count = 0;
+    static String __EVENTTARGET = "AspNetPager1";
+    static int __EVENTARGUMENT = index;
+    static String __EVENTVALIDATION = "/wEdAAKvVXD1oYELeveMr0vHCmYPNOl94asWmGCaF2LdqKNWh2llJ1q5QHivvzgg/3iFHWM214ccCG8N+P8o1exiW9Fq";
+    public static void getList() throws Exception {
+        while (true){
+            //列表url构建
+            String listUrl = String.format(indexUrl, index);
+            System.out.println(listUrl);
+            //采集列表页
+            String res = doPost(listUrl);
+            Document document = Jsoup.parse(res);
+            __EVENTVALIDATION = document.select("#__EVENTVALIDATION").first().val();
+            Elements items = document.select("div.in-news2 li");
+            for (int i = 0; i < items.size(); i++) {
+                Element item = items.get(i);
+                Element a = item.select("a").first();
+                String title = a.text().trim();
+                String href = a.attr("href");
+                String dateStr = item.select("span").first().text().trim();
+                String link  = urlClean(href,baseUrl,listUrl);
+                System.out.println(link);
+                System.out.println(title);
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                    prepareStatement.setString(1,link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                }
+                //详情
+                String detailRes = doGet(link);
+                Element content = Jsoup.parse(detailRes).select("div.ri-neir").first();
+                String detailTitle = content.select("div.detail.wow.dh3").first().text().trim();
+                //清洗
+                Elements imgs = content.select("img");
+                for (int j = 0; j < imgs.size(); j++) {
+                    Element img = imgs.get(j);
+                    img.attr("src",urlClean(img.attr("src"),baseUrl,link));
+                }
+                Elements as = content.select("a");
+                for (int j = 0; j < as.size(); j++) {
+                    a = as.get(j);
+                    a.attr("href",urlClean(a.attr("href"),baseUrl,link));
+                }
+                content.select("div.xx.wow.dh3").remove();
+                //save
+                //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+                String SOURCE_NAME = "南京市高淳中医院";
+                String DETAIL_LINK = link;
+                String DETAIL_TITLE = detailTitle;
+                String DETAIL_CONTENT = content.toString();
+                Matcher matcher = pattern.matcher(dateStr);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    dateStr = matcher.group(1) + "-" + month + "-" + day;
+                }
+                LocalDate PAGE_TIME = LocalDate.parse(dateStr);
+                LocalDate CREATE_TIME = LocalDate.now();
+                String LIST_TITLE = title;
+                String CREATE_BY = "黄略";
+
+                String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                    preparedStatement.setString(1, SOURCE_NAME);
+                    preparedStatement.setString(2, DETAIL_LINK);
+                    preparedStatement.setString(3, DETAIL_TITLE);
+                    preparedStatement.setString(4, DETAIL_CONTENT);
+                    preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                    preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                    preparedStatement.setString(7, LIST_TITLE);
+                    preparedStatement.setString(8, CREATE_BY);
+                    preparedStatement.executeUpdate();
+                }
+                count++;
+            }
+            //下一页
+            Element nextPage = document.select("#AspNetPager1 a:contains(下一页)").first();
+            if (Objects.nonNull(nextPage)) {
+                index++;
+            }else {
+                break;
+            }
+        }
+        System.out.println(count);
+    }
+
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    public static String doPost(String path) throws Exception {
+        String result = "";
+
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        // 创建http POST请求
+        HttpPost httpPost = new HttpPost(path);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpPost.setConfig(requestConfig);
+        List<NameValuePair> parameters = new ArrayList<NameValuePair>(0);
+        parameters.add(new BasicNameValuePair("__EVENTTARGET",__EVENTTARGET));
+        __EVENTARGUMENT = index;
+        parameters.add(new BasicNameValuePair("__EVENTARGUMENT",__EVENTARGUMENT+""));
+        parameters.add(new BasicNameValuePair("__EVENTVALIDATION",__EVENTVALIDATION));
+
+        // 构造一个form表单式的实体
+        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(parameters);
+        // 将请求实体设置到httpPost对象中
+        httpPost.setEntity(formEntity);
+        httpPost.setHeader("User-Agent",
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");
+        CloseableHttpResponse response = null;
+        try {
+            response = httpclient.execute(httpPost);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                return content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpclient.close();
+        }
+        return result;
+    }
+
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + url.substring(url.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+}

src/main/java/space/anyi/template/WebMagicTemplate.java

@@ -0,0 +1,191 @@
+package space.anyi.template;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import space.anyi.MyDao;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.model.HttpRequestBody;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.time.LocalDate;
+import java.util.HashMap;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class WebMagicTemplate implements PageProcessor {
+
+    //初始url
+    static String indexUrl = "http://www.njsgczyy.cn/lnys/lnys6.html?mtt=6";
+    static String baseUrl = "http://www.njsgczyy.cn";
+    static int total = 0;
+    static int index = 1;
+    static String __EVENTTARGET = "AspNetPager1";
+    static int __EVENTARGUMENT = index;
+    static String __EVENTVALIDATION = "/wEdAAKvVXD1oYELeveMr0vHCmYPNOl94asWmGCaF2LdqKNWh2llJ1q5QHivvzgg/3iFHWM214ccCG8N+P8o1exiW9Fq";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+    //配置
+    private Site site = new Site().setCharset(StandardCharsets.UTF_8.name())
+            .setTimeOut(100 * 1000)     //超时时间
+            .setRetryTimes(3)         //重试请求次数
+            .setRetrySleepTime(3000)  //重试间隔时间                  请求头
+            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + result.substring(result.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+
+    @Override
+    public void process(Page page) {
+        String currentUrl = page.getUrl().toString();
+        if (currentUrl.equals(indexUrl)) {
+            String res = page.getRawText();
+            Document document = Jsoup.parse(res);
+            __EVENTVALIDATION = document.select("#__EVENTVALIDATION").first().val();
+            Elements items = document.select("div.in-news2 li");
+            for (int i = 0; i < items.size(); i++) {
+                Element item = items.get(i);
+                Element a = item.select("a").first();
+                String title = a.text().trim();
+                String href = a.attr("href");
+                String dateStr = item.select("span").first().text().trim();
+                Matcher matcher = pattern.matcher(dateStr);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    dateStr = matcher.group(1) + "-" + month + "-" + day;
+                }
+                String link  = urlClean(href,baseUrl,currentUrl);
+                System.out.println(link);
+                System.out.println(title);
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                    prepareStatement.setString(1,link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                } catch (SQLException e) {
+                    throw new RuntimeException(e);
+                }
+                //详情请求
+                Request detailRequest = new Request(link);
+                detailRequest.putExtra("title",title);
+                detailRequest.putExtra("listDate",LocalDate.parse(dateStr));
+                page.addTargetRequest(detailRequest);
+            }
+            //下一页
+            Element nextPage = document.select("#AspNetPager1 a:contains(下一页)").first();
+            if (Objects.nonNull(nextPage)) {
+                index++;
+                HashMap<String, Object> params = new HashMap<>();
+                params.put("__EVENTTARGET",__EVENTTARGET);
+                __EVENTARGUMENT = index;
+                params.put("__EVENTARGUMENT",__EVENTARGUMENT);
+                params.put("__EVENTVALIDATION",__EVENTVALIDATION);
+                HttpRequestBody requestBody =  HttpRequestBody.form(params,StandardCharsets.UTF_8.name());
+                Request request = new Request(indexUrl).setMethod("post");
+                request.addHeader("content-type","application/x-www-form-urlencoded; charset=UTF-8");
+                request.setRequestBody(requestBody);
+                page.addTargetRequest(request);
+            }
+            page.setSkip(true);
+        } else {
+            //处理详情网址
+            String detailRes = page.getRawText();
+            Element content = Jsoup.parse(detailRes).select("div.ri-neir").first();
+            String detailTitle = content.select("div.detail.wow.dh3").first().text().trim();
+            //清洗
+            Elements imgs = content.select("img");
+            for (int j = 0; j < imgs.size(); j++) {
+                Element img = imgs.get(j);
+                img.attr("src",urlClean(img.attr("src"),baseUrl,currentUrl));
+            }
+            Elements as = content.select("a");
+            for (int j = 0; j < as.size(); j++) {
+                Element a = as.get(j);
+                a.attr("href",urlClean(a.attr("href"),baseUrl,currentUrl));
+            }
+            content.select("div.xx.wow.dh3").remove();
+
+            //save
+            //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+            String SOURCE_NAME = "南京市高淳中医院";
+            String DETAIL_LINK = currentUrl;
+            String DETAIL_TITLE = detailTitle;
+            String DETAIL_CONTENT = content.toString();
+            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
+            LocalDate CREATE_TIME = LocalDate.now();
+            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
+            String CREATE_BY = "黄略";
+
+            String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+            try(Connection connection = MyDao.getConnection();
+                PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                preparedStatement.setString(1, SOURCE_NAME);
+                preparedStatement.setString(2, DETAIL_LINK);
+                preparedStatement.setString(3, DETAIL_TITLE);
+                preparedStatement.setString(4, DETAIL_CONTENT);
+                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                preparedStatement.setString(7, LIST_TITLE);
+                preparedStatement.setString(8, CREATE_BY);
+                preparedStatement.executeUpdate();
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    //启动
+    public static void main(String[] args) {
+        HashMap<String, Object> params = new HashMap<>();
+        params.put("__EVENTTARGET",__EVENTTARGET);
+        params.put("__EVENTARGUMENT",__EVENTARGUMENT);
+        params.put("__EVENTVALIDATION",__EVENTVALIDATION);
+        HttpRequestBody requestBody =  HttpRequestBody.form(params,StandardCharsets.UTF_8.name());
+        Request request = new Request(indexUrl).setMethod("post");
+        request.addHeader("content-type","application/x-www-form-urlencoded; charset=UTF-8");
+        request.setRequestBody(requestBody);
+        Spider spider = Spider.create(new WebMagicTemplate())
+                .addRequest(request);
+        System.out.println("start");
+        spider.run();
+        System.out.println("end");
+    }
+}