feat:爬虫实习第三天

2026.10.20

@@ -1,2 +1,4 @@
 今日学习日报:
-学习采集动态参数校验的复杂网页,学习使用post请求携带校验参数采集数据;
+学习通正则表达式处理Jsoup无法处理的特殊情况;
+学习通过ojdbc6操作数据，将采集、清洗完成的数据入库;
+学习使用WebMagic框架进行数据采集;

2026.10.21

@@ -1,2 +1,3 @@
 今日学习日报:
-学习采集动态参数校验的复杂网页,学习使用post请求携带校验参数采集数据;
+学习使用post请求携带校验参数采集数据;
+学习采集json数据;

src/main/java/space/anyi/Application4.java

@@ -142,7 +142,7 @@ public class Application4 implements PageProcessor {
             String __VIEWSTATEGENERATOR = document.select("#__VIEWSTATEGENERATOR").first().val();
             String ctl00_myTreeView_ExpandState = document.select("#ctl00_myTreeView_ExpandState").first().val();
             String __VIEWSTATE = document.select("#__VIEWSTATE").first().val();
-            Element nextPage = document.getElementById("ctl00_ContentPlaceHolder1_LinkButtonNextPage");
+            Element nextPage = document.select("a#ctl00_ContentPlaceHolder1_LinkButtonNextPage").first();
             if (Objects.nonNull(nextPage)) {
                 Map<String, Object> params = new HashMap<>();
                 params.put("__EVENTVALIDATION",__EVENTVALIDATION);
@@ -162,15 +162,12 @@ public class Application4 implements PageProcessor {
             String rawText = page.getRawText();
             rawText = rawText.replace("display:none;", "");
             Document document = Jsoup.parse(rawText);
-            Element content = document.getElementById("form1");
+            Element content = document.select("form#form1").first();
             if (Objects.isNull(content)) {
                 content = document.select("#aspnetForm > div:nth-child(3) > table:nth-child(4) > tbody:nth-child(2) > tr > td:nth-child(3) > div > table").first();
             }
             //文件url清洗
-            Elements elements = content.select("a.lan");
-            if (Objects.nonNull(elements) && elements.size() > 0) {
-                elements.remove();
-            }
+            content.select("a.lan:contains(关闭窗口)").remove();
             Elements as = content.select("a");
             for (Element a : as) {
                 String href = a.attr("href");
@@ -204,9 +201,9 @@ public class Application4 implements PageProcessor {
             Map<String, Object> map = context.get(currentUrl);
             String SOURCE_NAME = "英山县公共资源交易中心-英山县政府采购中心";
             String DETAIL_LINK = currentUrl;
-            Element titleElement = content.getElementById("lblXmmc");
+            Element titleElement = content.select("span#lblXmmc").first();
             if (Objects.isNull(titleElement)) {
-                titleElement = content.getElementById("ctl00_ContentPlaceHolder1_TitleLabel");
+                titleElement = content.select("span#ctl00_ContentPlaceHolder1_TitleLabel").first();
             }
             String DETAIL_TITLE = titleElement.text().trim();
             String DETAIL_CONTENT = content.toString();

src/main/java/space/anyi/Application5.java

@@ -62,7 +62,7 @@ public class Application5 {
             __VIEWSTATEGENERATOR = doc.select("#__VIEWSTATEGENERATOR").first().val();
 //            ctl00_myTreeView_ExpandState = doc.select("#ctl00_myTreeView_ExpandState").first().val();
             __VIEWSTATE = doc.select("#__VIEWSTATE").first().val();
-            Element nextPage = doc.getElementById("ctl00_ContentPlaceHolder1_LinkButtonNextPage");
+            Element nextPage = doc.select("a#ctl00_ContentPlaceHolder1_LinkButtonNextPage").first();
             //列表
             Elements eachTags = doc.select("table.myGVClass tr:has(a)");
             if (eachTags != null) {
@@ -95,15 +95,12 @@ public class Application5 {
                         detail = detail.replace("display:none;", "");
                         Document document = Jsoup.parse(detail);
                         //#form1 > div:nth-child(3) > table > tbody > tr:nth-child(1) > td > table
-                        Element content = document.getElementById("form1");
+                        Element content = document.select("form#form1").first();
                         if (Objects.isNull(content)) {
                             content = document.select("#aspnetForm > div:nth-child(3) > table:nth-child(4) > tbody:nth-child(2) > tr > td:nth-child(3) > div > table").first();
                         }
                         //文件url清洗
-                        Elements elements = content.select("a.lan");
-                        if (Objects.nonNull(elements) && elements.size() > 0) {
-                            elements.remove();
-                        }
+                        content.select("a.lan:contains(关闭窗口)").remove();
                         Elements as = content.select("a");
                         for (Element a : as) {
                             href = a.attr("href");
@@ -136,9 +133,9 @@ public class Application5 {
                         //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
                         String SOURCE_NAME = "英山县公共资源交易中心-英山县政府采购中心";
                         String DETAIL_LINK = link;
-                        Element titleElement = content.getElementById("lblXmmc");
+                        Element titleElement = content.select("span#lblXmmc").first();
                         if (Objects.isNull(titleElement)) {
-                            titleElement = content.getElementById("ctl00_ContentPlaceHolder1_TitleLabel");
+                            titleElement = content.select("span#ctl00_ContentPlaceHolder1_TitleLabel").first();
                         }
                         String DETAIL_TITLE = titleElement.text().trim();
                         String DETAIL_CONTENT = content.toString();

src/main/java/space/anyi/Application6.java

@@ -0,0 +1,249 @@
+package space.anyi;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.model.HttpRequestBody;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.time.LocalDate;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application6 implements PageProcessor {
+
+    //初始url
+    static String indexUrl = "https://cg.bupt.edu.cn/sfw/e";
+    static int total = 0;
+    static int start = 1;
+    public static final String baseUrl = "https://cg.bupt.edu.cn";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+
+    private static final Map<String, Map<String,Object>> context = new HashMap<>();
+
+    //配置
+    private Site site = new Site()
+            .setTimeOut(100 * 1000)     //超时时间
+            .setRetryTimes(3)         //重试请求次数
+            .setRetrySleepTime(3000)  //重试间隔时间                  请求头
+            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + result.substring(result.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    @Override
+    public void process(Page page) {
+        String currentUrl = page.getUrl().toString();
+        if (currentUrl.equals(indexUrl)) {
+            String jsonStr = page.getRawText();
+            JSONObject jsonObject = new JSONObject(jsonStr);
+            //总页数
+            total = jsonObject.getInt("count");
+            JSONArray resultset = jsonObject.getJSONArray("resultset");
+            for (int i = 0; i < resultset.length(); i++) {
+                JSONObject item = resultset.getJSONObject(i);
+                String subject = item.get("subject").toString();
+                String fullUrl = item.get("fullUrl").toString();
+                String pdate = item.get("pdate").toString();
+                Matcher matcher = pattern.matcher(pdate);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    pdate = matcher.group(1) + "-" + month + "-" + day;
+                }
+                String link = fullUrl;
+                String title = subject;
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                    prepareStatement.setString(1,link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                } catch (SQLException e) {
+                    throw new RuntimeException(e);
+                }
+                Map<String, Object> map = new HashMap<>();
+                map.put("title",title);
+                map.put("listDate",LocalDate.parse(pdate));
+                context.put(link,map);
+                page.addTargetRequest(link);
+            }
+            //下一页
+            if (Application6.start <= Application6.total) {
+                start += 25;
+                HashMap<String, Object> params = new HashMap<>();
+                params.put("window_","json");
+                params.put("start",start);
+                params.put("limit",25);
+                params.put("cid",100166);
+                params.put("page","cms.portalArticle.listArticlesToList");
+                HttpRequestBody requestBody =  HttpRequestBody.form(params,StandardCharsets.UTF_8.name());
+                Request request = new Request("https://cg.bupt.edu.cn/sfw/e").setMethod("post");
+                request.setRequestBody(requestBody);
+                request.addHeader("content-type","application/x-www-form-urlencoded; charset=UTF-8");
+                page.addTargetRequest(request);
+            }
+            page.setSkip(true);
+        } else {
+            //处理详情网址
+            String rawText = page.getRawText();
+            rawText = rawText.replace("display:none;", "");
+            Document document = Jsoup.parse(rawText);
+            Element content = document.select("div#main").first();
+            //文件url补全
+            //<a href="/sfw/r/d/cms/a430c8e0805d4f57b9ff332b4b6951a3" target="_blank">2024批量集中采购配置标准.xlsx</a>
+            Elements uploadFileElements = content.select("div.up_file");
+            Iterator<Element> iterator = uploadFileElements.iterator();
+            while (iterator.hasNext()) {
+                Element uploadFileElement = iterator.next();
+                String urlKey = uploadFileElement.val();
+                urlKey = "/sfw/r/l/cms/" + urlKey;
+                urlKey = urlClean(urlKey, baseUrl, currentUrl);
+                try {
+                    String res = doGet(urlKey);
+                    String fileName = new JSONArray(res).getJSONObject(0).get("fileName").toString();
+                    Document fileLink = Jsoup.parse(String.format("<a href=\"%s\" target=\"_blank\">%s</a>", urlKey,fileName));
+                    uploadFileElement.appendChild(fileLink);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }
+            //文件url清洗
+            Elements as = content.select("a");
+            for (Element a : as) {
+                String href = a.attr("href");
+                href = urlClean(href,baseUrl,currentUrl);
+                a.attr("href", href);
+            }
+            //图片url清洗
+            Elements imgs = content.select("img");
+            for (Element img : imgs) {
+                String src = img.attr("src");
+                src = urlClean(src,baseUrl,currentUrl);
+                img.attr("src", src);
+            }
+            //清洗
+            content.select("div.title span").remove();
+            content.select("div.csu_foot").remove();
+            //save
+            Map<String, Object> map = context.get(currentUrl);
+            String SOURCE_NAME = "北京邮电大学-采购与招标办公室";
+            String DETAIL_LINK = currentUrl;
+            String DETAIL_TITLE = content.select("div.title").first().text().trim();
+            String DETAIL_CONTENT = content.toString();
+            LocalDate PAGE_TIME = ((LocalDate) map.get("listDate"));
+//            LocalDate PAGE_TIME = ((LocalDate) page.getRequest().getExtra("listDate"));
+            LocalDate CREATE_TIME = LocalDate.now();
+            String LIST_TITLE = ((String) map.get("title"));
+//            String LIST_TITLE = ((String) page.getRequest().getExtra("title"));
+            String CREATE_BY = "黄略";
+
+            String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+            try(Connection connection = MyDao.getConnection();
+                PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                preparedStatement.setString(1, SOURCE_NAME);
+                preparedStatement.setString(2, DETAIL_LINK);
+                preparedStatement.setString(3, DETAIL_TITLE);
+                preparedStatement.setString(4, DETAIL_CONTENT);
+                preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                preparedStatement.setString(7, LIST_TITLE);
+                preparedStatement.setString(8, CREATE_BY);
+                preparedStatement.executeUpdate();
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    //启动
+    public static void main(String[] args) {
+        HashMap<String, Object> params = new HashMap<>();
+        params.put("window_","json");
+        params.put("start",start);
+        params.put("limit",25);
+        params.put("cid",100166);
+        params.put("page","cms.portalArticle.listArticlesToList");
+        HttpRequestBody requestBody =  HttpRequestBody.form(params,StandardCharsets.UTF_8.name());
+        Request request = new Request("https://cg.bupt.edu.cn/sfw/e").setMethod("post");
+        request.addHeader("content-type","application/x-www-form-urlencoded; charset=UTF-8");
+        request.setRequestBody(requestBody);
+        Spider spider = Spider.create(new Application6())
+                .addRequest(request);
+        System.out.println("start");
+        spider.run();
+        System.out.println("end");
+    }
+}

src/main/java/space/anyi/Application7.java

@@ -0,0 +1,449 @@
+package space.anyi;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.parser.Parser;
+import org.jsoup.select.Elements;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application7 {
+    static String indexUrl = "https://auct.dgggzy.cn/zhjy/HomePage/findAllZHTradeList";
+    static String detailUrl_nz = "https://auct.dgggzy.cn/zhjy/HomePage/nzSubjectInfo?terderId=%s&subjectId=%s";
+    static String detailUrl_04 = "https://auct.dgggzy.cn/zhjy/HomePage/subjectInfo?terderId=%s&subjectId=%s";
+    static String fileDownloadUrl = "https://auct.dgggzy.cn/zhjy/HomePage/PublishInfo/downloadfile?id=%s";
+    static String baseUrl = "https://auct.dgggzy.cn";
+    public static Pattern pattern = Pattern.compile("(\\d{4})(年|/|-|\\.)(\\d{1,2})(月|/|-|\\.)(\\d{1,2})");
+
+    static MyDao myDao = new MyDao();
+    public static void main(String[] args) throws Exception {
+        getList();
+    }
+
+    static int total = 1;
+    static int index = 1;
+    static int pageSize = 10;
+    static int count = 0;
+    public static void getList() throws Exception {
+        while (true){
+            //列表url构建
+            String listUrl = indexUrl;
+            System.out.println(listUrl);
+            //采集列表页
+            String res = doPost(indexUrl);
+            //采集总页数
+            JSONObject response = new JSONObject(res);
+            total = response.getInt("total");
+
+            JSONArray items = response.getJSONArray("rows");
+            for (int i = 0; i < items.length(); i++) {
+                count++;
+                JSONObject item = items.getJSONObject(i);
+
+                String id = item.get("id").toString();
+                String subjectid = item.get("subjectid").toString();
+                String tenderid = item.get("tenderid").toString();
+                String fcProjectType = item.get("fcProjectType").toString();
+                //标的名称
+                String fcSubjectname = item.get("fcSubjectname").toString();
+                //公告标题
+                String fcInfotitle = fcSubjectname;
+                //是否多轮次竞价
+                String fcIsMultiWheel = item.get("fcIsMultiWheel").toString();
+                //申请要求
+                String fcApplyobject = item.get("fcApplyobject").toString();
+                //标的详细信息
+                String fcSubjectdetails = item.get("fcSubjectdetails").toString();
+                //公告内容
+                String fcBidComfirmation = item.get("fcBidComfirmation").toString();
+
+                String link;
+                if (fcProjectType.equals("nz")) {
+                    link = String.format(detailUrl_nz, tenderid,subjectid);
+                }else if (fcProjectType.equals("04")){
+                    link = String.format(detailUrl_04, tenderid,subjectid);
+                }else {
+                    System.err.println("未知的类型" + fcProjectType);
+                    continue;
+                }
+                System.out.println(fcSubjectname);
+                System.out.println(link);
+                //数据库是否已存在
+                String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                    prepareStatement.setString(1,link);
+                    ResultSet resultSet = prepareStatement.executeQuery();
+                    if (resultSet.next()) {
+                        System.out.println("跳过" + link);
+                        resultSet.close();
+                        continue;
+                    }
+                    resultSet.close();
+                }
+                //详情
+                String detailResStr = doGet(link);
+
+                Element content = Jsoup.parse(detailResStr).select("body > div.main > div > div > div.l_gpcr").first();
+                //文件url清洗
+                Elements as = content.select("a");
+                for (Element a : as) {
+                    a.attr("href", urlClean(a.attr("href"),baseUrl,link));
+                }
+                //图片url清洗
+                Elements imgs = content.select("img");
+                for (Element img : imgs) {
+                    img.attr("src", urlClean(img.attr("src"),baseUrl,link));
+                }
+                //头部清洗
+                content.select("div.left").remove();
+                //todo:表格清洗
+                //
+                Pattern p = Pattern.compile("subjectInfoResult\\s*:\\s*(\\{[^{}]*\\})", Pattern.DOTALL);
+                Matcher m = p.matcher(detailResStr);
+                if (m.find()) {
+                    JSONObject subjectInfoResult = new JSONObject(m.group(1));
+                    //公告标题
+                    fcInfotitle = subjectInfoResult.get("fcInfotitle").toString();
+//                    //公告开始时间
+//                    fdTradestartdate = subjectInfoResult.get("fdTradestartdate").toString();
+//                    //公告结束时间
+//                    fdTradeenddate = subjectInfoResult.get("fdTradeenddate").toString();
+//                    //报名截止时间
+//                    fdAuctionapplyenddate = subjectInfoResult.get("fdAuctionapplyenddate").toString();
+//                    //保证金截止时间
+//                    fdDepositendtime = subjectInfoResult.get("fdDepositendtime").toString();
+//                    //报价开始时间
+//                    fdListingstarttime = subjectInfoResult.get("fdListingstarttime").toString();
+//                    //报价截止时间
+//                    fdListingendtime = subjectInfoResult.get("fdListingendtime").toString();
+                }
+                //公告详情
+                if (fcBidComfirmation.equals("null")) {
+                    Pattern compile = Pattern.compile("notice\\s*:\\s*(\\{[^{}]*\\})");
+                    Matcher matcher1 = compile.matcher(detailResStr);
+                    if (matcher1.find()) {
+                        String group = matcher1.group(1);
+                        JSONObject jsonObject = new JSONObject(group);
+                        fcBidComfirmation = jsonObject.get("fcInfocontent").toString();
+                    }
+                }
+                String template = "<table class=\"tb02\" width=\"95%\" cellspacing=\"0\" cellpadding=\"0\" style=\"border-top: 1px solid #1d82ec\">\n" +
+                        "                      <tbody><tr style=\"border: 1px solid white;height:40px;\">\n" +
+                        "                          <td style=\"width:13%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              公告标题\n" +
+                        "                          </td>\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\" colspan=\"5\">\n" +
+                        "                              <span style=\"font-size: 20px;\" data-bind=\"text:data.notice.fcInfotitle\">" + fcInfotitle + "</span>\n" +
+                        "                          </td>\n" +
+                        "                      </tr>\n" +
+                        "                      <tr style=\"border: 1px solid white;height:40px;\">\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              标的编号\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:data.subjectInfoResult.fcSubjectsn\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+ item.get("fcProjectsn").toString() +"</td>\n" +
+                        "                          <td style=\"width:13%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              标的名称\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:data.subjectInfoResult.fcSubjectname\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+ fcSubjectname +"</td>\n" +
+                        "                          <td style=\"width:13%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              资审方式\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:viewModel.fun.getAuditType(data.subjectInfoResult.fcAudittype,viewModel.data.dictAudittypeList)\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+ getAudittype(item.get("fcAudittype").toString()) +"</td>\n" +
+                        "                      </tr>\n" +
+                        "                      <tr style=\"border: 1px solid white;height:40px;\">\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              公告开始时间\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:data.subjectInfoResult.fdTradestartdate\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+ item.get("fdTradestartdate").toString() +"</td>\n" +
+                        "                          <td style=\"width:13%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              公告结束时间\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:data.subjectInfoResult.fdTradeenddate\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+ item.get("fdTradeenddate").toString() +"</td>\n" +
+                        "                          <td style=\"width:13%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              报名截止时间\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:data.subjectInfoResult.fdAuctionapplyenddate\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+ item.get("fdAuctionapplyenddate").toString() +"</td>\n" +
+                        "                      </tr>\n" +
+                        "                      <tr style=\"border: 1px solid white;height:40px;\">\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              保证金（元）\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:showNo(numToCurrency(data.subjectInfoResult.fmDepositamt))\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+item.get("fmDepositamt").toString()+"</td>\n" +
+                        "\n" +
+                        "                          <td style=\"width:13%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              保证金截止时间\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:showNo(data.subjectInfoResult.fdDepositendtime)\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+item.get("fdDepositendtime").toString()+"</td>\n" +
+                        "                          <td style=\"width:13%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              报价开始时间\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:data.subjectInfoResult.fdListingstarttime\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+item.get("fdListingstarttime").toString()+"</td>\n" +
+                        "                      </tr>\n" +
+                        "                      <tr style=\"border: 1px solid white;height:40px;\">\n" +
+                        "                          <td style=\"width:13%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              报价截止时间\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:data.subjectInfoResult.fdListingendtime\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+item.get("fdListingendtime").toString()+"</td>\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              交易方式\n" +
+                        "                          </td>\n" +
+                        "                          <td data-bind=\"text:data.subjectInfoResult.fcTrademethod\" style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">"+getTradeMethod(item.get("fcTrademethod").toString())+"</td>\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\"></td>\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\"></td>\n" +
+                        "                      </tr>\n" +
+                        "\n" +
+                        "\n" +
+                        "                      \n" +
+                        "                      \n" +
+                        "                      <tr style=\"border: 1px solid white;height:40px; display: none\">\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              竞价规则\n" +
+                        "                          </td>\n" +
+                        "                          <td style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\" colspan=\"5\">\n" +
+                        "                              <table cellpadding=\"0\" cellspacing=\"0\" class=\"table_list table02\" style=\"border-collapse: collapse;font-size: 8pt;margin-bottom: 5px;width:99%;margin-left: 5px ; border: 1px solid #FF9E0D;\">\n" +
+                        "                                  <thead>\n" +
+                        "                                      <tr>\n" +
+                        "                                          <th style=\"width:5%;text-align:center;\">排序号</th>\n" +
+                        "                                          <th style=\"width:8%;text-align:center;\">起拍价</th>\n" +
+                        "                                          <th style=\"width:8%;text-align:center;\">竞价度量单位</th>\n" +
+                        "                                          <th style=\"width:8%;text-align:center;\">是否限最高价</th>\n" +
+                        "                                          <th style=\"width:8%;text-align:center;\">竞价规则类型</th>\n" +
+                        "                                          <th style=\"width:10%;text-align:center;\">是否有下一步骤</th>\n" +
+                        "                                          <th style=\"width:10%;text-align:center;\">进入下一步骤条件</th>\n" +
+                        "                                      </tr>\n" +
+                        "                                  </thead>\n" +
+                        "                                  <tbody id=\"dataBindId1\" data-bind=\"foreach: viewModel.data.auctionFlowList\"></tbody>\n" +
+                        "                              </table>\n" +
+                        "                          </td>\n" +
+                        "                      </tr>\n" +
+                        "                      \n" +
+                        "                      <tr style=\"border: 1px solid white; height: 40px; display: none;\" id=\"suppId\">\n" +
+                        "                          <td style=\"width:14%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\">\n" +
+                        "                              补充通知附件下载\n" +
+                        "                          </td>\n" +
+                        "                          <td style=\"width:20%;background: white;border: 1px solid #FF9E0D;margin-bottom: 10px;\" colspan=\"5\">\n" +
+                        "                              <table cellpadding=\"0\" cellspacing=\"0\" width=\"100%\" style=\"border-collapse: collapse;font-size: 12pt;margin-bottom: 10px;width:100%; border: 1px solid;\">\n" +
+                        "                                  <tbody data-bind=\"foreach: viewModel.data.addAttList\"></tbody>\n" +
+                        "                              </table>\n" +
+                        "                          </td>\n" +
+                        "                      </tr>\n" +
+                        "                  </tbody></table>";
+                content = Jsoup.parse(template);
+                fcBidComfirmation = Parser.unescapeEntities(fcBidComfirmation, false).replace("\\\"", "\"");
+                content.appendChild(Jsoup.parse(fcBidComfirmation));
+                //标的详细信息
+//                element = content.select("span[data-bind=text:data.subjectInfoResult.fcSubjectdetails]").first();
+//                if (Objects.isNull(element)) element = content.select("td[data-bind=text:data.subjectInfoResult.fcSubjectdetails]").first();
+//                if (Objects.nonNull(element)) {
+//                    element.text(fcSubjectdetails);
+//                }
+//                //公告内容
+//                element = content.select("div[data-bind=html:noticeContent(data.subjectInfoResult.fcBidComfirmation)]").first();
+//                if (Objects.isNull(element)) element = content.select("div[data-bind=html:noticeContent(data.notice.fcInfocontent)]").first();
+//                if (Objects.nonNull(element)) {
+////                    element.text(fcBidComfirmation);
+//                    //反转义 HTML 实体：&lt;div&gt; -> <div>
+//                    fcBidComfirmation = Parser.unescapeEntities(fcBidComfirmation, false).replace("\\\"", "\"");
+//                    element.appendChild(Jsoup.parse(fcBidComfirmation));
+//                }
+                //todo:附件
+                p = Pattern.compile("fileList\\s*:\\s*ko\\.observableArray\\(\\s*(\\[[\\s\\S]*?\\])\\s*\\)");
+                m = p.matcher(detailResStr);
+                if (m.find()) {
+                    JSONArray fileList = new JSONArray(m.group(1));
+                    for (int j = 0; j < fileList.length(); j++) {
+                        JSONObject file = fileList.getJSONObject(j);
+                        String fileId = file.get("id").toString();
+                        String fcFilename = file.get("fcFilename").toString();
+                        String fileUrl = String.format(fileDownloadUrl, fileId);
+                        Element element = Jsoup.parse(String.format("<div>点击查看公告内容：<a href='%s'>%s</a></div>", fileUrl, fcFilename));
+                        content.appendChild(element);
+                    }
+                }
+                //报价
+                p = Pattern.compile("priceList\\s*:\\s*ko\\.observableArray\\(\\s*(\\[[\\s\\S]*?\\])\\s*\\)");
+                m = p.matcher(detailResStr);
+                if (m.find()) {
+                    JSONArray priceList = new JSONArray(m.group(1));
+                    for (int j = 0; j < priceList.length(); j++) {
+                        JSONObject jsonObject = priceList.getJSONObject(j);
+                        String orderNum = jsonObject.get("orderNum").toString();
+                        String fnAuctionapplyno = jsonObject.get("fnAuctionapplyno").toString();
+                    }
+                }
+                //save
+                //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+                String SOURCE_NAME = "东莞市公共资源交易中心";
+                String DETAIL_LINK = link;
+                String DETAIL_TITLE = fcSubjectname;
+                String DETAIL_CONTENT = content.toString();
+                String dateStr = item.get("fdTradestartdate").toString();
+                Matcher matcher = pattern.matcher(dateStr);
+                if (matcher.find()) {
+                    String month = matcher.group(3).length() == 2 ? matcher.group(3) : "0" + matcher.group(3);
+                    String day = matcher.group(5).length() == 2 ? matcher.group(5) : "0" + matcher.group(5);
+                    dateStr = matcher.group(1) + "-" + month + "-" + day;
+                }
+                LocalDate PAGE_TIME = LocalDate.parse(dateStr);
+                LocalDate CREATE_TIME = LocalDate.now();
+                String LIST_TITLE = fcSubjectname;
+                String CREATE_BY = "黄略";
+
+                String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+                try(Connection connection = MyDao.getConnection();
+                    PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                    preparedStatement.setString(1, SOURCE_NAME);
+                    preparedStatement.setString(2, DETAIL_LINK);
+                    preparedStatement.setString(3, DETAIL_TITLE);
+                    preparedStatement.setString(4, DETAIL_CONTENT);
+                    preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                    preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                    preparedStatement.setString(7, LIST_TITLE);
+                    preparedStatement.setString(8, CREATE_BY);
+//                    preparedStatement.executeUpdate();
+                }
+            }
+
+            if (index<= (total % pageSize == 0 ? total / pageSize : (total / pageSize) + 1)) {
+                index++;
+            }else {
+                break;
+            }
+        }
+        System.out.println(count);
+    }
+
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+
+    public static String doPost(String path) throws Exception {
+        String result = "";
+
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        // 创建http POST请求
+        HttpPost httpPost = new HttpPost(path);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpPost.setConfig(requestConfig);
+        List<NameValuePair> parameters = new ArrayList<NameValuePair>(0);
+        parameters.add(new BasicNameValuePair("pagesize",pageSize+""));
+        parameters.add(new BasicNameValuePair("currentpage",index+""));
+        parameters.add(new BasicNameValuePair("search","{\"startDate\":\"\",\"endDate\":\"\",\"extVal\":\"\",\"extVal2\":\"\",\"extVal3\":\"\",\"extVal4\":\"\",\"extVal5\":\"\",\"extVal6\":-999999,\"extVal7\":-999999,\"extVal8\":null,\"extVal9\":\"02\",\"extVal10\":null,\"extVal11\":null,\"extVal12\":null,\"extVal13\":null,\"extVal14\":null,\"extVal15\":null,\"extVal16\":null,\"orderField\":\"\",\"orderType\":\"\",\"name\":\"\",\"code\":\"\"}"));
+
+        // 构造一个form表单式的实体
+        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(parameters);
+        // 将请求实体设置到httpPost对象中
+        httpPost.setEntity(formEntity);
+        httpPost.setHeader("User-Agent",
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");
+        CloseableHttpResponse response = null;
+        try {
+            response = httpclient.execute(httpPost);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                return content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpclient.close();
+        }
+        return result;
+    }
+
+    public static String urlClean(String url,String baseUrl,String currentUrl){
+        if (url.startsWith("http")) {
+            return url;
+        }
+        String result = "";
+        if (url.startsWith("/")) {
+            //根路径开始
+            result = baseUrl + url;
+        }else if (url.startsWith("./")){
+            //当前同级目录
+            result = currentUrl.substring(0, currentUrl.lastIndexOf("/")) + url.substring(url.indexOf("./") + 1);
+        }else if (url.startsWith("../")){
+            //多级目录，取最后的uri拼接到baseUrl，等价于所有 "../" 替换成 "/"
+            result = baseUrl + url.substring(url.lastIndexOf("../") + 2);
+        }else {
+            //等价于当前目录
+            result = currentUrl.substring(0,currentUrl.lastIndexOf("/") + 1) + url;
+        }
+        return result;
+    }
+
+    public static String getAudittype(String val){
+        switch (val){
+            case "10":
+                return "资格预审";
+            case "20":
+                return "资格后审";
+            default:
+                return val;
+        }
+    }
+
+    public static String getTradeMethod(String val){
+        switch (val){
+            case "01":
+                return "网上竞价/公开竞价模式";
+            case "02":
+                return "公开协商模式";
+            case "03":
+                return "小额简易交易模式";
+            case "04":
+                return "续约交易模式";
+            case "05":
+                return "底价交易";
+            default:
+                return val;
+        }
+    }
+}