feat:Java爬虫实习第一天

.gitignore

@@ -0,0 +1,2 @@
+.idea/
+target/

2026.10.09

@@ -0,0 +1,6 @@
+今日学习日报:
+学习采集天津市东丽区民政局政府公开信息的列表页和详情页;
+学习使用HTTP Client获取网页的html格式数据;
+学习使用Jsoup解析html数据,通过css选择器的方式进行数据预处理，提取需要的目标数据，例如标题、 正文、文件ur;
+学习通正则表达式处理Jsoup无法处理的特殊情况，例如总页数不在html元素显示，而是通过javascript动态渲染的方式显示，所以需要使用正则表达式提取混在js代码中的总页数数据;
+学习通过ojdbc6操作数据，预处理sql编写、sql参数动态填充、执行预编译sql,将采集、清洗完成的数据入库;

pom.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>space.anyi</groupId>
+    <artifactId>test</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>commons-httpclient</groupId>
+            <artifactId>commons-httpclient</artifactId>
+            <version>3.1</version>
+        </dependency>
+
+
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-core</artifactId>
+            <version>0.7.3</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-extension</artifactId>
+            <version>0.7.3</version>
+        </dependency>
+
+        <dependency>
+            <!-- oracle jdbc driver -->
+            <groupId>com.oracle</groupId>
+            <artifactId>ojdbc6</artifactId>
+            <version>11.2.0.3</version>
+            <scope>runtime</scope>
+        </dependency>
+
+
+        <dependency>
+            <groupId>org.json</groupId>
+            <artifactId>json</artifactId>
+            <version>20140107</version>
+        </dependency>
+    </dependencies>
+</project>

src/main/java/space/anyi/Application.java

@@ -0,0 +1,158 @@
+package space.anyi;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application {
+    static String indexUrl = "https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/index%s.html";
+    static String baseUrl = "https://www.tjdl.gov.cn";
+    public static void main(String[] args) throws Exception {
+//        String res = doGet(listUrl);
+//        System.out.println(res);
+        getList();
+    }
+
+    static int total = 1;
+    static int index = 0;
+    static int count = 0;
+    public static void getList() throws Exception {
+        while (index< total){
+            String listUrl = index == 0 ? String.format(indexUrl, "") : String.format(indexUrl, "_" + index);
+            System.out.println(listUrl);
+            String res = doGet(listUrl);
+            //var countPage = 5//共多少页
+            Pattern pattern = Pattern.compile("var countPage = (\\d+)//共多少页");
+            Matcher matcher = pattern.matcher(res);
+            if (matcher.find()) {
+                String totalStr = matcher.group(1);
+                total =  Integer.parseInt(totalStr);
+            }
+        Document doc = Jsoup.parse(res);
+
+//        Elements eachTags = doc.select("div[class=news-items] ul li:has(a)");
+        //#xxgk_right > div.zfxxgk_zd2.borderTop.zfxxgkzd.border0
+        Elements eachTags = doc.select("div.zfxxgk_zd2.borderTop.zfxxgkzd.border0 ul li:has(a)");
+        if (eachTags != null) {
+            if (eachTags.size() > 0) {
+                for (Element eachTag : eachTags) {
+                    count++;
+                    String id = eachTag.select("a").first().attr("href");
+                    id = id.substring(id.indexOf("./") + 1);
+                    String link = listUrl.substring(0, listUrl.lastIndexOf("/")) + id;
+                    String title = eachTag.select("a").first().text().trim();
+                    System.out.println(title);
+                    System.out.println(link);
+                    String detail = doGet(link);
+                    detail = detail.replace("display:none;", "");
+                    Document document = Jsoup.parse(detail);
+                    //body > div.bgf7f7f7 > div > div.listCon > div > div.contentCons > div.pages_content
+                    //body > div.bgf7f7f7 > div
+                    Element content = document.select("body > div.bgf7f7f7 > div").first();
+//                    Element pageContent = content.select("div.pages_content").first();
+                    //文件
+                    Elements as = content.select("a");
+                    for (Element a : as) {
+                        String href = a.attr("href");
+                        if (href.startsWith("./")) {
+                            href = link.substring(0, link.lastIndexOf('/')) + href.substring(href.indexOf("./") + 1);
+                            a.attr("href", href);
+                        } else if (href.startsWith("../")) {
+                            href = baseUrl + href.substring(href.lastIndexOf("../") + 2);
+                            a.attr("href", href);
+                        } else if (href.startsWith("/")) {
+                            href = baseUrl + href;
+                            a.attr("href", href);
+                        }
+//                        String fileTitle = a.text();
+//                        System.out.println(fileTitle);
+                    }
+                    //图片
+                    Elements imgs = content.select("img");
+                    for (Element img : imgs) {
+                        String src = img.attr("src");
+                        if (src.startsWith("./")) {
+                            //https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/202507/W020250723358692283424_ORIGIN.jpg
+                            src = link.substring(0, link.lastIndexOf('/')) + src.substring(src.indexOf("./") + 1);
+                            img.attr("src", src);
+                        } else if (src.startsWith("../")) {
+                            src = baseUrl + src.substring(src.lastIndexOf("../") + 2);
+                            img.attr("src", src);
+                        } else if (src.startsWith("/")) {
+                            src = baseUrl + src;
+                            img.attr("src", src);
+                        }
+                    }
+                    //头部
+                    content.select("div.dqwz").remove();
+                    content.select("div.contentMemo").remove();
+                    content.select("#articlePlayer").remove();
+                    content.select("#share").remove();
+                    content.select("dl.QrcodeBot").remove();
+                    //底部
+                    content.select("#file_document").remove();
+                    content.select("div.inRight").remove();
+                    content.select("script").remove();
+                    content.select("style").remove();
+                    String text = content.text();
+
+                    //save
+                    try (BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(Files.newOutputStream(Paths.get(System.getProperty("user.dir") + File.separator + "data" + File.separator + title.substring(2) + ".html"))))) {
+                        bufferedWriter.write(content.toString());
+                    }
+                }
+                }
+            }
+        index++;
+        }
+        System.out.println(count);
+    }
+
+    private static int getTotal(String res) {
+        //var countPage = 5//共多少页
+        Pattern pattern = Pattern.compile("var countPage = (\\d+)//共多少页");
+        Matcher matcher = pattern.matcher(res);
+        if (matcher.find()) {
+            String totalStr = matcher.group(1);
+            return Integer.parseInt(totalStr);
+        }
+        return total;
+    }
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+}

src/main/java/space/anyi/Application1.java

@@ -0,0 +1,190 @@
+package space.anyi;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.time.LocalDate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Application1 {
+    static String indexUrl = "https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/index%s.html";
+    static String baseUrl = "https://www.tjdl.gov.cn";
+    static MyDao myDao = new MyDao();
+    public static void main(String[] args) throws Exception {
+        getList();
+    }
+
+    static int total = 1;
+    static int index = 0;
+    static int count = 0;
+    public static void getList() throws Exception {
+        while (index< total){
+            //列表url构建
+            String listUrl = index == 0 ? String.format(indexUrl, "") : String.format(indexUrl, "_" + index);
+            System.out.println(listUrl);
+            //采集列表页
+            String res = doGet(listUrl);
+            //采集总页数
+            //var countPage = 5//共多少页
+            Pattern pattern = Pattern.compile("var countPage = (\\d+)//共多少页");
+            Matcher matcher = pattern.matcher(res);
+            if (matcher.find()) {
+                String totalStr = matcher.group(1);
+                total =  Integer.parseInt(totalStr);
+            }
+        Document doc = Jsoup.parse(res);
+
+//        Elements eachTags = doc.select("div[class=news-items] ul li:has(a)");
+        //#xxgk_right > div.zfxxgk_zd2.borderTop.zfxxgkzd.border0
+        Elements eachTags = doc.select("div.zfxxgk_zd2.borderTop.zfxxgkzd.border0 ul li:has(a)");
+        if (eachTags != null) {
+            if (eachTags.size() > 0) {
+                for (Element eachTag : eachTags) {
+                    count++;
+                    //解析详情页url
+                    String id = eachTag.select("a").first().attr("href");
+                    id = id.substring(id.indexOf("./") + 1);
+                    String link = listUrl.substring(0, listUrl.lastIndexOf("/")) + id;
+                    String title = eachTag.select("a").first().text().trim();
+                    System.out.println(title);
+                    System.out.println(link);
+                    //数据库是否已存在
+                    String querySql = "select DETAIL_LINK from XIN_XI_INFO_TEST where DETAIL_LINK = ?";
+                    try(Connection connection = MyDao.getConnection();
+                        PreparedStatement prepareStatement = connection.prepareStatement(querySql);){
+                        prepareStatement.setString(1,link);
+                        ResultSet resultSet = prepareStatement.executeQuery();
+                        if (resultSet.next()) {
+                            System.out.println("跳过" + link);
+                            resultSet.close();
+                            continue;
+                        }
+                        resultSet.close();
+                    }
+                    //采集详情页
+                    String detail = doGet(link);
+                    detail = detail.replace("display:none;", "");
+                    Document document = Jsoup.parse(detail);
+                    //body > div.bgf7f7f7 > div > div.listCon > div > div.contentCons > div.pages_content
+                    //body > div.bgf7f7f7 > div
+                    Element content = document.select("body > div.bgf7f7f7 > div").first();
+//                    Element pageContent = content.select("div.pages_content").first();
+                    //文件url清洗
+                    Elements as = content.select("a");
+                    for (Element a : as) {
+                        String href = a.attr("href");
+                        if (href.startsWith("./")) {
+                            href = link.substring(0, link.lastIndexOf('/')) + href.substring(href.indexOf("./") + 1);
+                            a.attr("href", href);
+                        } else if (href.startsWith("../")) {
+                            href = baseUrl + href.substring(href.lastIndexOf("../") + 2);
+                            a.attr("href", href);
+                        } else if (href.startsWith("/")) {
+                            href = baseUrl + href;
+                            a.attr("href", href);
+                        }
+                    }
+                    //图片url清洗
+                    Elements imgs = content.select("img");
+                    for (Element img : imgs) {
+                        String src = img.attr("src");
+                        if (src.startsWith("./")) {
+                            //https://www.tjdl.gov.cn/gongkai/zfxxgkzl/zfgbm/dlqmzj/gknr/czyjs/202507/W020250723358692283424_ORIGIN.jpg
+                            src = link.substring(0, link.lastIndexOf('/')) + src.substring(src.indexOf("./") + 1);
+                            img.attr("src", src);
+                        } else if (src.startsWith("../")) {
+                            src = baseUrl + src.substring(src.lastIndexOf("../") + 2);
+                            img.attr("src", src);
+                        } else if (src.startsWith("/")) {
+                            src = baseUrl + src;
+                            img.attr("src", src);
+                        }
+                    }
+                    //头部清洗
+                    content.select("div.dqwz").remove();
+                    content.select("div.contentMemo").remove();
+                    content.select("#articlePlayer").remove();
+                    content.select("#share").remove();
+                    content.select("dl.QrcodeBot").remove();
+                    //底部清洗
+                    content.select("#file_document").remove();
+                    content.select("div.inRight").remove();
+                    content.select("script").remove();
+                    content.select("style").remove();
+                    String text = content.text();
+
+                    //save
+//                    try (BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(Files.newOutputStream(Paths.get(System.getProperty("user.dir") + File.separator + "data" + File.separator + title.substring(2) + ".html"))))) {
+//                        bufferedWriter.write(content.toString());
+//                    }
+                    //入库
+                    //字段：ID，SOURCE_NAME（网站名称），DETAIL_LINK（详情链接），DETAIL_TITLE（详情标题），DETAIL_CONTENT（正文HTML），PAGE_TIME（列表时间），CREATE_TIME（创建时间），LIST_TITLE（列表标题），CREATE_BY（创建者名称）
+                    String SOURCE_NAME = "天津市东丽区民政局";
+                    String DETAIL_LINK = link;
+                    String DETAIL_TITLE = content.select("h1").first().text().trim();
+                    String DETAIL_CONTENT = content.toString();
+                    String dateStr = eachTag.select("b").first().text().trim();
+                    LocalDate PAGE_TIME = LocalDate.parse(dateStr);
+                    LocalDate CREATE_TIME = LocalDate.now();
+                    String LIST_TITLE = title.substring(2);
+                    String CREATE_BY = "黄略";
+
+                    String sql = "insert into " + MyDao.TABLE_NAME +"(SOURCE_NAME,DETAIL_LINK,DETAIL_TITLE,DETAIL_CONTENT,PAGE_TIME,CREATE_TIME,LIST_TITLE,CREATE_BY) VALUES (?,?,?,?,?,?,?,?)";
+                    try(Connection connection = MyDao.getConnection();
+                        PreparedStatement preparedStatement = connection.prepareStatement(sql);){
+                        preparedStatement.setString(1, SOURCE_NAME);
+                        preparedStatement.setString(2, DETAIL_LINK);
+                        preparedStatement.setString(3, DETAIL_TITLE);
+                        preparedStatement.setString(4, DETAIL_CONTENT);
+                        preparedStatement.setDate(5, Date.valueOf(PAGE_TIME));   // LocalDate -> java.sql.Date
+                        preparedStatement.setDate(6, Date.valueOf(CREATE_TIME));
+                        preparedStatement.setString(7, LIST_TITLE);
+                        preparedStatement.setString(8, CREATE_BY);
+                        preparedStatement.executeUpdate();
+                    }
+                }
+                }
+            }
+        index++;
+        }
+        System.out.println(count);
+    }
+
+
+    public static String doGet(String url) throws Exception {
+        String result = "";
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        HttpGet httpGet = new HttpGet(url);
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(10 * 1000).setSocketTimeout(60 * 1000)
+                .build();
+        httpGet.setConfig(requestConfig);
+        CloseableHttpResponse response = null;
+        try {
+            response = httpClient.execute(httpGet);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
+                result = content;
+            }
+        } finally {
+            if (response != null) {
+                response.close();
+            }
+            httpClient.hashCode();
+        }
+        return result;
+    }
+}

src/main/java/space/anyi/MyDao.java

@@ -0,0 +1,36 @@
+package space.anyi;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+public class MyDao {
+    private static final String url = "jdbc:oracle:thin:@192.168.2.42:1521:orcl";
+    private static final String host = "192.168.2.42";
+    private static final String name = "bxkc";
+    private static final String password = "bxkc";
+    public static final String TABLE_NAME = "XIN_XI_INFO_TEST";
+
+    static {
+        try {
+            Class.forName("oracle.jdbc.driver.OracleDriver");
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static Connection getConnection(){
+        try {
+            return DriverManager.getConnection(url, name, password);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static void close(Connection connection) throws SQLException {
+        if (!connection.isClosed()) {
+            connection.close();
+        }
+    }
+
+}