newCollection/src/test/java/space/anyi/MyListTest.java

package space.anyi;

import org.junit.jupiter.api.Test;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import space.anyi.cleaner.DetailCleaner;
import space.anyi.client.GzCmcClient;
import space.anyi.config.Config;
import space.anyi.db.Db;
import space.anyi.db.Mybatis;
import space.anyi.dto.ChannelAllContentsResponse;
import space.anyi.dto.NewsItem;
import space.anyi.dto.NewsItemData;
import space.anyi.entity.News;
import space.anyi.mapper.NewsMapper;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;

/**
 * 20 条数据的完整流程测试。
 *
 * <p>覆盖两个阶段的整条链路：列表采集 -> 去重入库 -> 详情抓取 -> 清洗 ->
 * 记者/编辑解析 -> 更新，用于快速验证全流程与可重复运行。</p>
 */
public class MyListTest {

    /**
     * 执行 20 条的完整采集流程。
     *
     * @throws Exception 采集过程中未预期的异常
     */
    @Test
    public void fullCollectionFlow() throws Exception {
        Config config = new Config();
        Mybatis mybatis = new Mybatis(Db.init(config));
        GzCmcClient client = new GzCmcClient(config);
        NewsMapper mapper = mybatis.getMapper(NewsMapper.class);
        try {
            // 阶段一: 列表采集 20 条并去重入库
            ChannelAllContentsResponse resp = client.search(config.keywords().get(0), 1, 20);
            List<News> newsList = new ArrayList<>();
            for (NewsItem item : resp.getList()) {
                NewsItemData data = item.getData();
                if (data == null || data.getId() == null) {
                    continue;
                }
                News news = new News();
                news.setId(data.getId());
                news.setTitle(data.getTitle());
                news.setUrl(data.getUrl());
                news.setPublishTime(LocalDateTime.parse(data.getPublishTime(), Config.TIME_FORMATTER));
                news.setChannelId(data.getChannelId());
                news.setChannelName(data.getChannelName());
                news.setEditor(data.getUserName());
                newsList.add(news);
            }
            int inserted = 0;
            for (News news : newsList) {
                if (mapper.selectById(news.getId()) == null) {
                    mapper.insert(news);
                    inserted++;
                }
            }
            System.out.printf("[20条流程] 列表采集=%d 新增入库=%d%n", newsList.size(), inserted);

            // 阶段二: 逐条抓详情 -> 清洗 -> 解析 -> 更新
            int ok = 0;
            for (News news : newsList) {
                try {
                    String html = client.fetchDetail(news.getUrl());
                    String content = DetailCleaner.clean(html);
                    Document doc = Jsoup.parse(content);
                    DetailCleaner.PersonInfo person = DetailCleaner.parsePersons(DetailCleaner.articleContentText(doc));

                    News update = new News();
                    update.setId(news.getId());
                    update.setContent(content);
                    update.setEditor(person.editor());
                    update.setTextReporters(person.textReporters().isEmpty() ? null : String.join(",", person.textReporters()));
                    update.setImageReporters(person.imageReporters().isEmpty() ? null : String.join(",", person.imageReporters()));
                    update.setCorrespondents(person.correspondents().isEmpty() ? null : String.join(",", person.correspondents()));
                    update.setContentFetchedAt(LocalDateTime.now());
                    mapper.updateById(update);
                    ok++;
                    System.out.printf("[20条流程] %s 编辑=%s 文字记者=%s 图片记者=%s 通讯员=%s%n",
                            news.getId(), person.editor(), person.textReporters(), person.imageReporters(), person.correspondents());
                } catch (Exception e) {
                    System.err.printf("[20条流程] 失败 id=%s url=%s err=%s%n", news.getId(), news.getUrl(), e.getMessage());
                }
                Thread.sleep(config.sleepMs());
            }
            System.out.printf("[20条流程] 完成: 成功=%d 失败=%d%n", ok, newsList.size() - ok);
        } finally {
            mybatis.close();
        }
    }
}