package space.anyi; import org.junit.jupiter.api.Test; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import space.anyi.cleaner.DetailCleaner; import space.anyi.client.GzCmcClient; import space.anyi.config.Config; import space.anyi.db.Db; import space.anyi.db.Mybatis; import space.anyi.dto.ChannelAllContentsResponse; import space.anyi.dto.NewsItem; import space.anyi.dto.NewsItemData; import space.anyi.entity.News; import space.anyi.mapper.NewsMapper; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; /** * 20 条数据的完整流程测试。 * *

覆盖两个阶段的整条链路:列表采集 -> 去重入库 -> 详情抓取 -> 清洗 -> * 记者/编辑解析 -> 更新,用于快速验证全流程与可重复运行。

*/ public class MyListTest { /** * 执行 20 条的完整采集流程。 * * @throws Exception 采集过程中未预期的异常 */ @Test public void fullCollectionFlow() throws Exception { Config config = new Config(); Mybatis mybatis = new Mybatis(Db.init(config)); GzCmcClient client = new GzCmcClient(config); NewsMapper mapper = mybatis.getMapper(NewsMapper.class); try { // 阶段一: 列表采集 20 条并去重入库 ChannelAllContentsResponse resp = client.search(config.keywords().get(0), 1, 20); List newsList = new ArrayList<>(); for (NewsItem item : resp.getList()) { NewsItemData data = item.getData(); if (data == null || data.getId() == null) { continue; } News news = new News(); news.setId(data.getId()); news.setTitle(data.getTitle()); news.setUrl(data.getUrl()); news.setPublishTime(LocalDateTime.parse(data.getPublishTime(), Config.TIME_FORMATTER)); news.setChannelId(data.getChannelId()); news.setChannelName(data.getChannelName()); news.setEditor(data.getUserName()); newsList.add(news); } int inserted = 0; for (News news : newsList) { if (mapper.selectById(news.getId()) == null) { mapper.insert(news); inserted++; } } System.out.printf("[20条流程] 列表采集=%d 新增入库=%d%n", newsList.size(), inserted); // 阶段二: 逐条抓详情 -> 清洗 -> 解析 -> 更新 int ok = 0; for (News news : newsList) { try { String html = client.fetchDetail(news.getUrl()); String content = DetailCleaner.clean(html); Document doc = Jsoup.parse(content); DetailCleaner.PersonInfo person = DetailCleaner.parsePersons(DetailCleaner.articleContentText(doc)); News update = new News(); update.setId(news.getId()); update.setContent(content); update.setEditor(person.editor()); update.setTextReporters(person.textReporters().isEmpty() ? null : String.join(",", person.textReporters())); update.setImageReporters(person.imageReporters().isEmpty() ? null : String.join(",", person.imageReporters())); update.setCorrespondents(person.correspondents().isEmpty() ? null : String.join(",", person.correspondents())); update.setContentFetchedAt(LocalDateTime.now()); mapper.updateById(update); ok++; System.out.printf("[20条流程] %s 编辑=%s 文字记者=%s 图片记者=%s 通讯员=%s%n", news.getId(), person.editor(), person.textReporters(), person.imageReporters(), person.correspondents()); } catch (Exception e) { System.err.printf("[20条流程] 失败 id=%s url=%s err=%s%n", news.getId(), news.getUrl(), e.getMessage()); } Thread.sleep(config.sleepMs()); } System.out.printf("[20条流程] 完成: 成功=%d 失败=%d%n", ok, newsList.size() - ok); } finally { mybatis.close(); } } }