package space.anyi;
import org.junit.jupiter.api.Test;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import space.anyi.cleaner.DetailCleaner;
import space.anyi.client.GzCmcClient;
import space.anyi.config.Config;
import space.anyi.db.Db;
import space.anyi.db.Mybatis;
import space.anyi.dto.ChannelAllContentsResponse;
import space.anyi.dto.NewsItem;
import space.anyi.dto.NewsItemData;
import space.anyi.entity.News;
import space.anyi.mapper.NewsMapper;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
/**
* 20 条数据的完整流程测试。
*
*
覆盖两个阶段的整条链路:列表采集 -> 去重入库 -> 详情抓取 -> 清洗 ->
* 记者/编辑解析 -> 更新,用于快速验证全流程与可重复运行。
*/
public class MyListTest {
/**
* 执行 20 条的完整采集流程。
*
* @throws Exception 采集过程中未预期的异常
*/
@Test
public void fullCollectionFlow() throws Exception {
Config config = new Config();
Mybatis mybatis = new Mybatis(Db.init(config));
GzCmcClient client = new GzCmcClient(config);
NewsMapper mapper = mybatis.getMapper(NewsMapper.class);
try {
// 阶段一: 列表采集 20 条并去重入库
ChannelAllContentsResponse resp = client.search(config.keywords().get(0), 1, 20);
List newsList = new ArrayList<>();
for (NewsItem item : resp.getList()) {
NewsItemData data = item.getData();
if (data == null || data.getId() == null) {
continue;
}
News news = new News();
news.setId(data.getId());
news.setTitle(data.getTitle());
news.setUrl(data.getUrl());
news.setPublishTime(LocalDateTime.parse(data.getPublishTime(), Config.TIME_FORMATTER));
news.setChannelId(data.getChannelId());
news.setChannelName(data.getChannelName());
news.setEditor(data.getUserName());
newsList.add(news);
}
int inserted = 0;
for (News news : newsList) {
if (mapper.selectById(news.getId()) == null) {
mapper.insert(news);
inserted++;
}
}
System.out.printf("[20条流程] 列表采集=%d 新增入库=%d%n", newsList.size(), inserted);
// 阶段二: 逐条抓详情 -> 清洗 -> 解析 -> 更新
int ok = 0;
for (News news : newsList) {
try {
String html = client.fetchDetail(news.getUrl());
String content = DetailCleaner.clean(html);
Document doc = Jsoup.parse(content);
DetailCleaner.PersonInfo person = DetailCleaner.parsePersons(DetailCleaner.articleContentText(doc));
News update = new News();
update.setId(news.getId());
update.setContent(content);
update.setEditor(person.editor());
update.setTextReporters(person.textReporters().isEmpty() ? null : String.join(",", person.textReporters()));
update.setImageReporters(person.imageReporters().isEmpty() ? null : String.join(",", person.imageReporters()));
update.setCorrespondents(person.correspondents().isEmpty() ? null : String.join(",", person.correspondents()));
update.setContentFetchedAt(LocalDateTime.now());
mapper.updateById(update);
ok++;
System.out.printf("[20条流程] %s 编辑=%s 文字记者=%s 图片记者=%s 通讯员=%s%n",
news.getId(), person.editor(), person.textReporters(), person.imageReporters(), person.correspondents());
} catch (Exception e) {
System.err.printf("[20条流程] 失败 id=%s url=%s err=%s%n", news.getId(), news.getUrl(), e.getMessage());
}
Thread.sleep(config.sleepMs());
}
System.out.printf("[20条流程] 完成: 成功=%d 失败=%d%n", ok, newsList.size() - ok);
} finally {
mybatis.close();
}
}
}