newCollection/src/test/java/space/anyi/MyListTest.java

package space.anyi;

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.Test;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import space.anyi.cleaner.DetailCleaner;
import space.anyi.client.GzCmcClient;
import space.anyi.config.Config;
import space.anyi.db.Db;
import space.anyi.db.Mybatis;
import space.anyi.dto.ChannelAllContentsResponse;
import space.anyi.dto.NewsItem;
import space.anyi.dto.NewsItemData;
import space.anyi.entity.News;
import space.anyi.mapper.NewsMapper;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * 20 条数据的完整流程测试。
 *
 * <p>覆盖两个阶段的整条链路：列表采集 -> 去重入库 -> 详情抓取 -> 清洗 ->
 * 记者/编辑解析 -> 更新，用于快速验证全流程与可重复运行。</p>
 */
public class MyListTest {
    private final static Logger log = LoggerFactory.getLogger(MyListTest.class);

    /**
     * 执行 20 条的完整采集流程。
     *
     * @throws Exception 采集过程中未预期的异常
     */
    @Test
    public void fullCollectionFlow() throws Exception {
        Config config = new Config();
        Mybatis mybatis = new Mybatis(Db.init(config));
        GzCmcClient client = new GzCmcClient(config);
        NewsMapper mapper = mybatis.getMapper(NewsMapper.class);
        try {
            // 阶段一: 列表采集 20 条并去重入库
            ChannelAllContentsResponse resp = client.search(config.keywords().get(0), config.channelIds().get(0), 1, 20);
            List<News> newsList = new ArrayList<>();
            for (NewsItem item : resp.getList()) {
                NewsItemData data = item.getData();
                if (data == null || data.getId() == null) {
                    continue;
                }
                News news = new News();
                news.setId(data.getId());
                news.setTitle(DetailCleaner.titleClean(data.getTitle()));
                news.setUrl(data.getUrl());
                news.setPublishTime(LocalDateTime.parse(data.getPublishTime(), Config.TIME_FORMATTER));
                news.setChannelId(data.getChannelId());
                news.setChannelName(data.getChannelName());
                news.setEditor(data.getUserName());
                newsList.add(news);
            }
            int inserted = 0;
            for (News news : newsList) {
                if (mapper.selectById(news.getId()) == null) {
                    mapper.insert(news);
                    inserted++;
                }
            }
            log.info("[20条流程] 列表采集={} 新增入库={}", newsList.size(), inserted);

            // 阶段二: 逐条抓详情 -> 清洗 -> 解析 -> 更新
            int ok = 0;
            for (News news : newsList) {
                try {
                    String html = client.fetchDetail(news.getUrl());
                    String content = DetailCleaner.clean(html);
                    Document doc = Jsoup.parse(content);
                    DetailCleaner.PersonInfo person = DetailCleaner.parsePersons(DetailCleaner.articleContentText(doc));

                    News update = new News();
                    update.setId(news.getId());
                    update.setContent(content);
                    update.setEditor(person.editor());
                    update.setTextReporters(person.textReporters().isEmpty() ? null : String.join(",", person.textReporters()));
                    update.setImageReporters(person.imageReporters().isEmpty() ? null : String.join(",", person.imageReporters()));
                    update.setCorrespondents(person.correspondents().isEmpty() ? null : String.join(",", person.correspondents()));
                    update.setContentFetchedAt(LocalDateTime.now());
                    mapper.updateById(update);
                    ok++;
                    log.info("[20条流程] {} 编辑={} 文字记者={} 图片记者={} 通讯员={}",
                            news.getId(), person.editor(), person.textReporters(), person.imageReporters(), person.correspondents());
                } catch (Exception e) {
                    log.error("[20条流程] 失败 id={} url={} err={}", news.getId(), news.getUrl(), e.getMessage());
                }
                Thread.sleep(config.sleepMs());
            }
            log.info("[20条流程] 完成: 成功={} 失败={}", ok, newsList.size() - ok);
        } finally {
            mybatis.close();
        }
    }

    @Test
    public void cleanHtmlTest() {
        String html = """
                <!DOCTYPE html>
                <html lang="en" style="font-size: 18px;">
                <head>
                  <meta charset="UTF-8">
                  <link rel="icon" href="https://oss.gz-cmc.com/default-image/400x400.png" />
                  <title>畅通版权交易渠道，漫博会推动国产IP出海与海外IP引进落地</title>
                  <meta name="viewport" content="width=device-width,initial-scale=1,minimum-scale=1">
                  <meta name="keywords" content="">
                  <meta name="author" content="莫斯其格">
                  <meta name="description" content="">
                  <meta property="og:type" content="article">
                  <meta property="og:title" content="畅通版权交易渠道，漫博会推动国产IP出海与海外IP引进落地">
                  <meta property="og:description" content="畅通版权交易渠道，漫博会推动国产IP出海与海外IP引进落地">
                  <meta property="og:url" content>
                  <meta property="og:image" itemprop="image" content="https://oss.gz-cmc.com/default-image/default-share-image.jpg">
                  <meta property="article:author" content="莫斯其格">
                      <meta property="article:published_time" content="2026-07-24">
                  <style name="launch-app-style">
                      .download-link {
                          text-decoration: none;
                          color: #333 !important;
                      }
                
                      .guide-image {
                          width: 100vw;
                      }
                  </style>
                  <style name="launch-swiper-style">
                      .swiper-news-box {
                          display: flex;
                          align-items: center;
                          justify-content: space-between;
                          padding: 10px;
                          width: 100vw;
                          box-sizing: border-box;
                          height: 62px;
                      }
                
                      .swiper-news-image {
                          width: 64px;
                          height: 36px;
                          object-fit: cover;
                          border-radius: 5px;
                          flex-shrink: 0;
                      }
                
                      .swiper-news-title {
                          font-size: 14px;
                          text-align: justify;
                          margin-left: 10px;
                          line-height: 1.5;
                          word-break: break-all;
                          overflow: hidden;
                          display: -webkit-box;
                          -webkit-line-clamp: 2;
                          -webkit-box-orient: vertical;
                          flex-grow: 1;
                      }
                
                      .swiper-news-button {
                          width: 65px;
                          height: 24px;
                          background: linear-gradient(90deg, #ef4243 0%, #fa7419 100%);
                          border-radius: 12px;
                          color: #ffffff;
                          font-size: 14px;
                          margin-left: 10px;
                          flex-shrink: 0;
                          display: flex;
                          align-items: center;
                          justify-content: center;
                      }
                  </style>
                  <style name="like-unlike-style">
                      .comment-unlike-icon,
                      .comment-like-icon {
                          width: 18px;
                          height: 18px;
                          cursor: pointer;
                      }
                      .sub-comment-unlike-icon,
                      .sub-comment-like-icon {
                          width: 16px;
                          height: 16px;
                          cursor: pointer;
                      }
                  </style>
                  <style name="comment-reply-style">
                      .comment-reply {
                          height: 30px;
                          border-radius: 15px;
                          background-color: #e5e5e5;
                          display: flex;
                          align-items: center;
                          width: 100px;
                          justify-content: center;
                          font-size: 14px;
                          padding-left: 5px;
                      }
                  </style>
                  <style name="comment-more-style">
                      .more-comment {
                          height: 44px;
                          background: rgba(248, 110, 29, 0.1);
                          border-radius: 22px;
                          border: 1px solid #f04640;
                          width: 250px;
                          margin: 15px auto 0 auto;
                          display: flex;
                          align-items: center;
                          justify-content: center;
                          color: #e60213;
                          cursor: pointer;
                      }
                  </style>
                  <style name="article-launch-app-style">
                    .text-launch-app {
                      width: 100%;
                      height: 40px;
                      text-align: center;
                      background: linear-gradient(90deg, #EF4243 0%, #FA7419 100%);
                      border-radius: 20px;
                      font-size: 14px;
                      color: #fff;
                      display: flex;
                      align-items: center;
                      justify-content: center;
                    }
                
                    .text-huacheng-icon {
                      width: 20px;
                      height: 20px;
                      margin-right: 5px;
                    }
                  </style>
                  <style name="article-operate-style">
                    .article-operate-list {
                      display: flex;
                      align-items: center;
                      justify-content: space-between;
                    }
                
                    .article-operate-input {
                      height: 28px;
                      background: #F5F5F5;
                      border-radius: 14px;
                      width: 50%;
                      padding: 0 15px;
                      font-size: 12px;
                      color: #999;
                      line-height: 28px;
                    }
                
                    .article-operate-comment,
                    .article-operate-collect,
                    .article-operate-like,
                    .article-operate-share {
                      width: 22px; \s
                      height: 22px;
                    }
                  </style>
                  <link href="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/css/86-6b9ab229.css" rel="stylesheet">
                  <link href="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/css/textDetail-43f87c2a.css" rel="stylesheet">
                </head>
                
                <body>
                <div class="container">
                      <div class="header-guide-box" id="header-guide-box">
                        <div class="launch-app-btn" name="launch-app-btn" data-style="launch-app-style">
                          <a name="download-link" class="download-link">
                            <img name="guide-image" alt="" class="guide-image"></a>
                        </div>
                        <img class="close-guide"
                             name="close-guide"
                             src="https://oss.gz-cmc.com/news-static/huacheng/2024-03-25/images/guide-close-cc491637.png"
                             alt="关闭引导">
                      </div>
                    <div class="article-title" id="article-title">畅通版权交易渠道，漫博会推动国产IP出海与海外IP引进落地</div>
                      <div class="not-exist-media-leader">
                        <div class="article-source-time">
                            <div class="article-time">2026-07-24 15:43:26</div>
                            <div class="article-source">广州日报新花城</div>
                        </div>
                      </div>
                
                      <div class="article-video-fixed">
                        <img src="https://oss.gz-cmc.com/news-static/huacheng/2025-06-10/images/close-3e4f193c.png"
                          class="article-video-fixed-close" alt="">
                        <video id="sticky-player" class="video-js vjs-16-9" webkit-playsinline="true" playsinline="true"
                          x5-video-player-type="h5" controls="false">
                        </video>
                      </div>
                      <div class="article-content" id="article-content">
                      <p style="text-align: justify;">7月24日，记者从第十六届中国国际影视动漫版权保护和贸易博览会新闻发布会上获悉，第十六届中国国际动漫博览会将于2026年8月6日至10日在东莞“中国潮玩之都·漫博中心”举行。据悉，本届展会现场设立版权服务工作站，提供版权登记咨询、快速登记、维权指引等一站式服务，为参展企业原创IP、新品设计提供即时版权保护。同时建立展会版权快速维权机制，严厉打击侵权盗版行为，大力营造尊重原创、保护版权的良好展会氛围。</p><p style="text-align: justify;"><span class="insert-img-container" style="display:inline-block;line-height: 1.3em; "><img style="vertical-align: bottom;" src="https://oss.gz-cmc.com/pgcr/root/huacheng/upload/news/image/2026/07/24/640aebfbf44e4649954551fbd309dd04.jpg?x-oss-process=style/content" class="uedito-cusimg" title="微信图片_20260724110823_7734_327" alt="微信图片_20260724110823_7734_327"/><br/></span></p><p style="text-align: justify;">本届展会畅通版权交易渠道，激活版权市场价值。展会进一步打造专业化版权交易对接平台，设置IP产业融合馆，配套举办AI漫剧产业对接及出海合作活动、IP品牌授权精准对接交流活动、文博IP跨界合作发布会、非遗潮玩创新合作发布会等产业活动，推动版权方与制造企业、品牌方、渠道商精准对接，促进版权成果转化落地。展会还将联动中阿合作中心、西班牙商会等数十家海外商会、协会、外事机构，组织跨境采购团，搭建中外版权合作桥梁，推动国产IP出海与海外IP引进落地，构建双向循环的版权贸易格局。</p><p style="text-align: justify;">展会将创新版权金融服务，赋能企业健康发展。展会现场设置版权金融咨询专区，为中小微动漫潮玩企业提供融资对接服务，破解企业发展难点。同时开展版权金融政策宣讲，普及质押融资流程等知识，引导企业用好版权金融工具，将版权资产转化为发展动能。</p><p style="text-align: justify;">文/广州日报新花城记者：莫斯其格 实习生：谭斯文</p><p style="text-align: justify;">图/广州日报新花城记者：杨泽彬</p><p style="text-align: justify;">广州日报新花城编辑：吴嘉丽</p>
                      </div>
                    <div class="read-like-number hidden" id="read-like-number">
                      <div class="read-number">浏览量：<span id="read-number-text"></span></div>
                      <div class="like-number">点赞量：<span id="like-number-text"></span></div>
                    </div>
                      <div class="article-belong-topics"></div>
                  <div class="article-copyright">
                    <div class="article-copyright-line"></div>
                    <div>@新花城 版权所有 转载需经授权</div>
                    <div class="article-copyright-line"></div>
                  </div>
                    <div class="media-list swiper" id="media-list">
                      <div class="swiper-wrapper">
                                <a href="https://huacheng.gz-cmc.com/html/mediaDetail.html?siteId=5e88c884e2ed4e7a9a8d5225c299f707&mediaId=3dc715eda4514b24b3031931713c3f18" data-id="3dc715eda4514b24b3031931713c3f18" class="media-box-link swiper-slide">
                                  <div class="media-box">
                                    <img
                                      src="https://oss.gz-cmc.com/default-image/default-avatar.png"
                                      data-src=""
                                      alt="" onerror="this.src='https://oss.gz-cmc.com/default-image/default-avatar.png'" class="media-avatar lazyload">
                                    <div class="media-name">吴嘉丽</div>
                                    <div class="media-more">更多文章</div>
                                  </div>
                                </a>
                                <a href="https://huacheng.gz-cmc.com/html/mediaDetail.html?siteId=5e88c884e2ed4e7a9a8d5225c299f707&mediaId=d37d7de896e84e2598b55396aa6143a5" data-id="d37d7de896e84e2598b55396aa6143a5" class="media-box-link swiper-slide">
                                  <div class="media-box">
                                    <img
                                      src="https://oss.gz-cmc.com/default-image/default-avatar.png"
                                      data-src=""
                                      alt="" onerror="this.src='https://oss.gz-cmc.com/default-image/default-avatar.png'" class="media-avatar lazyload">
                                    <div class="media-name">杨泽彬</div>
                                    <div class="media-more">更多文章</div>
                                  </div>
                                </a>
                                <a href="https://huacheng.gz-cmc.com/html/mediaDetail.html?siteId=5e88c884e2ed4e7a9a8d5225c299f707&mediaId=ea916e08c213441093623cb4bbd79a99" data-id="ea916e08c213441093623cb4bbd79a99" class="media-box-link swiper-slide">
                                  <div class="media-box">
                                    <img
                                      src="https://oss.gz-cmc.com/default-image/default-avatar.png"
                                      data-src="https://ugcoss.gz-cmc.com/userr/root/huacheng/user/mp/media/icon/2023/11/02/22d2c55b55264b658a65e4d4cb70c20c.jpg"
                                      alt="" onerror="this.src='https://oss.gz-cmc.com/default-image/default-avatar.png'" class="media-avatar lazyload">
                                    <div class="media-name">莫斯其格</div>
                                    <div class="media-more">更多文章</div>
                                  </div>
                                </a>
                      </div>
                    </div>
                  <div class="article-launch-app-box" id="article-launch-app-box">
                    <div class="launch-app-btn" name="launch-app-btn" data-style="article-launch-app-style">
                      <a name="download-link" class="download-link">
                        <div class="text-launch-app">
                          <img class="text-huacheng-icon"
                            src="https://oss.gz-cmc.com/news-static/huacheng/2024-03-25/images/text-huacheng-icon-2f415b25.png"
                            alt="打开app">打开广州日报新花城，享受流畅体验
                        </div>
                      </a>
                    </div>
                  </div>
                  <div class="article-comment" id="article-comment">
                    <div class="article-empty-line"></div>
                    <div class="related-title">热门评论</div>
                    <div id="comment-list" class="comment-list">
                      <img src="https://oss.gz-cmc.com/news-static/huacheng/2024-03-25/images/comment-empty-5a1fe672.png"
                        class="comment-empty-img" alt="">
                    </div>
                    <div class="more-comment-box" id="more-comment-box">
                      <div class="launch-app-btn" name="launch-app-btn" data-style="comment-more-style">
                        <a class="download-link" name="download-link">
                          <div class="more-comment">查看更多评论</div>
                        </a>
                      </div>
                    </div>
                  </div>
                  <div class="related-content" id="related-content">
                    <div class="article-empty-line"></div>
                    <div class="related-title">相关推荐</div>
                    <div id="content-list" class="content-list"></div>
                  </div>
                  <div class="related-channel" id="related-channel">
                    <div class="article-empty-line"></div>
                    <div class="related-title">相关频道推荐</div>
                    <div id="channel-list" class="channel-list"></div>
                  </div>
                  <div class="article-operate" id="article-operate">
                    <div class="launch-app-btn" name="launch-app-btn" data-style="article-operate-style">
                      <a class="download-link" name="download-link">
                        <div class="article-operate-list">
                          <div class="article-operate-input">说点什么</div>
                          <img src="https://oss.gz-cmc.com/news-static/huacheng/2024-03-25/images/text-comment-e8e04213.png"
                            alt="评论" class="article-operate-comment">
                          <img src="https://oss.gz-cmc.com/news-static/huacheng/2024-03-25/images/text-collect-b1054ae5.png"
                            alt="收藏" class="article-operate-collect">
                          <img src="https://oss.gz-cmc.com/news-static/huacheng/2024-03-25/images/text-like-67f9fd7d.png"
                            alt="点赞" class="article-operate-like">
                          <img src="https://oss.gz-cmc.com/news-static/huacheng/2024-03-25/images/text-share-9200469e.png"
                            alt="分享" class="article-operate-share">
                        </div>
                      </a>
                    </div>
                  </div>
                </div>
                <div class="hidden-box" id="hidden-box">
                  <div hidden name="mainDomain">https://www.gz-cmc.com</div>
                  <div hidden name="subDomain">https://huacheng.gz-cmc.com</div>
                  <div hidden name="newsId">70290505ac3f439cbe7a96d971aae131</div>
                  <div hidden name="channelId">903d342af9af43a59cf7cd9d5342be0b</div>
                  <div hidden name="siteId">5e88c884e2ed4e7a9a8d5225c299f707</div>
                  <div hidden name="shareTitle">畅通版权交易渠道，漫博会推动国产IP出海与海外IP引进落地</div>
                  <div hidden name="shareDesc">分享来自广州日报新花城客户端，请点击打开更多精彩...</div>
                  <div hidden name="shareImgUrl">https://oss.gz-cmc.com/default-image/default-share-image.jpg</div>
                  <div hidden name="staticType">1</div>
                  <div hidden name="contentType">1</div>
                  <div hidden name="mListpattern">1</div>
                </div>
                  <script defer src="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/js/681-ddb2f1b4.js"></script>
                  <script defer src="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/js/86-f0059e14.js"></script>
                  <script defer src="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/js/textDetail-e2c6dbe5.js"></script>
                  <script defer src="https://oss.gz-cmc.com/news-static/huacheng/config/config.js"></script>
                </body>
                </html>
                """;
        Document doc = Jsoup.parse(html);

        html = """
                <!DOCTYPE html>
                <html lang="en" style="font-size: 18px;">
                <head></head>
                <body>
                </body>
                </html>
                """;
        Document result = Jsoup.parse(html);

        Elements style = doc.select("link[rel='stylesheet']");
        Element head = result.head();
        head.insertChildren(0, style);

        Element container = doc.select("div.container").first();
        Elements keep = container.children().select("div.article-title, div.not-exist-media-leader, div.article-content");
        keep.select("video").remove();
        result.body().appendChildren(keep);

        log.info("{}", result.html());

    }

    @Test
    public void html2pdfTest() throws Exception {
        String html = """
                <!doctype html>
                <html lang="en" style="font-size: 18px;">
                 <head>
                  <link href="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/css/86-6b9ab229.css" rel="stylesheet">
                  <link href="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/css/textDetail-43f87c2a.css" rel="stylesheet">
                 </head>
                 <body>
                  <div class="article-title" id="article-title">
                   畅通版权交易渠道，漫博会推动国产IP出海与海外IP引进落地
                  </div>
                  <div class="not-exist-media-leader">
                   <div class="article-source-time">
                    <div class="article-time">
                     2026-07-24 15:43:26
                    </div>
                    <div class="article-source">
                     广州日报新花城
                    </div>
                   </div>
                  </div>
                  <div class="article-content" id="article-content">
                   <p style="text-align: justify;">7月24日，记者从第十六届中国国际影视动漫版权保护和贸易博览会新闻发布会上获悉，第十六届中国国际动漫博览会将于2026年8月6日至10日在东莞“中国潮玩之都·漫博中心”举行。据悉，本届展会现场设立版权服务工作站，提供版权登记咨询、快速登记、维权指引等一站式服务，为参展企业原创IP、新品设计提供即时版权保护。同时建立展会版权快速维权机制，严厉打击侵权盗版行为，大力营造尊重原创、保护版权的良好展会氛围。</p>
                   <p style="text-align: justify;"><span class="insert-img-container" style="display:inline-block;line-height: 1.3em; "><img style="vertical-align: bottom;" src="https://oss.gz-cmc.com/pgcr/root/huacheng/upload/news/image/2026/07/24/640aebfbf44e4649954551fbd309dd04.jpg?x-oss-process=style/content" class="uedito-cusimg" title="微信图片_20260724110823_7734_327" alt="微信图片_20260724110823_7734_327"><br></span></p>
                   <p style="text-align: justify;">本届展会畅通版权交易渠道，激活版权市场价值。展会进一步打造专业化版权交易对接平台，设置IP产业融合馆，配套举办AI漫剧产业对接及出海合作活动、IP品牌授权精准对接交流活动、文博IP跨界合作发布会、非遗潮玩创新合作发布会等产业活动，推动版权方与制造企业、品牌方、渠道商精准对接，促进版权成果转化落地。展会还将联动中阿合作中心、西班牙商会等数十家海外商会、协会、外事机构，组织跨境采购团，搭建中外版权合作桥梁，推动国产IP出海与海外IP引进落地，构建双向循环的版权贸易格局。</p>
                   <p style="text-align: justify;">展会将创新版权金融服务，赋能企业健康发展。展会现场设置版权金融咨询专区，为中小微动漫潮玩企业提供融资对接服务，破解企业发展难点。同时开展版权金融政策宣讲，普及质押融资流程等知识，引导企业用好版权金融工具，将版权资产转化为发展动能。</p>
                   <p style="text-align: justify;">文/广州日报新花城记者：莫斯其格 实习生：谭斯文</p>
                   <p style="text-align: justify;">图/广州日报新花城记者：杨泽彬</p>
                   <p style="text-align: justify;">广州日报新花城编辑：吴嘉丽</p>
                  </div>
                 </body>
                </html>
                """;

        File outDir = new File("output");
        Files.createDirectories(outDir.toPath());
        File pdf = new File(outDir, LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) + ".pdf");

        DetailCleaner.html2pdf(html, pdf.toPath());
        log.info("[html2pdf] 生成成功: {}", pdf.getAbsolutePath());
    }

    @Test
    public void html2plaintextTest(){
        String html = """
                <!doctype html>
                <html lang="en" style="font-size: 18px;">
                 <head>
                  <link href="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/css/86-6b9ab229.css" rel="stylesheet">
                  <link href="https://oss.gz-cmc.com/news-static/huacheng/2026-06-03/css/textDetail-43f87c2a.css" rel="stylesheet">
                 </head>
                 <body>
                  <div class="article-title" id="article-title">
                   畅通版权交易渠道，漫博会推动国产IP出海与海外IP引进落地
                  </div>
                  <div class="not-exist-media-leader">
                   <div class="article-source-time">
                    <div class="article-time">
                     2026-07-24 15:43:26
                    </div>
                    <div class="article-source">
                     广州日报新花城
                    </div>
                   </div>
                  </div>
                  <div class="article-content" id="article-content">
                   <p style="text-align: justify;">7月24日，记者从第十六届中国国际影视动漫版权保护和贸易博览会新闻发布会上获悉，第十六届中国国际动漫博览会将于2026年8月6日至10日在东莞“中国潮玩之都·漫博中心”举行。据悉，本届展会现场设立版权服务工作站，提供版权登记咨询、快速登记、维权指引等一站式服务，为参展企业原创IP、新品设计提供即时版权保护。同时建立展会版权快速维权机制，严厉打击侵权盗版行为，大力营造尊重原创、保护版权的良好展会氛围。</p>
                   <p style="text-align: justify;"><span class="insert-img-container" style="display:inline-block;line-height: 1.3em; "><img style="vertical-align: bottom;" src="https://oss.gz-cmc.com/pgcr/root/huacheng/upload/news/image/2026/07/24/640aebfbf44e4649954551fbd309dd04.jpg?x-oss-process=style/content" class="uedito-cusimg" title="微信图片_20260724110823_7734_327" alt="微信图片_20260724110823_7734_327"><br></span></p>
                   <p style="text-align: justify;">本届展会畅通版权交易渠道，激活版权市场价值。展会进一步打造专业化版权交易对接平台，设置IP产业融合馆，配套举办AI漫剧产业对接及出海合作活动、IP品牌授权精准对接交流活动、文博IP跨界合作发布会、非遗潮玩创新合作发布会等产业活动，推动版权方与制造企业、品牌方、渠道商精准对接，促进版权成果转化落地。展会还将联动中阿合作中心、西班牙商会等数十家海外商会、协会、外事机构，组织跨境采购团，搭建中外版权合作桥梁，推动国产IP出海与海外IP引进落地，构建双向循环的版权贸易格局。</p>
                   <p style="text-align: justify;">展会将创新版权金融服务，赋能企业健康发展。展会现场设置版权金融咨询专区，为中小微动漫潮玩企业提供融资对接服务，破解企业发展难点。同时开展版权金融政策宣讲，普及质押融资流程等知识，引导企业用好版权金融工具，将版权资产转化为发展动能。</p>
                   <p style="text-align: justify;">文/广州日报新花城记者：莫斯其格 实习生：谭斯文</p>
                   <p style="text-align: justify;">图/广州日报新花城记者：杨泽彬</p>
                   <p style="text-align: justify;">广州日报新花城编辑：吴嘉丽</p>
                  </div>
                 </body>
                </html>
                """;
        String text = Jsoup.parse(html).text();
        log.info("{}", text);
    }

    /**
     * 为库中已采集详情（content 非空）的新闻批量生成 PDF。
     *
     * <p>取发布时间最新的 100 条，逐条渲染到 output/ 目录，单条失败不中断整体。</p>
     *
     * @throws Exception 数据库初始化等未预期的异常
     */
    @Test
    public void test() throws Exception {
        Config config = new Config();
        Mybatis mybatis = new Mybatis(Db.init(config));
        try {
            NewsMapper mapper = mybatis.getMapper(NewsMapper.class);

            //获取一百条已采集详情的记录
            QueryWrapper<News> wrapper = new QueryWrapper<News>()
                    .isNotNull("content")
                    .eq("channel_id","903d342af9af43a59cf7cd9d5342be0b")
                    .orderByDesc("publish_time")
                    .last("LIMIT 100");
            List<News> pending = mapper.selectList(wrapper);
            log.info("获取数据成功:{}条", pending.size());

            File outDir = new File("output");
            Files.createDirectories(outDir.toPath());
            int ok = 0;
            for (News news : pending) {
                try {
                    String publicTime = news.getPublishTime().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"));
                    String editor = news.getEditor() == null ? "" : news.getEditor();
                    String title = news.getTitle() == null ? "" : news.getTitle();
                    String fileName = sanitizeFileName(String.format("%s-%s-%s-%s.pdf", publicTime, title, editor, news.getId()));
                    File file = new File(outDir, fileName);
                    DetailCleaner.html2pdf(news.getContent(), file.toPath());
                    ok++;
                } catch (Exception e) {
                    log.error("[生成PDF] 失败 id={} err={}", news.getId(), e.getMessage());
                }
            }
            log.info("[生成PDF] 完成: 成功={} 失败={}", ok, pending.size() - ok);
        } finally {
            mybatis.close();
        }
    }

    /**
     * 清理文件名中的非法字符并截断超长标题。
     *
     * <p>截断按 UTF-8 字节数计算（文件名上限 255 字节，中文每字占 3 字节），
     * 并在字符边界处切断，避免截出乱码。</p>
     *
     * @param name 原始文件名
     * @return 可安全落盘的文件名（最长 200 字节）
     */
    private static String sanitizeFileName(String name) {
        String s = name.replaceAll("[\\\\/:*?\"<>|\\s]", "_");
        byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
        if (bytes.length <= 200) {
            return s;
        }
        int cut = 200;
        while (cut > 0 && (bytes[cut - 1] & 0xC0) == 0x80) {
            cut--;
        }
        return new String(bytes, 0, cut, StandardCharsets.UTF_8);
    }

    @Test
    public void titleCleanTest() throws Exception {
        Config config = new Config();
        Mybatis mybatis = new Mybatis(Db.init(config));
        try {
            NewsMapper mapper = mybatis.getMapper(NewsMapper.class);
            List<News> all = mapper.selectList(new QueryWrapper<News>().like("title", "<"));
            int updated = 0;
            for (News news : all) {
                String cleaned = DetailCleaner.titleClean(news.getTitle());
                if (cleaned != null && !cleaned.equals(news.getTitle())) {
                    News update = new News();
                    update.setId(news.getId());
                    update.setTitle(cleaned);
                    mapper.updateById(update);
                    updated++;
                    log.info("[标题清洗] {} 原:{}  新:{}", news.getId(), news.getTitle(), cleaned);
                }
            }
            log.info("[标题清洗] 完成: 总数={} 更新={}", all.size(), updated);
        } finally {
            mybatis.close();
        }
    }

    /**
     * 统计日期区间中新闻数量和去除包含视频新闻后的数量
     * 20260913
     */
    @Test
    public void videoCountTest() throws Exception {
        //1.从数据库获取符合日期要求的新闻
        //select * from news where (publish_time >= '2024-01-01' and publish_time < '2024-03-01') or (publish_time >= '2024-09-01' and publish_time < '2024-10-01') and content is not null;
        Config config = new Config();
        Mybatis mybatis = new Mybatis(Db.init(config));
        try {
            NewsMapper mapper = mybatis.getMapper(NewsMapper.class);
            List<News> newsList = mapper.selectList(new QueryWrapper<News>()
                    .and(w -> w.ge("publish_time", LocalDateTime.of(2024, 1, 1, 0, 0))
                            .lt("publish_time", LocalDateTime.of(2024, 3, 1, 0, 0)))
                    .or(w -> w.ge("publish_time", LocalDateTime.of(2024, 9, 1, 0, 0))
                            .lt("publish_time", LocalDateTime.of(2024, 10, 1, 0, 0))
                            .isNotNull("content")));
            log.info("[视频统计] 获取新闻={}条", newsList.size());

            //2.统计包含视频的新闻(通过URL获取新闻的html，包含 video 标签即为包含视频，使用Jsoup解析html进行判断)
            GzCmcClient client = new GzCmcClient(config);
            Map<String, Boolean> videoMap = new LinkedHashMap<>();
            int videoCount = 0;
            for (News news : newsList) {
                boolean hasVideo = false;
                try {
                    String html = client.fetchDetail(news.getUrl());
                    hasVideo = !Jsoup.parse(html).select("video").isEmpty();
                    if (hasVideo) {
                        videoCount++;
                    }
                    log.info("[视频统计] {} 视频={}", news.getId(), hasVideo);
                } catch (Exception e) {
                    log.error("[视频统计] 失败 id={} url={} err={}", news.getId(), news.getUrl(), e.getMessage());
                }
                videoMap.put(news.getId(), hasVideo);
                Thread.sleep(config.sleepMs());
            }

            //3.输出结果(新闻总数，去掉包含视频新闻的数量)
            log.info("[视频统计] 完成: 总数={} 包含视频={} 去除视频后={}",
                    newsList.size(), videoCount, newsList.size() - videoCount);

            //4.统计结果存储到CSV(仅明细: id,是否包含视频)
            List<String> lines = new ArrayList<>();
            lines.add("id,video");
            for (Map.Entry<String, Boolean> e : videoMap.entrySet()) {
                lines.add(e.getKey() + "," + e.getValue());
            }
            File csv = new File("videoCountResult.csv");
            Files.write(csv.toPath(), lines, StandardCharsets.UTF_8);
            log.info("[视频统计] 已写入: {} (共{}行)", csv.getAbsolutePath(), lines.size());
        } finally {
            mybatis.close();
        }
    }

    /**
     * 二次筛选
     * - 标题出现一次关键字或正文出现一个关键词两次
     * - 正文中包含视频
     * 记录符合要求的id和命中原因（使用Map）
     * 输出二次筛选符合的数量
     * 20260913
     */
    @Test
    public void secondSelectTest() throws Exception {
        /**
         * 1.从文件 videoCountResult.csv 中选出包含视频的新闻id(仅video=true的记录)
         * 2.从数据库中获取新闻
         * 3.进行筛选,正文需要使用Jsoup进行存文本解析，数据库存储的是HTML
         * 4.输出符合的记录和数量
         * 5.筛选结果存储到result.csv文件中
         */
        Set<String> titleKeys = Set.of("非遗","非物质文化遗产","非遗保护","遗产保护","传承","保护项目","文化政策","文化多样性","非遗创新","文化传统","非遗价值","非遗传承人","非遗活动");
        Set<String> contentKeys = Set.of("非物质文化遗产","非遗保护","遗产保护","传承","保护项目","文化政策","文化多样性","非遗创新","文化传统","非遗价值","非遗传承人","非遗活动");

        //1.从CSV中选出包含视频的新闻id
        Set<String> videoIds = new HashSet<>();
        for (String line : Files.readAllLines(Path.of("videoCountResult.csv"), StandardCharsets.UTF_8)) {
            if (line.isBlank() || line.startsWith("id,")) {
                continue;
            }
            String[] cols = line.split(",", -1);
            if (cols.length >= 2 && "true".equals(cols[1].trim())) {
                videoIds.add(cols[0].trim());
            }
        }
        log.info("[二次筛选] CSV视频新闻={}条", videoIds.size());

        //2.从数据库中获取新闻
        Config config = new Config();
        Mybatis mybatis = new Mybatis(Db.init(config));
        try {
            NewsMapper mapper = mybatis.getMapper(NewsMapper.class);
            List<News> newsList = videoIds.isEmpty() ? List.of() : mapper.selectBatchIds(videoIds);
            log.info("[二次筛选] 数据库命中={}条", newsList.size());

            //3.进行标题判断和正文判断（正文存储为HTML，需先用Jsoup解析为纯文本）
            Map<String, String> matched = new LinkedHashMap<>();
            for (News news : newsList) {
                String reason = matchReason(news, titleKeys, contentKeys);
                if (reason != null) {
                    matched.put(news.getId(), reason);
                }
            }

            //4.输出符合的记录和数量
            matched.forEach((id, reason) -> log.info("[二次筛选] {} 命中={}", id, reason));
            log.info("[二次筛选] 符合: {}条", matched.size());

            //5.筛选结果存储到result.csv文件中
            List<String> lines = new ArrayList<>();
            lines.add("id,title,reason");
            for (News news : newsList) {
                String reason = matched.get(news.getId());
                if (reason != null) {
                    lines.add(csvEscape(news.getId()) + "," + csvEscape(news.getTitle()) + "," + csvEscape(reason));
                }
            }
            File csv = new File("result1.csv");
            Files.write(csv.toPath(), lines, StandardCharsets.UTF_8);
            log.info("[二次筛选] 已写入: {}", csv.getAbsolutePath());
        } finally {
            mybatis.close();
        }
    }

    /**
     * 判断新闻是否命中二次筛选规则：标题出现任一关键字一次，或正文出现任一关键字两次。
     *
     * @param news        新闻对象
     * @param titleKeys   标题关键字集合
     * @param contentKeys 正文关键字集合
     * @return 命中原因；未命中返回 null
     */
    private static String matchReason(News news, Set<String> titleKeys, Set<String> contentKeys) {
        String title = news.getTitle() == null ? "" : news.getTitle();
        List<String> parts = new ArrayList<>();
        for (String key : titleKeys) {
            int cnt = countOccurrences(title, key);
            if (cnt >= 1) {
                parts.add("标题'" + key + "'x" + cnt);
            }
        }
        String content = news.getContent() == null ? "" : DetailCleaner.articleContentText(Jsoup.parse(news.getContent()));
        //篇幅控制，字数小于1500剔除
        //if (content.length() > 1500) {
        //    return null;
        //}
        for (String key : contentKeys) {
            int cnt = countOccurrences(content, key);
            if (cnt >= 2) {
                parts.add("正文'" + key + "'x" + cnt);
            }
        }
        return parts.isEmpty() ? null : String.join(" | ", parts);
    }

    /**
     * 统计关键字在文本中的出现次数（非重叠）。
     */
    private static int countOccurrences(String text, String key) {
        int count = 0;
        int idx = 0;
        while ((idx = text.indexOf(key, idx)) >= 0) {
            count++;
            idx += key.length();
        }
        return count;
    }

    /**
     * CSV 字段转义：含逗号/引号/换行时加引号包裹，内部引号加倍。
     */
    private static String csvEscape(String field) {
        if (field == null) {
            return "";
        }
        String s = field.replace("\"", "\"\"");
        return s.contains(",") || s.contains("\"") || s.contains("\n") || s.contains("\r")
                ? "\"" + s + "\""
                : s;
    }

    /**
     * 整理结果新闻列表和新闻详情pdf(符合要求的新闻存储在result.csv文件中)
     * csv格式的表格(序号,id,发布日期,标题,链接,频道名称,作者,篇幅字数)
     * 正文通过URL获取;正文含视频时下载到 doc/序号/ 下(1.ext、2.ext...)
     * pdf和视频都保存到 doc/序号/ 子目录, pdf命名:序号-发布时间-新闻标题-编辑.pdf
     * 20260913
     */
    @Test
    public void resultCollectTest() throws Exception {
        Config config = new Config();
        Mybatis mybatis = new Mybatis(Db.init(config));
        try {
            NewsMapper mapper = mybatis.getMapper(NewsMapper.class);

            //1.读取 result.csv 中的 id 并查库
            List<String> ids = new ArrayList<>();
            for (String line : Files.readAllLines(Path.of("result1.csv"), StandardCharsets.UTF_8)) {
                if (line.isBlank() || line.startsWith("id,")) {
                    continue;
                }
                int comma = line.indexOf(',');
                String id = comma < 0 ? line.trim() : line.substring(0, comma).trim();
                if (id.startsWith("\"") && id.endsWith("\"") && id.length() >= 2) {
                    id = id.substring(1, id.length() - 1).replace("\"\"", "\"");
                }
                ids.add(id);
            }
            List<News> newsList = ids.isEmpty() ? new ArrayList<News>() : new ArrayList<>(mapper.selectBatchIds(ids));
            log.info("[结果整理] result.csv记录={}条 数据库命中={}条", ids.size(), newsList.size());

            //按发布时间升序排列，保证序号稳定
            newsList.sort(Comparator.comparing(News::getPublishTime, Comparator.nullsLast(Comparator.naturalOrder())));

            File docDir = new File("doc1");
            Files.createDirectories(docDir.toPath());
            GzCmcClient client = new GzCmcClient(config);
            HttpClient videoHttp = HttpClient.newHttpClient();

            //2.逐条处理: 正文通过URL获取 -> 下载视频 -> 清洗 -> 生成PDF, 并组装csv行
            List<String> lines = new ArrayList<>();
            lines.add("序号,id,发布日期,标题,链接,频道名称,作者,篇幅字数");
            int ok = 0;
            int seq = 0;
            for (News news : newsList) {
                seq++;
                try {
                    String html;
                    boolean fetched = true;
                    try {
                        html = client.fetchDetail(news.getUrl());
                    } catch (Exception ef) {
                        fetched = false;
                        if (news.getContent() != null) {
                            log.warn("[结果整理] URL抓取失败回退库内content id={} err={}", news.getId(), ef.getMessage());
                            html = news.getContent();
                        } else {
                            log.error("[结果整理] 抓取失败且无库内content id={} url={} err={}", news.getId(), news.getUrl(), ef.getMessage());
                            lines.add(resultCsvLine(seq, news, 0));
                            continue;
                        }
                    }

                    //序号子目录(时机: 需要清洗正文时创建)
                    File seqDir = new File(docDir, String.format("%03d", seq));

                    //下载正文中的视频到序号目录: 1.ext、2.ext...
                    //视频采集行为控制
                    //if (false) {
                    if (fetched) {
                        Files.createDirectories(seqDir.toPath());
                        List<String> videoSrcs = extractVideoSrcs(html);
                        for (int i = 0; i < videoSrcs.size(); i++) {
                            try {
                                String url = videoSrcs.get(i);
                                Path target = seqDir.toPath().resolve((i + 1) + "." + extOf(url));
                                int status = downloadVideo(videoHttp, url, target);
                                log.info("[结果整理] 视频下载 序号={} id={} {}/{} code={} 文件={}", seq, news.getId(), i + 1, videoSrcs.size(), status, target.getFileName());
                            } catch (Exception e) {
                                log.error("[结果整理] 视频下载失败 序号={} id={} url={} err={}", seq, news.getId(), videoSrcs.get(i), e.getMessage());
                            }
                        }
                    } else {
                        log.warn("[结果整理] 回退库内content, 跳过视频 序号={} id={}", seq, news.getId());
                    }

                    //清洗正文并统计字数
                    String content = fetched ? DetailCleaner.clean(html) : html;
                    String contentText = DetailCleaner.articleContentText(Jsoup.parse(content));
                    lines.add(resultCsvLine(seq, news, contentText.length()));

                    //生成PDF到序号目录
                    Files.createDirectories(seqDir.toPath());
                    String publicTime = news.getPublishTime().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"));
                    String editor = news.getEditor() == null ? "" : news.getEditor();
                    String title = news.getTitle() == null ? "" : news.getTitle();
                    String fileName = sanitizeFileName(String.format("%03d-%s-%s-%s.pdf", seq, publicTime, title, editor));
                    DetailCleaner.html2pdf(content, new File(seqDir, fileName).toPath());
                    ok++;
                    log.info("[结果整理] PDF生成 序号={} id={} 文件={}", seq, news.getId(), fileName);
                } catch (Exception e) {
                    log.error("[结果整理] 失败 序号={} id={} err={}", seq, news.getId(), e.getMessage());
                }
            }

            //3.目标csv存储到doc目录
            File csv = new File(docDir, "result.csv");
            Files.write(csv.toPath(), lines, StandardCharsets.UTF_8);
            log.info("[结果整理] 目标csv已写入: {}", csv.getAbsolutePath());

            log.info("[结果整理] 完成: 成功={} 失败={}", ok, newsList.size() - ok);
        } finally {
            mybatis.close();
        }
    }

    /**
     * 组装结果csv行: 序号,id,发布日期,标题,链接,频道名称,作者,篇幅字数
     */
    private static String resultCsvLine(int seq, News news, int wordCount) {
        return seq + "," + csvEscape(news.getId()) + ","
                + csvEscape(news.getPublishTime() == null ? "" : news.getPublishTime().toLocalDate().toString()) + ","
                + csvEscape(news.getTitle()) + "," + csvEscape(news.getUrl()) + ","
                + csvEscape(news.getChannelName()) + "," + csvEscape(news.getEditor()) + ","
                + wordCount;
    }

    /**
     * 提取正文中视频的下载地址: video[src] 或其子 source[src]，按文档顺序去重。
     */
    private static List<String> extractVideoSrcs(String html) {
        Document doc = Jsoup.parse(html);
        Set<String> srcs = new LinkedHashSet<>();
        for (Element video : doc.select("video")) {
            String src = video.attr("src");
            if (!src.isBlank()) {
                srcs.add(src);
            }
            for (Element source : video.select("source[src]")) {
                String s = source.attr("src");
                if (!s.isBlank()) {
                    srcs.add(s);
                }
            }
        }
        return new ArrayList<>(srcs);
    }

    /**
     * 从视频URL路径中提取扩展名，取不到时默认 mp4。
     */
    private static String extOf(String url) {
        try {
            String path = URI.create(url).getPath();
            int dot = path.lastIndexOf('.');
            if (dot >= 0 && dot < path.length() - 1) {
                String ext = path.substring(dot + 1);
                if (ext.matches("[a-zA-Z0-9]{1,10}")) {
                    return ext;
                }
            }
        } catch (Exception ignored) {
        }
        return "mp4";
    }

    /**
     * 参考 mTest: 用 JDK HttpClient 直接下载视频到目标文件。
     */
    private static int downloadVideo(HttpClient http, String url, Path target) throws IOException, InterruptedException {
        HttpRequest request = HttpRequest.newBuilder(URI.create(url)).GET().build();
        HttpResponse<Path> resp = http.send(request, HttpResponse.BodyHandlers.ofFile(target));
        return resp.statusCode();
    }

}