diff --git a/project/AnalyzeCommand.java b/project/AnalyzeCommand.java new file mode 100644 index 0000000..18059ce --- /dev/null +++ b/project/AnalyzeCommand.java @@ -0,0 +1,91 @@ +package com.example.datacollect.command; + +import com.example.datacollect.exception.CrawlerException; +import com.example.datacollect.exception.ErrorCode; +import com.example.datacollect.model.Article; +import com.example.datacollect.repository.ArticleRepository; +import com.example.datacollect.strategy.CrawlStrategy; +import com.example.datacollect.strategy.StrategyFactory; +import com.example.datacollect.view.ConsoleView; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import java.io.IOException; +import java.util.List; + +public class AnalyzeCommand implements Command { + + private final ConsoleView view; + private final StrategyFactory strategyFactory; + + public AnalyzeCommand(ConsoleView view, StrategyFactory strategyFactory) { + this.view = view; + this.strategyFactory = strategyFactory; + } + + @Override + public String getName() { + return "analyze"; + } + + @Override + public void execute(String[] args, ArticleRepository repository) { + if (args.length < 2) { + view.printError("[" + ErrorCode.INVALID_ARGUMENT.getCode() + "] " + ErrorCode.INVALID_ARGUMENT.getMessage() + ",格式:analyze "); + return; + } + String url = args[1]; + + CrawlStrategy strategy = strategyFactory.getStrategy(url); + if (strategy == null) { + throw new CrawlerException(ErrorCode.STRATEGY_NOT_FOUND, "不支持解析该URL:" + url); + } + + Document doc; + try { + view.printInfo("正在分析: " + url); + doc = Jsoup.connect(url) + .timeout(30000) + .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8") + .header("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") + .header("Accept-Encoding", "gzip, deflate, br") + .header("Connection", "keep-alive") + .header("Cache-Control", "max-age=0") + .get(); + } catch (IOException e) { + throw new CrawlerException(ErrorCode.NETWORK_ERROR, "网页爬取失败:" + e.getMessage(), e); + } + + List
articles; + try { + articles = strategy.parse(url, doc); + } catch (Exception e) { + throw new CrawlerException(ErrorCode.PARSE_ERROR, "网页解析失败:" + e.getMessage(), e); + } + + printAnalysisResult(articles, url); + } + + private void printAnalysisResult(List
articles, String url) { + view.printInfo("\n===== 网页分析结果(不存储数据) ====="); + view.printInfo("分析URL:" + url); + view.printInfo("解析到文章数量:" + articles.size()); + + if (!articles.isEmpty()) { + int totalTitleLength = 0; + for (Article article : articles) { + totalTitleLength += article.getTitle().length(); + } + double avgTitleLength = (double) totalTitleLength / articles.size(); + view.printInfo(String.format("文章标题平均长度:%.2f 字符", avgTitleLength)); + + view.printInfo("\n前3篇文章标题预览:"); + for (int i = 0; i < Math.min(3, articles.size()); i++) { + view.printInfo((i + 1) + ". " + articles.get(i).getTitle()); + } + } else { + view.printInfo("未解析到任何文章内容"); + } + view.printInfo("=====================================\n"); + } +} \ No newline at end of file diff --git a/project/Command.java b/project/Command.java new file mode 100644 index 0000000..27a2c70 --- /dev/null +++ b/project/Command.java @@ -0,0 +1,8 @@ +package com.example.datacollect.command; + +import com.example.datacollect.repository.ArticleRepository; + +public interface Command { + String getName(); + void execute(String[] args, ArticleRepository repository); +} diff --git a/project/CrawlCommand.java b/project/CrawlCommand.java new file mode 100644 index 0000000..857022c --- /dev/null +++ b/project/CrawlCommand.java @@ -0,0 +1,53 @@ +package com.example.datacollect.command; + +import com.example.datacollect.repository.ArticleRepository; +import com.example.datacollect.strategy.CrawlStrategy; +import com.example.datacollect.strategy.StrategyFactory; +import com.example.datacollect.view.ConsoleView; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +public class CrawlCommand implements Command { + private final ConsoleView view; + private final StrategyFactory strategyFactory; + + public CrawlCommand(ConsoleView view, StrategyFactory strategyFactory) { + this.view = view; + this.strategyFactory = strategyFactory; + } + + @Override + public String getName() { + return "crawl"; + } + + @Override + public void execute(String[] args, ArticleRepository repository) { + if (args.length < 2) { + view.printError("Usage: crawl "); + return; + } + String url = args[1]; + + CrawlStrategy strategy = strategyFactory.getStrategy(url); + if (strategy == null) { + view.printError("No strategy found for: " + url); + return; + } + + try { + view.printInfo("Crawling: " + url); + Document doc = Jsoup.connect(url) + .timeout(10000) + .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + .get(); + var articles = strategy.parse(url, doc); + for (var article : articles) { + repository.add(article); + } + view.printSuccess("Crawled " + articles.size() + " articles."); + } catch (Exception e) { + view.printError("Failed to crawl: " + e.getMessage()); + } + } +} diff --git a/project/ExitCommand.java b/project/ExitCommand.java new file mode 100644 index 0000000..5763f71 --- /dev/null +++ b/project/ExitCommand.java @@ -0,0 +1,23 @@ +package com.example.datacollect.command; + +import com.example.datacollect.repository.ArticleRepository; +import com.example.datacollect.view.ConsoleView; + +public class ExitCommand implements Command { + private final ConsoleView view; + + public ExitCommand(ConsoleView view) { + this.view = view; + } + + @Override + public String getName() { + return "exit"; + } + + @Override + public void execute(String[] args, ArticleRepository repository) { + view.printSuccess("Bye!"); + System.exit(0); + } +} diff --git a/project/Main.java b/project/Main.java new file mode 100644 index 0000000..dc7ab06 --- /dev/null +++ b/project/Main.java @@ -0,0 +1,21 @@ +package com.example.datacollect; + +import com.example.datacollect.controller.CrawlerController; +import com.example.datacollect.repository.ArticleRepository; +import com.example.datacollect.strategy.StrategyFactory; +import com.example.datacollect.view.ConsoleView; + +public class Main { + + public static void main(String[] args) { + ConsoleView view = new ConsoleView(); + ArticleRepository repository = new ArticleRepository(); + StrategyFactory strategyFactory = new StrategyFactory(); + CrawlerController controller = new CrawlerController(view, repository, strategyFactory); + + view.printSuccess("Welcome to CLI Crawler (w10_3)! Type help for commands."); + while (true) { + controller.handle(view.readLine()); + } + } +}