16 changed files with 566 additions and 71 deletions
@ -0,0 +1,71 @@ |
|||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" |
||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
<groupId>com.example</groupId> |
||||
|
<artifactId>datacollect-cli</artifactId> |
||||
|
<version>0.1.0</version> |
||||
|
<properties> |
||||
|
<maven.compiler.source>11</maven.compiler.source> |
||||
|
<maven.compiler.target>11</maven.compiler.target> |
||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
||||
|
<logback.version>1.4.11</logback.version> |
||||
|
<slf4j.version>2.0.9</slf4j.version> |
||||
|
<jsoup.version>1.17.2</jsoup.version> |
||||
|
</properties> |
||||
|
|
||||
|
<dependencies> |
||||
|
<dependency> |
||||
|
<groupId>ch.qos.logback</groupId> |
||||
|
<artifactId>logback-classic</artifactId> |
||||
|
<version>${logback.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.slf4j</groupId> |
||||
|
<artifactId>slf4j-api</artifactId> |
||||
|
<version>${slf4j.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.jsoup</groupId> |
||||
|
<artifactId>jsoup</artifactId> |
||||
|
<version>${jsoup.version}</version> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<build> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-compiler-plugin</artifactId> |
||||
|
<version>3.8.1</version> |
||||
|
<configuration> |
||||
|
<encoding>UTF-8</encoding> |
||||
|
</configuration> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-assembly-plugin</artifactId> |
||||
|
<version>3.3.0</version> |
||||
|
<configuration> |
||||
|
<archive> |
||||
|
<manifest> |
||||
|
<mainClass>com.example.datacollect.Main</mainClass> |
||||
|
</manifest> |
||||
|
</archive> |
||||
|
<descriptorRefs> |
||||
|
<descriptorRef>jar-with-dependencies</descriptorRef> |
||||
|
</descriptorRefs> |
||||
|
</configuration> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>make-assembly</id> |
||||
|
<phase>package</phase> |
||||
|
<goals> |
||||
|
<goal>single</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
</project> |
||||
@ -1,19 +1,28 @@ |
|||||
package com.example.datacollect; |
package com.example.datacollect; |
||||
|
|
||||
import com.example.datacollect.controller.CrawlerController; |
import com.example.datacollect.controller.CrawlerController; |
||||
import com.example.datacollect.repository.ArticleRepository; |
|
||||
import com.example.datacollect.view.ConsoleView; |
import com.example.datacollect.view.ConsoleView; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
public class Main { |
public class Main { |
||||
|
private static final Logger logger = LoggerFactory.getLogger(Main.class); |
||||
|
|
||||
public static void main(String[] args) { |
public static void main(String[] args) { |
||||
|
logger.info("启动 CLI Crawler 程序"); |
||||
|
|
||||
ConsoleView view = new ConsoleView(); |
ConsoleView view = new ConsoleView(); |
||||
ArticleRepository repository = new ArticleRepository(); |
CrawlerController controller = new CrawlerController(view); |
||||
CrawlerController controller = new CrawlerController(view, repository); |
|
||||
|
view.printSuccess("Welcome to CLI Crawler! Type help for commands."); |
||||
|
|
||||
view.printSuccess("Welcome to CLI Crawler (w10)! Type help for commands."); |
try { |
||||
while (true) { |
while (true) { |
||||
controller.handle(view.readLine()); |
controller.handle(view.readLine()); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
logger.error("程序异常退出: {}", e.getMessage(), e); |
||||
|
view.printError("程序异常退出: " + e.getMessage()); |
||||
} |
} |
||||
} |
} |
||||
} |
} |
||||
@ -1,8 +1,10 @@ |
|||||
package com.example.datacollect.command; |
package com.example.datacollect.command; |
||||
|
|
||||
import com.example.datacollect.repository.ArticleRepository; |
import java.util.List; |
||||
|
|
||||
|
import com.example.datacollect.model.Article; |
||||
|
|
||||
public interface Command { |
public interface Command { |
||||
String getName(); |
String getName(); |
||||
void execute(String[] args, ArticleRepository repository); |
void execute(String[] args, List<Article> articles); |
||||
} |
} |
||||
@ -1,6 +1,6 @@ |
|||||
package com.example.crawler.exception; |
package com.example.datacollect.exception; |
||||
|
|
||||
public class CrawlerException extends Exception { |
public class CrawlerException extends RuntimeException { |
||||
public CrawlerException(String message) { |
public CrawlerException(String message) { |
||||
super(message); |
super(message); |
||||
} |
} |
||||
@ -0,0 +1,29 @@ |
|||||
|
package com.example.datacollect.exception; |
||||
|
|
||||
|
public class NetworkException extends CrawlerException { |
||||
|
private final String url; |
||||
|
|
||||
|
public NetworkException(String message) { |
||||
|
super(message); |
||||
|
this.url = null; |
||||
|
} |
||||
|
|
||||
|
public NetworkException(String message, String url) { |
||||
|
super(message); |
||||
|
this.url = url; |
||||
|
} |
||||
|
|
||||
|
public NetworkException(String message, Throwable cause) { |
||||
|
super(message, cause); |
||||
|
this.url = null; |
||||
|
} |
||||
|
|
||||
|
public NetworkException(String message, String url, Throwable cause) { |
||||
|
super(message, cause); |
||||
|
this.url = url; |
||||
|
} |
||||
|
|
||||
|
public String getUrl() { |
||||
|
return url; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,29 @@ |
|||||
|
package com.example.datacollect.exception; |
||||
|
|
||||
|
public class ParseException extends CrawlerException { |
||||
|
private final String source; |
||||
|
|
||||
|
public ParseException(String message) { |
||||
|
super(message); |
||||
|
this.source = null; |
||||
|
} |
||||
|
|
||||
|
public ParseException(String message, String source) { |
||||
|
super(message); |
||||
|
this.source = source; |
||||
|
} |
||||
|
|
||||
|
public ParseException(String message, Throwable cause) { |
||||
|
super(message, cause); |
||||
|
this.source = null; |
||||
|
} |
||||
|
|
||||
|
public ParseException(String message, String source, Throwable cause) { |
||||
|
super(message, cause); |
||||
|
this.source = source; |
||||
|
} |
||||
|
|
||||
|
public String getSource() { |
||||
|
return source; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,29 @@ |
|||||
|
package com.example.datacollect.exception; |
||||
|
|
||||
|
public class UrlFormatException extends RuntimeException { |
||||
|
private final String url; |
||||
|
|
||||
|
public UrlFormatException(String message) { |
||||
|
super(message); |
||||
|
this.url = null; |
||||
|
} |
||||
|
|
||||
|
public UrlFormatException(String message, String url) { |
||||
|
super(message); |
||||
|
this.url = url; |
||||
|
} |
||||
|
|
||||
|
public UrlFormatException(String message, Throwable cause) { |
||||
|
super(message, cause); |
||||
|
this.url = null; |
||||
|
} |
||||
|
|
||||
|
public UrlFormatException(String message, String url, Throwable cause) { |
||||
|
super(message, cause); |
||||
|
this.url = url; |
||||
|
} |
||||
|
|
||||
|
public String getUrl() { |
||||
|
return url; |
||||
|
} |
||||
|
} |
||||
@ -1,42 +1,86 @@ |
|||||
package com.example.datacollect.repository; |
package com.example.datacollect.repository; |
||||
|
|
||||
|
import com.example.datacollect.exception.UrlFormatException; |
||||
import com.example.datacollect.model.Article; |
import com.example.datacollect.model.Article; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
import java.util.ArrayList; |
import java.util.ArrayList; |
||||
import java.util.Collections; |
|
||||
import java.util.List; |
import java.util.List; |
||||
import java.util.Optional; |
|
||||
|
|
||||
public class ArticleRepository { |
public class ArticleRepository { |
||||
private final List<Article> articles = new ArrayList<>(); |
private static final Logger logger = LoggerFactory.getLogger(ArticleRepository.class); |
||||
|
private final List<Article> articles; |
||||
|
|
||||
|
public ArticleRepository() { |
||||
|
this.articles = new ArrayList<>(); |
||||
|
} |
||||
|
|
||||
public void add(Article article) { |
public void add(Article article) { |
||||
if (article != null) { |
// 防御检查：标题非空
|
||||
articles.add(article); |
if (article.getTitle() == null || article.getTitle().trim().isEmpty()) { |
||||
|
logger.error("文章标题不能为空"); |
||||
|
throw new IllegalArgumentException("文章标题不能为空"); |
||||
|
} |
||||
|
|
||||
|
// 防御检查：URL非空且格式正确
|
||||
|
if (article.getUrl() == null || article.getUrl().trim().isEmpty()) { |
||||
|
logger.error("文章URL不能为空"); |
||||
|
throw new IllegalArgumentException("文章URL不能为空"); |
||||
|
} |
||||
|
|
||||
|
// URL格式校验
|
||||
|
String url = article.getUrl().trim(); |
||||
|
if (!url.startsWith("http://") && !url.startsWith("https://")) { |
||||
|
logger.warn("URL格式不正确，缺少协议头: {}", url); |
||||
|
throw new UrlFormatException("URL格式不正确，必须以 http:// 或 https:// 开头", url); |
||||
} |
} |
||||
|
|
||||
|
// 验证URL是否为有效格式
|
||||
|
try { |
||||
|
java.net.URL validUrl = new java.net.URL(url); |
||||
|
String host = validUrl.getHost(); |
||||
|
if (host == null || host.isEmpty()) { |
||||
|
logger.error("URL主机名无效: {}", url); |
||||
|
throw new UrlFormatException("URL主机名无效", url); |
||||
|
} |
||||
|
} catch (java.net.MalformedURLException e) { |
||||
|
logger.error("URL格式错误: {}", url); |
||||
|
throw new UrlFormatException("URL格式错误: " + e.getMessage(), url, e); |
||||
|
} |
||||
|
|
||||
|
articles.add(article); |
||||
|
logger.info("成功添加文章: {}", article.getTitle()); |
||||
} |
} |
||||
|
|
||||
public void addAll(List<Article> newArticles) { |
public void addAll(List<Article> articles) { |
||||
if (newArticles != null && !newArticles.isEmpty()) { |
for (Article article : articles) { |
||||
articles.addAll(newArticles); |
add(article); |
||||
} |
} |
||||
} |
} |
||||
|
|
||||
public List<Article> getAll() { |
public List<Article> getAll() { |
||||
return Collections.unmodifiableList(articles); |
return new ArrayList<>(articles); |
||||
} |
} |
||||
|
|
||||
public Optional<Article> findByTitle(String title) { |
public Article get(int index) { |
||||
return articles.stream() |
if (index < 0 || index >= articles.size()) { |
||||
.filter(a -> a.getTitle().equals(title)) |
logger.error("索引越界: {}", index); |
||||
.findFirst(); |
throw new IndexOutOfBoundsException("索引越界: " + index); |
||||
|
} |
||||
|
return articles.get(index); |
||||
} |
} |
||||
|
|
||||
public int count() { |
public int size() { |
||||
return articles.size(); |
return articles.size(); |
||||
} |
} |
||||
|
|
||||
|
public boolean isEmpty() { |
||||
|
return articles.isEmpty(); |
||||
|
} |
||||
|
|
||||
public void clear() { |
public void clear() { |
||||
articles.clear(); |
articles.clear(); |
||||
|
logger.info("文章列表已清空"); |
||||
} |
} |
||||
} |
} |
||||
@ -0,0 +1,89 @@ |
|||||
|
package com.example.datacollect.util; |
||||
|
|
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.util.function.Supplier; |
||||
|
|
||||
|
public class RetryUtils { |
||||
|
private static final Logger logger = LoggerFactory.getLogger(RetryUtils.class); |
||||
|
|
||||
|
private static final int DEFAULT_MAX_RETRIES = 3; |
||||
|
private static final long DEFAULT_BASE_WAIT_MS = 500; |
||||
|
|
||||
|
private final int maxRetries; |
||||
|
private final long baseWaitMs; |
||||
|
|
||||
|
public RetryUtils() { |
||||
|
this(DEFAULT_MAX_RETRIES, DEFAULT_BASE_WAIT_MS); |
||||
|
} |
||||
|
|
||||
|
public RetryUtils(int maxRetries, long baseWaitMs) { |
||||
|
this.maxRetries = maxRetries; |
||||
|
this.baseWaitMs = baseWaitMs; |
||||
|
} |
||||
|
|
||||
|
public <T> T executeWithRetry(Supplier<T> supplier, String operationName) { |
||||
|
Exception lastException = null; |
||||
|
|
||||
|
for (int attempt = 0; attempt <= maxRetries; attempt++) { |
||||
|
try { |
||||
|
logger.debug("[{}] 第 {} 次尝试", operationName, attempt + 1); |
||||
|
T result = supplier.get(); |
||||
|
|
||||
|
if (result != null) { |
||||
|
logger.info("[{}] 第 {} 次尝试成功", operationName, attempt + 1); |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
logger.warn("[{}] 第 {} 次尝试返回空结果", operationName, attempt + 1); |
||||
|
|
||||
|
} catch (Exception e) { |
||||
|
lastException = e; |
||||
|
logger.warn("[{}] 第 {} 次尝试失败: {}", operationName, attempt + 1, e.getMessage()); |
||||
|
|
||||
|
if (attempt < maxRetries) { |
||||
|
long waitTime = calculateWaitTime(attempt); |
||||
|
logger.warn("[{}] 将在 {} ms 后重试", operationName, waitTime); |
||||
|
|
||||
|
try { |
||||
|
Thread.sleep(waitTime); |
||||
|
} catch (InterruptedException ie) { |
||||
|
Thread.currentThread().interrupt(); |
||||
|
logger.error("[{}] 重试等待被中断", operationName); |
||||
|
throw new RuntimeException("重试等待被中断", ie); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
logger.error("[{}] 已重试 {} 次，全部失败", operationName, maxRetries); |
||||
|
throw new RuntimeException("操作失败，已重试 " + maxRetries + " 次", lastException); |
||||
|
} |
||||
|
|
||||
|
private long calculateWaitTime(int attempt) { |
||||
|
// 指数退避: wait = base * 2^attempt
|
||||
|
double waitTime = baseWaitMs * Math.pow(2, attempt); |
||||
|
return (long) waitTime; |
||||
|
} |
||||
|
|
||||
|
public static <T> T execute(Supplier<T> supplier, String operationName) { |
||||
|
return new RetryUtils().executeWithRetry(supplier, operationName); |
||||
|
} |
||||
|
|
||||
|
public static <T> T execute(Supplier<T> supplier, String operationName, int maxRetries) { |
||||
|
return new RetryUtils(maxRetries, DEFAULT_BASE_WAIT_MS).executeWithRetry(supplier, operationName); |
||||
|
} |
||||
|
|
||||
|
public static <T> T execute(Supplier<T> supplier, String operationName, int maxRetries, long baseWaitMs) { |
||||
|
return new RetryUtils(maxRetries, baseWaitMs).executeWithRetry(supplier, operationName); |
||||
|
} |
||||
|
|
||||
|
public int getMaxRetries() { |
||||
|
return maxRetries; |
||||
|
} |
||||
|
|
||||
|
public long getBaseWaitMs() { |
||||
|
return baseWaitMs; |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue