You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
47 lines
1.7 KiB
47 lines
1.7 KiB
package com.example.datacollect.strategy;
|
|
|
|
import com.example.datacollect.exception.ParseException;
|
|
import com.example.datacollect.model.Article;
|
|
import org.jsoup.nodes.Document;
|
|
import org.jsoup.nodes.Element;
|
|
import org.jsoup.select.Elements;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
import java.util.regex.Pattern;
|
|
|
|
public class BlogStrategy implements CrawlStrategy {
|
|
private static final Logger logger = LoggerFactory.getLogger(BlogStrategy.class);
|
|
private static final Pattern URL_PATTERN = Pattern.compile(".*blog\\.example\\.com.*");
|
|
|
|
@Override
|
|
public boolean supports(String url) {
|
|
boolean isSupported = URL_PATTERN.matcher(url).matches();
|
|
logger.debug("URL {} support status: {}", url, isSupported);
|
|
return isSupported;
|
|
}
|
|
|
|
@Override
|
|
public List<Article> parse(String url, Document doc) throws ParseException {
|
|
try {
|
|
logger.info("Start parsing blog articles from url: {}", url);
|
|
List<Article> articles = new ArrayList<>();
|
|
Elements titles = doc.select(".post-title");
|
|
for (Element e : titles) {
|
|
articles.add(new Article(e.text(), url, ""));
|
|
}
|
|
logger.debug("Parsed {} blog articles from url: {}", articles.size(), url);
|
|
return articles;
|
|
} catch (Exception e) {
|
|
logger.error("Failed to parse blog articles from url: {}", url, e);
|
|
throw new ParseException("Blog article parse failed for url: " + url, e);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public int getPriority() {
|
|
return 10; // 优先级高于默认策略
|
|
}
|
|
}
|
|
|