Browse Source

删除 'project/java-cli/src/main/java/com/example/datacollect/strategy/BlogStrategy.java'

main
LeiJuntao 3 weeks ago
parent
commit
b76d506dec
  1. 59
      project/java-cli/src/main/java/com/example/datacollect/strategy/BlogStrategy.java

59
project/java-cli/src/main/java/com/example/datacollect/strategy/BlogStrategy.java

@ -1,59 +0,0 @@
package com.example.datacollect.strategy;
import com.example.datacollect.exception.ParseException;
import com.example.datacollect.model.Article;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.ArrayList;
import java.util.List;
public class BlogStrategy implements CrawlStrategy {
@Override
public boolean supports(String url) {
return url.contains("blog.example.com");
}
@Override
public List<Article> parse(String url, Document doc) throws ParseException {
List<Article> articles = new ArrayList<>();
Elements listItems = doc.select("ul.blog-list li");
for (Element li : listItems) {
Element link = li.selectFirst("a");
if (link == null) continue;
String articleUrl = link.attr("href");
if (!articleUrl.startsWith("http")) {
if (articleUrl.startsWith("//")) {
articleUrl = "https:" + articleUrl;
} else if (articleUrl.startsWith("/")) {
articleUrl = "https://blog.example.com" + articleUrl;
} else {
articleUrl = "https://blog.example.com/" + articleUrl;
}
}
String title = "";
Element titleEl = link.selectFirst("h3.post-title");
if (titleEl != null) {
title = titleEl.text().trim();
}
if (title.isEmpty()) {
title = link.text().trim();
}
String content = "";
Element contentEl = li.selectFirst("p.post-excerpt");
if (contentEl != null) {
content = contentEl.text().trim();
}
if (!title.isEmpty()) {
articles.add(new Article(title, articleUrl, content));
}
}
return articles;
}
}
Loading…
Cancel
Save