Browse Source

上传文件至 'project'

main
wuqiuyu 3 weeks ago
parent
commit
8f51818043
  1. 42
      project/ConsoleView.java
  2. 10
      project/CrawlStrategy.java
  3. 54
      project/HnuNewsStrategy.java
  4. 25
      project/NewsStrategy.java
  5. 28
      project/StrategyFactory.java

42
project/ConsoleView.java

@ -0,0 +1,42 @@
package com.example.datacollect.view;
import com.example.datacollect.model.Article;
import java.util.List;
import java.util.Scanner;
public class ConsoleView {
private static final String ANSI_RESET = "\u001B[0m";
private static final String ANSI_GREEN = "\u001B[32m";
private static final String ANSI_RED = "\u001B[31m";
private static final String ANSI_BLUE = "\u001B[34m";
private final Scanner scanner = new Scanner(System.in);
public String readLine() {
System.out.print("> ");
return scanner.nextLine();
}
public void printSuccess(String msg) {
System.out.println(ANSI_GREEN + msg + ANSI_RESET);
}
public void printError(String msg) {
System.out.println(ANSI_RED + msg + ANSI_RESET);
}
public void printInfo(String msg) {
System.out.println(ANSI_BLUE + msg + ANSI_RESET);
}
public void display(List<Article> articles) {
if (articles.isEmpty()) {
printInfo("暂无文章,请先执行 crawl。");
return;
}
for (int i = 0; i < articles.size(); i++) {
Article a = articles.get(i);
System.out.println((i + 1) + ". " + a.getTitle() + " | " + a.getUrl());
}
}
}

10
project/CrawlStrategy.java

@ -0,0 +1,10 @@
package com.example.datacollect.strategy;
import com.example.datacollect.model.Article;
import org.jsoup.nodes.Document;
import java.util.List;
public interface CrawlStrategy {
List<Article> parse(String url, Document doc);
boolean supports(String url);
}

54
project/HnuNewsStrategy.java

@ -0,0 +1,54 @@
package com.example.datacollect.strategy;
import com.example.datacollect.model.Article;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.ArrayList;
import java.util.List;
public class HnuNewsStrategy implements CrawlStrategy {
@Override
public boolean supports(String url) {
return url.contains("news.hnu.edu.cn");
}
@Override
public List<Article> parse(String url, Document doc) {
List<Article> articles = new ArrayList<>();
Elements links = doc.select("a[href*='/info/']");
for (Element link : links) {
String articleUrl = link.attr("abs:href");
if (!articleUrl.startsWith("http")) {
articleUrl = "https://news.hnu.edu.cn" + link.attr("href").replace("..", "");
}
String title = link.text().trim();
if (title.isEmpty()) {
Element titleEl = link.selectFirst("h2, h3, h4, .title");
if (titleEl != null) {
title = titleEl.text().trim();
}
}
String content = "";
Element parent = link.parent();
if (parent != null) {
Element contentEl = parent.selectFirst("p, .summary, .description");
if (contentEl != null) {
content = contentEl.text().trim();
}
}
if (!title.isEmpty() && articleUrl.contains("/info/")) {
articles.add(new Article(title, articleUrl, content));
}
}
return articles;
}
}

25
project/NewsStrategy.java

@ -0,0 +1,25 @@
package com.example.datacollect.strategy;
import com.example.datacollect.model.Article;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.ArrayList;
import java.util.List;
public class NewsStrategy implements CrawlStrategy {
@Override
public boolean supports(String url) {
return url.contains("news.example.com");
}
@Override
public List<Article> parse(String url, Document doc) {
List<Article> articles = new ArrayList<>();
Elements items = doc.select(".article-headline");
for (Element e : items) {
articles.add(new Article(e.text(), url, ""));
}
return articles;
}
}

28
project/StrategyFactory.java

@ -0,0 +1,28 @@
package com.example.datacollect.strategy;
import java.util.ArrayList;
import java.util.List;
public class StrategyFactory {
private final List<CrawlStrategy> strategies = new ArrayList<>();
public StrategyFactory() {
strategies.add(new HnuNewsStrategy());
strategies.add(new BlogStrategy());
strategies.add(new NewsStrategy());
strategies.add(new BaiduBaikeStrategy());
}
public CrawlStrategy getStrategy(String url) {
for (CrawlStrategy s : strategies) {
if (s.supports(url)) {
return s;
}
}
return null;
}
public void register(CrawlStrategy strategy) {
strategies.add(strategy);
}
}
Loading…
Cancel
Save