1 changed files with 0 additions and 41 deletions
@ -1,41 +0,0 @@ |
|||||
package com.example.datacollect.strategy; |
|
||||
|
|
||||
import com.example.datacollect.exception.ParseException; |
|
||||
import com.example.datacollect.model.Article; |
|
||||
import org.jsoup.nodes.Document; |
|
||||
import org.jsoup.nodes.Element; |
|
||||
import org.jsoup.select.Elements; |
|
||||
import java.util.ArrayList; |
|
||||
import java.util.List; |
|
||||
|
|
||||
public class HnuNewsStrategy implements CrawlStrategy { |
|
||||
@Override |
|
||||
public boolean supports(String url) { |
|
||||
return url.contains("news.hnu.edu.cn"); |
|
||||
} |
|
||||
|
|
||||
@Override |
|
||||
public List<Article> parse(String url, Document doc) throws ParseException { |
|
||||
List<Article> articles = new ArrayList<>(); |
|
||||
Elements listItems = doc.select("ul.list3 li, ul.list6 li"); |
|
||||
|
|
||||
for (Element li : listItems) { |
|
||||
Element link = li.selectFirst("a"); |
|
||||
if (link == null) continue; |
|
||||
|
|
||||
String articleUrl = link.attr("href"); |
|
||||
if (!articleUrl.startsWith("http")) { |
|
||||
articleUrl = "https://news.hnu.edu.cn" + articleUrl.replace("..", ""); |
|
||||
} |
|
||||
|
|
||||
String title = link.text().trim(); |
|
||||
String content = ""; |
|
||||
|
|
||||
if (!title.isEmpty()) { |
|
||||
articles.add(new Article(title, articleUrl, content)); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
return articles; |
|
||||
} |
|
||||
} |
|
||||
Loading…
Reference in new issue