1 changed files with 0 additions and 59 deletions
@ -1,59 +0,0 @@ |
|||
package com.example.datacollect.strategy; |
|||
|
|||
import com.example.datacollect.exception.ParseException; |
|||
import com.example.datacollect.model.Article; |
|||
import org.jsoup.nodes.Document; |
|||
import org.jsoup.nodes.Element; |
|||
import org.jsoup.select.Elements; |
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
|
|||
public class NewsStrategy implements CrawlStrategy { |
|||
@Override |
|||
public boolean supports(String url) { |
|||
return url.contains("news.example.com"); |
|||
} |
|||
|
|||
@Override |
|||
public List<Article> parse(String url, Document doc) throws ParseException { |
|||
List<Article> articles = new ArrayList<>(); |
|||
Elements listItems = doc.select("ul.news-list li"); |
|||
|
|||
for (Element li : listItems) { |
|||
Element link = li.selectFirst("a"); |
|||
if (link == null) continue; |
|||
|
|||
String articleUrl = link.attr("href"); |
|||
if (!articleUrl.startsWith("http")) { |
|||
if (articleUrl.startsWith("//")) { |
|||
articleUrl = "https:" + articleUrl; |
|||
} else if (articleUrl.startsWith("/")) { |
|||
articleUrl = "https://news.example.com" + articleUrl; |
|||
} else { |
|||
articleUrl = "https://news.example.com/" + articleUrl; |
|||
} |
|||
} |
|||
|
|||
String title = ""; |
|||
Element titleEl = link.selectFirst("h3.article-headline"); |
|||
if (titleEl != null) { |
|||
title = titleEl.text().trim(); |
|||
} |
|||
if (title.isEmpty()) { |
|||
title = link.text().trim(); |
|||
} |
|||
|
|||
String content = ""; |
|||
Element contentEl = li.selectFirst("p.article-summary"); |
|||
if (contentEl != null) { |
|||
content = contentEl.text().trim(); |
|||
} |
|||
|
|||
if (!title.isEmpty()) { |
|||
articles.add(new Article(title, articleUrl, content)); |
|||
} |
|||
} |
|||
|
|||
return articles; |
|||
} |
|||
} |
|||
Loading…
Reference in new issue