You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
59 lines
2.3 KiB
59 lines
2.3 KiB
import java.io.FileOutputStream;
|
|
import java.io.IOException;
|
|
import java.io.OutputStreamWriter;
|
|
import java.io.PrintWriter;
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
|
|
public class CrawlerContext {
|
|
private CrawlStrategy<CrawlResult> strategy;
|
|
|
|
public CrawlerContext(CrawlStrategy<CrawlResult> strategy) {
|
|
this.strategy = strategy;
|
|
}
|
|
|
|
public void setStrategy(CrawlStrategy<CrawlResult> strategy) {
|
|
this.strategy = strategy;
|
|
}
|
|
|
|
public List<CrawlResult> executeCrawl(int startPage, int endPage) {
|
|
List<CrawlResult> allResults = new ArrayList<CrawlResult>();
|
|
for (int page = startPage; page <= endPage; page++) {
|
|
try {
|
|
List<CrawlResult> pageResults = strategy.crawlPage(page);
|
|
allResults.addAll(pageResults);
|
|
System.out.println("Crawling " + strategy.getBaseUrl() + " Page " + page + ": " + pageResults.size() + " items");
|
|
Thread.sleep(500);
|
|
} catch (IOException e) {
|
|
System.err.println("Error crawling page " + page + ": " + e.getMessage());
|
|
} catch (InterruptedException e) {
|
|
Thread.currentThread().interrupt();
|
|
break;
|
|
}
|
|
}
|
|
return allResults;
|
|
}
|
|
|
|
public void executeCrawlAndSave(int startPage, int endPage, String filename) {
|
|
List<CrawlResult> results = executeCrawl(startPage, endPage);
|
|
saveToFile(results, filename);
|
|
System.out.println("Crawl completed, " + results.size() + " items saved to " + filename);
|
|
}
|
|
|
|
private void saveToFile(List<CrawlResult> results, String filename) {
|
|
try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(filename), "UTF-8"))) {
|
|
writer.println("Title,Price,OriginalPrice,Discount,ImageUrl,AuthorOrSeller");
|
|
for (CrawlResult result : results) {
|
|
writer.printf("%s,%.2f,%.2f,%.1f,%s,%s%n",
|
|
result.getTitle(),
|
|
result.getPrice(),
|
|
result.getOriginalPrice(),
|
|
result.getDiscount(),
|
|
result.getImageUrl(),
|
|
result.getAuthorOrSeller());
|
|
}
|
|
} catch (IOException e) {
|
|
System.err.println("Failed to save file: " + e.getMessage());
|
|
}
|
|
}
|
|
}
|