java/继承+多态爬虫/SpiderFramework.java

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

// 爬虫基类
abstract class Spider {
    protected String baseUrl;
    
    public Spider(String baseUrl) {
        this.baseUrl = baseUrl;
    }
    
    // 发送HTTP请求获取页面内容
    protected Document fetchPage(String url) throws IOException {
        return Jsoup.connect(url)
                .userAgent("Mozilla/5.0")
                .timeout(10000)
                .get();
    }
    
    // 抽象方法：爬取数据
    public abstract List<?> crawl() throws IOException;
    
    // 抽象方法：解析页面
    protected abstract List<?> parsePage(Document doc);
    
    // 重载方法：解析页面（用于原生HTML解析）
    protected List<?> parsePage(String html) {
        // 默认实现，子类可以选择性重写
        return new ArrayList<>();
    }
}

// 电影数据模型
class Movie {
    private String title;
    private String rating;
    private String url;
    
    public Movie(String title, String rating, String url) {
        this.title = title;
        this.rating = rating;
        this.url = url;
    }
    
    @Override
    public String toString() {
        return "Movie{" +
                "title='" + title + '\'' +
                ", rating='" + rating + '\'' +
                ", url='" + url + '\'' +
                '}';
    }
}

// 书籍数据模型
class Book {
    private String title;
    private String author;
    private String rating;
    private String url;
    
    public Book(String title, String author, String rating, String url) {
        this.title = title;
        this.author = author;
        this.rating = rating;
        this.url = url;
    }
    
    @Override
    public String toString() {
        return "Book{" +
                "title='" + title + '\'' +
                ", author='" + author + '\'' +
                ", rating='" + rating + '\'' +
                ", url='" + url + '\'' +
                '}';
    }
}

// 天气数据模型
class Weather {
    private String province;
    private String city;
    private String temperature;
    private String weatherCondition;
    private String url;
    
    public Weather(String province, String city, String temperature, String weatherCondition, String url) {
        this.province = province;
        this.city = city;
        this.temperature = temperature;
        this.weatherCondition = weatherCondition;
        this.url = url;
    }
    
    @Override
    public String toString() {
        return "Weather{" +
                "province='" + province + '\'' +
                ", city='" + city + '\'' +
                ", temperature='" + temperature + '\'' +
                ", weatherCondition='" + weatherCondition + '\'' +
                ", url='" + url + '\'' +
                '}';
    }
}

// 豆瓣电影爬虫 - 爬取Top250
class DoubanMovieSpider extends Spider {
    public DoubanMovieSpider() {
        super("https://movie.douban.com/top250");
    }
    
    @Override
    public List<Movie> crawl() throws IOException {
        List<Movie> allMovies = new ArrayList<>();
        
        // 豆瓣Top250有10页，每页25部电影
        for (int start = 0; start < 250; start += 25) {
            String pageUrl = baseUrl + "?start=" + start;
            System.out.println("正在爬取第 " + (start / 25 + 1) + " 页...");
            Document doc = fetchPage(pageUrl);
            List<Movie> movies = parsePage(doc);
            allMovies.addAll(movies);
            
            // 添加延迟，避免请求过快
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }
        
        return allMovies;
    }
    
    @Override
    protected List<Movie> parsePage(Document doc) {
        List<Movie> movies = new ArrayList<>();
        
        // 使用Jsoup选择器解析页面
        Elements movieItems = doc.select(".info");
        for (Element item : movieItems) {
            // 提取标题
            Element titleElement = item.selectFirst(".title");
            if (titleElement != null) {
                String title = titleElement.text().trim();
                
                // 提取评分
                Element ratingElement = item.selectFirst(".rating_num");
                String rating = "";
                if (ratingElement != null) {
                    rating = ratingElement.text().trim();
                }
                
                // 提取链接
                Element linkElement = item.selectFirst("a");
                String url = "";
                if (linkElement != null) {
                    url = linkElement.attr("href");
                }
                
                movies.add(new Movie(title, rating, url));
            }
        }
        
        return movies;
    }
}

// 豆瓣读书爬虫 - 爬取评分前100
class DoubanBookSpider extends Spider {
    public DoubanBookSpider() {
        super("https://book.douban.com/top250");
    }
    
    @Override
    public List<Book> crawl() throws IOException {
        List<Book> allBooks = new ArrayList<>();
        
        // 豆瓣读书Top250，每页25本，爬取前4页（100本）
        for (int start = 0; start < 100; start += 25) {
            String pageUrl = baseUrl + "?start=" + start;
            System.out.println("正在爬取第 " + (start / 25 + 1) + " 页...");
            Document doc = fetchPage(pageUrl);
            List<Book> books = parsePage(doc);
            allBooks.addAll(books);
            
            // 添加延迟，避免请求过快
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }
        
        return allBooks;
    }
    
    @Override
    protected List<Book> parsePage(Document doc) {
        List<Book> books = new ArrayList<>();
        
        // 使用Jsoup选择器解析页面
        Elements bookItems = doc.select("tr.item");
        for (Element item : bookItems) {
            // 提取链接
            Element linkElement = item.selectFirst(".nbg");
            if (linkElement != null) {
                String url = linkElement.attr("href");
                
                // 提取标题
                Element titleElement = linkElement.selectFirst("img");
                String title = "";
                if (titleElement != null) {
                    title = titleElement.attr("alt").trim();
                }
                
                // 提取作者信息
                Element authorElement = item.selectFirst(".pl");
                String author = "";
                if (authorElement != null) {
                    author = authorElement.text().trim();
                }
                
                // 提取评分
                Element ratingElement = item.selectFirst(".rating_nums");
                String rating = "";
                if (ratingElement != null) {
                    rating = ratingElement.text().trim();
                }
                
                books.add(new Book(title, author, rating, url));
            }
        }
        
        return books;
    }
}

// 中国天气网爬虫 - 爬取各省份天气
class WeatherSpider extends Spider {
    // 中国各省份代码映射
    private static final String[] PROVINCES = {
        "北京", "上海", "天津", "重庆", "河北", "山西", "辽宁", "吉林", "黑龙江",
        "江苏", "浙江", "安徽", "福建", "江西", "山东", "河南", "湖北", "湖南",
        "广东", "海南", "四川", "贵州", "云南", "陕西", "甘肃", "青海", "台湾",
        "内蒙古", "广西", "西藏", "宁夏", "新疆", "香港", "澳门"
    };
    
    public WeatherSpider() {
        super("http://www.weather.com.cn");
    }
    
    @Override
    public List<Weather> crawl() throws IOException {
        List<Weather> allWeather = new ArrayList<>();
        
        System.out.println("正在爬取中国各省份天气数据...");
        
        // 爬取主要城市天气（使用中国天气网的API）
        String[] majorCities = {
            "101010100", // 北京
            "101020100", // 上海
            "101030100", // 天津
            "101040100", // 重庆
            "101050101", // 哈尔滨
            "101060101", // 长春
            "101070101", // 沈阳
            "101080101", // 呼和浩特
            "101090101", // 石家庄
            "101100101", // 太原
            "101110101", // 西安
            "101120101", // 济南
            "101130101", // 乌鲁木齐
            "101140101", // 拉萨
            "101150101", // 西宁
            "101160101", // 兰州
            "101170101", // 银川
            "101180101", // 郑州
            "101190101", // 南京
            "101200101", // 武汉
            "101210101", // 杭州
            "101220101", // 合肥
            "101230101", // 福州
            "101240101", // 南昌
            "101250101", // 长沙
            "101260101", // 贵阳
            "101270101", // 成都
            "101280101", // 广州
            "101290101", // 昆明
            "101300101", // 南宁
            "101310101", // 海口
            "101320101", // 香港
            "101330101", // 澳门
            "101340101"  // 台北
        };
        
        String[] cityNames = {
            "北京", "上海", "天津", "重庆", "哈尔滨", "长春", "沈阳", "呼和浩特",
            "石家庄", "太原", "西安", "济南", "乌鲁木齐", "拉萨", "西宁", "兰州",
            "银川", "郑州", "南京", "武汉", "杭州", "合肥", "福州", "南昌",
            "长沙", "贵阳", "成都", "广州", "昆明", "南宁", "海口", "香港", "澳门", "台北"
        };
        
        for (int i = 0; i < majorCities.length; i++) {
            String cityCode = majorCities[i];
            String cityName = cityNames[i];
            String province = getProvinceByCity(cityName);
            
            try {
                String weatherUrl = "http://www.weather.com.cn/weather1d/" + cityCode + ".shtml";
                Document doc = fetchPage(weatherUrl);
                Weather weather = parseWeatherPage(doc, province, cityName, weatherUrl);
                if (weather != null) {
                    allWeather.add(weather);
                    System.out.println("已获取 " + cityName + " 的天气数据");
                }
                
                // 添加延迟，避免请求过快
                Thread.sleep(500);
            } catch (Exception e) {
                System.err.println("获取 " + cityName + " 天气数据失败: " + e.getMessage());
            }
        }
        
        return allWeather;
    }
    
    private String getProvinceByCity(String city) {
        // 简化的省份映射
        if (city.equals("北京") || city.equals("上海") || city.equals("天津") || city.equals("重庆")) {
            return city;
        } else if (city.equals("哈尔滨") || city.equals("长春") || city.equals("沈阳")) {
            return "东北";
        } else if (city.equals("呼和浩特") || city.equals("石家庄") || city.equals("太原")) {
            return "华北";
        } else if (city.equals("西安") || city.equals("兰州") || city.equals("西宁") || city.equals("银川")) {
            return "西北";
        } else if (city.equals("济南") || city.equals("郑州") || city.equals("南京")) {
            return "华东";
        } else if (city.equals("武汉") || city.equals("长沙") || city.equals("南昌")) {
            return "华中";
        } else if (city.equals("杭州") || city.equals("合肥") || city.equals("福州")) {
            return "华东";
        } else if (city.equals("成都") || city.equals("贵阳") || city.equals("昆明")) {
            return "西南";
        } else if (city.equals("广州") || city.equals("南宁") || city.equals("海口")) {
            return "华南";
        } else if (city.equals("乌鲁木齐") || city.equals("拉萨")) {
            return "西北";
        } else if (city.equals("香港") || city.equals("澳门") || city.equals("台北")) {
            return "港澳台";
        }
        return "其他";
    }
    
    private Weather parseWeatherPage(Document doc, String province, String city, String url) {
        try {
            // 提取温度
            Element tempElement = doc.selectFirst(".tem");
            String temperature = "";
            if (tempElement != null) {
                temperature = tempElement.text().trim();
            }
            
            // 提取天气状况
            Element weatherElement = doc.selectFirst(".wea");
            String weatherCondition = "";
            if (weatherElement != null) {
                weatherCondition = weatherElement.text().trim();
            }
            
            return new Weather(province, city, temperature, weatherCondition, url);
        } catch (Exception e) {
            System.err.println("解析 " + city + " 天气数据失败: " + e.getMessage());
            return null;
        }
    }
    
    @Override
    protected List<Weather> parsePage(Document doc) {
        // 这个方法在crawl中被直接调用，不需要实现
        return new ArrayList<>();
    }
}

// 测试类
public class SpiderFramework {
    public static void main(String[] args) {
        List<Spider> spiders = new ArrayList<>();
        spiders.add(new DoubanMovieSpider());
        spiders.add(new DoubanBookSpider());
        spiders.add(new WeatherSpider());
        
        for (Spider spider : spiders) {
            try {
                System.out.println("\n爬取 " + spider.getClass().getSimpleName() + " 数据:");
                List<?> data = spider.crawl();
                System.out.println("共爬取 " + data.size() + " 条数据");
                
                // 只显示前5条数据
                int displayCount = Math.min(5, data.size());
                System.out.println("显示前 " + displayCount + " 条:");
                for (int i = 0; i < displayCount; i++) {
                    System.out.println(data.get(i));
                }
            } catch (IOException e) {
                System.err.println("爬取失败: " + e.getMessage());
            }
        }
    }
}