10 changed files with 0 additions and 810 deletions
|
@ -1,30 +0,0 @@ |
|||
import project.bean.Movie; |
|||
import project.crawler.MovieCrawler; |
|||
import project.utils.DataStorage; |
|||
import project.display.ResultDisplay; |
|||
|
|||
import java.util.List; |
|||
|
|||
public class Main { |
|||
public static void main(String[] args) { |
|||
try { |
|||
System.out.println("Starting to crawl movie data..."); |
|||
List<Movie> movies = MovieCrawler.crawlMovies(10); // Crawl 10 pages of data
|
|||
System.out.println("Crawling completed, obtained " + movies.size() + " movies data"); |
|||
|
|||
System.out.println("Saving data to CSV file..."); |
|||
DataStorage.saveToCsv(movies, "project/movies.csv"); |
|||
System.out.println("Data saved successfully"); |
|||
|
|||
System.out.println("Analyzing data..."); |
|||
ResultDisplay.displayResults(movies); |
|||
|
|||
System.out.println("Generating charts..."); |
|||
ResultDisplay.generateCharts(movies); |
|||
System.out.println("Chart generation completed, saved to project directory"); |
|||
|
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
} |
|||
@ -1,101 +0,0 @@ |
|||
import org.jsoup.Jsoup; |
|||
import org.jsoup.Connection; |
|||
import org.json.JSONArray; |
|||
import org.json.JSONObject; |
|||
import org.apache.commons.csv.*; |
|||
|
|||
import java.io.FileWriter; |
|||
import java.io.IOException; |
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
|
|||
public class JobSpider { |
|||
|
|||
// ⚠️ 注意：这个 URL 可能会随时间变化，请务必按上面的步骤在 F12 中确认最新的 URL
|
|||
// 这里的参数 keyword=Java, page=1 是示例，实际需要根据网站调整
|
|||
private static final String API_URL = "https://www.iguopin.com/api/job/search?keyword=&page=1&pageSize=20"; |
|||
|
|||
public static void main(String[] args) { |
|||
List<String[]> jobList = new ArrayList<>(); |
|||
|
|||
try { |
|||
System.out.println("🚀 开始连接国聘网数据接口..."); |
|||
|
|||
// 1. 构造请求，必须伪装 Header，否则会被拒绝
|
|||
String jsonResponse = Jsoup.connect(API_URL) |
|||
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") |
|||
.header("Accept", "application/json, text/plain, */*") |
|||
.header("Referer", "https://www.iguopin.com/") // 假装是从首页跳过来的
|
|||
.timeout(5000) |
|||
.ignoreContentType(true) // 重要！允许接收非 HTML 内容 (即 JSON)
|
|||
.execute() |
|||
.body(); |
|||
|
|||
// 2. 解析 JSON 数据
|
|||
JSONObject root = new JSONObject(jsonResponse); |
|||
|
|||
// ⚠️ 关键：你需要根据 F12 看到的实际 JSON 结构调整这里的键名 (key)
|
|||
// 假设数据结构是 { "data": { "list": [...] } } 或者 { "result": [...] }
|
|||
// 下面是一个通用的猜测逻辑，请根据实际打印结果修改！
|
|||
|
|||
JSONArray jobsArray = null; |
|||
|
|||
// 尝试几种常见的结构 (你需要打印 root.toString() 来确认到底是哪一层)
|
|||
if (root.has("data")) { |
|||
JSONObject dataObj = root.getJSONObject("data"); |
|||
if (dataObj.has("list")) jobsArray = dataObj.getJSONArray("list"); |
|||
else if (dataObj.has("jobs")) jobsArray = dataObj.getJSONArray("jobs"); |
|||
} else if (root.has("result")) { |
|||
jobsArray = root.getJSONArray("result"); |
|||
} else if (root.has("jobs")) { |
|||
jobsArray = root.getJSONArray("jobs"); |
|||
} |
|||
|
|||
if (jobsArray == null) { |
|||
System.err.println("❌ 未找到职位列表数据。JSON 结构可能已变更，请打印查看：\n" + jsonResponse); |
|||
return; |
|||
} |
|||
|
|||
System.out.println("✅ 解析成功，共发现 " + jobsArray.length() + " 个职位。"); |
|||
|
|||
// 3. 提取具体字段
|
|||
for (int i = 0; i < jobsArray.length(); i++) { |
|||
JSONObject job = jobsArray.getJSONObject(i); |
|||
|
|||
// ⚠️ 再次强调：这里的 "jobName", "companyName" 必须和你 F12 里看到的一模一样！
|
|||
String title = job.optString("jobName", "未知职位"); |
|||
String company = job.optString("companyName", "未知公司"); |
|||
String salary = job.optString("salary", "面议"); |
|||
String location = job.optString("workLocation", "未知地点"); |
|||
String link = "https://www.iguopin.com/job/detail/" + job.optString("id"); // 拼接详情页链接
|
|||
|
|||
jobList.add(new String[]{title, company, salary, location, link}); |
|||
System.out.println("[" + (i+1) + "] " + title + " | " + company); |
|||
} |
|||
|
|||
// 4. 保存到 CSV
|
|||
saveToCsv(jobList, "guopin_jobs.csv"); |
|||
System.out.println("💾 数据已保存至 guopin_jobs.csv"); |
|||
|
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
System.err.println("❌ 网络请求失败：可能是接口地址变了，或者被反爬拦截。"); |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
System.err.println("❌ JSON 解析失败：请检查代码中的 key 名称是否与网页返回的一致。"); |
|||
} |
|||
} |
|||
|
|||
private static void saveToCsv(List<String[]> data, String fileName) throws IOException { |
|||
FileWriter out = new FileWriter(fileName); |
|||
// 定义表头
|
|||
CSVFormat format = CSVFormat.DEFAULT.withHeader("职位名称", "公司名称", "薪资", "地点", "链接"); |
|||
CSVPrinter printer = new CSVPrinter(out, format); |
|||
|
|||
for (String[] row : data) { |
|||
printer.printRecord(row); |
|||
} |
|||
printer.close(); |
|||
out.close(); |
|||
} |
|||
} |
|||
@ -1,42 +0,0 @@ |
|||
package project.analysis; |
|||
|
|||
import project.bean.Movie; |
|||
|
|||
import java.util.*; |
|||
import java.util.stream.Collectors; |
|||
|
|||
public class MovieAnalyzer { |
|||
public static Map<Double, Long> getRatingDistribution(List<Movie> movies) { |
|||
return movies.stream() |
|||
.collect(Collectors.groupingBy(Movie::getRating, Collectors.counting())); |
|||
} |
|||
|
|||
public static Map<Integer, Double> getYearRatingCorrelation(List<Movie> movies) { |
|||
return movies.stream() |
|||
.collect(Collectors.groupingBy(Movie::getYear, |
|||
Collectors.averagingDouble(Movie::getRating))); |
|||
} |
|||
|
|||
public static Map<String, Long> getDirectorMovieCount(List<Movie> movies) { |
|||
return movies.stream() |
|||
.collect(Collectors.groupingBy(Movie::getDirector, Collectors.counting())) |
|||
.entrySet().stream() |
|||
.filter(entry -> entry.getValue() > 1) |
|||
.sorted(Map.Entry.<String, Long>comparingByValue().reversed()) |
|||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new)); |
|||
} |
|||
|
|||
public static double getAverageRating(List<Movie> movies) { |
|||
return movies.stream() |
|||
.mapToDouble(Movie::getRating) |
|||
.average() |
|||
.orElse(0.0); |
|||
} |
|||
|
|||
public static List<Movie> getTopRatedMovies(List<Movie> movies, int count) { |
|||
return movies.stream() |
|||
.sorted(Comparator.comparingDouble(Movie::getRating).reversed()) |
|||
.limit(count) |
|||
.collect(Collectors.toList()); |
|||
} |
|||
} |
|||
@ -1,60 +0,0 @@ |
|||
package project.bean; |
|||
|
|||
public class Movie { |
|||
private String title; |
|||
private double rating; |
|||
private int year; |
|||
private String director; |
|||
|
|||
public Movie() { |
|||
} |
|||
|
|||
public Movie(String title, double rating, int year, String director) { |
|||
this.title = title; |
|||
this.rating = rating; |
|||
this.year = year; |
|||
this.director = director; |
|||
} |
|||
|
|||
public String getTitle() { |
|||
return title; |
|||
} |
|||
|
|||
public void setTitle(String title) { |
|||
this.title = title; |
|||
} |
|||
|
|||
public double getRating() { |
|||
return rating; |
|||
} |
|||
|
|||
public void setRating(double rating) { |
|||
this.rating = rating; |
|||
} |
|||
|
|||
public int getYear() { |
|||
return year; |
|||
} |
|||
|
|||
public void setYear(int year) { |
|||
this.year = year; |
|||
} |
|||
|
|||
public String getDirector() { |
|||
return director; |
|||
} |
|||
|
|||
public void setDirector(String director) { |
|||
this.director = director; |
|||
} |
|||
|
|||
@Override |
|||
public String toString() { |
|||
return "Movie{" + |
|||
"title='" + title + '\'' + |
|||
", rating=" + rating + |
|||
", year=" + year + |
|||
", director='" + director + '\'' + |
|||
'}'; |
|||
} |
|||
} |
|||
@ -1,194 +0,0 @@ |
|||
package project.crawler; |
|||
|
|||
import project.bean.Movie; |
|||
import project.utils.DataCleaner; |
|||
import project.utils.HttpUtils; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
import java.util.regex.Matcher; |
|||
import java.util.regex.Pattern; |
|||
|
|||
public class MovieCrawler { |
|||
public static List<Movie> crawlMovies(int pageCount) throws Exception { |
|||
List<Movie> movies = new ArrayList<>(); |
|||
|
|||
for (int page = 1; page <= pageCount; page++) { |
|||
String url = "https://movie.douban.com/top250?start=" + (page - 1) * 25; |
|||
System.out.println("Crawling page " + page + " from " + url); |
|||
try { |
|||
String html = HttpUtils.getHtml(url); |
|||
System.out.println("Got HTML content, length: " + html.length()); |
|||
|
|||
// 打印 HTML 内容的前 500 个字符，了解实际结构
|
|||
if (html.length() > 500) { |
|||
System.out.println("HTML preview: " + html.substring(0, 500) + "..."); |
|||
} |
|||
|
|||
List<Movie> pageMovies = parseMovies(html); |
|||
System.out.println("Parsed " + pageMovies.size() + " movies from page " + page); |
|||
movies.addAll(pageMovies); |
|||
} catch (Exception e) { |
|||
System.out.println("Error crawling page " + page + ": " + e.getMessage()); |
|||
} |
|||
Thread.sleep(1000); // 控制请求频率
|
|||
} |
|||
|
|||
System.out.println("Total movies crawled: " + movies.size()); |
|||
return movies; |
|||
} |
|||
|
|||
private static List<Movie> parseMovies(String html) { |
|||
List<Movie> movies = new ArrayList<>(); |
|||
|
|||
// Find all movie items by looking for <div class="item"> and matching until </div> at the same nesting level
|
|||
int startIndex = 0; |
|||
int count = 0; |
|||
|
|||
while (true) { |
|||
int itemStart = html.indexOf("<div class=\"item\">", startIndex); |
|||
if (itemStart < 0) break; |
|||
|
|||
// Find the matching </div> by counting nested divs
|
|||
int pos = itemStart + "<div class=\"item\">".length(); |
|||
int depth = 1; |
|||
int itemEnd = -1; |
|||
|
|||
while (pos < html.length() && depth > 0) { |
|||
int nextOpen = html.indexOf("<div", pos); |
|||
int nextClose = html.indexOf("</div>", pos); |
|||
|
|||
if (nextClose < 0) break; // No closing tag found
|
|||
|
|||
if (nextOpen >= 0 && nextOpen < nextClose) { |
|||
// Found an opening div before closing
|
|||
depth++; |
|||
pos = nextOpen + 4; |
|||
} else { |
|||
// Found a closing div
|
|||
depth--; |
|||
if (depth == 0) { |
|||
itemEnd = nextClose + 6; |
|||
} |
|||
pos = nextClose + 6; |
|||
} |
|||
} |
|||
|
|||
if (itemEnd > itemStart) { |
|||
count++; |
|||
String movieHtml = html.substring(itemStart, itemEnd); |
|||
// Don't print movie HTML to avoid excessive output
|
|||
Movie movie = parseMovie(movieHtml); |
|||
if (movie != null) { |
|||
movies.add(movie); |
|||
} |
|||
startIndex = itemEnd; |
|||
} else { |
|||
break; |
|||
} |
|||
} |
|||
|
|||
System.out.println("Found " + count + " movie items, parsed " + movies.size() + " valid movies"); |
|||
return movies; |
|||
} |
|||
|
|||
private static Movie parseMovie(String movieHtml) { |
|||
try { |
|||
// Extract title from img alt attribute
|
|||
String title = ""; |
|||
int altIndex = movieHtml.indexOf("alt="); |
|||
if (altIndex > 0) { |
|||
int start = movieHtml.indexOf('"', altIndex); |
|||
int end = movieHtml.indexOf('"', start + 1); |
|||
if (start > 0 && end > 0) { |
|||
title = movieHtml.substring(start + 1, end).trim(); |
|||
} |
|||
} |
|||
|
|||
// Extract rating
|
|||
double rating = 0.0; |
|||
int ratingIndex = movieHtml.indexOf("rating_num"); |
|||
if (ratingIndex > 0) { |
|||
int start = movieHtml.indexOf('>', ratingIndex); |
|||
int end = movieHtml.indexOf("</span>", start); |
|||
if (start > 0 && end > 0) { |
|||
String ratingStr = movieHtml.substring(start + 1, end).trim(); |
|||
try { |
|||
rating = Double.parseDouble(ratingStr); |
|||
} catch (NumberFormatException e) { |
|||
rating = 0.0; |
|||
} |
|||
} |
|||
} |
|||
|
|||
// Extract year and director from movie info
|
|||
int year = 0; |
|||
String director = "Unknown"; |
|||
|
|||
// Find the info section which contains year and director
|
|||
// Look for <p> tag without class or with specific class
|
|||
int infoStart = -1; |
|||
int pStart = movieHtml.indexOf("<p>"); |
|||
int pClassStart = movieHtml.indexOf("<p class=\"\">"); |
|||
|
|||
if (pStart >= 0) { |
|||
infoStart = pStart; |
|||
} |
|||
if (pClassStart >= 0 && (pStart < 0 || pClassStart < pStart)) { |
|||
infoStart = pClassStart; |
|||
} |
|||
|
|||
if (infoStart > 0) { |
|||
int infoEnd = movieHtml.indexOf("</p>", infoStart); |
|||
if (infoEnd > infoStart) { |
|||
String infoSection = movieHtml.substring(infoStart, infoEnd); |
|||
|
|||
// Extract year - look for 4-digit year after <br> tag
|
|||
int brIndex = infoSection.indexOf("<br>"); |
|||
if (brIndex > 0) { |
|||
String afterBr = infoSection.substring(brIndex + 4).trim(); |
|||
// Find first 4-digit number
|
|||
for (int i = 0; i <= afterBr.length() - 4; i++) { |
|||
String possibleYear = afterBr.substring(i, i + 4); |
|||
if (possibleYear.matches("\\d{4}")) { |
|||
try { |
|||
year = Integer.parseInt(possibleYear); |
|||
break; |
|||
} catch (NumberFormatException e) { |
|||
// Continue
|
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
// Extract director - director info is between "导演:" and " "
|
|||
// Look for the pattern: 导演: [director name]
|
|||
int directorLabelIdx = infoSection.indexOf("\u5bfc\u6f14:"); // Unicode for "导演:"
|
|||
if (directorLabelIdx >= 0) { |
|||
int directorStart = directorLabelIdx + 3; // Skip "导演:"
|
|||
int directorEnd = infoSection.indexOf(" ", directorStart); |
|||
if (directorEnd > directorStart) { |
|||
director = infoSection.substring(directorStart, directorEnd).trim(); |
|||
// Clean up any remaining HTML
|
|||
director = director.replaceAll("<[^>]*>", "").trim(); |
|||
// Extract only Chinese name (before space)
|
|||
int spaceIdx = director.indexOf(" "); |
|||
if (spaceIdx > 0) { |
|||
director = director.substring(0, spaceIdx).trim(); |
|||
} |
|||
if (director.isEmpty()) director = "Unknown"; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
// If title and rating are valid, create movie object
|
|||
if (!title.isEmpty() && rating > 0) { |
|||
return new Movie(title, rating, year, director); |
|||
} |
|||
} catch (Exception e) { |
|||
// Silently handle exceptions
|
|||
} |
|||
return null; |
|||
} |
|||
} |
|||
@ -1,47 +0,0 @@ |
|||
package project.display; |
|||
|
|||
import project.bean.Movie; |
|||
import project.analysis.MovieAnalyzer; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
public class ResultDisplay { |
|||
public static void displayResults(List<Movie> movies) { |
|||
System.out.println("===== Movie Data Analysis Results ====="); |
|||
System.out.println("Total movies: " + movies.size()); |
|||
System.out.printf("Average rating: %.2f\n\n", MovieAnalyzer.getAverageRating(movies)); |
|||
|
|||
System.out.println("===== Rating Distribution ====="); |
|||
Map<Double, Long> ratingDistribution = MovieAnalyzer.getRatingDistribution(movies); |
|||
ratingDistribution.entrySet().stream() |
|||
.sorted(Map.Entry.comparingByKey()) |
|||
.forEach(entry -> System.out.printf("Rating %.1f: %d movies\n", entry.getKey(), entry.getValue())); |
|||
|
|||
System.out.println("\n===== Year-Rating Correlation ====="); |
|||
Map<Integer, Double> yearRating = MovieAnalyzer.getYearRatingCorrelation(movies); |
|||
yearRating.entrySet().stream() |
|||
.sorted(Map.Entry.comparingByKey()) |
|||
.forEach(entry -> System.out.printf("%d: %.2f\n", entry.getKey(), entry.getValue())); |
|||
|
|||
System.out.println("\n===== Director Movie Count Ranking ====="); |
|||
Map<String, Long> directorCount = MovieAnalyzer.getDirectorMovieCount(movies); |
|||
directorCount.entrySet().stream() |
|||
.limit(10) |
|||
.forEach(entry -> System.out.printf("%s: %d movies\n", entry.getKey(), entry.getValue())); |
|||
|
|||
System.out.println("\n===== Top 10 Highest Rated Movies ====="); |
|||
List<Movie> topRated = MovieAnalyzer.getTopRatedMovies(movies, 10); |
|||
for (int i = 0; i < topRated.size(); i++) { |
|||
Movie movie = topRated.get(i); |
|||
System.out.printf("%d. %s (%.1f) - %d - Director: %s\n", |
|||
i + 1, movie.getTitle(), movie.getRating(), movie.getYear(), movie.getDirector()); |
|||
} |
|||
} |
|||
|
|||
public static void generateCharts(List<Movie> movies) throws Exception { |
|||
System.out.println("\n===== Chart Generation ====="); |
|||
System.out.println("Due to environment limitations, chart generation is not implemented"); |
|||
System.out.println("Suggest using JFreeChart or other chart libraries for visualization"); |
|||
} |
|||
} |
|||
@ -1,29 +0,0 @@ |
|||
package project.utils; |
|||
|
|||
public class DataCleaner { |
|||
public static String cleanText(String text) { |
|||
if (text == null) return ""; |
|||
return text.trim() |
|||
.replaceAll("<[^>]*>", "") |
|||
.replaceAll("\\s+", " ") |
|||
.replaceAll("[\\r\\n]", ""); |
|||
} |
|||
|
|||
public static double parseRating(String ratingStr) { |
|||
if (ratingStr == null || ratingStr.isEmpty()) return 0.0; |
|||
try { |
|||
return Double.parseDouble(ratingStr.trim()); |
|||
} catch (NumberFormatException e) { |
|||
return 0.0; |
|||
} |
|||
} |
|||
|
|||
public static int parseYear(String yearStr) { |
|||
if (yearStr == null || yearStr.isEmpty()) return 0; |
|||
try { |
|||
return Integer.parseInt(yearStr.replaceAll("[^0-9]", "")); |
|||
} catch (NumberFormatException e) { |
|||
return 0; |
|||
} |
|||
} |
|||
} |
|||
@ -1,26 +0,0 @@ |
|||
package project.utils; |
|||
|
|||
import project.bean.Movie; |
|||
|
|||
import java.io.OutputStreamWriter; |
|||
import java.io.FileOutputStream; |
|||
import java.io.IOException; |
|||
import java.util.List; |
|||
|
|||
public class DataStorage { |
|||
public static void saveToCsv(List<Movie> movies, String filePath) throws IOException { |
|||
// Use UTF-8 encoding to properly handle Chinese characters
|
|||
OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(filePath), "UTF-8"); |
|||
writer.write("Title,Rating,Year,Director\n"); |
|||
|
|||
for (Movie movie : movies) { |
|||
writer.write(String.format("%s,%.1f,%d,%s\n", |
|||
movie.getTitle(), |
|||
movie.getRating(), |
|||
movie.getYear(), |
|||
movie.getDirector())); |
|||
} |
|||
|
|||
writer.close(); |
|||
} |
|||
} |
|||
@ -1,30 +0,0 @@ |
|||
package project.utils; |
|||
|
|||
import java.io.BufferedReader; |
|||
import java.io.InputStreamReader; |
|||
import java.net.HttpURLConnection; |
|||
import java.net.URL; |
|||
|
|||
public class HttpUtils { |
|||
public static String getHtml(String url) throws Exception { |
|||
URL obj = new URL(url); |
|||
HttpURLConnection con = (HttpURLConnection) obj.openConnection(); |
|||
con.setRequestMethod("GET"); |
|||
con.setRequestProperty("User-Agent", "Mozilla/5.0"); |
|||
|
|||
int responseCode = con.getResponseCode(); |
|||
if (responseCode != HttpURLConnection.HTTP_OK) { |
|||
throw new Exception("HTTP error code: " + responseCode); |
|||
} |
|||
|
|||
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), "UTF-8")); |
|||
String inputLine; |
|||
StringBuilder html = new StringBuilder(); |
|||
|
|||
while ((inputLine = in.readLine()) != null) { |
|||
html.append(inputLine); |
|||
} |
|||
in.close(); |
|||
return html.toString(); |
|||
} |
|||
} |
|||
Loading…
Reference in new issue