上传文件至 'project'

project/DataAnalyzer.java

@@ -0,0 +1,142 @@
+package java01;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class DataAnalyzer {
+    private DataCleaner cleaner;
+    
+    public DataAnalyzer() {
+        this.cleaner = new DataCleaner();
+    }
+    
+    // 统计技能词频
+    public Map<String, Integer> analyzeSkillFrequency(List<Job> jobs) {
+        Map<String, Integer> skillMap = new HashMap<>();
+        
+        for (Job job : jobs) {
+            String skills = job.getSkills();
+            if (skills != null && !skills.isEmpty()) {
+                String[] skillArray = skills.split(" ");
+                for (String skill : skillArray) {
+                    if (!skill.isEmpty()) {
+                        skillMap.put(skill, skillMap.getOrDefault(skill, 0) + 1);
+                    }
+                }
+            }
+        }
+        
+        // 按词频排序
+        return skillMap.entrySet().stream()
+                .sorted(Map.Entry.<String, Integer>comparingByValue().reversed())
+                .collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        Map.Entry::getValue,
+                        (e1, e2) -> e1,
+                        LinkedHashMap::new
+                ));
+    }
+    
+    // 分析薪资与经验的关系
+    public Map<String, Double> analyzeSalaryByExperience(List<Job> jobs) {
+        Map<String, List<Integer>> experienceSalaryMap = new HashMap<>();
+        
+        for (Job job : jobs) {
+            String experience = job.getExperience();
+            int minSalary = cleaner.extractMinSalary(job.getSalary());
+            int maxSalary = cleaner.extractMaxSalary(job.getSalary());
+            int avgSalary = (minSalary + maxSalary) / 2;
+            
+            if (!experience.isEmpty() && avgSalary > 0) {
+                experienceSalaryMap.computeIfAbsent(experience, k -> new ArrayList<>()).add(avgSalary);
+            }
+        }
+        
+        // 计算每个经验级别的平均薪资
+        Map<String, Double> result = new HashMap<>();
+        for (Map.Entry<String, List<Integer>> entry : experienceSalaryMap.entrySet()) {
+            double avgSalary = entry.getValue().stream().mapToInt(Integer::intValue).average().orElse(0);
+            result.put(entry.getKey(), avgSalary);
+        }
+        
+        return result;
+    }
+    
+    // 分析薪资与学历的关系
+    public Map<String, Double> analyzeSalaryByEducation(List<Job> jobs) {
+        Map<String, List<Integer>> educationSalaryMap = new HashMap<>();
+        
+        for (Job job : jobs) {
+            String education = job.getEducation();
+            int minSalary = cleaner.extractMinSalary(job.getSalary());
+            int maxSalary = cleaner.extractMaxSalary(job.getSalary());
+            int avgSalary = (minSalary + maxSalary) / 2;
+            
+            if (!education.isEmpty() && avgSalary > 0) {
+                educationSalaryMap.computeIfAbsent(education, k -> new ArrayList<>()).add(avgSalary);
+            }
+        }
+        
+        // 计算每个学历级别的平均薪资
+        Map<String, Double> result = new HashMap<>();
+        for (Map.Entry<String, List<Integer>> entry : educationSalaryMap.entrySet()) {
+            double avgSalary = entry.getValue().stream().mapToInt(Integer::intValue).average().orElse(0);
+            result.put(entry.getKey(), avgSalary);
+        }
+        
+        return result;
+    }
+    
+    // 分析不同地点的薪资水平
+    public Map<String, Double> analyzeSalaryByLocation(List<Job> jobs) {
+        Map<String, List<Integer>> locationSalaryMap = new HashMap<>();
+        
+        for (Job job : jobs) {
+            String location = job.getLocation();
+            int minSalary = cleaner.extractMinSalary(job.getSalary());
+            int maxSalary = cleaner.extractMaxSalary(job.getSalary());
+            int avgSalary = (minSalary + maxSalary) / 2;
+            
+            if (!location.isEmpty() && avgSalary > 0) {
+                locationSalaryMap.computeIfAbsent(location, k -> new ArrayList<>()).add(avgSalary);
+            }
+        }
+        
+        // 计算每个地点的平均薪资
+        Map<String, Double> result = new HashMap<>();
+        for (Map.Entry<String, List<Integer>> entry : locationSalaryMap.entrySet()) {
+            double avgSalary = entry.getValue().stream().mapToInt(Integer::intValue).average().orElse(0);
+            result.put(entry.getKey(), avgSalary);
+        }
+        
+        return result;
+    }
+    
+    // 获取薪资分布
+    public Map<String, Integer> analyzeSalaryDistribution(List<Job> jobs) {
+        Map<String, Integer> salaryDistribution = new HashMap<>();
+        
+        for (Job job : jobs) {
+            int avgSalary = (cleaner.extractMinSalary(job.getSalary()) + cleaner.extractMaxSalary(job.getSalary())) / 2;
+            
+            String salaryRange;
+            if (avgSalary < 5000) {
+                salaryRange = "5K以下";
+            } else if (avgSalary < 10000) {
+                salaryRange = "5K-10K";
+            } else if (avgSalary < 15000) {
+                salaryRange = "10K-15K";
+            } else if (avgSalary < 20000) {
+                salaryRange = "15K-20K";
+            } else if (avgSalary < 30000) {
+                salaryRange = "20K-30K";
+            } else {
+                salaryRange = "30K以上";
+            }
+            
+            salaryDistribution.put(salaryRange, salaryDistribution.getOrDefault(salaryRange, 0) + 1);
+        }
+        
+        return salaryDistribution;
+    }
+}

project/DataCleaner.java

@@ -0,0 +1,124 @@
+package java01;
+
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class DataCleaner {
+    
+    public List<Job> cleanJobs(List<Job> jobs) {
+        for (Job job : jobs) {
+            cleanJob(job);
+        }
+        return jobs;
+    }
+    
+    private void cleanJob(Job job) {
+        // 清理职位标题
+        if (job.getTitle() != null) {
+            job.setTitle(job.getTitle().trim());
+        }
+        
+        // 清理公司名称
+        if (job.getCompany() != null) {
+            job.setCompany(job.getCompany().trim());
+        }
+        
+        // 清理薪资
+        if (job.getSalary() != null) {
+            job.setSalary(job.getSalary().trim());
+        }
+        
+        // 清理地点
+        if (job.getLocation() != null) {
+            job.setLocation(job.getLocation().trim());
+        }
+        
+        // 清理经验
+        if (job.getExperience() != null) {
+            job.setExperience(job.getExperience().trim());
+        }
+        
+        // 清理学历
+        if (job.getEducation() != null) {
+            job.setEducation(job.getEducation().trim());
+        }
+        
+        // 清理技能
+        if (job.getSkills() != null) {
+            job.setSkills(job.getSkills().trim());
+        }
+    }
+    
+    // 提取薪资范围的最小值
+    public int extractMinSalary(String salary) {
+        if (salary == null || salary.isEmpty()) {
+            return 0;
+        }
+        
+        // 匹配薪资范围，如"10K-20K"
+        Pattern pattern = Pattern.compile("(\\d+)K-(\\d+)K");
+        Matcher matcher = pattern.matcher(salary);
+        
+        if (matcher.find()) {
+            return Integer.parseInt(matcher.group(1)) * 1000;
+        }
+        
+        // 匹配固定薪资，如"15K"
+        pattern = Pattern.compile("(\\d+)K");
+        matcher = pattern.matcher(salary);
+        if (matcher.find()) {
+            return Integer.parseInt(matcher.group(1)) * 1000;
+        }
+        
+        return 0;
+    }
+    
+    // 提取薪资范围的最大值
+    public int extractMaxSalary(String salary) {
+        if (salary == null || salary.isEmpty()) {
+            return 0;
+        }
+        
+        // 匹配薪资范围，如"10K-20K"
+        Pattern pattern = Pattern.compile("(\\d+)K-(\\d+)K");
+        Matcher matcher = pattern.matcher(salary);
+        
+        if (matcher.find()) {
+            return Integer.parseInt(matcher.group(2)) * 1000;
+        }
+        
+        // 匹配固定薪资，如"15K"
+        pattern = Pattern.compile("(\\d+)K");
+        matcher = pattern.matcher(salary);
+        if (matcher.find()) {
+            return Integer.parseInt(matcher.group(1)) * 1000;
+        }
+        
+        return 0;
+    }
+    
+    // 提取经验年限
+    public int extractExperienceYears(String experience) {
+        if (experience == null || experience.isEmpty()) {
+            return 0;
+        }
+        
+        // 匹配经验年限，如"3-5年"
+        Pattern pattern = Pattern.compile("(\\d+)-(\\d+)年");
+        Matcher matcher = pattern.matcher(experience);
+        
+        if (matcher.find()) {
+            return Integer.parseInt(matcher.group(1));
+        }
+        
+        // 匹配固定经验，如"3年以上"
+        pattern = Pattern.compile("(\\d+)年");
+        matcher = pattern.matcher(experience);
+        if (matcher.find()) {
+            return Integer.parseInt(matcher.group(1));
+        }
+        
+        return 0;
+    }
+}

project/DataStorage.java

@@ -0,0 +1,44 @@
+package java01;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+
+public class DataStorage {
+    
+    public void writeJobsToCSV(List<Job> jobs, String fileName) {
+        try (FileWriter writer = new FileWriter(fileName)) {
+            // 写入表头
+            writer.write("职位标题,公司名称,薪资,地点,经验,学历,技能要求\n");
+            
+            for (Job job : jobs) {
+                // 写入数据行，处理逗号和引号
+                writer.write(escapeCsvField(job.getTitle()) + ",");
+                writer.write(escapeCsvField(job.getCompany()) + ",");
+                writer.write(escapeCsvField(job.getSalary()) + ",");
+                writer.write(escapeCsvField(job.getLocation()) + ",");
+                writer.write(escapeCsvField(job.getExperience()) + ",");
+                writer.write(escapeCsvField(job.getEducation()) + ",");
+                writer.write(escapeCsvField(job.getSkills()) + "\n");
+            }
+            
+            System.out.println("数据已成功写入CSV文件: " + fileName);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+    
+    private String escapeCsvField(String field) {
+        if (field == null) {
+            return "";
+        }
+        // 如果字段包含逗号、引号或换行符，需要用引号包围
+        if (field.contains(",") || field.contains("\"") || field.contains("\n")) {
+            // 转义字段中的引号
+            field = field.replace("\"", "\"\"");
+            // 用引号包围字段
+            return "\"" + field + "\"";
+        }
+        return field;
+    }
+}

project/Job.java

@@ -0,0 +1,90 @@
+package java01;
+
+public class Job {
+    private String title;
+    private String company;
+    private String salary;
+    private String location;
+    private String experience;
+    private String education;
+    private String skills;
+    
+    public Job(String title, String company, String salary, String location, String experience, String education, String skills) {
+        this.title = title;
+        this.company = company;
+        this.salary = salary;
+        this.location = location;
+        this.experience = experience;
+        this.education = education;
+        this.skills = skills;
+    }
+    
+    public String getTitle() {
+        return title;
+    }
+    
+    public void setTitle(String title) {
+        this.title = title;
+    }
+    
+    public String getCompany() {
+        return company;
+    }
+    
+    public void setCompany(String company) {
+        this.company = company;
+    }
+    
+    public String getSalary() {
+        return salary;
+    }
+    
+    public void setSalary(String salary) {
+        this.salary = salary;
+    }
+    
+    public String getLocation() {
+        return location;
+    }
+    
+    public void setLocation(String location) {
+        this.location = location;
+    }
+    
+    public String getExperience() {
+        return experience;
+    }
+    
+    public void setExperience(String experience) {
+        this.experience = experience;
+    }
+    
+    public String getEducation() {
+        return education;
+    }
+    
+    public void setEducation(String education) {
+        this.education = education;
+    }
+    
+    public String getSkills() {
+        return skills;
+    }
+    
+    public void setSkills(String skills) {
+        this.skills = skills;
+    }
+    
+    @Override
+    public String toString() {
+        return "Job{" +
+                "title='" + title + '\'' +
+                ", company='" + company + '\'' +
+                ", salary='" + salary + '\'' +
+                ", location='" + location + '\'' +
+                ", experience='" + experience + '\'' +
+                ", education='" + education + '\'' +
+                ", skills='" + skills + '\'' +
+                '}';
+    }
+}

project/JobCrawler.java

@@ -0,0 +1,86 @@
+package java01;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class JobCrawler {
+    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36";
+    
+    public List<Job> crawlJobs(String keyword, int pageCount) {
+        List<Job> jobs = new ArrayList<>();
+        
+        try {
+            for (int page = 1; page <= pageCount; page++) {
+                String encodedKeyword = URLEncoder.encode(keyword, "UTF-8");
+                String url = String.format("https://sou.zhaopin.com/?jl=489&kw=%s&p=%d", encodedKeyword, page);
+                System.out.println("爬取页面: " + url);
+                
+                // 发送HTTP请求
+                String html = sendHttpRequest(url);
+                
+                // 解析HTML，提取职位信息
+                List<Job> pageJobs = extractJobsFromHtml(html);
+                jobs.addAll(pageJobs);
+                
+                // 添加延迟，避免被反爬虫
+                Thread.sleep(2000);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        
+        return jobs;
+    }
+    
+    private String sendHttpRequest(String urlString) throws Exception {
+        URL url = new URL(urlString);
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setRequestMethod("GET");
+        conn.setRequestProperty("User-Agent", USER_AGENT);
+        conn.setConnectTimeout(10000);
+        conn.setReadTimeout(10000);
+        
+        BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));
+        StringBuilder response = new StringBuilder();
+        String inputLine;
+        
+        while ((inputLine = in.readLine()) != null) {
+            response.append(inputLine);
+        }
+        in.close();
+        conn.disconnect();
+        
+        return response.toString();
+    }
+    
+    private List<Job> extractJobsFromHtml(String html) {
+        List<Job> jobs = new ArrayList<>();
+        
+        // 简化的正则表达式，实际项目中可能需要更复杂的解析
+        // 这里使用模拟数据，因为实际解析HTML需要更复杂的逻辑
+        // 在真实环境中，建议使用Jsoup等库进行HTML解析
+        
+        // 模拟数据
+        for (int i = 1; i <= 10; i++) {
+            Job job = new Job(
+                    "Java开发工程师" + i,
+                    "科技公司" + i,
+                    (10 + i) + "K-" + (20 + i) + "K",
+                    "北京",
+                    "3-5年",
+                    "本科",
+                    "Java Spring Boot MySQL Redis"
+            );
+            jobs.add(job);
+        }
+        
+        return jobs;
+    }
+}

project/jobs.csv

@@ -0,0 +1,31 @@
+职位标题,公司名称,薪资,地点,经验,学历,技能要求
+Java开发工程师1,科技公司1,11K-21K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师2,科技公司2,12K-22K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师3,科技公司3,13K-23K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师4,科技公司4,14K-24K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师5,科技公司5,15K-25K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师6,科技公司6,16K-26K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师7,科技公司7,17K-27K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师8,科技公司8,18K-28K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师9,科技公司9,19K-29K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师10,科技公司10,20K-30K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师1,科技公司1,11K-21K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师2,科技公司2,12K-22K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师3,科技公司3,13K-23K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师4,科技公司4,14K-24K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师5,科技公司5,15K-25K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师6,科技公司6,16K-26K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师7,科技公司7,17K-27K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师8,科技公司8,18K-28K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师9,科技公司9,19K-29K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师10,科技公司10,20K-30K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师1,科技公司1,11K-21K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师2,科技公司2,12K-22K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师3,科技公司3,13K-23K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师4,科技公司4,14K-24K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师5,科技公司5,15K-25K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师6,科技公司6,16K-26K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师7,科技公司7,17K-27K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师8,科技公司8,18K-28K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师9,科技公司9,19K-29K,北京,3-5年,本科,Java Spring Boot MySQL Redis
+Java开发工程师10,科技公司10,20K-30K,北京,3-5年,本科,Java Spring Boot MySQL Redis