Compare commits
1 Commits
| Author | SHA1 | Date |
|---|---|---|
 张馨月
|
74a2cd6319 | 3 weeks ago |
3 changed files with 397 additions and 0 deletions
Binary file not shown.
@ -0,0 +1,149 @@ |
|||
import org.jsoup.Jsoup; |
|||
import org.jsoup.nodes.Document; |
|||
import org.jsoup.nodes.Element; |
|||
import org.jsoup.select.Elements; |
|||
|
|||
import java.io.BufferedWriter; |
|||
import java.io.FileWriter; |
|||
import java.io.IOException; |
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 中国非物质文化遗产爬虫 |
|||
* 爬取 Wikipedia 上的国家级非遗项目列表（1557项） |
|||
*/ |
|||
public class IntangibleHeritageCrawler { |
|||
|
|||
// Wikipedia 非遗列表页面
|
|||
private static final String WIKIPEDIA_URL = "https://zh.wikipedia.org/wiki/国家级非物质文化遗产代表性项目名录"; |
|||
// 输出文件路径
|
|||
private static final String OUTPUT_FILE = "intangible_heritage.csv"; |
|||
|
|||
public static void main(String[] args) { |
|||
System.out.println("开始爬取国家级非物质文化遗产项目数据..."); |
|||
System.out.println("目标：1557个非遗项目\n"); |
|||
|
|||
int totalCount = 0; |
|||
|
|||
try (BufferedWriter writer = new BufferedWriter(new FileWriter(OUTPUT_FILE))) { |
|||
// 写入CSV表头
|
|||
writer.write("项目名称,类别,地区,简介"); |
|||
writer.newLine(); |
|||
|
|||
System.out.println("正在访问 Wikipedia 页面..."); |
|||
|
|||
Document doc = Jsoup.connect(WIKIPEDIA_URL) |
|||
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") |
|||
.timeout(30000) |
|||
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") |
|||
.header("Accept-Language", "zh-CN,zh;q=0.9") |
|||
.followRedirects(true) |
|||
.get(); |
|||
|
|||
System.out.println("页面标题：" + doc.title()); |
|||
|
|||
// Wikipedia 的表格通常有特定的class
|
|||
Elements tables = doc.select("table.wikitable, table.sortable"); |
|||
System.out.println("找到 " + tables.size() + " 个表格\n"); |
|||
|
|||
// 遍历所有表格
|
|||
for (Element table : tables) { |
|||
// 提取表格标题（类别）
|
|||
String category = ""; |
|||
Element caption = table.selectFirst("caption"); |
|||
if (caption != null) { |
|||
category = caption.text().trim(); |
|||
} |
|||
|
|||
// 提取表格行
|
|||
Elements rows = table.select("tr"); |
|||
System.out.println("表格：" + category + " - 共 " + rows.size() + " 行"); |
|||
|
|||
int tableCount = 0; |
|||
for (Element row : rows) { |
|||
try { |
|||
// 提取单元格
|
|||
Elements cells = row.select("td"); |
|||
|
|||
if (cells.size() >= 2) { |
|||
// 第一列通常是项目名称
|
|||
String name = cells.get(0).text().trim(); |
|||
|
|||
// 第二列通常是地区
|
|||
String region = cells.get(1).text().trim(); |
|||
|
|||
// 如果有第三列，可能是简介或批次
|
|||
String description = ""; |
|||
if (cells.size() >= 3) { |
|||
description = cells.get(2).text().trim(); |
|||
} |
|||
|
|||
// 清理数据
|
|||
name = cleanCsvField(name); |
|||
category = cleanCsvField(category); |
|||
region = cleanCsvField(region); |
|||
description = cleanCsvField(description); |
|||
|
|||
// 如果项目名称不为空且不是表头，则写入CSV
|
|||
if (!name.isEmpty() && |
|||
!name.equals("项目名称") && |
|||
!name.equals("名称") && |
|||
!name.equals("序号") && |
|||
name.length() > 1) { |
|||
|
|||
writer.write(String.format("%s,%s,%s,%s", name, category, region, description)); |
|||
writer.newLine(); |
|||
tableCount++; |
|||
totalCount++; |
|||
|
|||
if (totalCount % 100 == 0) { |
|||
System.out.println(" 已爬取 " + totalCount + " 个项目..."); |
|||
} |
|||
} |
|||
} |
|||
} catch (Exception e) { |
|||
System.err.println(" 解析行时出错：" + e.getMessage()); |
|||
} |
|||
} |
|||
|
|||
System.out.println(" 该表格爬取完成！共 " + tableCount + " 个项目\n"); |
|||
|
|||
// 延时，避免请求过快
|
|||
Thread.sleep(1000); |
|||
} |
|||
|
|||
System.out.println("\n========================================"); |
|||
System.out.println("全部爬取完成！"); |
|||
System.out.println("共爬取 " + totalCount + " 个非遗项目"); |
|||
System.out.println("目标：1557个项目"); |
|||
System.out.println("完成率：" + String.format("%.2f", (totalCount / 1557.0) * 100) + "%"); |
|||
System.out.println("========================================"); |
|||
System.out.println("数据已保存到：" + OUTPUT_FILE); |
|||
|
|||
} catch (IOException e) { |
|||
System.err.println("爬取失败：" + e.getMessage()); |
|||
e.printStackTrace(); |
|||
} catch (InterruptedException e) { |
|||
Thread.currentThread().interrupt(); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 清理CSV字段中的特殊字符 |
|||
*/ |
|||
private static String cleanCsvField(String field) { |
|||
if (field == null) { |
|||
return ""; |
|||
} |
|||
// 移除换行符和制表符
|
|||
field = field.replace("\n", " ").replace("\r", " ").replace("\t", " "); |
|||
// 移除引用标记
|
|||
field = field.replace("[", "").replace("]", ""); |
|||
// 如果包含逗号，用双引号包裹
|
|||
if (field.contains(",")) { |
|||
field = "\"" + field.replace("\"", "\"\"") + "\""; |
|||
} |
|||
return field; |
|||
} |
|||
} |
|||
|
Loading…
Reference in new issue