You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

236 lines
9.2 KiB

# Test Script for CLI Crawler - Data Import/Export Features
# This script automates the test sequence
$ErrorActionPreference = "Stop"
$env:JAVA_HOME = "C:\Program Files\Java\latest\jdk-25"
$APP_JAR = "target\datacollect-cli-0.1.0-jar-with-dependencies.jar"
$TEST_EXPORT_FILE = "data\test_export.json"
$USERPROFILE_PATH = "$env:USERPROFILE\.datacollect"
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "CLI Crawler - Import/Export Test Suite" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host ""
# Clean up function
function Clean-Up {
Write-Host "[CLEANUP] Removing old data files..." -ForegroundColor Yellow
if (Test-Path $USERPROFILE_PATH) {
Remove-Item "$USERPROFILE_PATH\*" -Force -Recurse -ErrorAction SilentlyContinue
}
if (Test-Path $TEST_EXPORT_FILE) {
Remove-Item $TEST_EXPORT_FILE -Force -ErrorAction SilentlyContinue
}
}
# Run CLI command function
function Run-CLI {
param([string]$Commands)
$commandsArray = $Commands -split "`n"
foreach ($cmd in $commandsArray) {
$cmd = $cmd.Trim()
if ($cmd -ne "") {
Write-Host "[CLI] $cmd" -ForegroundColor Gray
$result = & java -jar $APP_JAR $cmd 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
}
}
}
# Step 1: Initial Cleanup
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 1: Initial Cleanup" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Clean-Up
Write-Host ""
# Step 2: Crawl some data
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 2: Crawl Data (CSDN)" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: crawl https://www.csdn.net/" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "crawl https://www.csdn.net/" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
Start-Sleep -Seconds 2
# Step 3: List articles
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 3: List Articles" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: list" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "list" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
Start-Sleep -Seconds 1
# Step 4: Export to JSON
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 4: Export to JSON" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: export data\test_export.json --format json" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "export data\test_export.json --format json" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
Start-Sleep -Seconds 1
# Step 5: Check JSON file
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 5: Check Exported JSON File" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
if (Test-Path $TEST_EXPORT_FILE) {
Write-Host "[SUCCESS] JSON file created: $TEST_EXPORT_FILE" -ForegroundColor Green
Write-Host ""
Write-Host "JSON File Content Preview (first 1500 chars):" -ForegroundColor Cyan
$content = Get-Content $TEST_EXPORT_FILE -Raw
if ($content.Length -gt 1500) {
Write-Host ($content.Substring(0, 1500) + "...") -ForegroundColor White
} else {
Write-Host $content -ForegroundColor White
}
# Check for crawledAt field
if ($content -match "crawledAt") {
Write-Host ""
Write-Host "[SUCCESS] crawledAt field found in JSON!" -ForegroundColor Green
} else {
Write-Host ""
Write-Host "[ERROR] crawledAt field NOT found in JSON!" -ForegroundColor Red
}
# Check for metadata
if ($content -match "metadata") {
Write-Host "[SUCCESS] metadata field found in JSON!" -ForegroundColor Green
} else {
Write-Host "[WARNING] metadata field NOT found in JSON!" -ForegroundColor Yellow
}
} else {
Write-Host "[ERROR] JSON file NOT created!" -ForegroundColor Red
}
Write-Host ""
# Step 6: Get article count before clear
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 6: Get Article Count Before Clear" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: list" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "list" 2>&1
Write-Host $result -ForegroundColor Green
# Count articles
$articleCount = 0
$lines = $result -split "`n"
foreach ($line in $lines) {
if ($line -match "Total: (\d+) articles") {
$articleCount = [int]$matches[1]
break
}
}
Write-Host ""
Write-Host "Current article count: $articleCount" -ForegroundColor Cyan
Write-Host ""
Start-Sleep -Seconds 1
# Step 7: Clear all data
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 7: Clear All Data" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: clear" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "clear" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
Start-Sleep -Seconds 1
# Step 8: Verify data is cleared
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 8: Verify Data Cleared" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: list" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "list" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
Start-Sleep -Seconds 1
# Step 9: Import data from JSON
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 9: Import Data from JSON" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: import data\test_export.json" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "import data\test_export.json" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
Start-Sleep -Seconds 1
# Step 10: Verify data restored
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 10: Verify Data Restored" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: list" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "list" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
# Count articles after import
$articleCountAfterImport = 0
$lines = $result -split "`n"
foreach ($line in $lines) {
if ($line -match "Total: (\d+) articles") {
$articleCountAfterImport = [int]$matches[1]
break
}
}
if ($articleCountAfterImport -eq $articleCount) {
Write-Host "[SUCCESS] Data restored successfully! Article count matches: $articleCountAfterImport" -ForegroundColor Green
} else {
Write-Host "[WARNING] Article count mismatch. Before: $articleCount, After: $articleCountAfterImport" -ForegroundColor Yellow
}
Write-Host ""
# Step 11: Test duplicate import (should not duplicate)
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 11: Test Duplicate Import (No Duplication)" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: import data\test_export.json (second time)" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "import data\test_export.json" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
Start-Sleep -Seconds 1
# Step 12: Final article count
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "STEP 12: Final Article Count" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Command: list" -ForegroundColor Yellow
$result = & java -jar $APP_JAR "list" 2>&1
Write-Host $result -ForegroundColor Green
Write-Host ""
# Final count
$finalCount = 0
$lines = $result -split "`n"
foreach ($line in $lines) {
if ($line -match "Total: (\d+) articles") {
$finalCount = [int]$matches[1]
break
}
}
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "TEST SUMMARY" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Articles after first import: $articleCountAfterImport" -ForegroundColor White
Write-Host "Articles after second import: $finalCount" -ForegroundColor White
Write-Host ""
if ($finalCount -eq $articleCountAfterImport) {
Write-Host "[SUCCESS] Duplicate import correctly skipped! No duplication occurred." -ForegroundColor Green
} else {
Write-Host "[ERROR] Duplicate import created duplicates! Count increased from $articleCountAfterImport to $finalCount" -ForegroundColor Red
}
Write-Host ""
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "ALL TESTS COMPLETED" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan