364 lines
9.2 KiB
Bash
Executable File
364 lines
9.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
##############################################################################
|
|
# SCP Wiki Downloader
|
|
# Downloads SCP articles from the wiki and converts them to PDF/EPUB
|
|
##############################################################################
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
OUTPUT_DIR="${OUTPUT_DIR:-./scp_downloads}"
|
|
FORMAT="${FORMAT:-pdf}" # pdf, epub, or both
|
|
CONCURRENT_DOWNLOADS=3
|
|
RETRY_ATTEMPTS=3
|
|
RETRY_DELAY=2
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
##############################################################################
|
|
# Functions
|
|
##############################################################################
|
|
|
|
print_usage() {
|
|
cat << EOF
|
|
Usage: $0 <start> <end> [options]
|
|
|
|
Arguments:
|
|
start Starting SCP number (e.g., 2)
|
|
end Ending SCP number (e.g., 99)
|
|
|
|
Options:
|
|
-f, --format Output format: pdf, epub, or both (default: pdf)
|
|
-o, --output Output directory (default: ./scp_downloads)
|
|
-c, --concurrent Number of concurrent downloads (default: 3)
|
|
-h, --help Show this help message
|
|
|
|
Examples:
|
|
$0 2 99
|
|
$0 100 199 --format both
|
|
$0 1 50 --output ~/scp_collection --concurrent 5
|
|
|
|
EOF
|
|
}
|
|
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[✓]${NC} $1"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARN]${NC} $1"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[✗]${NC} $1"
|
|
}
|
|
|
|
check_dependencies() {
|
|
local missing=0
|
|
|
|
for cmd in curl pandoc; do
|
|
if ! command -v "$cmd" &> /dev/null; then
|
|
log_error "Missing dependency: $cmd"
|
|
missing=1
|
|
fi
|
|
done
|
|
|
|
if [ $missing -eq 1 ]; then
|
|
log_error "Please install missing dependencies:"
|
|
echo " Ubuntu/Debian: sudo apt-get install curl pandoc"
|
|
echo " macOS: brew install curl pandoc"
|
|
echo " Fedora: sudo dnf install curl pandoc"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
get_scp_url() {
|
|
local num=$1
|
|
echo "https://scp-wiki.wikidot.com/scp-$(printf "%03d" "$num")"
|
|
#echo "https://scp-cs.wikidot.com/scp-$(printf "%03d" "$num")"
|
|
}
|
|
|
|
download_scp_html() {
|
|
local num=$1
|
|
local url=$(get_scp_url "$num")
|
|
local html_file="$OUTPUT_DIR/html/scp-$(printf "%03d" "$num").html"
|
|
|
|
mkdir -p "$OUTPUT_DIR/html"
|
|
|
|
for attempt in $(seq 1 $RETRY_ATTEMPTS); do
|
|
if curl -s -L -A "Mozilla/5.0" "$url" -o "$html_file" 2>/dev/null; then
|
|
# Check if we got a valid page (not 404)
|
|
if grep -q "SCP-$(printf "%03d" "$num")" "$html_file" 2>/dev/null; then
|
|
return 0
|
|
else
|
|
rm -f "$html_file"
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
if [ $attempt -lt $RETRY_ATTEMPTS ]; then
|
|
log_warning "Download attempt $attempt failed for SCP-$(printf "%03d" "$num"), retrying..."
|
|
sleep $RETRY_DELAY
|
|
fi
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
extract_page_content() {
|
|
local html_file=$1
|
|
local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
|
|
local output_html="${html_file%.html}_content.html"
|
|
|
|
# Extract only the page-content div and wrap it in proper HTML
|
|
cat > /tmp/extract_content.awk << 'AWKSCRIPT'
|
|
BEGIN {
|
|
in_content = 0
|
|
depth = 0
|
|
}
|
|
/<div[^>]*id="page-content"[^>]*>/ {
|
|
in_content = 1
|
|
depth = 1
|
|
print "<!DOCTYPE html>"
|
|
print "<html><head><meta charset=\"UTF-8\"><title>SCP Document</title></head><body>"
|
|
print $0
|
|
next
|
|
}
|
|
in_content {
|
|
if (/<div[^>]*>/) depth++
|
|
if (/<\/div>/) {
|
|
depth--
|
|
if (depth == 0) {
|
|
print $0
|
|
print "</body></html>"
|
|
in_content = 0
|
|
exit
|
|
}
|
|
}
|
|
print $0
|
|
}
|
|
AWKSCRIPT
|
|
|
|
awk -f /tmp/extract_content.awk "$html_file" > "$output_html" 2>/dev/null || {
|
|
log_warning "Could not extract content from SCP-$scp_num, using full HTML"
|
|
cp "$html_file" "$output_html"
|
|
return 1
|
|
}
|
|
|
|
rm -f /tmp/extract_content.awk
|
|
return 0
|
|
}
|
|
|
|
convert_to_markdown() {
|
|
local html_file=$1
|
|
local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
|
|
local md_file="${html_file%.html}.md"
|
|
|
|
if pandoc -f html -t markdown "$html_file" -o "$md_file" 2>/dev/null; then
|
|
return 0
|
|
else
|
|
log_warning "Could not convert SCP-$scp_num to markdown"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
get_scp_title() {
|
|
local html_file=$1
|
|
|
|
# Try to extract title from h1 or meta tags
|
|
local title=$(grep -oP '<h1[^>]*>\K[^<]*' "$html_file" | head -1)
|
|
|
|
if [ -z "$title" ]; then
|
|
title=$(grep -oP '(?<=<title>)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//')
|
|
fi
|
|
|
|
# Clean up title
|
|
title=$(echo "$title" | sed 's/^[^S]*//; s/[<>:"\/\\|?*]//g; s/^[[:space:]]*//; s/[[:space:]]*$//')
|
|
|
|
if [ -z "$title" ]; then
|
|
local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
|
|
title="SCP-$scp_num"
|
|
fi
|
|
|
|
echo "$title"
|
|
}
|
|
|
|
convert_to_pdf() {
|
|
local md_file=$1
|
|
# Extract the SCP number from the markdown filename (e.g., scp-017_content.md -> 017)
|
|
local scp_num=$(basename "$md_file" | grep -oP 'scp-\K\d+' | head -1)
|
|
local title=$(get_scp_title "$OUTPUT_DIR/html/scp-$scp_num.html")
|
|
local pdf_file="$OUTPUT_DIR/pdf/SCP-$scp_num - ${title}.pdf"
|
|
|
|
mkdir -p "$OUTPUT_DIR/pdf"
|
|
|
|
if pandoc "$md_file" -o "$pdf_file" \
|
|
-V colorlinks=true \
|
|
-V linkcolor=blue \
|
|
-V geometry=margin=1in \
|
|
2>/dev/null; then
|
|
log_success "Converted SCP-$scp_num to PDF"
|
|
return 0
|
|
else
|
|
log_error "Failed to convert SCP-$scp_num to PDF"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
convert_to_epub() {
|
|
local md_file=$1
|
|
# Extract the SCP number from the markdown filename (e.g., scp-017_content.md -> 017)
|
|
local scp_num=$(basename "$md_file" | grep -oP 'scp-\K\d+' | head -1)
|
|
local title=$(get_scp_title "$OUTPUT_DIR/html/scp-$scp_num.html")
|
|
local epub_file="$OUTPUT_DIR/epub/SCP-$scp_num - ${title}.epub"
|
|
|
|
mkdir -p "$OUTPUT_DIR/epub"
|
|
|
|
if pandoc "$md_file" -o "$epub_file" \
|
|
--metadata author="SCP Foundation" \
|
|
2>/dev/null; then
|
|
log_success "Converted SCP-$scp_num to EPUB"
|
|
return 0
|
|
else
|
|
log_error "Failed to convert SCP-$scp_num to EPUB"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
download_and_convert() {
|
|
local num=$1
|
|
local scp_num=$(printf "%03d" "$num")
|
|
|
|
log_info "Processing SCP-$scp_num..."
|
|
|
|
if ! download_scp_html "$num"; then
|
|
log_error "Failed to download SCP-$scp_num"
|
|
return 1
|
|
fi
|
|
|
|
# Extract page content
|
|
local html_file="$OUTPUT_DIR/html/scp-$scp_num.html"
|
|
local content_html="${html_file%.html}_content.html"
|
|
|
|
if ! extract_page_content "$html_file"; then
|
|
log_warning "Using full HTML for SCP-$scp_num"
|
|
content_html="$html_file"
|
|
fi
|
|
|
|
# Convert to markdown
|
|
if ! convert_to_markdown "$content_html"; then
|
|
log_error "Failed to convert SCP-$scp_num to markdown"
|
|
return 1
|
|
fi
|
|
|
|
local md_file="${content_html%.html}.md"
|
|
|
|
# Convert to desired format(s)
|
|
if [ "$FORMAT" = "pdf" ] || [ "$FORMAT" = "both" ]; then
|
|
convert_to_pdf "$md_file"
|
|
fi
|
|
|
|
if [ "$FORMAT" = "epub" ] || [ "$FORMAT" = "both" ]; then
|
|
convert_to_epub "$md_file"
|
|
fi
|
|
|
|
log_success "SCP-$scp_num complete"
|
|
}
|
|
|
|
##############################################################################
|
|
# Main
|
|
##############################################################################
|
|
|
|
main() {
|
|
local start=$1
|
|
local end=$2
|
|
|
|
# Validate arguments
|
|
if [ -z "$start" ] || [ -z "$end" ]; then
|
|
print_usage
|
|
exit 1
|
|
fi
|
|
|
|
if ! [[ "$start" =~ ^[0-9]+$ ]] || ! [[ "$end" =~ ^[0-9]+$ ]]; then
|
|
log_error "Start and end must be valid numbers"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$start" -gt "$end" ]; then
|
|
log_error "Start must be less than or equal to end"
|
|
exit 1
|
|
fi
|
|
|
|
check_dependencies
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
log_info "Downloading SCPs from $start to $end"
|
|
log_info "Format: $FORMAT"
|
|
log_info "Output directory: $OUTPUT_DIR"
|
|
log_info "Concurrent downloads: $CONCURRENT_DOWNLOADS"
|
|
|
|
local count=0
|
|
local pids=()
|
|
|
|
for num in $(seq "$start" "$end"); do
|
|
download_and_convert "$num"
|
|
done
|
|
|
|
log_success "Download and conversion complete!"
|
|
log_info "Files saved to: $OUTPUT_DIR"
|
|
}
|
|
|
|
# Parse command line arguments
|
|
START=""
|
|
END=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
-h|--help)
|
|
print_usage
|
|
exit 0
|
|
;;
|
|
-f|--format)
|
|
FORMAT="$2"
|
|
shift 2
|
|
;;
|
|
-o|--output)
|
|
OUTPUT_DIR="$2"
|
|
shift 2
|
|
;;
|
|
-c|--concurrent)
|
|
CONCURRENT_DOWNLOADS="$2"
|
|
shift 2
|
|
;;
|
|
-*)
|
|
log_error "Unknown option: $1"
|
|
print_usage
|
|
exit 1
|
|
;;
|
|
*)
|
|
if [ -z "$START" ]; then
|
|
START="$1"
|
|
elif [ -z "$END" ]; then
|
|
END="$1"
|
|
else
|
|
log_error "Too many positional arguments"
|
|
print_usage
|
|
exit 1
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
main "$START" "$END"
|