initial release
This commit is contained in:
odweta
Executable
+299
@@ -0,0 +1,299 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
# SCP Wiki Downloader
|
||||||
|
# Downloads SCP articles from the wiki and converts them to PDF/EPUB
|
||||||
|
##############################################################################
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Configuration
|
||||||
|
OUTPUT_DIR="${OUTPUT_DIR:-./scp_downloads}"
|
||||||
|
FORMAT="${FORMAT:-pdf}" # pdf, epub, or both
|
||||||
|
CONCURRENT_DOWNLOADS=3
|
||||||
|
RETRY_ATTEMPTS=3
|
||||||
|
RETRY_DELAY=2
|
||||||
|
|
||||||
|
# Colors for output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
BLUE='\033[0;34m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
# Functions
|
||||||
|
##############################################################################
|
||||||
|
|
||||||
|
print_usage() {
|
||||||
|
cat << EOF
|
||||||
|
Usage: $0 <start> <end> [options]
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
start Starting SCP number (e.g., 2)
|
||||||
|
end Ending SCP number (e.g., 99)
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-f, --format Output format: pdf, epub, or both (default: pdf)
|
||||||
|
-o, --output Output directory (default: ./scp_downloads)
|
||||||
|
-c, --concurrent Number of concurrent downloads (default: 3)
|
||||||
|
-h, --help Show this help message
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
$0 2 99
|
||||||
|
$0 100 199 --format both
|
||||||
|
$0 1 50 --output ~/scp_collection --concurrent 5
|
||||||
|
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
log_info() {
|
||||||
|
echo -e "${BLUE}[INFO]${NC} $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_success() {
|
||||||
|
echo -e "${GREEN}[✓]${NC} $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_warning() {
|
||||||
|
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_error() {
|
||||||
|
echo -e "${RED}[✗]${NC} $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
check_dependencies() {
|
||||||
|
local missing=0
|
||||||
|
|
||||||
|
for cmd in curl pandoc; do
|
||||||
|
if ! command -v "$cmd" &> /dev/null; then
|
||||||
|
log_error "Missing dependency: $cmd"
|
||||||
|
missing=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ $missing -eq 1 ]; then
|
||||||
|
log_error "Please install missing dependencies:"
|
||||||
|
echo " Ubuntu/Debian: sudo apt-get install curl pandoc"
|
||||||
|
echo " macOS: brew install curl pandoc"
|
||||||
|
echo " Fedora: sudo dnf install curl pandoc"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
get_scp_url() {
|
||||||
|
local num=$1
|
||||||
|
echo "https://scp-wiki.wikidot.com/scp-$(printf "%03d" "$num")"
|
||||||
|
}
|
||||||
|
|
||||||
|
sanitize_filename() {
|
||||||
|
echo "$1" | sed 's/[^a-zA-Z0-9._-]/_/g'
|
||||||
|
}
|
||||||
|
|
||||||
|
download_scp_html() {
|
||||||
|
local num=$1
|
||||||
|
local url=$(get_scp_url "$num")
|
||||||
|
local html_file="$OUTPUT_DIR/html/scp-$(printf "%03d" "$num").html"
|
||||||
|
|
||||||
|
mkdir -p "$OUTPUT_DIR/html"
|
||||||
|
|
||||||
|
for attempt in $(seq 1 $RETRY_ATTEMPTS); do
|
||||||
|
if curl -s -L -A "Mozilla/5.0" "$url" -o "$html_file" 2>/dev/null; then
|
||||||
|
# Check if we got a valid page (not 404)
|
||||||
|
if grep -q "SCP-$(printf "%03d" "$num")" "$html_file" 2>/dev/null; then
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
rm -f "$html_file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $attempt -lt $RETRY_ATTEMPTS ]; then
|
||||||
|
log_warning "Download attempt $attempt failed for SCP-$(printf "%03d" "$num"), retrying..."
|
||||||
|
sleep $RETRY_DELAY
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
extract_scp_content() {
|
||||||
|
local html_file=$1
|
||||||
|
local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
|
||||||
|
|
||||||
|
# Extract title and content using pandoc
|
||||||
|
# This is a basic extraction; you might need to adjust selectors
|
||||||
|
|
||||||
|
local title=$(grep -oP '(?<=<title>)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//')
|
||||||
|
|
||||||
|
if [ -z "$title" ]; then
|
||||||
|
title="SCP-$scp_num"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create a markdown file from HTML
|
||||||
|
pandoc -f html -t markdown "$html_file" -o "${html_file%.html}.md" 2>/dev/null || {
|
||||||
|
log_warning "Could not convert SCP-$scp_num to markdown"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "$title"
|
||||||
|
}
|
||||||
|
|
||||||
|
convert_to_pdf() {
|
||||||
|
local md_file=$1
|
||||||
|
local scp_num=$(basename "$md_file" .md | sed 's/scp-//')
|
||||||
|
local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num")
|
||||||
|
local pdf_file="$OUTPUT_DIR/pdf/SCP-$(printf "%03d" "$scp_num") - ${title}.pdf"
|
||||||
|
|
||||||
|
mkdir -p "$OUTPUT_DIR/pdf"
|
||||||
|
|
||||||
|
if pandoc "$md_file" -o "$pdf_file" \
|
||||||
|
-V colorlinks=true \
|
||||||
|
-V linkcolor=blue \
|
||||||
|
-V geometry=margin=1in \
|
||||||
|
2>/dev/null; then
|
||||||
|
log_success "Converted SCP-$(printf "%03d" "$scp_num") to PDF"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to PDF"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
convert_to_epub() {
|
||||||
|
local md_file=$1
|
||||||
|
local scp_num=$(basename "$md_file" .md | sed 's/scp-//')
|
||||||
|
local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num")
|
||||||
|
local epub_file="$OUTPUT_DIR/epub/SCP-$(printf "%03d" "$scp_num") - ${title}.epub"
|
||||||
|
|
||||||
|
mkdir -p "$OUTPUT_DIR/epub"
|
||||||
|
|
||||||
|
if pandoc "$md_file" -o "$epub_file" \
|
||||||
|
--metadata author="SCP Foundation" \
|
||||||
|
2>/dev/null; then
|
||||||
|
log_success "Converted SCP-$(printf "%03d" "$scp_num") to EPUB"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to EPUB"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
download_and_convert() {
|
||||||
|
local num=$1
|
||||||
|
local scp_num=$(printf "%03d" "$num")
|
||||||
|
|
||||||
|
log_info "Processing SCP-$scp_num..."
|
||||||
|
|
||||||
|
if ! download_scp_html "$num"; then
|
||||||
|
log_error "Failed to download SCP-$scp_num"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local md_file="$OUTPUT_DIR/html/scp-$scp_num.md"
|
||||||
|
|
||||||
|
if [ "$FORMAT" = "pdf" ] || [ "$FORMAT" = "both" ]; then
|
||||||
|
convert_to_pdf "$md_file" &
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$FORMAT" = "epub" ] || [ "$FORMAT" = "both" ]; then
|
||||||
|
convert_to_epub "$md_file" &
|
||||||
|
fi
|
||||||
|
|
||||||
|
wait
|
||||||
|
}
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
# Main
|
||||||
|
##############################################################################
|
||||||
|
|
||||||
|
main() {
|
||||||
|
local start=$1
|
||||||
|
local end=$2
|
||||||
|
|
||||||
|
# Validate arguments
|
||||||
|
if [ -z "$start" ] || [ -z "$end" ]; then
|
||||||
|
print_usage
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! [[ "$start" =~ ^[0-9]+$ ]] || ! [[ "$end" =~ ^[0-9]+$ ]]; then
|
||||||
|
log_error "Start and end must be valid numbers"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$start" -gt "$end" ]; then
|
||||||
|
log_error "Start must be less than or equal to end"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
check_dependencies
|
||||||
|
mkdir -p "$OUTPUT_DIR"
|
||||||
|
|
||||||
|
log_info "Downloading SCPs from $start to $end"
|
||||||
|
log_info "Format: $FORMAT"
|
||||||
|
log_info "Output directory: $OUTPUT_DIR"
|
||||||
|
|
||||||
|
local count=0
|
||||||
|
for num in $(seq "$start" "$end"); do
|
||||||
|
download_and_convert "$num" &
|
||||||
|
|
||||||
|
# Limit concurrent downloads
|
||||||
|
((count++))
|
||||||
|
if [ $((count % CONCURRENT_DOWNLOADS)) -eq 0 ]; then
|
||||||
|
wait
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Wait for remaining processes
|
||||||
|
wait
|
||||||
|
|
||||||
|
log_success "Download and conversion complete!"
|
||||||
|
log_info "Files saved to: $OUTPUT_DIR"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Parse command line arguments
|
||||||
|
START=""
|
||||||
|
END=""
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case $1 in
|
||||||
|
-h|--help)
|
||||||
|
print_usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
-f|--format)
|
||||||
|
FORMAT="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-o|--output)
|
||||||
|
OUTPUT_DIR="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-c|--concurrent)
|
||||||
|
CONCURRENT_DOWNLOADS="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-*)
|
||||||
|
log_error "Unknown option: $1"
|
||||||
|
print_usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
if [ -z "$START" ]; then
|
||||||
|
START="$1"
|
||||||
|
elif [ -z "$END" ]; then
|
||||||
|
END="$1"
|
||||||
|
else
|
||||||
|
log_error "Too many positional arguments"
|
||||||
|
print_usage
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
main "$START" "$END"
|
||||||
|
|
||||||
Reference in New Issue
Block a user