From ae513d20af3e3c1ca3d0e9f126bd96ea311acb62 Mon Sep 17 00:00:00 2001 From: odweta Date: Wed, 9 Sep 2026 22:09:11 +0200 Subject: [PATCH] initial release --- scp-downloader.sh | 299 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100755 scp-downloader.sh diff --git a/scp-downloader.sh b/scp-downloader.sh new file mode 100755 index 0000000..d25b31a --- /dev/null +++ b/scp-downloader.sh @@ -0,0 +1,299 @@ +#!/bin/bash + +############################################################################## +# SCP Wiki Downloader +# Downloads SCP articles from the wiki and converts them to PDF/EPUB +############################################################################## + +set -euo pipefail + +# Configuration +OUTPUT_DIR="${OUTPUT_DIR:-./scp_downloads}" +FORMAT="${FORMAT:-pdf}" # pdf, epub, or both +CONCURRENT_DOWNLOADS=3 +RETRY_ATTEMPTS=3 +RETRY_DELAY=2 + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +############################################################################## +# Functions +############################################################################## + +print_usage() { + cat << EOF +Usage: $0 [options] + +Arguments: + start Starting SCP number (e.g., 2) + end Ending SCP number (e.g., 99) + +Options: + -f, --format Output format: pdf, epub, or both (default: pdf) + -o, --output Output directory (default: ./scp_downloads) + -c, --concurrent Number of concurrent downloads (default: 3) + -h, --help Show this help message + +Examples: + $0 2 99 + $0 100 199 --format both + $0 1 50 --output ~/scp_collection --concurrent 5 + +EOF +} + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[✓]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[✗]${NC} $1" +} + +check_dependencies() { + local missing=0 + + for cmd in curl pandoc; do + if ! command -v "$cmd" &> /dev/null; then + log_error "Missing dependency: $cmd" + missing=1 + fi + done + + if [ $missing -eq 1 ]; then + log_error "Please install missing dependencies:" + echo " Ubuntu/Debian: sudo apt-get install curl pandoc" + echo " macOS: brew install curl pandoc" + echo " Fedora: sudo dnf install curl pandoc" + exit 1 + fi +} + +get_scp_url() { + local num=$1 + echo "https://scp-wiki.wikidot.com/scp-$(printf "%03d" "$num")" +} + +sanitize_filename() { + echo "$1" | sed 's/[^a-zA-Z0-9._-]/_/g' +} + +download_scp_html() { + local num=$1 + local url=$(get_scp_url "$num") + local html_file="$OUTPUT_DIR/html/scp-$(printf "%03d" "$num").html" + + mkdir -p "$OUTPUT_DIR/html" + + for attempt in $(seq 1 $RETRY_ATTEMPTS); do + if curl -s -L -A "Mozilla/5.0" "$url" -o "$html_file" 2>/dev/null; then + # Check if we got a valid page (not 404) + if grep -q "SCP-$(printf "%03d" "$num")" "$html_file" 2>/dev/null; then + return 0 + else + rm -f "$html_file" + return 1 + fi + fi + + if [ $attempt -lt $RETRY_ATTEMPTS ]; then + log_warning "Download attempt $attempt failed for SCP-$(printf "%03d" "$num"), retrying..." + sleep $RETRY_DELAY + fi + done + + return 1 +} + +extract_scp_content() { + local html_file=$1 + local scp_num=$(basename "$html_file" .html | sed 's/scp-//') + + # Extract title and content using pandoc + # This is a basic extraction; you might need to adjust selectors + + local title=$(grep -oP '(?<=)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//') + + if [ -z "$title" ]; then + title="SCP-$scp_num" + fi + + # Create a markdown file from HTML + pandoc -f html -t markdown "$html_file" -o "${html_file%.html}.md" 2>/dev/null || { + log_warning "Could not convert SCP-$scp_num to markdown" + return 1 + } + + echo "$title" +} + +convert_to_pdf() { + local md_file=$1 + local scp_num=$(basename "$md_file" .md | sed 's/scp-//') + local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num") + local pdf_file="$OUTPUT_DIR/pdf/SCP-$(printf "%03d" "$scp_num") - ${title}.pdf" + + mkdir -p "$OUTPUT_DIR/pdf" + + if pandoc "$md_file" -o "$pdf_file" \ + -V colorlinks=true \ + -V linkcolor=blue \ + -V geometry=margin=1in \ + 2>/dev/null; then + log_success "Converted SCP-$(printf "%03d" "$scp_num") to PDF" + return 0 + else + log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to PDF" + return 1 + fi +} + +convert_to_epub() { + local md_file=$1 + local scp_num=$(basename "$md_file" .md | sed 's/scp-//') + local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num") + local epub_file="$OUTPUT_DIR/epub/SCP-$(printf "%03d" "$scp_num") - ${title}.epub" + + mkdir -p "$OUTPUT_DIR/epub" + + if pandoc "$md_file" -o "$epub_file" \ + --metadata author="SCP Foundation" \ + 2>/dev/null; then + log_success "Converted SCP-$(printf "%03d" "$scp_num") to EPUB" + return 0 + else + log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to EPUB" + return 1 + fi +} + +download_and_convert() { + local num=$1 + local scp_num=$(printf "%03d" "$num") + + log_info "Processing SCP-$scp_num..." + + if ! download_scp_html "$num"; then + log_error "Failed to download SCP-$scp_num" + return 1 + fi + + local md_file="$OUTPUT_DIR/html/scp-$scp_num.md" + + if [ "$FORMAT" = "pdf" ] || [ "$FORMAT" = "both" ]; then + convert_to_pdf "$md_file" & + fi + + if [ "$FORMAT" = "epub" ] || [ "$FORMAT" = "both" ]; then + convert_to_epub "$md_file" & + fi + + wait +} + +############################################################################## +# Main +############################################################################## + +main() { + local start=$1 + local end=$2 + + # Validate arguments + if [ -z "$start" ] || [ -z "$end" ]; then + print_usage + exit 1 + fi + + if ! [[ "$start" =~ ^[0-9]+$ ]] || ! [[ "$end" =~ ^[0-9]+$ ]]; then + log_error "Start and end must be valid numbers" + exit 1 + fi + + if [ "$start" -gt "$end" ]; then + log_error "Start must be less than or equal to end" + exit 1 + fi + + check_dependencies + mkdir -p "$OUTPUT_DIR" + + log_info "Downloading SCPs from $start to $end" + log_info "Format: $FORMAT" + log_info "Output directory: $OUTPUT_DIR" + + local count=0 + for num in $(seq "$start" "$end"); do + download_and_convert "$num" & + + # Limit concurrent downloads + ((count++)) + if [ $((count % CONCURRENT_DOWNLOADS)) -eq 0 ]; then + wait + fi + done + + # Wait for remaining processes + wait + + log_success "Download and conversion complete!" + log_info "Files saved to: $OUTPUT_DIR" +} + +# Parse command line arguments +START="" +END="" + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_usage + exit 0 + ;; + -f|--format) + FORMAT="$2" + shift 2 + ;; + -o|--output) + OUTPUT_DIR="$2" + shift 2 + ;; + -c|--concurrent) + CONCURRENT_DOWNLOADS="$2" + shift 2 + ;; + -*) + log_error "Unknown option: $1" + print_usage + exit 1 + ;; + *) + if [ -z "$START" ]; then + START="$1" + elif [ -z "$END" ]; then + END="$1" + else + log_error "Too many positional arguments" + print_usage + exit 1 + fi + shift + ;; + esac +done + +main "$START" "$END" +