diff --git a/scp-downloader.sh b/scp-downloader.sh index d25b31a..f742492 100755 --- a/scp-downloader.sh +++ b/scp-downloader.sh @@ -85,10 +85,7 @@ check_dependencies() { get_scp_url() { local num=$1 echo "https://scp-wiki.wikidot.com/scp-$(printf "%03d" "$num")" -} - -sanitize_filename() { - echo "$1" | sed 's/[^a-zA-Z0-9._-]/_/g' + #echo "https://scp-cs.wikidot.com/scp-$(printf "%03d" "$num")" } download_scp_html() { @@ -118,33 +115,90 @@ download_scp_html() { return 1 } -extract_scp_content() { +extract_page_content() { local html_file=$1 local scp_num=$(basename "$html_file" .html | sed 's/scp-//') + local output_html="${html_file%.html}_content.html" - # Extract title and content using pandoc - # This is a basic extraction; you might need to adjust selectors + # Extract only the page-content div and wrap it in proper HTML + cat > /tmp/extract_content.awk << 'AWKSCRIPT' +BEGIN { + in_content = 0 + depth = 0 +} +/]*id="page-content"[^>]*>/ { + in_content = 1 + depth = 1 + print "" + print "SCP Document" + print $0 + next +} +in_content { + if (/]*>/) depth++ + if (/<\/div>/) { + depth-- + if (depth == 0) { + print $0 + print "" + in_content = 0 + exit + } + } + print $0 +} +AWKSCRIPT - local title=$(grep -oP '(?<=)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//') - - if [ -z "$title" ]; then - title="SCP-$scp_num" - fi - - # Create a markdown file from HTML - pandoc -f html -t markdown "$html_file" -o "${html_file%.html}.md" 2>/dev/null || { - log_warning "Could not convert SCP-$scp_num to markdown" + awk -f /tmp/extract_content.awk "$html_file" > "$output_html" 2>/dev/null || { + log_warning "Could not extract content from SCP-$scp_num, using full HTML" + cp "$html_file" "$output_html" return 1 } + rm -f /tmp/extract_content.awk + return 0 +} + +convert_to_markdown() { + local html_file=$1 + local scp_num=$(basename "$html_file" .html | sed 's/scp-//') + local md_file="${html_file%.html}.md" + + if pandoc -f html -t markdown "$html_file" -o "$md_file" 2>/dev/null; then + return 0 + else + log_warning "Could not convert SCP-$scp_num to markdown" + return 1 + fi +} + +get_scp_title() { + local html_file=$1 + + # Try to extract title from h1 or meta tags + local title=$(grep -oP '<h1[^>]*>\K[^<]*' "$html_file" | head -1) + + if [ -z "$title" ]; then + title=$(grep -oP '(?<=<title>)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//') + fi + + # Clean up title + title=$(echo "$title" | sed 's/^[^S]*//; s/[<>:"\/\\|?*]//g; s/^[[:space:]]*//; s/[[:space:]]*$//') + + if [ -z "$title" ]; then + local scp_num=$(basename "$html_file" .html | sed 's/scp-//') + title="SCP-$scp_num" + fi + echo "$title" } convert_to_pdf() { local md_file=$1 - local scp_num=$(basename "$md_file" .md | sed 's/scp-//') - local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num") - local pdf_file="$OUTPUT_DIR/pdf/SCP-$(printf "%03d" "$scp_num") - ${title}.pdf" + # Extract the SCP number from the markdown filename (e.g., scp-017_content.md -> 017) + local scp_num=$(basename "$md_file" | grep -oP 'scp-\K\d+' | head -1) + local title=$(get_scp_title "$OUTPUT_DIR/html/scp-$scp_num.html") + local pdf_file="$OUTPUT_DIR/pdf/SCP-$scp_num - ${title}.pdf" mkdir -p "$OUTPUT_DIR/pdf" @@ -153,29 +207,30 @@ convert_to_pdf() { -V linkcolor=blue \ -V geometry=margin=1in \ 2>/dev/null; then - log_success "Converted SCP-$(printf "%03d" "$scp_num") to PDF" + log_success "Converted SCP-$scp_num to PDF" return 0 else - log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to PDF" + log_error "Failed to convert SCP-$scp_num to PDF" return 1 fi } convert_to_epub() { local md_file=$1 - local scp_num=$(basename "$md_file" .md | sed 's/scp-//') - local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num") - local epub_file="$OUTPUT_DIR/epub/SCP-$(printf "%03d" "$scp_num") - ${title}.epub" + # Extract the SCP number from the markdown filename (e.g., scp-017_content.md -> 017) + local scp_num=$(basename "$md_file" | grep -oP 'scp-\K\d+' | head -1) + local title=$(get_scp_title "$OUTPUT_DIR/html/scp-$scp_num.html") + local epub_file="$OUTPUT_DIR/epub/SCP-$scp_num - ${title}.epub" mkdir -p "$OUTPUT_DIR/epub" if pandoc "$md_file" -o "$epub_file" \ --metadata author="SCP Foundation" \ 2>/dev/null; then - log_success "Converted SCP-$(printf "%03d" "$scp_num") to EPUB" + log_success "Converted SCP-$scp_num to EPUB" return 0 else - log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to EPUB" + log_error "Failed to convert SCP-$scp_num to EPUB" return 1 fi } @@ -191,17 +246,33 @@ download_and_convert() { return 1 fi - local md_file="$OUTPUT_DIR/html/scp-$scp_num.md" + # Extract page content + local html_file="$OUTPUT_DIR/html/scp-$scp_num.html" + local content_html="${html_file%.html}_content.html" + if ! extract_page_content "$html_file"; then + log_warning "Using full HTML for SCP-$scp_num" + content_html="$html_file" + fi + + # Convert to markdown + if ! convert_to_markdown "$content_html"; then + log_error "Failed to convert SCP-$scp_num to markdown" + return 1 + fi + + local md_file="${content_html%.html}.md" + + # Convert to desired format(s) if [ "$FORMAT" = "pdf" ] || [ "$FORMAT" = "both" ]; then - convert_to_pdf "$md_file" & + convert_to_pdf "$md_file" fi if [ "$FORMAT" = "epub" ] || [ "$FORMAT" = "both" ]; then - convert_to_epub "$md_file" & + convert_to_epub "$md_file" fi - wait + log_success "SCP-$scp_num complete" } ############################################################################## @@ -234,20 +305,14 @@ main() { log_info "Downloading SCPs from $start to $end" log_info "Format: $FORMAT" log_info "Output directory: $OUTPUT_DIR" + log_info "Concurrent downloads: $CONCURRENT_DOWNLOADS" local count=0 - for num in $(seq "$start" "$end"); do - download_and_convert "$num" & - - # Limit concurrent downloads - ((count++)) - if [ $((count % CONCURRENT_DOWNLOADS)) -eq 0 ]; then - wait - fi - done + local pids=() - # Wait for remaining processes - wait + for num in $(seq "$start" "$end"); do + download_and_convert "$num" + done log_success "Download and conversion complete!" log_info "Files saved to: $OUTPUT_DIR" @@ -296,4 +361,3 @@ while [[ $# -gt 0 ]]; do done main "$START" "$END" -