second better working version (without concurrency though)

scp-downloader.sh

@@ -85,10 +85,7 @@ check_dependencies() {
 get_scp_url() {
     local num=$1
     echo "https://scp-wiki.wikidot.com/scp-$(printf "%03d" "$num")"
-}
+    #echo "https://scp-cs.wikidot.com/scp-$(printf "%03d" "$num")"
-
-sanitize_filename() {
-    echo "$1" | sed 's/[^a-zA-Z0-9._-]/_/g'
 }
 
 download_scp_html() {
@@ -118,33 +115,90 @@ download_scp_html() {
     return 1
 }
 
-extract_scp_content() {
+extract_page_content() {
     local html_file=$1
     local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
+    local output_html="${html_file%.html}_content.html"
     
-    # Extract title and content using pandoc
+    # Extract only the page-content div and wrap it in proper HTML
-    # This is a basic extraction; you might need to adjust selectors
+    cat > /tmp/extract_content.awk << 'AWKSCRIPT'
+BEGIN {
+    in_content = 0
+    depth = 0
+}
+/<div[^>]*id="page-content"[^>]*>/ {
+    in_content = 1
+    depth = 1
+    print "<!DOCTYPE html>"
+    print "<html><head><meta charset=\"UTF-8\"><title>SCP Document</title></head><body>"
+    print $0
+    next
+}
+in_content {
+    if (/<div[^>]*>/) depth++
+    if (/<\/div>/) {
+        depth--
+        if (depth == 0) {
+            print $0
+            print "</body></html>"
+            in_content = 0
+            exit
+        }
+    }
+    print $0
+}
+AWKSCRIPT
     
-    local title=$(grep -oP '(?<=<title>)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//')
+    awk -f /tmp/extract_content.awk "$html_file" > "$output_html" 2>/dev/null || {
-    
+        log_warning "Could not extract content from SCP-$scp_num, using full HTML"
-    if [ -z "$title" ]; then
+        cp "$html_file" "$output_html"
-        title="SCP-$scp_num"
-    fi
-    
-    # Create a markdown file from HTML
-    pandoc -f html -t markdown "$html_file" -o "${html_file%.html}.md" 2>/dev/null || {
-        log_warning "Could not convert SCP-$scp_num to markdown"
         return 1
     }
     
+    rm -f /tmp/extract_content.awk
+    return 0
+}
+
+convert_to_markdown() {
+    local html_file=$1
+    local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
+    local md_file="${html_file%.html}.md"
+    
+    if pandoc -f html -t markdown "$html_file" -o "$md_file" 2>/dev/null; then
+        return 0
+    else
+        log_warning "Could not convert SCP-$scp_num to markdown"
+        return 1
+    fi
+}
+
+get_scp_title() {
+    local html_file=$1
+    
+    # Try to extract title from h1 or meta tags
+    local title=$(grep -oP '<h1[^>]*>\K[^<]*' "$html_file" | head -1)
+    
+    if [ -z "$title" ]; then
+        title=$(grep -oP '(?<=<title>)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//')
+    fi
+    
+    # Clean up title
+    title=$(echo "$title" | sed 's/^[^S]*//; s/[<>:"\/\\|?*]//g; s/^[[:space:]]*//; s/[[:space:]]*$//')
+    
+    if [ -z "$title" ]; then
+        local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
+        title="SCP-$scp_num"
+    fi
+    
     echo "$title"
 }
 
 convert_to_pdf() {
     local md_file=$1
-    local scp_num=$(basename "$md_file" .md | sed 's/scp-//')
+    # Extract the SCP number from the markdown filename (e.g., scp-017_content.md -> 017)
-    local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num")
+    local scp_num=$(basename "$md_file" | grep -oP 'scp-\K\d+' | head -1)
-    local pdf_file="$OUTPUT_DIR/pdf/SCP-$(printf "%03d" "$scp_num") - ${title}.pdf"
+    local title=$(get_scp_title "$OUTPUT_DIR/html/scp-$scp_num.html")
+    local pdf_file="$OUTPUT_DIR/pdf/SCP-$scp_num - ${title}.pdf"
     
     mkdir -p "$OUTPUT_DIR/pdf"
     
@@ -153,29 +207,30 @@ convert_to_pdf() {
         -V linkcolor=blue \
         -V geometry=margin=1in \
         2>/dev/null; then
-        log_success "Converted SCP-$(printf "%03d" "$scp_num") to PDF"
+        log_success "Converted SCP-$scp_num to PDF"
         return 0
     else
-        log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to PDF"
+        log_error "Failed to convert SCP-$scp_num to PDF"
         return 1
     fi
 }
 
 convert_to_epub() {
     local md_file=$1
-    local scp_num=$(basename "$md_file" .md | sed 's/scp-//')
+    # Extract the SCP number from the markdown filename (e.g., scp-017_content.md -> 017)
-    local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num")
+    local scp_num=$(basename "$md_file" | grep -oP 'scp-\K\d+' | head -1)
-    local epub_file="$OUTPUT_DIR/epub/SCP-$(printf "%03d" "$scp_num") - ${title}.epub"
+    local title=$(get_scp_title "$OUTPUT_DIR/html/scp-$scp_num.html")
+    local epub_file="$OUTPUT_DIR/epub/SCP-$scp_num - ${title}.epub"
     
     mkdir -p "$OUTPUT_DIR/epub"
     
     if pandoc "$md_file" -o "$epub_file" \
         --metadata author="SCP Foundation" \
         2>/dev/null; then
-        log_success "Converted SCP-$(printf "%03d" "$scp_num") to EPUB"
+        log_success "Converted SCP-$scp_num to EPUB"
         return 0
     else
-        log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to EPUB"
+        log_error "Failed to convert SCP-$scp_num to EPUB"
         return 1
     fi
 }
@@ -191,17 +246,33 @@ download_and_convert() {
         return 1
     fi
     
-    local md_file="$OUTPUT_DIR/html/scp-$scp_num.md"
+    # Extract page content
+    local html_file="$OUTPUT_DIR/html/scp-$scp_num.html"
+    local content_html="${html_file%.html}_content.html"
     
+    if ! extract_page_content "$html_file"; then
+        log_warning "Using full HTML for SCP-$scp_num"
+        content_html="$html_file"
+    fi
+    
+    # Convert to markdown
+    if ! convert_to_markdown "$content_html"; then
+        log_error "Failed to convert SCP-$scp_num to markdown"
+        return 1
+    fi
+    
+    local md_file="${content_html%.html}.md"
+    
+    # Convert to desired format(s)
     if [ "$FORMAT" = "pdf" ] || [ "$FORMAT" = "both" ]; then
-        convert_to_pdf "$md_file" &
+        convert_to_pdf "$md_file"
     fi
     
     if [ "$FORMAT" = "epub" ] || [ "$FORMAT" = "both" ]; then
-        convert_to_epub "$md_file" &
+        convert_to_epub "$md_file"
     fi
     
-    wait
+    log_success "SCP-$scp_num complete"
 }
 
 ##############################################################################
@@ -234,20 +305,14 @@ main() {
     log_info "Downloading SCPs from $start to $end"
     log_info "Format: $FORMAT"
     log_info "Output directory: $OUTPUT_DIR"
+    log_info "Concurrent downloads: $CONCURRENT_DOWNLOADS"
     
     local count=0
-    for num in $(seq "$start" "$end"); do
+    local pids=()
-        download_and_convert "$num" &
-        
-        # Limit concurrent downloads
-        ((count++))
-        if [ $((count % CONCURRENT_DOWNLOADS)) -eq 0 ]; then
-            wait
-        fi
-    done
     
-    # Wait for remaining processes
+    for num in $(seq "$start" "$end"); do
-    wait
+        download_and_convert "$num"
+    done
     
     log_success "Download and conversion complete!"
     log_info "Files saved to: $OUTPUT_DIR"
@@ -296,4 +361,3 @@ while [[ $# -gt 0 ]]; do
 done
 
 main "$START" "$END"
-