diff --git a/scp-downloader.sh b/scp-downloader.sh
index d25b31a..f742492 100755
--- a/scp-downloader.sh
+++ b/scp-downloader.sh
@@ -85,10 +85,7 @@ check_dependencies() {
get_scp_url() {
local num=$1
echo "https://scp-wiki.wikidot.com/scp-$(printf "%03d" "$num")"
-}
-
-sanitize_filename() {
- echo "$1" | sed 's/[^a-zA-Z0-9._-]/_/g'
+ #echo "https://scp-cs.wikidot.com/scp-$(printf "%03d" "$num")"
}
download_scp_html() {
@@ -118,33 +115,90 @@ download_scp_html() {
return 1
}
-extract_scp_content() {
+extract_page_content() {
local html_file=$1
local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
+ local output_html="${html_file%.html}_content.html"
- # Extract title and content using pandoc
- # This is a basic extraction; you might need to adjust selectors
+ # Extract only the page-content div and wrap it in proper HTML
+ cat > /tmp/extract_content.awk << 'AWKSCRIPT'
+BEGIN {
+ in_content = 0
+ depth = 0
+}
+/
]*id="page-content"[^>]*>/ {
+ in_content = 1
+ depth = 1
+ print ""
+ print "
SCP Document"
+ print $0
+ next
+}
+in_content {
+ if (/
]*>/) depth++
+ if (/<\/div>/) {
+ depth--
+ if (depth == 0) {
+ print $0
+ print ""
+ in_content = 0
+ exit
+ }
+ }
+ print $0
+}
+AWKSCRIPT
- local title=$(grep -oP '(?<=
)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//')
-
- if [ -z "$title" ]; then
- title="SCP-$scp_num"
- fi
-
- # Create a markdown file from HTML
- pandoc -f html -t markdown "$html_file" -o "${html_file%.html}.md" 2>/dev/null || {
- log_warning "Could not convert SCP-$scp_num to markdown"
+ awk -f /tmp/extract_content.awk "$html_file" > "$output_html" 2>/dev/null || {
+ log_warning "Could not extract content from SCP-$scp_num, using full HTML"
+ cp "$html_file" "$output_html"
return 1
}
+ rm -f /tmp/extract_content.awk
+ return 0
+}
+
+convert_to_markdown() {
+ local html_file=$1
+ local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
+ local md_file="${html_file%.html}.md"
+
+ if pandoc -f html -t markdown "$html_file" -o "$md_file" 2>/dev/null; then
+ return 0
+ else
+ log_warning "Could not convert SCP-$scp_num to markdown"
+ return 1
+ fi
+}
+
+get_scp_title() {
+ local html_file=$1
+
+ # Try to extract title from h1 or meta tags
+ local title=$(grep -oP ']*>\K[^<]*' "$html_file" | head -1)
+
+ if [ -z "$title" ]; then
+ title=$(grep -oP '(?<=)[^<]*' "$html_file" | head -1 | sed 's/ - SCP Foundation.*//')
+ fi
+
+ # Clean up title
+ title=$(echo "$title" | sed 's/^[^S]*//; s/[<>:"\/\\|?*]//g; s/^[[:space:]]*//; s/[[:space:]]*$//')
+
+ if [ -z "$title" ]; then
+ local scp_num=$(basename "$html_file" .html | sed 's/scp-//')
+ title="SCP-$scp_num"
+ fi
+
echo "$title"
}
convert_to_pdf() {
local md_file=$1
- local scp_num=$(basename "$md_file" .md | sed 's/scp-//')
- local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num")
- local pdf_file="$OUTPUT_DIR/pdf/SCP-$(printf "%03d" "$scp_num") - ${title}.pdf"
+ # Extract the SCP number from the markdown filename (e.g., scp-017_content.md -> 017)
+ local scp_num=$(basename "$md_file" | grep -oP 'scp-\K\d+' | head -1)
+ local title=$(get_scp_title "$OUTPUT_DIR/html/scp-$scp_num.html")
+ local pdf_file="$OUTPUT_DIR/pdf/SCP-$scp_num - ${title}.pdf"
mkdir -p "$OUTPUT_DIR/pdf"
@@ -153,29 +207,30 @@ convert_to_pdf() {
-V linkcolor=blue \
-V geometry=margin=1in \
2>/dev/null; then
- log_success "Converted SCP-$(printf "%03d" "$scp_num") to PDF"
+ log_success "Converted SCP-$scp_num to PDF"
return 0
else
- log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to PDF"
+ log_error "Failed to convert SCP-$scp_num to PDF"
return 1
fi
}
convert_to_epub() {
local md_file=$1
- local scp_num=$(basename "$md_file" .md | sed 's/scp-//')
- local title=$(extract_scp_content "$OUTPUT_DIR/html/scp-$scp_num.html" 2>/dev/null || echo "SCP-$scp_num")
- local epub_file="$OUTPUT_DIR/epub/SCP-$(printf "%03d" "$scp_num") - ${title}.epub"
+ # Extract the SCP number from the markdown filename (e.g., scp-017_content.md -> 017)
+ local scp_num=$(basename "$md_file" | grep -oP 'scp-\K\d+' | head -1)
+ local title=$(get_scp_title "$OUTPUT_DIR/html/scp-$scp_num.html")
+ local epub_file="$OUTPUT_DIR/epub/SCP-$scp_num - ${title}.epub"
mkdir -p "$OUTPUT_DIR/epub"
if pandoc "$md_file" -o "$epub_file" \
--metadata author="SCP Foundation" \
2>/dev/null; then
- log_success "Converted SCP-$(printf "%03d" "$scp_num") to EPUB"
+ log_success "Converted SCP-$scp_num to EPUB"
return 0
else
- log_error "Failed to convert SCP-$(printf "%03d" "$scp_num") to EPUB"
+ log_error "Failed to convert SCP-$scp_num to EPUB"
return 1
fi
}
@@ -191,17 +246,33 @@ download_and_convert() {
return 1
fi
- local md_file="$OUTPUT_DIR/html/scp-$scp_num.md"
+ # Extract page content
+ local html_file="$OUTPUT_DIR/html/scp-$scp_num.html"
+ local content_html="${html_file%.html}_content.html"
+ if ! extract_page_content "$html_file"; then
+ log_warning "Using full HTML for SCP-$scp_num"
+ content_html="$html_file"
+ fi
+
+ # Convert to markdown
+ if ! convert_to_markdown "$content_html"; then
+ log_error "Failed to convert SCP-$scp_num to markdown"
+ return 1
+ fi
+
+ local md_file="${content_html%.html}.md"
+
+ # Convert to desired format(s)
if [ "$FORMAT" = "pdf" ] || [ "$FORMAT" = "both" ]; then
- convert_to_pdf "$md_file" &
+ convert_to_pdf "$md_file"
fi
if [ "$FORMAT" = "epub" ] || [ "$FORMAT" = "both" ]; then
- convert_to_epub "$md_file" &
+ convert_to_epub "$md_file"
fi
- wait
+ log_success "SCP-$scp_num complete"
}
##############################################################################
@@ -234,20 +305,14 @@ main() {
log_info "Downloading SCPs from $start to $end"
log_info "Format: $FORMAT"
log_info "Output directory: $OUTPUT_DIR"
+ log_info "Concurrent downloads: $CONCURRENT_DOWNLOADS"
local count=0
- for num in $(seq "$start" "$end"); do
- download_and_convert "$num" &
-
- # Limit concurrent downloads
- ((count++))
- if [ $((count % CONCURRENT_DOWNLOADS)) -eq 0 ]; then
- wait
- fi
- done
+ local pids=()
- # Wait for remaining processes
- wait
+ for num in $(seq "$start" "$end"); do
+ download_and_convert "$num"
+ done
log_success "Download and conversion complete!"
log_info "Files saved to: $OUTPUT_DIR"
@@ -296,4 +361,3 @@ while [[ $# -gt 0 ]]; do
done
main "$START" "$END"
-