furaffinity-dl/furaffinity-dl

#!/bin/bash
# shellcheck disable=SC2001
set -e

# Default options
outdir="."
prefix="https:"
metadata=true
rename=true
maxsavefiles="0"
maxduplicatecount="0"
overwrite=false
textmeta=false
classic=false

# Helper functions
help() {
	echo "Usage: $0 [ARGUMENTS] SECTION/USER
Downloads the entire gallery/scraps/favorites of any furaffinity user.

Arguments:
 -h (H)elp screen
 -i Use an (I)nsecure connection when downloading
 -o The (O)utput directory to put files in
 -c If you need to download restricted content
    you can provide a path to a (C)ookie file
 -p (P)lain file without any additional metadata
 -r Don't (R)ename files, just give them the same
    filename as on d.furaffinity.net
 -n (N)umber of images to download, starting from
    the most recent submission
 -d Number of consecutive (D)uplicate files to register 
    before exiting
 -w Over(Write) files if they already exist
 -s (S)eperate metadata files, to make sure all
    metadata is downloaded regardless of file
 -t Not using the \"beta\" (T)heme

Examples:
 $0 gallery/mylafox
 $0 -o mylasArt gallery/mylafox
 $0 -o koulsFavs favorites/koul

You can also log in to FurAffinity to download restricted content, like this:
 $0 -c /path/to/your/cookies.txt gallery/gonnaneedabiggerboat

DISCLAIMER: It is your own responsibility to check whether batch downloading is allowed by FurAffinity terms of service and to abide by them."
	exit 1
}

# Display help if no arguments given
[[ $# -eq 0 ]] && help

# Options via arguments
while getopts 'o:c:n:d:iphrwst' flag; do
	case "${flag}" in
		t) classic=true;;
		w) overwrite=true;;
		o) outdir=${OPTARG};;
		c) cookiefile=${OPTARG};;
		i) prefix="http:";;
		p) metadata=false;;
		r) rename=false;;
		n) maxsavefiles=${OPTARG};;
		d) maxduplicatecount=${OPTARG};;
		h) help;;
		s) textmeta=true;;
		*) help;;
	esac
done

# Detect installed metadata injectors
eyed3=true
if [ -z "$(command -v eyeD3)" ]; then
	eyed3=false
	echo "INFO: eyed3 is not installed, no metadata will be injected into music files."
fi

exiftool=true
if [ -z "$(command -v exiftool)" ]; then
	exiftool=false
	echo "INFO: exiftool is not installed, no metadata will be injected into pictures."
fi

cleanup() {
	rm -f "$tempfile"
}

# Attempt to create the output directory
mkdir -p -- "$outdir"

# Setup temporarily file with 600 perms
tempfile="$(umask u=rwx,g=,o= && mktemp)"

# Call cleanup function on exit
trap cleanup EXIT

if [ -z "$cookiefile" ]; then
	# Set wget with a custom user agent
	fwget() {
		wget --quiet --user-agent="Mozilla/5.0 furaffinity-dl (https://github.com/Xerbo/furaffinity-dl)" "$@"
	}
else
	# Set wget with a custom user agent and cookies
	fwget() {
		wget --quiet --user-agent="Mozilla/5.0 furaffinity-dl (https://github.com/Xerbo/furaffinity-dl)" --load-cookies "$cookiefile" "$@"
	}
fi

url="https://www.furaffinity.net/${*: -1}"
download_count="0"
duplicate_count="0"

# Iterate over the gallery pages with thumbnails and links to artwork view pages
while true; do
	fwget "$url" -O "$tempfile"
	if [ -n "$cookiefile" ] && grep -q 'furaffinity.net/login/' "$tempfile"; then
		echo "ERROR: You have provided a cookies file, but it does not contain valid cookies.

If this file used to work, this means that the cookies have expired;
you will have to log in to FurAffinity from your web browser and export the cookies again.

If this is the first time you're trying to use cookies, make sure you have exported them
in Netscape format (this is normally done through \"cookie export\" browser extensions)
and supplied the correct path to the cookies.txt file to this script.

If that doesn't resolve the issue, please report the problem at
https://github.com/Xerbo/furaffinity-dl/issues" >&2
		exit 1
	fi

	# Get URL for next page out of "Next" button. Required for favorites, pages of which are not numbered
	if [ $classic = true ]; then
		next_page_url="$(grep '<a class="button-link right" href="' "$tempfile" | grep '">Next &nbsp;&#x276f;&#x276f;</a>' | cut -d '"' -f 4 | sort -u)"
	else
		next_page_url="$(grep -B 1 --max-count=1 'type="submit">Next' "$tempfile" | grep form | cut -d '"' -f 2)"
	fi

	# Extract links to pages with individual artworks and iterate over them
	artwork_pages="$(grep '<a href="/view/' "$tempfile" | grep -E --only-matching '/view/[[:digit:]]+/' | uniq)"
	for page in $artwork_pages; do
		# Download the submission page
		fwget -O "$tempfile" "https://www.furaffinity.net$page"

		if grep -q "System Message" "$tempfile"; then
			echo "WARNING: $page seems to be inaccessible, skipping."
			continue
		fi

		# Get the full size image URL.
		# This will be a d.furaffinity.net link, we will default to HTTPS
		# but this can be disabled with -i or --http for specific reasons
		image_url="$prefix$(grep --only-matching --max-count=1 ' href="//d.furaffinity.net/art/[^"]\+">Download' "$tempfile" | cut -d '"' -f 2)" 

		# Get metadata
		description="$(grep 'og:description" content="' "$tempfile" | cut -d '"' -f 4)"
		if [ $classic = true ]; then
			title="$(grep -Eo '<h2>.*</h2>' "$tempfile" | awk -F "<h2>" '{print $2}' | awk -F "</h2>" '{print $1}')"
		else
			title="$(grep -Eo '<h2><p>.*</p></h2>' "$tempfile" | awk -F "<p>" '{print $2}' | awk -F "</p>" '{print $1}')"
		fi
		
		file_type="${image_url##*.}"
		file_name="$(echo "$image_url" | cut -d "/" -f 7)"
		if [[ "$file_name" =~ ^[0-9]{0,12}$ ]]; then
			file_name="$(echo "$image_url" | cut -d "/" -f 8)"
		fi

		# Choose the output path
		if [ $rename = true ]; then
			# FIXME titles that are just a single emoji get changed to " " and overwrite eachother
			file="$outdir/$(echo "$title" | sed -e 's/[^A-Za-z0-9._-]/ /g').$file_type"
		else
			file="$outdir/$file_name"
		fi

		# Download the image
		if [ ! -f "$file" ] || [ $overwrite = true ] ; then
			wget --quiet --show-progress "$image_url" -O "$file"
			# reset the duplicate counter, another non-duplicate file has been found
			duplicate_count=0
		else
			echo "File already exists, skipping. Use -w to skip this check"
			# increment the duplicate counter
			duplicate_count="$((duplicate_count + 1))"
			# If we've reached the max number of consecutive duplicates, 
			# should output message and exit
			if [ "$maxduplicatecount" -ne "0" ] && [ "$duplicate_count" -ge "$maxduplicatecount" ]; then
				echo "Reached set maximum of consecutive duplicate files"
				exit 0
			fi
		fi

		mime_type="$(file -- "$file")"

		if [ $textmeta = true ]; then
			 echo -ne "Title: $title\nURL: $page\nFilename: $file_name\nDescription: $description" > "$file.meta"
		fi

		# Add metadata
		if [[ $mime_type ==  *"audio"* ]]; then
			# Use eyeD3 for injecting metadata into audio files (if it's installed)
			if [ $eyed3 = true ] && [ $metadata = true ]; then
				if [ -z "$description" ]; then
					eyeD3 -t "$title" -- "$file" || true
				else
					# HACK: eyeD3 throws an error if a description containing a ":"
					eyeD3 -t "$title" --add-comment "${description//:/\\:}" -- "$file" || true
				fi
			fi
		elif [[ $mime_type == *"image"* ]]; then
			# Use exiftool for injecting metadata into pictures (if it's installed)
			if [ $exiftool = true ] && [ $metadata = true ]; then
				cat -- "$file" | exiftool -description="$description" -title="$title" -overwrite_original - > "$tempfile" && mv -- "$tempfile" "$file" || true
			fi
		fi

		# If there is a file download limit then keep track of it
		if [ "$maxsavefiles" -ne "0" ]; then
			download_count="$((download_count + 1))"

			if [ "$download_count" -ge "$maxsavefiles" ]; then
				echo "Reached set file download limit."
				exit 0
			fi
		fi
	done

	[ -z "$next_page_url" ] && break
	url='https://www.furaffinity.net'"$next_page_url"
done