From d28fac0fd30b1317ba9b9c7820a236aeac8a4812 Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 4 Apr 2019 18:14:52 +0100 Subject: [PATCH 01/10] Update and rename furaffinity-dl to furaffinity-dl-ng Supports many new features over the original. - Better argument parsing - Choose between HTTP and HTTPS - Titles files properly - Metadata support (description and title) --- furaffinity-dl | 91 --------------------------- furaffinity-dl-ng | 153 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 91 deletions(-) delete mode 100755 furaffinity-dl create mode 100755 furaffinity-dl-ng diff --git a/furaffinity-dl b/furaffinity-dl deleted file mode 100755 index 6c7e331..0000000 --- a/furaffinity-dl +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -set -e - -if [ "$1" = "" ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then - echo "Usage: $0 SECTION/USER [YOUR_USERNAME] -Downloads the entire gallery/scraps/favorites of any furaffinity.net user. - -Examples: - $0 gallery/kodardragon - $0 scraps/--- - $0 favorites/kivuli - -You can also log in to FurAffinity and download restricted content, like this: - $0 gallery/gonnaneedabiggerboat /path/to/your/cookies.txt" - exit 1 -fi - -runtime_dir="$HOME"'/.cache/furaffinity-dl' -mkdir -p "$runtime_dir" -tempfile="$(umask u=rwx,g=,o= && mktemp $runtime_dir/fa-dl.XXXXXXXXXX)" - -cleanup() { - rm -r "$tempfile" -} -trap cleanup EXIT - -COOKIES_FILE="$2" -if [ "$COOKIES_FILE" = "" ]; then - # set a wget wrapper with custom user agent - fwget() { - wget --user-agent="Mozilla/5.0 furaffinity-dl (https://github.com/Shnatsel/furaffinity-dl)" $* - } -else - # set a wget wrapper with custom user agent and cookies - fwget() { - wget --user-agent="Mozilla/5.0 furaffinity-dl (https://github.com/Shnatsel/furaffinity-dl)" \ - --load-cookies "$COOKIES_FILE" $* - } -fi - -url=https://www.furaffinity.net/"$1" - -# Iterate over the gallery pages with thumbnails and links to artwork view pages -while true; do - fwget -O "$tempfile" "$url" - if [ "$COOKIES_FILE" != "" ] && grep -q 'furaffinity.net/login/' "$tempfile"; then - echo "-------------- - -ERROR: You have provided a cookies file, but it does not contain valid cookies. - -If this file used to work, this means that the cookies have expired; -you will have to log in to FurAffinity from your web browser and export the cookies again. - -If this is the first time you're trying to use cookies, make sure you have exported them -in Netscape format (this is normally done through \"cookie export\" browser extensions) -and supplied the correct path to the cookies.txt file to this script. - -If that doesn't resolve the issue, please report the problem at -https://github.com/Shnatsel/furaffinity-dl/issues" >&2 - exit 1 - fi - - # Get URL for next page out of "Next" button. Required for favorites, pages of which are not numbered - next_page_url="$(grep 'Next  ❯❯' | cut -d '"' -f 4 | sort -u)" - - # Extract links to pages with individual artworks and iterate over them - artwork_pages=$(grep 'Download' "$tempfile" | cut -d '"' -f 2) - - # TODO: Get the submission title out of the page - # this trick may come in handy for avoiding slashes in filenames: - # | tr '/' '∕' - - wget --timestamping "$image_url" - done - - [ "$next_page_url" = "" ] && break - url='https://www.furaffinity.net'"$next_page_url" -done diff --git a/furaffinity-dl-ng b/furaffinity-dl-ng new file mode 100755 index 0000000..e0285c6 --- /dev/null +++ b/furaffinity-dl-ng @@ -0,0 +1,153 @@ +#!/bin/bash +set -e + +# Detect installed applications +if [ -f /usr/bin/eyeD3 ]; then + eyed3=true +else + eyed3=false + echo "INFO: eyed3 is not installed, no metadata will be injected into music files." +fi +if [ -f /usr/bin/exiftool ]; then + exiftool=true +else + exiftool=false + echo "INFO: exiftool is not installed, no metadata will be injected into pictures." +fi + +# Helper functions +help() { + echo "Usage: $0 SECTION/USER [ARGUMENTS] +Downloads the entire gallery/scraps/favorites of any furaffinity user. + +Arguments: + -h --help This text + -i --http Use an insecure connection + -o --out The directory to put files in + -c --cookiefile If you need to download restricted content + you can provide a path to a cookie file + -p --nometa Plain file without any additional metadata + +Examples: + $0 gallery/mylafox + $0 -o=myla gallery/mylafox + $0 --out=koul favorites/koul + +You can also log in to FurAffinity to download restricted content, like this: + $0 gallery/gonnaneedabiggerboat -c=/path/to/your/cookies.txt + +DISCLAIMER: It is your own responsibility to check whether batch downloading is allowed by FurAffinity terms of service and to abide by them." + exit 1 +} +cleanup() { + rm -r "$tempfile" +} + +# Arguments +if [ $# -eq 0 ]; then + help +fi + +prefix="https:" +outdir="." +metadata=true; +case "$1" in + # Help + -h|--help) help;; + + # HTTP / HTTPS + -i|--http) prefix="http:";; + + # Output directory + -o=*|--out=*) outdir="${1#*=}"; shift 1;; + + # Cookie file + -c=*|--cookiefile=*) cookiefile="${1#*=}"; shift 1;; + + # Metadata + -p|--nometa) meta=false;; +esac +mkdir -p "$outdir" + +runtime_dir="$HOME"'/.cache/furaffinity-dl-ng' +mkdir -p "$runtime_dir" +tempfile="$(umask u=rwx,g=,o= && mktemp $runtime_dir/fa-dl.XXXXXXXXXX)" + +trap cleanup EXIT + +if [ "$cookiefile" = "" ]; then + # Set wget with a custom user agent + fwget() { + wget -nv --user-agent="Mozilla/5.0 furaffinity-dl-ng (https://github.com/Xerbo/furaffinity-dl-ng)" $* + } +else + # Set wget with a custom user agent and cookies + fwget() { + wget -nv --user-agent="Mozilla/5.0 furaffinity-dl-ng (https://github.com/Xerbo/furaffinity-dl-ng)" --load-cookies "$cookiefile" $* + } +fi + +url="https://www.furaffinity.net/${@: -1}" + +# Iterate over the gallery pages with thumbnails and links to artwork view pages +while true; do + fwget "$url" -O "$tempfile" + if [ "$cookiefile" != "" ] && grep -q 'furaffinity.net/login/' "$tempfile"; then + echo "ERROR: You have provided a cookies file, but it does not contain valid cookies. + +If this file used to work, this means that the cookies have expired; +you will have to log in to FurAffinity from your web browser and export the cookies again. + +If this is the first time you're trying to use cookies, make sure you have exported them +in Netscape format (this is normally done through \"cookie export\" browser extensions) +and supplied the correct path to the cookies.txt file to this script. + +If that doesn't resolve the issue, please report the problem at +https://github.com/Xerbo/furaffinity-dl-ng/issues" >&2 + exit 1 + fi + + # Get URL for next page out of "Next" button. Required for favorites, pages of which are not numbered + next_page_url="$(grep 'Next  ❯❯' | cut -d '"' -f 4 | sort -u)" + + # Extract links to pages with individual artworks and iterate over them + artwork_pages=$(grep 'Download' "$tempfile" | cut -d '"' -f 2) + + # Get metadata + description=$(grep 'og:description" content="' "$tempfile" | cut -d '"' -f4) + title=$(grep 'og:title" content="' "$tempfile" | cut -d '"' -f4) + file_type=${image_url##*.} + file="$outdir/$title.$file_type" + + wget "$image_url" -O "$file" + + # Add metadata + if [[ $(file $file) =~ (MPEG|audio) ]]; then + # Use exiftool for injecting metadata into pictures (if it's installed) + if [ $exiftool ] && [ $metadata ]; then + exiftool "$file" -description="$description" -title="$title" + fi + else + # Use eyeD3 for injecting injecting metadata into audio files (if it's installed) + if [ $eyed3 ] && [ $metadata ]; then + eyeD3 -t "$title" --add-comment "$description" "$file" + fi + fi + done + + [ "$next_page_url" = "" ] && break + url='https://www.furaffinity.net'"$next_page_url" +done From 039f84f640047d60836618814bcf03b65558c55e Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 4 Apr 2019 18:20:40 +0100 Subject: [PATCH 02/10] Update README.md --- README.md | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index bdcb852..7fb8314 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,36 @@ -# FurAffinity Downloader -**furaffinity-dl** is a bash script for batch downloading of galleries and favorites from furaffinity.net users. -I've written it for preservation of culture, to counter the people nuking their galleries every once a while. +# FurAffinity Downloader - Next Gen +**furaffinity-dl-ng** is a bash script for batch downloading of galleries and favorites from furaffinity.net users. +It was written for preservation of culture, to counter the people nuking their galleries every once a while. -Supports all known submission types: images, texts and audio. Sorts downloaded files in chronological order. -I'd like to eventually expand it to download the description pages as well. Patches are welcome! +Supports all known submission types: images, texts and audio. ## Requirements +Coreutils, bash and wget are the only dependencies. However if you want to embed metadata into files you will need eyed3 and exiftool -Coreutils, bash and wget are the only dependencies. - -furaffinity-dl was tested only on Linux. It should also work on Mac and BSDs. -Windows users can get it to work via Microsoft's "BASH on Windows". Cygwin is not supported. +furaffinity-dl-ng was tested only on Linux. It should also work on Mac and BSDs. +Windows users can get it to work via Microsoft's [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10). Cygwin is not supported. ## Usage - `furaffinity-dl section/username` +Make it executable with + `chmod +x faraffinity-dl-ng` +And run it with + `./faraffinity-dl-ng section/username` All files from the given section and user will be downloaded to the current directory. ### Examples - `furaffinity-dl gallery/kodardragon` + `./faraffinity-dl-ng gallery/mylafox` - `furaffinity-dl scraps/---` + `./faraffinity-dl-ng -o=myla gallery/mylafox` - `furaffinity-dl favorites/kivuli` + `./faraffinity-dl-ng --out=koul favorites/koul` You can also log in to download restricted content. To do that, log in to FurAffinity in your web browser, export cookies to a file from your web browser in Netscape format (there are extensions to do that [for Firefox](https://addons.mozilla.org/en-US/firefox/addon/ganbo/) and [for Chrome/Vivaldi](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg)) and pass them to the script as a second parameter, like this: `furaffinity-dl gallery/gonnaneedabiggerboat /path/to/your/cookies.txt` ## TODO - * Download author's description of the artwork, and ideally the entire description page along with user comments + * Download user bio, post tags and ideally user comments ## Disclaimer It is your own responsibility to check whether batch downloading is allowed by FurAffinity terms of service and to abide by them. For further disclaimers see LICENSE. - -## See also - -There is a similar downloader for VCL art library, see https://github.com/Shnatsel/vcl-dl From 86e19f2089b5a971b9bc7a3db470a3fabe891244 Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 4 Apr 2019 19:09:10 +0100 Subject: [PATCH 03/10] fix a bug eyeD3 throws an error if a description containing a ":". so escape it first --- furaffinity-dl-ng | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/furaffinity-dl-ng b/furaffinity-dl-ng index e0285c6..64f2b9c 100755 --- a/furaffinity-dl-ng +++ b/furaffinity-dl-ng @@ -143,7 +143,12 @@ https://github.com/Xerbo/furaffinity-dl-ng/issues" >&2 else # Use eyeD3 for injecting injecting metadata into audio files (if it's installed) if [ $eyed3 ] && [ $metadata ]; then - eyeD3 -t "$title" --add-comment "$description" "$file" + if [ -z "$description" ]; then + eyeD3 -t "$title" "$file" + else + # HACK eyeD3 throws an error if a description containing a ":" + eyeD3 -t "$title" --add-comment "${description//:/\\:}" "$file" + fi fi fi done From f57671f31747a3eabddd2e8bd1a1e729ba09b478 Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 4 Apr 2019 19:36:36 +0100 Subject: [PATCH 04/10] buggggggz File gets stuck sometimes --- furaffinity-dl-ng | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/furaffinity-dl-ng b/furaffinity-dl-ng index 64f2b9c..5e77ed2 100755 --- a/furaffinity-dl-ng +++ b/furaffinity-dl-ng @@ -135,12 +135,7 @@ https://github.com/Xerbo/furaffinity-dl-ng/issues" >&2 wget "$image_url" -O "$file" # Add metadata - if [[ $(file $file) =~ (MPEG|audio) ]]; then - # Use exiftool for injecting metadata into pictures (if it's installed) - if [ $exiftool ] && [ $metadata ]; then - exiftool "$file" -description="$description" -title="$title" - fi - else + if [ $file_type == "mp3" ] || [ $file_type == "wav" ] || [ $file_type == "wmv" ] || [ $file_type == "ogg" ] || [ $file_type == "flac" ]; then # Use eyeD3 for injecting injecting metadata into audio files (if it's installed) if [ $eyed3 ] && [ $metadata ]; then if [ -z "$description" ]; then @@ -150,6 +145,11 @@ https://github.com/Xerbo/furaffinity-dl-ng/issues" >&2 eyeD3 -t "$title" --add-comment "${description//:/\\:}" "$file" fi fi + else + # Use exiftool for injecting metadata into pictures (if it's installed) + if [ $exiftool ] && [ $metadata ]; then + exiftool "$file" -description="$description" -title="$title" + fi fi done From 1b5e61e4d6c53de93c51f6432518561be49abed3 Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 5 Apr 2019 09:46:13 +0100 Subject: [PATCH 05/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7fb8314..37fe93c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ All files from the given section and user will be downloaded to the current dire You can also log in to download restricted content. To do that, log in to FurAffinity in your web browser, export cookies to a file from your web browser in Netscape format (there are extensions to do that [for Firefox](https://addons.mozilla.org/en-US/firefox/addon/ganbo/) and [for Chrome/Vivaldi](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg)) and pass them to the script as a second parameter, like this: - `furaffinity-dl gallery/gonnaneedabiggerboat /path/to/your/cookies.txt` + `furaffinity-dl -c=/path/to/your/cookies.txt gallery/gonnaneedabiggerboat` ## TODO * Download user bio, post tags and ideally user comments From 4814cd713a37992e723d0791d06f628063284d3b Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 5 Apr 2019 12:49:08 +0100 Subject: [PATCH 06/10] better support for text --- furaffinity-dl-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/furaffinity-dl-ng b/furaffinity-dl-ng index 5e77ed2..1cda72b 100755 --- a/furaffinity-dl-ng +++ b/furaffinity-dl-ng @@ -145,7 +145,7 @@ https://github.com/Xerbo/furaffinity-dl-ng/issues" >&2 eyeD3 -t "$title" --add-comment "${description//:/\\:}" "$file" fi fi - else + elif [ $file_type == "png" ] || [ $file_type == "jpg" ] || [ $file_type == "jpeg" ]; then # Use exiftool for injecting metadata into pictures (if it's installed) if [ $exiftool ] && [ $metadata ]; then exiftool "$file" -description="$description" -title="$title" From 3a097bb24c9d38e0ff734f430ff2f7a818357de9 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 7 Apr 2019 18:40:47 +0100 Subject: [PATCH 07/10] fix help text --- furaffinity-dl-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/furaffinity-dl-ng b/furaffinity-dl-ng index 1cda72b..ddedc7c 100755 --- a/furaffinity-dl-ng +++ b/furaffinity-dl-ng @@ -34,7 +34,7 @@ Examples: $0 --out=koul favorites/koul You can also log in to FurAffinity to download restricted content, like this: - $0 gallery/gonnaneedabiggerboat -c=/path/to/your/cookies.txt + $0 -c=/path/to/your/cookies.txt gallery/gonnaneedabiggerboat DISCLAIMER: It is your own responsibility to check whether batch downloading is allowed by FurAffinity terms of service and to abide by them." exit 1 From 8d257cca4e0556ff7ee2a95ffd8c2ca1c8f7ac19 Mon Sep 17 00:00:00 2001 From: Xerbo Date: Sat, 20 Jul 2019 22:08:22 +0100 Subject: [PATCH 08/10] Preparing for migration --- furaffinity-dl-ng => furaffinity-dl | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename furaffinity-dl-ng => furaffinity-dl (100%) diff --git a/furaffinity-dl-ng b/furaffinity-dl similarity index 100% rename from furaffinity-dl-ng rename to furaffinity-dl From c42b06d3fc7f7e1d609f7c1a06bfce464dabab9f Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 20 Jul 2019 23:32:29 +0000 Subject: [PATCH 09/10] README changes --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 37fe93c..2e25258 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# FurAffinity Downloader - Next Gen -**furaffinity-dl-ng** is a bash script for batch downloading of galleries and favorites from furaffinity.net users. +# FurAffinity Downloader +**furaffinity-dl** is a bash script for batch downloading of galleries and favorites from furaffinity.net users. It was written for preservation of culture, to counter the people nuking their galleries every once a while. Supports all known submission types: images, texts and audio. @@ -7,7 +7,7 @@ Supports all known submission types: images, texts and audio. ## Requirements Coreutils, bash and wget are the only dependencies. However if you want to embed metadata into files you will need eyed3 and exiftool -furaffinity-dl-ng was tested only on Linux. It should also work on Mac and BSDs. +furaffinity-dl was tested only on Linux. It should also work on Mac and BSDs. Windows users can get it to work via Microsoft's [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10). Cygwin is not supported. ## Usage @@ -21,13 +21,13 @@ All files from the given section and user will be downloaded to the current dire ### Examples `./faraffinity-dl-ng gallery/mylafox` - `./faraffinity-dl-ng -o=myla gallery/mylafox` + `./faraffinity-dl-ng -o=mylasArt gallery/mylafox` - `./faraffinity-dl-ng --out=koul favorites/koul` + `./faraffinity-dl-ng --out=koulsFavs favorites/koul` You can also log in to download restricted content. To do that, log in to FurAffinity in your web browser, export cookies to a file from your web browser in Netscape format (there are extensions to do that [for Firefox](https://addons.mozilla.org/en-US/firefox/addon/ganbo/) and [for Chrome/Vivaldi](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg)) and pass them to the script as a second parameter, like this: - `furaffinity-dl -c=/path/to/your/cookies.txt gallery/gonnaneedabiggerboat` + `./furaffinity-dl -c=/path/to/your/cookies.txt gallery/gonnaneedabiggerboat` ## TODO * Download user bio, post tags and ideally user comments From ea4c4ed1646263a9f4a190139d7cc7f3c8e4b133 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 21 Jul 2019 00:38:20 +0100 Subject: [PATCH 10/10] Spelling + grammer --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2e25258..d7fe36e 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,16 @@ Windows users can get it to work via Microsoft's [WSL](https://docs.microsoft.co Make it executable with `chmod +x faraffinity-dl-ng` And run it with - `./faraffinity-dl-ng section/username` + `./furaffinity-dl-ng section/username` All files from the given section and user will be downloaded to the current directory. ### Examples - `./faraffinity-dl-ng gallery/mylafox` + `./furaffinity-dl-ng gallery/mylafox` - `./faraffinity-dl-ng -o=mylasArt gallery/mylafox` + `./furaffinity-dl-ng -o=mylasArt gallery/mylafox` - `./faraffinity-dl-ng --out=koulsFavs favorites/koul` + `./furaffinity-dl-ng --out=koulsFavs favorites/koul` You can also log in to download restricted content. To do that, log in to FurAffinity in your web browser, export cookies to a file from your web browser in Netscape format (there are extensions to do that [for Firefox](https://addons.mozilla.org/en-US/firefox/addon/ganbo/) and [for Chrome/Vivaldi](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg)) and pass them to the script as a second parameter, like this: @@ -33,4 +33,4 @@ You can also log in to download restricted content. To do that, log in to FurAff * Download user bio, post tags and ideally user comments ## Disclaimer -It is your own responsibility to check whether batch downloading is allowed by FurAffinity terms of service and to abide by them. For further disclaimers see LICENSE. +It is your own responsibility to check whether batch downloading is allowed by FurAffinity's terms of service and to abide by them. For further disclaimers see LICENSE.