From c4400d4f78b27cee0fb61b525e0ec2e914a1c358 Mon Sep 17 00:00:00 2001 From: Radiquum Date: Fri, 24 Jun 2022 17:51:07 +0500 Subject: [PATCH] some cosmetic changes: - add shorter arguments - metadata file saving will now be in same folder as an image (it's same as it was before) - print output path when downloaded a specific file --- README.md | 27 +++++++++++++------------- furaffinity-dl.py | 49 +++++++++++++++++++++++++++-------------------- 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 0168714..ebc392c 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,9 @@ When downloading a folder make sure to put everything after **/folder/**, for ex ```help -usage: furaffinity-dl.py [-h] [--submissions] [--folder FOLDER [FOLDER ...]] [--cookies COOKIES [COOKIES ...]] - [--user-agent USER_AGENT [USER_AGENT ...]] [--start START [START ...]] [--stop STOP [STOP ...]] [--dont-redownload] - [--interval INTERVAL [INTERVAL ...]] [--rating] [--filter] [--metadata] [--download DOWNLOAD] [--json-description] - [--login] +usage: furaffinity-dl.py [-h] [-sub] [-f FOLDER [FOLDER ...]] [-c COOKIES [COOKIES ...]] [-ua USER_AGENT [USER_AGENT ...]] [--start START [START ...]] + [--stop STOP [STOP ...]] [--redownload] [--interval INTERVAL [INTERVAL ...]] [--rating] [--filter] [--metadata] [--download DOWNLOAD] + [-jd] [--login] [username] [category] Downloads the entire gallery/scraps/folder/favorites of a furaffinity user, or your submissions notifications @@ -38,36 +37,36 @@ positional arguments: options: -h, --help show this help message and exit - --submissions download your submissions - --folder FOLDER [FOLDER ...] + -sub, --submissions download your submissions + -f FOLDER [FOLDER ...], --folder FOLDER [FOLDER ...] full path of the furaffinity gallery folder. for instance 123456/Folder-Name-Here - --cookies COOKIES [COOKIES ...], -c COOKIES [COOKIES ...] + -c COOKIES [COOKIES ...], --cookies COOKIES [COOKIES ...] path to a NetScape cookies file - --user-agent USER_AGENT [USER_AGENT ...] + -ua USER_AGENT [USER_AGENT ...], --user-agent USER_AGENT [USER_AGENT ...] Your browser's useragent, may be required, depending on your luck --start START [START ...], -s START [START ...] page number to start from --stop STOP [STOP ...], -S STOP [STOP ...] Page number to stop on. Specify the full URL after the username: for favorites pages (1234567890/next) or for submissions pages: (new~123456789@48) - --dont-redownload, -d - Allow to redownload files that have been downloaded already + --redownload, -rd Redownload files that have been downloaded already --interval INTERVAL [INTERVAL ...], -i INTERVAL [INTERVAL ...] delay between downloading pages in seconds [default: 0] --rating, -r disable rating separation --filter disable submission filter - --metadata, -m enable downloading of metadata + --metadata, -m enable metadata saving --download DOWNLOAD download a specific submission /view/12345678/ - --json-description download description as a JSON list + -jd, --json-description + download description as a JSON list --login extract furaffinity cookies directly from your browser Examples: python3 furaffinity-dl.py koul -> will download gallery of user koul python3 furaffinity-dl.py koul scraps -> will download scraps of user koul - python3 furaffinity-dl.py mylafox favorites -> will download favorites of user mylafox + python3 furaffinity-dl.py mylafox favorites -> will download favorites of user mylafox You can also log in to FurAffinity in a web browser and load cookies to download age restricted content or submissions: python3 furaffinity-dl.py letodoesart -c cookies.txt -> will download gallery of user letodoesart including Mature and Adult submissions - python3 furaffinity-dl.py --submissions -c cookies.txt -> will download your submissions notifications + python3 furaffinity-dl.py --submissions -c cookies.txt -> will download your submissions notifications DISCLAIMER: It is your own responsibility to check whether batch downloading is allowed by FurAffinity terms of service and to abide by them. diff --git a/furaffinity-dl.py b/furaffinity-dl.py index 3b5eb57..2abe8d0 100644 --- a/furaffinity-dl.py +++ b/furaffinity-dl.py @@ -40,17 +40,19 @@ parser.add_argument( default="gallery", ) parser.add_argument( - "--submissions", action="store_true", help="download your submissions" + "-sub", "--submissions", action="store_true", help="download your submissions" ) parser.add_argument( + "-f", "--folder", nargs="+", help="full path of the furaffinity gallery folder. for instance 123456/Folder-Name-Here", ) parser.add_argument( - "--cookies", "-c", nargs="+", help="path to a NetScape cookies file" + "-c", "--cookies", nargs="+", help="path to a NetScape cookies file" ) parser.add_argument( + "-ua", "--user-agent", dest="user_agent", nargs="+", @@ -70,10 +72,11 @@ parser.add_argument( help="Page number to stop on. Specify the full URL after the username: for favorites pages (1234567890/next) or for submissions pages: (new~123456789@48)", ) parser.add_argument( - "--dont-redownload", - "-d", + "--redownload", + "-rd", + dest="dont_redownload", action="store_false", - help="Allow to redownload files that have been downloaded already", + help="Redownload files that have been downloaded already", ) parser.add_argument( "--interval", @@ -98,13 +101,14 @@ parser.add_argument( "--metadata", "-m", action="store_true", - help="enable downloading of metadata", + help="enable metadata saving", ) parser.add_argument( "--download", help="download a specific submission /view/12345678/", ) parser.add_argument( + "-jd", "--json-description", dest="json_description", action="store_true", @@ -163,7 +167,7 @@ def download_file(url, fname, desc): total = int(r.headers.get("Content-Length", 0)) with open(fname, "wb") as file, tqdm( - desc=desc.ljust(60)[:60], + desc=desc.ljust(40)[:40], total=total, miniters=100, unit="b", @@ -216,7 +220,7 @@ def download(path): exit() image = s.find(class_="download").find("a").attrs.get("href") - title = s.find(class_="submission-title").find("p").contents[0] + " " + title = f' {s.find(class_="submission-title").find("p").contents[0]} ' description = ( s.find(class_="submission-description").text.strip().replace("\r\n", "\n") ) @@ -246,7 +250,7 @@ def download(path): "comments": [], } - if args.submissions is True: + if args.submissions is True or args.download is not None: global output output = f"furaffinity-dl/gallery/{data.get('author')}" @@ -258,33 +262,40 @@ def download(path): ) if match is not None and title == match.string: print( - f"{YELLOW} post {title} was filtered and will not be downloaded - {data.get('url')}{END}" + f"{YELLOW} post:{title}was filtered and will not be downloaded - {data.get('url')}{END}" ) return True image_url = f"https:{image}" os.makedirs(output, exist_ok=True) + global output_path output_path = f"{output}/{filename}" if args.rating is True: os.makedirs(f'{output}/{data.get("rating")}', exist_ok=True) output_path = f'{output}/{data.get("rating")}/{filename}' if args.dont_redownload is True and os.path.isfile(output_path): - print(f'{YELLOW} Skipping "{title}", since it\'s already downloaded{END}') + print(f"{YELLOW} Skipping:{title} since it's already downloaded{END}") else: download_file(image_url, output_path, title) if args.metadata is True: - metadata = f"{output}/metadata" + metadata = output_path # Extract description as list if args.json_description is True: for desc in s.find("div", class_="submission-description").stripped_strings: - if re.search("[<>/]", desc) is True: - desc = desc.replace("<", "").replace(">", "").replace("/", "") + if re.search("<", desc) is True: + desc = desc.replace("<", "") + + if re.search(">", desc) is True: + desc = desc.replace(">", "") + + if re.search("/", desc) is True: + desc = desc.replace("/", "") data["description"].append(desc) @@ -294,7 +305,7 @@ def download(path): for tag in s.find(class_="tags-row").findAll(class_="tags"): data["tags"].append(tag.find("a").text) except AttributeError: - print(f'{YELLOW} post: "{title}", has no tags{END}') + print(f"{YELLOW} post:{title} has no tags{END}") # Extract comments for comment in s.findAll(class_="comment_container"): @@ -319,19 +330,15 @@ def download(path): ) # Write a UTF-8 encoded JSON file for metadata - os.makedirs(metadata, exist_ok=True) - with open( - os.path.join(metadata, f"{filename}.json"), "w", encoding="utf-8" - ) as f: + with open(f"{metadata}.json", "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=4) return True if args.download is not None: - output = "furaffinity-dl/downloaded/" download(args.download) - print(f"{GREEN} File downloaded{END}") + print(f"{GREEN} File saved as {output_path} {END}") exit() # Main function