changelog:

ability to add comments in username list with "#"
autoremoval of "_" in usernames
speedup filter checking
add basic indexing -> speedup existing file checking for newer files
other small changes

furaffinity-dl.py

@@ -9,10 +9,14 @@ from bs4 import BeautifulSoup
 
 import Modules.config as config
 from Modules.download import download
+from Modules.functions import check_filter
 from Modules.functions import download_complete
 from Modules.functions import login
 from Modules.functions import next_button
+from Modules.functions import requests_retry_session
 from Modules.functions import system_message_handler
+from Modules.index import check_file
+from Modules.index import start_indexing
 
 # get session
 session = requests.session()
@@ -31,12 +35,13 @@ def main():
         while True:
             if config.stop == page_num:
                 print(
-                    f'{config.WARN_COLOR}Reached page "{config.stop}", stopping.{config.END}'
+                    f'{config.WARN_COLOR}Reached page "{config.stop}", \
+stopping.{config.END}'
                 )
                 break
 
             page_url = f"{download_url}/{page_num}"
-            response = session.get(page_url)
+            response = requests_retry_session(session=session).get(page_url)
             s = BeautifulSoup(response.text, "html.parser")
 
             # System messages
@@ -50,7 +55,30 @@ def main():
 
             # Download all images on the page
             for img in s.findAll("figure"):
-                download(img.find("a").attrs.get("href"))
+                title = img.find("figcaption").contents[0].text
+                img_url = img.find("a").attrs.get("href")
+
+                if config.submission_filter is True and check_filter(title) is True:
+                    print(
+                        f'{config.WARN_COLOR}"{title}" was filtered and will not be \
+downloaded - {config.BASE_URL}{img_url}{config.END}'
+                    )
+                    continue
+
+                if config.dont_redownload is True and check_file(img_url) is True:
+                    if config.check is True:
+                        print(
+                            f'{config.SUCCESS_COLOR}Downloaded all recent files of \
+"{config.username[0]}"{config.END}'
+                        )
+                        raise download_complete
+                    print(
+                        f'{config.WARN_COLOR}Skipping "{title}" since \
+it\'s already downloaded{config.END}'
+                    )
+                    continue
+
+                download(img_url)
                 sleep(config.interval)
 
             page_num = next_button(page_url)
@@ -58,13 +86,18 @@ def main():
 
 if __name__ == "__main__":
     if config.login is True:
-        login(config.user_agent)
+        login()
+        exit()
+
+    if config.index is True:
+        if os.path.isfile(f"{config.output_folder}/index.idx"):
+            os.remove(f"{config.output_folder}/index.idx")
+        start_indexing(config.output_folder)
+        print(f"{config.SUCCESS_COLOR}indexing finished{config.END}")
+        exit()
 
     try:
-        response = session.get(config.BASE_URL)
-    except ConnectionError:
-        print(f"{config.ERROR_COLOR}Connection failed{config.END}")
-        exit()
+        response = requests_retry_session(session=session).get(config.BASE_URL)
     except KeyboardInterrupt:
         print(f"{config.WARN_COLOR}Aborted by user{config.END}")
         exit()
@@ -72,14 +105,18 @@ if __name__ == "__main__":
     s = BeautifulSoup(response.text, "html.parser")
     if s.find(class_="loggedin_user_avatar") is not None:
         account_username = s.find(class_="loggedin_user_avatar").attrs.get("alt")
-        print(f'{config.SUCCESS_COLOR}Logged in as "{account_username}"{config.END}')
+        print(
+            f'{config.SUCCESS_COLOR}Logged in as \
+"{account_username}"{config.END}'
+        )
     else:
         print(
-            f"{config.WARN_COLOR}Not logged in, NSFW content is inaccessible{config.END}"
+            f"{config.WARN_COLOR}Not logged in, NSFW content \
+is inaccessible{config.END}"
         )
 
     if config.download is not None:
-        download(config.download)
+        download(f"/view/{config.download}/")
         exit()
 
     if config.submissions is True:
@@ -109,15 +146,29 @@ downloading "{config.folder[1]}"{config.END}'
         )
         exit()
 
-    if os.path.exists(config.username[0]):
-        data = open(config.username[0]).read()
-        config.username = filter(None, data.split("\n"))
+    try:
+        if os.path.exists(config.username[0]):
+            data = open(config.username[0]).read()
+            config.username = filter(None, data.split("\n"))
+    except TypeError or AttributeError:
+        print(
+            f"{config.ERROR_COLOR}Please enter a username \
+or provide a file with usernames (1 username per line){config.END}"
+        )
+        exit()
 
     for username in config.username:
-        print(f'{config.SUCCESS_COLOR}Now downloading "{username}"{config.END}')
-        download_url = f"{config.BASE_URL}/{config.category}/{username}"
-        main()
-        print(
-            f'{config.SUCCESS_COLOR}Finished \
-downloading "{username}"{config.END}'
+        username = username.split("#")[0].translate(
+            str.maketrans(config.username_replace_chars)
         )
+        if username != "":
+            print(f'{config.SUCCESS_COLOR}Now downloading "{username}"{config.END}')
+            download_url = f"{config.BASE_URL}/{config.category}/{username}"
+            main()
+            print(
+                f'{config.SUCCESS_COLOR}Finished \
+downloading "{username}"{config.END}'
+            )
+    if os.path.isfile(f"{config.output_folder}/index.idx"):
+        os.remove(f"{config.output_folder}/index.idx")
+    start_indexing(config.output_folder)