v1.9.0 (#114)

* IMGUR API is no longer used

* --skip now accepts file types instead of domain

* --skip-domain added

* --no-download added

* --no-dupe now supports YouTube

* Duplicates of older posts will not be dowloaded if --no-dupe and --downloaded-posts options are given together

* Invalid characters in MacOS and Linux platforms are removed from filenames

* Bug fixes

README.md

@@ -22,14 +22,6 @@ OR, regardless of your operating system, you can fire up the program from the **
 See the [Interpret from source code](docs/INTERPRET_FROM_SOURCE.md) page for more information.
 
 ## 🔨 Setting up the program
-### 🖼 IMGUR API
-
-You need to create an imgur developer app in order API to work. Go to https://api.imgur.com/oauth2/addclient and login.
-
-IMGUR will redirect you to homepage instead of API form page. After you log in, open the above link manually. Fill the form in the link (It does not really matter what you fill it with. You can write www.google.com to the callback url)
-
-After you send the form, it will redirect you to a page where it shows your **imgur_client_id** and **imgur_client_secret**. Type in those values into program respectively.
-
 ### 📽 ffmpeg Library
   
 Program needs **ffmpeg software** to add audio to some video files. However, installing it is **voluntary**. Although the program can still run with no errors without the ffmpeg library, some video files might have no sound.
@@ -113,9 +105,14 @@ Example usage: **`--limit 500`**
 ---
 
 ## **`--skip`**
-Takes a number of domains as a parameter to skip the posts from those domains. Use self to imply text posts.
+Takes a number of file types as a parameter to skip the posts from those domains. Valid file types are `images`, `videos`, `gifs`, `self`
   
-Example usage: **`--skip v.redd.it youtube.com youtu.be self`**
+Example usage: **`--skip self videos`**
+  
+## **`--skip-domain`**
+Takes a number of domains as a parameter to skip the posts from those domains.
+
+Example usage: **`--skip v.redd.it youtube.com youtu.be`**
   
 ## **`--quit`**
 Automatically quits the application after it finishes. Otherwise, it will wait for an input to quit.
@@ -167,8 +164,13 @@ Skips the same posts in different subreddits. Does not take any parameter.
 
 Example usage: **`--no-dupes`**
   
+## **`--no-download`**
+Quits the program without downloading the posts. Does not take any parameter
+
+Example usage: **`--no-download`**
+  
 ## **`--downloaded-posts`**
-Takes a file directory as a parameter and skips the posts if it matches with the post IDs inside the file. It also saves the newly downloaded posts to the given file. Does not take any parameter.
+Takes a file directory as a parameter and skips the posts if it matches with the post IDs inside the file. It also saves the newly downloaded posts to the given file.
 
 Example usage: **`--downloaded-posts D:\bdfr\ALL_POSTS.txt`**
   

requirements.txt

@@ -1,5 +1,4 @@
 bs4
 requests
 praw
-imgurpython
 youtube-dl

script.py

@@ -24,7 +24,7 @@ from src.downloaders.selfPost import SelfPost
 from src.downloaders.vreddit import VReddit
 from src.downloaders.youtube import Youtube
 from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
-from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, DomainInSkip, full_exc_info
+from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info
 from src.parser import LinkDesigner
 from src.searcher import getPosts
 from src.utils import (GLOBAL, createLogFile, nameCorrector,
@@ -38,7 +38,7 @@ from src.store import Store
 
 __author__ = "Ali Parlakci"
 __license__ = "GPL"
-__version__ = "1.8.0"
+__version__ = "1.9.0"
 __maintainer__ = "Ali Parlakci"
 __email__ = "parlakciali@gmail.com"
 
@@ -84,9 +84,6 @@ def isPostExists(POST,directory):
 
 def downloadPost(SUBMISSION,directory):
 
-    global lastRequestTime
-    lastRequestTime = 0
-
     downloaders = {
         "imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":SelfPost,
         "redgifs":Redgifs, "gifdeliverynetwork": GifDeliveryNetwork,
@@ -95,55 +92,7 @@ def downloadPost(SUBMISSION,directory):
 
     print()
     if SUBMISSION['TYPE'] in downloaders:
-
-        # WORKAROUND FOR IMGUR API LIMIT
-        if SUBMISSION['TYPE'] == "imgur":
-            
-            while int(time.time() - lastRequestTime) <= 2:
-                pass
-
-            credit = Imgur.get_credits()
-
-            IMGUR_RESET_TIME = credit['UserReset']-time.time()
-            USER_RESET = ("after " \
-                            + str(int(IMGUR_RESET_TIME/60)) \
-                            + " Minutes " \
-                            + str(int(IMGUR_RESET_TIME%60)) \
-                            + " Seconds") 
-            
-            if credit['ClientRemaining'] < 25 or credit['UserRemaining'] < 25:
-                printCredit = {"noPrint":False}
-            else:
-                printCredit = {"noPrint":True}
-
-            print(
-                "==> Client: {} - User: {} - Reset {}\n".format(
-                    credit['ClientRemaining'],
-                    credit['UserRemaining'],
-                    USER_RESET
-                ),end="",**printCredit
-            )
-
-            if not (credit['UserRemaining'] == 0 or \
-                    credit['ClientRemaining'] == 0):
-
-                """This block of code is needed for API workaround
-                """
-                while int(time.time() - lastRequestTime) <= 2:
-                    pass
-
-                lastRequestTime = time.time()
-
-            else:
-                if credit['UserRemaining'] == 0:
-                    KEYWORD = "user"
-                elif credit['ClientRemaining'] == 0:
-                    KEYWORD = "client"
-
-                raise ImgurLimitError('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()))
-
         downloaders[SUBMISSION['TYPE']] (directory,SUBMISSION)
-
     else:
         raise NoSuitablePost
 
@@ -154,8 +103,6 @@ def download(submissions):
     to download each one, catch errors, update the log files
     """
 
-    global lastRequestTime
-    lastRequestTime = 0
     downloadedCount = 0
     duplicates = 0
 
@@ -164,7 +111,6 @@ def download(submissions):
     if GLOBAL.arguments.unsave:
         reddit = Reddit(GLOBAL.config['credentials']['reddit']).begin()
 
-    submissions = list(filter(lambda x: x['POSTID'] not in GLOBAL.downloadedPosts(), submissions))
     subsLenght = len(submissions)
         
     for i in range(len(submissions)):
@@ -177,13 +123,24 @@ def download(submissions):
               end="")
         print(f" – {submissions[i]['TYPE'].upper()}",end="",noPrint=True)
 
-        details = {**submissions[i], **{"TITLE": nameCorrector(submissions[i]['TITLE'])}}
+        directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**submissions[i])
-        directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**details)
+        details = {
+            **submissions[i], 
+            **{
+                "TITLE": nameCorrector(
+                    submissions[i]['TITLE'],
+                    reference = str(directory)
+                                + GLOBAL.config['filename'].format(**submissions[i])
+                                + ".ext"
+                )
+            }
+        }
+        filename = GLOBAL.config['filename'].format(**details)
 
         if isPostExists(details,directory):
             print()
             print(directory)
-            print(GLOBAL.config['filename'].format(**details))
+            print(filename)
             print("It already exists")
             duplicates += 1
             continue
@@ -227,7 +184,7 @@ def download(submissions):
 
         except NotADownloadableLinkError as exception:
             print(
-                "{class_name}: {info} See CONSOLE_LOG.txt for more information".format(
+                "{class_name}: {info}".format(
                     class_name=exception.__class__.__name__,info=str(exception)
                 )
             )
@@ -238,27 +195,40 @@ def download(submissions):
                 submissions[i]
             ]})
 
+        except TypeInSkip:
+            print()
+            print(submissions[i]['CONTENTURL'])
+            print("Skipping post...")
+
         except DomainInSkip:
             print()
             print(submissions[i]['CONTENTURL'])
-            print("Domain found in skip domains, skipping post...")
+            print("Skipping post...")
 
         except NoSuitablePost:
             print("No match found, skipping...")
 
         except FailedToDownload:
             print("Failed to download the posts, skipping...")
+        except AlbumNotDownloadedCompletely:
+            print("Album did not downloaded completely.")
+            FAILED_FILE.add({int(i+1):[
+                "{class_name}: {info}".format(
+                    class_name=exc.__class__.__name__,info=str(exc)
+                ),
+                submissions[i]
+            ]})
         
         except Exception as exc:
             print(
-                "{class_name}: {info} See CONSOLE_LOG.txt for more information".format(
+                "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
                     class_name=exc.__class__.__name__,info=str(exc)
                 )
             )
 
             logging.error(sys.exc_info()[0].__name__,
                           exc_info=full_exc_info(sys.exc_info()))
-            print(log_stream.getvalue(),noPrint=True)
+            print(GLOBAL.log_stream.getvalue(),noPrint=True)
 
             FAILED_FILE.add({int(i+1):[
                 "{class_name}: {info}".format(
@@ -355,7 +325,7 @@ def main():
     except Exception as exc:
         logging.error(sys.exc_info()[0].__name__,
                       exc_info=full_exc_info(sys.exc_info()))
-        print(log_stream.getvalue(),noPrint=True)
+        print(GLOBAL.log_stream.getvalue(),noPrint=True)
         print(exc)
         sys.exit()
 
@@ -363,12 +333,13 @@ def main():
         print("I could not find any posts in that URL")
         sys.exit()
 
-    download(posts)
+    if GLOBAL.arguments.no_download: pass
+    else: download(posts)
 
 if __name__ == "__main__":
 
-    log_stream = StringIO()    
+    GLOBAL.log_stream = StringIO()    
-    logging.basicConfig(stream=log_stream, level=logging.INFO)
+    logging.basicConfig(stream=GLOBAL.log_stream, level=logging.INFO)
 
     try:
         VanillaPrint = print
@@ -388,6 +359,7 @@ if __name__ == "__main__":
             GLOBAL.directory = Path("..\\")
         logging.error(sys.exc_info()[0].__name__,
                       exc_info=full_exc_info(sys.exc_info()))
-        print(log_stream.getvalue())
+        print(GLOBAL.log_stream.getvalue())
 
     if not GLOBAL.arguments.quit: input("\nPress enter to quit\n")
+              

setup.py

@@ -8,7 +8,7 @@ from script import __version__
 options = {
     "build_exe": {
         "packages":[
-            "idna","imgurpython", "praw", "requests", "multiprocessing"
+            "idna", "praw", "requests", "multiprocessing"
         ]
     }
 }

src/arguments.py

@@ -102,6 +102,13 @@ class Arguments:
                             type=str)
 
         parser.add_argument("--skip",
+                            nargs="+",
+                            help="Skip posts with given type",
+                            type=str,
+                            choices=["images","videos","gifs","self"],
+                            default=[])   
+
+        parser.add_argument("--skip-domain",
                             nargs="+",
                             help="Skip posts with given domain",
                             type=str,
@@ -142,6 +149,12 @@ class Arguments:
                             type=str
                             )
 
+        parser.add_argument("--no-download",
+                            action="store_true",
+                            help="Just saved posts into a the POSTS.json file without downloading"
+                            )
+   
+
         if arguments == []:
             return parser.parse_args()
         else:

src/config.py

@@ -5,6 +5,7 @@ import random
 
 from src.reddit import Reddit
 from src.jsonHelper import JsonFile
+from src.utils import nameCorrector
 
 class Config():
 
@@ -36,7 +37,7 @@ For example: {FLAIR}_{SUBREDDIT}_{REDDITOR}
 
 Existing filename template:""", None if "filename" not in self.file.read() else self.file.read()["filename"])
 
-        filename = input(">> ").upper()
+        filename = nameCorrector(input(">> ").upper())
         self.file.add({
             "filename": filename
         })
@@ -68,7 +69,7 @@ For example: {REDDITOR}/{SUBREDDIT}/{FLAIR}
 
 Existing folder structure""", None if "folderpath" not in self.file.read() else self.file.read()["folderpath"])
 
-        folderpath = input(">> ").strip("\\").strip("/").upper()
+        folderpath = nameCorrector(input(">> ").strip("\\").strip("/").upper())
 
         self.file.add({
             "folderpath": folderpath
@@ -105,8 +106,6 @@ Existing default options:""", None if "options" not in self.file.read() else sel
     def _validateCredentials(self):
         """Read credentials from config.json file"""
 
-        keys = ['imgur_client_id',
-                'imgur_client_secret']
         try:
             content = self.file.read()["credentials"]
         except:
@@ -120,24 +119,6 @@ Existing default options:""", None if "options" not in self.file.read() else sel
         else:
             Reddit().begin()
             
-        if not all(content.get(key,False) for key in keys):
-            print(
-                "---Setting up the Imgur API---\n\n" \
-                "Go to this URL and fill the form:\n" \
-                "https://api.imgur.com/oauth2/addclient\n" \
-                "Then, enter the client id and client secret here\n" \
-                "Press Enter to open the link in the browser"
-            )
-            input()
-            webbrowser.open("https://api.imgur.com/oauth2/addclient",new=2)
-
-        for key in keys:
-            try:
-                if content[key] == "":
-                    raise KeyError
-            except KeyError:
-                self.file.add({key:input("\t"+key+": ")},
-                              "credentials")
         print()
 
     def setDefaultDirectory(self):

src/downloaders/Imgur.py

@@ -1,93 +1,95 @@
+import urllib
+import json
 import os
+import time
+import requests
 
-import imgurpython
-
-from src.downloaders.downloaderUtils import getExtension, getFile
-from src.errors import (AlbumNotDownloadedCompletely, FileAlreadyExistsError,
-                        FileNameTooLong)
 from src.utils import GLOBAL, nameCorrector
 from src.utils import printToFile as print
-
+from src.downloaders.Direct import Direct
+from src.downloaders.downloaderUtils import getFile
+from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, ExtensionError, NotADownloadableLinkError, TypeInSkip
 
 class Imgur:
-    def __init__(self,directory,post):
-        self.imgurClient = self.initImgur()
 
-        imgurID = self.getId(post['CONTENTURL'])
+    IMGUR_IMAGE_DOMAIN = "https://i.imgur.com/"
-        content = self.getLink(imgurID)
 
-        if not os.path.exists(directory): os.makedirs(directory)
+    def __init__(self,directory, post):
 
-        if content['type'] == 'image':
+        link = post['CONTENTURL']
 
-            try:
+        if link.endswith(".gifv"):
-                post['MEDIAURL'] = content['object'].mp4
+            link = link.replace(".gifv",".mp4")
-            except AttributeError:
+            Direct(directory, {**post, 'CONTENTURL': link})
-                post['MEDIAURL'] = content['object'].link
+            return None
 
-            post['EXTENSION'] = getExtension(post['MEDIAURL'])
+        self.rawData = self.getData(link)
 
-            filename = GLOBAL.config['filename'].format(**post)+post["EXTENSION"]
+        self.directory = directory
-            shortFilename = post['POSTID']+post['EXTENSION']
+        self.post = post
 
-            getFile(filename,shortFilename,directory,post['MEDIAURL'])
+        if self.isAlbum:
+            if self.rawData["album_images"]["count"] != 1:
+                self.downloadAlbum(self.rawData["album_images"])
+            else:
+                self.download(self.rawData["album_images"]["images"][0])
+        else:
+            self.download(self.rawData)
 
-        elif content['type'] == 'album':
+    def downloadAlbum(self, images):
-            images = content['object'].images
+        folderName = GLOBAL.config['filename'].format(**self.post)
-            imagesLenght = len(images)
+        folderDir = self.directory / folderName
-            howManyDownloaded = imagesLenght
+
+        imagesLenght = images["count"]
+        howManyDownloaded = 0
         duplicates = 0
 
-            filename = GLOBAL.config['filename'].format(**post)
-
-            print(filename)
-
-            folderDir = directory / filename
-
         try:
             if not os.path.exists(folderDir):
                 os.makedirs(folderDir)
         except FileNotFoundError:
-                folderDir = directory / post['POSTID']
+            folderDir = self.directory / self.post['POSTID']
             os.makedirs(folderDir)
 
+        print(folderName)
+
         for i in range(imagesLenght):
-                try:
-                    imageURL = images[i]['mp4']
-                except KeyError:
-                    imageURL = images[i]['link']
 
-                images[i]['Ext'] = getExtension(imageURL)
+            extension = self.validateExtension(images["images"][i]["ext"])
 
-                filename = (str(i+1)
+            imageURL = self.IMGUR_IMAGE_DOMAIN + images["images"][i]["hash"] + extension
-                            + "_"
-                            + nameCorrector(str(images[i]['title']))
-                            + "_"
-                            + images[i]['id'])
 
-                shortFilename = (str(i+1) + "_" + images[i]['id'])
+            filename = "_".join([
+                str(i+1), nameCorrector(images["images"][i]['title']), images["images"][i]['hash']
+            ]) + extension
+            shortFilename = str(i+1) + "_" + images["images"][i]['hash']
 
             print("\n  ({}/{})".format(i+1,imagesLenght))
 
             try:
                 getFile(filename,shortFilename,folderDir,imageURL,indent=2)
+                howManyDownloaded += 1
                 print()
+
             except FileAlreadyExistsError:
                 print("  The file already exists" + " "*10,end="\n\n")
                 duplicates += 1
-                    howManyDownloaded -= 1
+
+            except TypeInSkip:
+                print("  Skipping...")
+                howManyDownloaded += 1
 
             except Exception as exception:
                 print("\n  Could not get the file")
                 print(
                     "  "
-                        + "{class_name}: {info}".format(
+                    + "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
                         class_name=exception.__class__.__name__,
                         info=str(exception)
                     )
                     + "\n"
                 )
-                    howManyDownloaded -= 1
+                print(GLOBAL.log_stream.getvalue(),noPrint=True)
 
         if duplicates == imagesLenght:
             raise FileAlreadyExistsError
@@ -96,42 +98,45 @@ class Imgur:
                 "Album Not Downloaded Completely"
             )           
 
+    def download(self, image):        
+        extension = self.validateExtension(image["ext"])
+        imageURL = self.IMGUR_IMAGE_DOMAIN + image["hash"] + extension
+
+        filename = GLOBAL.config['filename'].format(**self.post) + extension
+        shortFilename = self.post['POSTID']+extension
+        
+        getFile(filename,shortFilename,self.directory,imageURL)
+
+    @property
+    def isAlbum(self):
+        return "album_images" in self.rawData
+
     @staticmethod 
-    def initImgur():
+    def getData(link):
-        """Initialize imgur api"""
         
-        config = GLOBAL.config
+        cookies = {"over18": "1"}
-        return imgurpython.ImgurClient(
+        res = requests.get(link, cookies=cookies)
-            config["credentials"]['imgur_client_id'],
+        if res.status_code != 200: raise ImageNotFound(f"Server responded with {res.status_code} to {link}")
-            config["credentials"]['imgur_client_secret']
+        pageSource = requests.get(link, cookies=cookies).text
-        )
-    def getId(self,submissionURL):
-        """Extract imgur post id
-        and determine if its a single image or album
-        """
 
-        if submissionURL[-1] == "/":
+        STARTING_STRING = "image               : "
-            submissionURL = submissionURL[:-1]
+        ENDING_STRING = "group               :"
 
-        if "a/" in submissionURL or "gallery/" in submissionURL:
+        STARTING_STRING_LENGHT = len(STARTING_STRING)
-            albumId = submissionURL.split("/")[-1]
+        try:
-            return {'id':albumId, 'type':'album'}
+            startIndex = pageSource.index(STARTING_STRING) + STARTING_STRING_LENGHT
+            endIndex = pageSource.index(ENDING_STRING)
+        except ValueError:
+            raise NotADownloadableLinkError(f"Could not read the page source on {link}")
 
-        else:
+        data = pageSource[startIndex:endIndex].strip()[:-1]
-            url = submissionURL.replace('.','/').split('/')
-            imageId = url[url.index('com')+1]
-            return {'id':imageId, 'type':'image'}
 
-    def getLink(self,identity):
+        return json.loads(data)
-        """Request imgur object from imgur api
-        """
 
-        if identity['type'] == 'image':
-            return {'object':self.imgurClient.get_image(identity['id']),
-                    'type':'image'}
-        elif identity['type'] == 'album':
-            return {'object':self.imgurClient.get_album(identity['id']),
-                    'type':'album'}
     @staticmethod
-    def get_credits():
+    def validateExtension(string):
-        return Imgur.initImgur().get_credits()
+        POSSIBLE_EXTENSIONS = [".jpg", ".png", ".mp4", ".gif"]
+
+        for extension in POSSIBLE_EXTENSIONS:
+            if extension in string: return extension
+        else: raise ExtensionError(f"\"{string}\" is not recognized as a valid extension.")

src/downloaders/downloaderUtils.py

@@ -8,7 +8,7 @@ import hashlib
 
 from src.utils import nameCorrector, GLOBAL
 from src.utils import printToFile as print
-from src.errors import FileAlreadyExistsError, FileNameTooLong, FailedToDownload, DomainInSkip
+from src.errors import FileAlreadyExistsError, FileNameTooLong, FailedToDownload, TypeInSkip, DomainInSkip
 
 def dlProgress(count, blockSize, totalSize):
     """Function for writing download progress to console
@@ -37,7 +37,18 @@ def getExtension(link):
 
 def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
 
-    if any(domain in imageURL for domain in GLOBAL.arguments.skip):
+    FORMATS = {
+        "videos": [".mp4", ".webm"],
+        "images": [".jpg",".jpeg",".png",".bmp"],
+        "gifs": [".gif"]
+    }
+
+    for type in GLOBAL.arguments.skip:
+        for extension in FORMATS[type]:
+            if extension in filename:
+                raise TypeInSkip
+
+    if any(domain in imageURL for domain in GLOBAL.arguments.skip_domain):
         raise DomainInSkip
 
     headers = [
@@ -52,13 +63,13 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
         ("Connection", "keep-alive")
     ]
 
+    if not os.path.exists(folderDir): os.makedirs(folderDir)
+
     opener = urllib.request.build_opener()
     if not "imgur" in imageURL:
         opener.addheaders = headers
     urllib.request.install_opener(opener)
 
-    filename = nameCorrector(filename)
-
     if not silent: print(" "*indent + str(folderDir),
                          " "*indent + str(filename),
                          sep="\n")
@@ -74,12 +85,12 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
                                            tempDir,
                                            reporthook=dlProgress)
 
-                if GLOBAL.arguments.no_dupes:
                 fileHash = createHash(tempDir)
-                    if fileHash in GLOBAL.hashList:
+                if GLOBAL.arguments.no_dupes:
+                    if fileHash in GLOBAL.downloadedPosts():
                         os.remove(tempDir)
                         raise FileAlreadyExistsError
-                    GLOBAL.hashList.add(fileHash)
+                GLOBAL.downloadedPosts.add(fileHash)
 
                 os.rename(tempDir,fileDir)
                 if not silent: print(" "*indent+"Downloaded"+" "*10)

src/downloaders/selfPost.py

@@ -2,7 +2,7 @@ import io
 import os
 from pathlib import Path
 
-from src.errors import FileAlreadyExistsError
+from src.errors import FileAlreadyExistsError, TypeInSkip
 from src.utils import GLOBAL
 
 VanillaPrint = print
@@ -10,6 +10,9 @@ from src.utils import printToFile as print
 
 class SelfPost:
     def __init__(self,directory,post):
+
+        if "self" in GLOBAL.arguments.skip: raise TypeInSkip
+
         if not os.path.exists(directory): os.makedirs(directory)
 
         filename = GLOBAL.config['filename'].format(**post)

src/downloaders/youtube.py

@@ -36,10 +36,10 @@ class Youtube:
                 fileHash = createHash(location)
             except FileNotFoundError:
                 return None
-            if fileHash in GLOBAL.hashList:
+            if fileHash in GLOBAL.downloadedPosts():
                 os.remove(location)
                 raise FileAlreadyExistsError
-            GLOBAL.hashList.add(fileHash)
+            GLOBAL.downloadedPosts.add(fileHash)
         
     @staticmethod
     def _hook(d):

src/errors.py

@@ -99,5 +99,14 @@ class InvalidJSONFile(Exception):
 class FailedToDownload(Exception):
     pass
 
+class TypeInSkip(Exception):
+    pass
+
 class DomainInSkip(Exception):
     pass
+
+class ImageNotFound(Exception):
+    pass
+
+class ExtensionError(Exception):
+    pass

src/searcher.py

@@ -201,7 +201,7 @@ def extractDetails(posts,SINGLE_POST=False):
     """
 
     postList = []
-    postCount = 0
+    postCount = 1
 
     allPosts = {}
 
@@ -227,18 +227,17 @@ def extractDetails(posts,SINGLE_POST=False):
         except AttributeError:
             pass
 
+        if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain):
             result = matchWithDownloader(submission)
 
             if result is not None:
                 details = {**details, **result}
                 postList.append(details)
-
                 postsFile.add({postCount:details})
 
     else:
         try:
             for submission in posts:
-                postCount += 1
 
                 if postCount % 100 == 0:
                     sys.stdout.write("• ")
@@ -264,6 +263,9 @@ def extractDetails(posts,SINGLE_POST=False):
                 except AttributeError:
                     continue
 
+                if details['POSTID'] in GLOBAL.downloadedPosts(): continue
+
+                if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain):
                     result = matchWithDownloader(submission)
 
                     if result is not None:
@@ -271,6 +273,8 @@ def extractDetails(posts,SINGLE_POST=False):
                         postList.append(details)
                     
                     allPosts[postCount] = details
+                    postCount += 1
+                
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt",noPrint=True)
         
@@ -284,6 +288,11 @@ def extractDetails(posts,SINGLE_POST=False):
 
 def matchWithDownloader(submission):
 
+    directLink = extractDirectLink(submission.url)
+    if directLink:
+         return {'TYPE': 'direct',
+                 'CONTENTURL': directLink}
+
     if 'v.redd.it' in submission.domain:
         bitrates = ["DASH_1080","DASH_720","DASH_600", \
                     "DASH_480","DASH_360","DASH_240"]
@@ -327,12 +336,6 @@ def matchWithDownloader(submission):
         return {'TYPE': 'self',
                 'CONTENT': submission.selftext}
 
-    try:
-        return {'TYPE': 'direct',
-                'CONTENTURL': extractDirectLink(submission.url)}
-    except DirectLinkNotFound:
-        return None        
-
 def extractDirectLink(URL):
     """Check if link is a direct image link.
     If so, return URL,
@@ -346,26 +349,8 @@ def extractDirectLink(URL):
     if "i.reddituploads.com" in URL:
         return URL
 
-    elif "v.redd.it" in URL:
-        bitrates = ["DASH_1080","DASH_720","DASH_600", \
-                    "DASH_480","DASH_360","DASH_240"]
-                    
-        for bitrate in bitrates:
-            videoURL = URL+"/"+bitrate
-
-            try:
-                responseCode = urllib.request.urlopen(videoURL).getcode()
-            except urllib.error.HTTPError:
-                responseCode = 0
-
-            if responseCode == 200:
-                return videoURL
-
-        else:
-            raise DirectLinkNotFound
-
     for extension in imageTypes:
         if extension in URL.split("/")[-1]:
             return URL
     else:
-        raise DirectLinkNotFound
+        return  None

src/store.py

@@ -17,8 +17,8 @@ class Store:
     def __call__(self):
         return self.list
 
-    def add(self, filehash):
+    def add(self, data):
-        self.list.append(filehash)
+        self.list.append(data)
         if self.directory:
             with open(self.directory, 'a') as f:
-                f.write("{filehash}\n".format(filehash=filehash))
+                f.write("{data}\n".format(data=data))

src/utils.py

@@ -18,10 +18,11 @@ class GLOBAL:
     configDirectory = ""
     reddit_client_id = "U-6gk4ZCh3IeNQ"
     reddit_client_secret = "7CZHY6AmKweZME5s50SfDGylaPg"
-    hashList = set()
     downloadedPosts = lambda: []
     printVanilla = print
 
+    log_stream= None
+
 def createLogFile(TITLE):
     """Create a log file with given name
     inside a folder time stampt in its name and
@@ -63,34 +64,32 @@ def printToFile(*args, noPrint=False,**kwargs):
         ) as FILE:
             print(*args, file=FILE, **kwargs) 
 
-def nameCorrector(string):
+def nameCorrector(string,reference=None):
     """Swap strange characters from given string 
     with underscore (_) and shorten it.
     Return the string
     """
 
-    stringLenght = len(string)
+    LIMIT = 247
-    if stringLenght > 200:
-        string = string[:200]
-    stringLenght = len(string)
-    spacesRemoved = []
 
-    for b in range(stringLenght):
+    stringLength = len(string)
-        if string[b] == " ":
+
-            spacesRemoved.append("_")
+    if reference:
+        referenceLenght = len(reference)
+        totalLenght = referenceLenght
     else:
-            spacesRemoved.append(string[b])
+        totalLenght = stringLength
 
-    string = ''.join(spacesRemoved)
+    if totalLenght > LIMIT:
+        limit = LIMIT - referenceLenght
+        string = string[:limit-1]
+
+    string = string.replace(" ", "_")
     
     if len(string.split('\n')) > 1:
         string = "".join(string.split('\n'))
     
-    BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','#']
+    BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','#', '.', '@' ,'“', '’', '\'', '!']
-    
+    string = "".join([i if i not in BAD_CHARS else "_" for i in string])
-    if any(x in string for x in BAD_CHARS):
-        for char in string:
-            if char in BAD_CHARS:
-                string = string.replace(char,"_")
 
     return string