v1.9.0 (#114)

* IMGUR API is no longer used

* --skip now accepts file types instead of domain

* --skip-domain added

* --no-download added

* --no-dupe now supports YouTube

* Duplicates of older posts will not be dowloaded if --no-dupe and --downloaded-posts options are given together

* Invalid characters in MacOS and Linux platforms are removed from filenames

* Bug fixes

src/searcher.py

@@ -201,7 +201,7 @@ def extractDetails(posts,SINGLE_POST=False):
     """
 
     postList = []
-    postCount = 0
+    postCount = 1
 
     allPosts = {}
 
@@ -227,18 +227,17 @@ def extractDetails(posts,SINGLE_POST=False):
         except AttributeError:
             pass
 
-        result = matchWithDownloader(submission)
+        if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain):
+            result = matchWithDownloader(submission)
 
-        if result is not None:
-            details = {**details, **result}
-            postList.append(details)
-
-        postsFile.add({postCount:details})
+            if result is not None:
+                details = {**details, **result}
+                postList.append(details)
+                postsFile.add({postCount:details})
 
     else:
         try:
             for submission in posts:
-                postCount += 1
 
                 if postCount % 100 == 0:
                     sys.stdout.write("• ")
@@ -264,13 +263,18 @@ def extractDetails(posts,SINGLE_POST=False):
                 except AttributeError:
                     continue
 
-                result = matchWithDownloader(submission)
+                if details['POSTID'] in GLOBAL.downloadedPosts(): continue
 
-                if result is not None:
-                    details = {**details, **result}
-                    postList.append(details)
+                if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain):
+                    result = matchWithDownloader(submission)
 
-                allPosts[postCount] = details
+                    if result is not None:
+                        details = {**details, **result}
+                        postList.append(details)
+                    
+                    allPosts[postCount] = details
+                    postCount += 1
+                
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt",noPrint=True)
         
@@ -284,6 +288,11 @@ def extractDetails(posts,SINGLE_POST=False):
 
 def matchWithDownloader(submission):
 
+    directLink = extractDirectLink(submission.url)
+    if directLink:
+         return {'TYPE': 'direct',
+                 'CONTENTURL': directLink}
+
     if 'v.redd.it' in submission.domain:
         bitrates = ["DASH_1080","DASH_720","DASH_600", \
                     "DASH_480","DASH_360","DASH_240"]
@@ -291,7 +300,7 @@ def matchWithDownloader(submission):
         for bitrate in bitrates:
             videoURL = submission.url+"/"+bitrate
 
-            try:
+            try:    
                 responseCode = urllib.request.urlopen(videoURL).getcode()
             except urllib.error.HTTPError:
                 responseCode = 0
@@ -327,12 +336,6 @@ def matchWithDownloader(submission):
         return {'TYPE': 'self',
                 'CONTENT': submission.selftext}
 
-    try:
-        return {'TYPE': 'direct',
-                'CONTENTURL': extractDirectLink(submission.url)}
-    except DirectLinkNotFound:
-        return None        
-
 def extractDirectLink(URL):
     """Check if link is a direct image link.
     If so, return URL,
@@ -346,26 +349,8 @@ def extractDirectLink(URL):
     if "i.reddituploads.com" in URL:
         return URL
 
-    elif "v.redd.it" in URL:
-        bitrates = ["DASH_1080","DASH_720","DASH_600", \
-                    "DASH_480","DASH_360","DASH_240"]
-                    
-        for bitrate in bitrates:
-            videoURL = URL+"/"+bitrate
-
-            try:
-                responseCode = urllib.request.urlopen(videoURL).getcode()
-            except urllib.error.HTTPError:
-                responseCode = 0
-
-            if responseCode == 200:
-                return videoURL
-
-        else:
-            raise DirectLinkNotFound
-
     for extension in imageTypes:
         if extension in URL.split("/")[-1]:
             return URL
     else:
-        raise DirectLinkNotFound
+        return  None