v1.9.0 (#114)
* IMGUR API is no longer used * --skip now accepts file types instead of domain * --skip-domain added * --no-download added * --no-dupe now supports YouTube * Duplicates of older posts will not be dowloaded if --no-dupe and --downloaded-posts options are given together * Invalid characters in MacOS and Linux platforms are removed from filenames * Bug fixes
This commit is contained in:
Ali Parlakçı
110
script.py
110
script.py
@@ -24,7 +24,7 @@ from src.downloaders.selfPost import SelfPost
|
||||
from src.downloaders.vreddit import VReddit
|
||||
from src.downloaders.youtube import Youtube
|
||||
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
|
||||
from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, DomainInSkip, full_exc_info
|
||||
from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info
|
||||
from src.parser import LinkDesigner
|
||||
from src.searcher import getPosts
|
||||
from src.utils import (GLOBAL, createLogFile, nameCorrector,
|
||||
@@ -38,7 +38,7 @@ from src.store import Store
|
||||
|
||||
__author__ = "Ali Parlakci"
|
||||
__license__ = "GPL"
|
||||
__version__ = "1.8.0"
|
||||
__version__ = "1.9.0"
|
||||
__maintainer__ = "Ali Parlakci"
|
||||
__email__ = "parlakciali@gmail.com"
|
||||
|
||||
@@ -84,9 +84,6 @@ def isPostExists(POST,directory):
|
||||
|
||||
def downloadPost(SUBMISSION,directory):
|
||||
|
||||
global lastRequestTime
|
||||
lastRequestTime = 0
|
||||
|
||||
downloaders = {
|
||||
"imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":SelfPost,
|
||||
"redgifs":Redgifs, "gifdeliverynetwork": GifDeliveryNetwork,
|
||||
@@ -95,55 +92,7 @@ def downloadPost(SUBMISSION,directory):
|
||||
|
||||
print()
|
||||
if SUBMISSION['TYPE'] in downloaders:
|
||||
|
||||
# WORKAROUND FOR IMGUR API LIMIT
|
||||
if SUBMISSION['TYPE'] == "imgur":
|
||||
|
||||
while int(time.time() - lastRequestTime) <= 2:
|
||||
pass
|
||||
|
||||
credit = Imgur.get_credits()
|
||||
|
||||
IMGUR_RESET_TIME = credit['UserReset']-time.time()
|
||||
USER_RESET = ("after " \
|
||||
+ str(int(IMGUR_RESET_TIME/60)) \
|
||||
+ " Minutes " \
|
||||
+ str(int(IMGUR_RESET_TIME%60)) \
|
||||
+ " Seconds")
|
||||
|
||||
if credit['ClientRemaining'] < 25 or credit['UserRemaining'] < 25:
|
||||
printCredit = {"noPrint":False}
|
||||
else:
|
||||
printCredit = {"noPrint":True}
|
||||
|
||||
print(
|
||||
"==> Client: {} - User: {} - Reset {}\n".format(
|
||||
credit['ClientRemaining'],
|
||||
credit['UserRemaining'],
|
||||
USER_RESET
|
||||
),end="",**printCredit
|
||||
)
|
||||
|
||||
if not (credit['UserRemaining'] == 0 or \
|
||||
credit['ClientRemaining'] == 0):
|
||||
|
||||
"""This block of code is needed for API workaround
|
||||
"""
|
||||
while int(time.time() - lastRequestTime) <= 2:
|
||||
pass
|
||||
|
||||
lastRequestTime = time.time()
|
||||
|
||||
else:
|
||||
if credit['UserRemaining'] == 0:
|
||||
KEYWORD = "user"
|
||||
elif credit['ClientRemaining'] == 0:
|
||||
KEYWORD = "client"
|
||||
|
||||
raise ImgurLimitError('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()))
|
||||
|
||||
downloaders[SUBMISSION['TYPE']] (directory,SUBMISSION)
|
||||
|
||||
else:
|
||||
raise NoSuitablePost
|
||||
|
||||
@@ -154,8 +103,6 @@ def download(submissions):
|
||||
to download each one, catch errors, update the log files
|
||||
"""
|
||||
|
||||
global lastRequestTime
|
||||
lastRequestTime = 0
|
||||
downloadedCount = 0
|
||||
duplicates = 0
|
||||
|
||||
@@ -164,7 +111,6 @@ def download(submissions):
|
||||
if GLOBAL.arguments.unsave:
|
||||
reddit = Reddit(GLOBAL.config['credentials']['reddit']).begin()
|
||||
|
||||
submissions = list(filter(lambda x: x['POSTID'] not in GLOBAL.downloadedPosts(), submissions))
|
||||
subsLenght = len(submissions)
|
||||
|
||||
for i in range(len(submissions)):
|
||||
@@ -177,13 +123,24 @@ def download(submissions):
|
||||
end="")
|
||||
print(f" – {submissions[i]['TYPE'].upper()}",end="",noPrint=True)
|
||||
|
||||
details = {**submissions[i], **{"TITLE": nameCorrector(submissions[i]['TITLE'])}}
|
||||
directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**details)
|
||||
directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**submissions[i])
|
||||
details = {
|
||||
**submissions[i],
|
||||
**{
|
||||
"TITLE": nameCorrector(
|
||||
submissions[i]['TITLE'],
|
||||
reference = str(directory)
|
||||
+ GLOBAL.config['filename'].format(**submissions[i])
|
||||
+ ".ext"
|
||||
)
|
||||
}
|
||||
}
|
||||
filename = GLOBAL.config['filename'].format(**details)
|
||||
|
||||
if isPostExists(details,directory):
|
||||
print()
|
||||
print(directory)
|
||||
print(GLOBAL.config['filename'].format(**details))
|
||||
print(filename)
|
||||
print("It already exists")
|
||||
duplicates += 1
|
||||
continue
|
||||
@@ -227,7 +184,7 @@ def download(submissions):
|
||||
|
||||
except NotADownloadableLinkError as exception:
|
||||
print(
|
||||
"{class_name}: {info} See CONSOLE_LOG.txt for more information".format(
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__,info=str(exception)
|
||||
)
|
||||
)
|
||||
@@ -238,27 +195,40 @@ def download(submissions):
|
||||
submissions[i]
|
||||
]})
|
||||
|
||||
except TypeInSkip:
|
||||
print()
|
||||
print(submissions[i]['CONTENTURL'])
|
||||
print("Skipping post...")
|
||||
|
||||
except DomainInSkip:
|
||||
print()
|
||||
print(submissions[i]['CONTENTURL'])
|
||||
print("Domain found in skip domains, skipping post...")
|
||||
print("Skipping post...")
|
||||
|
||||
except NoSuitablePost:
|
||||
print("No match found, skipping...")
|
||||
|
||||
except FailedToDownload:
|
||||
print("Failed to download the posts, skipping...")
|
||||
except AlbumNotDownloadedCompletely:
|
||||
print("Album did not downloaded completely.")
|
||||
FAILED_FILE.add({int(i+1):[
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exc.__class__.__name__,info=str(exc)
|
||||
),
|
||||
submissions[i]
|
||||
]})
|
||||
|
||||
except Exception as exc:
|
||||
print(
|
||||
"{class_name}: {info} See CONSOLE_LOG.txt for more information".format(
|
||||
"{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
|
||||
class_name=exc.__class__.__name__,info=str(exc)
|
||||
)
|
||||
)
|
||||
|
||||
logging.error(sys.exc_info()[0].__name__,
|
||||
exc_info=full_exc_info(sys.exc_info()))
|
||||
print(log_stream.getvalue(),noPrint=True)
|
||||
print(GLOBAL.log_stream.getvalue(),noPrint=True)
|
||||
|
||||
FAILED_FILE.add({int(i+1):[
|
||||
"{class_name}: {info}".format(
|
||||
@@ -355,7 +325,7 @@ def main():
|
||||
except Exception as exc:
|
||||
logging.error(sys.exc_info()[0].__name__,
|
||||
exc_info=full_exc_info(sys.exc_info()))
|
||||
print(log_stream.getvalue(),noPrint=True)
|
||||
print(GLOBAL.log_stream.getvalue(),noPrint=True)
|
||||
print(exc)
|
||||
sys.exit()
|
||||
|
||||
@@ -363,12 +333,13 @@ def main():
|
||||
print("I could not find any posts in that URL")
|
||||
sys.exit()
|
||||
|
||||
download(posts)
|
||||
if GLOBAL.arguments.no_download: pass
|
||||
else: download(posts)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
log_stream = StringIO()
|
||||
logging.basicConfig(stream=log_stream, level=logging.INFO)
|
||||
GLOBAL.log_stream = StringIO()
|
||||
logging.basicConfig(stream=GLOBAL.log_stream, level=logging.INFO)
|
||||
|
||||
try:
|
||||
VanillaPrint = print
|
||||
@@ -388,6 +359,7 @@ if __name__ == "__main__":
|
||||
GLOBAL.directory = Path("..\\")
|
||||
logging.error(sys.exc_info()[0].__name__,
|
||||
exc_info=full_exc_info(sys.exc_info()))
|
||||
print(log_stream.getvalue())
|
||||
print(GLOBAL.log_stream.getvalue())
|
||||
|
||||
if not GLOBAL.arguments.quit: input("\nPress enter to quit\n")
|
||||
if not GLOBAL.arguments.quit: input("\nPress enter to quit\n")
|
||||
|
||||
Reference in New Issue
Block a user