Pep8 format (#184)
* Format file to be PEP8 compliant * Remove unused imports * Format file to PEP8 * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Format file to PEP8 * Format file to PEP8 * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Format file to PEP8 * Remove unused imports * Format file to PEP8 * Remove unused imports * Condense spacing
This commit is contained in:
Serene
183
script.py
183
script.py
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
This program downloads imgur, gfycat and direct image and video links of
|
||||
This program downloads imgur, gfycat and direct image and video links of
|
||||
saved posts from a reddit account. It is written in Python 3.
|
||||
"""
|
||||
import logging
|
||||
@@ -12,26 +12,28 @@ from io import StringIO
|
||||
from pathlib import Path
|
||||
from prawcore.exceptions import InsufficientScope
|
||||
|
||||
from src.arguments import Arguments
|
||||
from src.config import Config
|
||||
from src.downloaders.Direct import Direct
|
||||
from src.downloaders.Erome import Erome
|
||||
from src.downloaders.gallery import Gallery
|
||||
from src.downloaders.Gfycat import Gfycat
|
||||
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
|
||||
from src.downloaders.Imgur import Imgur
|
||||
from src.downloaders.Gallery import Gallery
|
||||
from src.downloaders.redgifs import Redgifs
|
||||
from src.downloaders.selfPost import SelfPost
|
||||
from src.downloaders.vreddit import VReddit
|
||||
from src.downloaders.youtube import Youtube
|
||||
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
|
||||
from src.errors import ImgurLimitError, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info
|
||||
from src.searcher import getPosts
|
||||
from src.utils import (GLOBAL, createLogFile, nameCorrector,
|
||||
printToFile)
|
||||
from src.errors import (AlbumNotDownloadedCompletely, DomainInSkip, FailedToDownload, FileAlreadyExistsError,
|
||||
ImgurLimitError, ImgurLoginError, InvalidJSONFile, NoSuitablePost, NotADownloadableLinkError,
|
||||
TypeInSkip, full_exc_info)
|
||||
from src.jsonHelper import JsonFile
|
||||
from src.config import Config
|
||||
from src.arguments import Arguments
|
||||
from src.programMode import ProgramMode
|
||||
from src.reddit import Reddit
|
||||
from src.searcher import getPosts
|
||||
from src.store import Store
|
||||
from src.utils import GLOBAL, createLogFile, nameCorrector, printToFile
|
||||
|
||||
from time import sleep
|
||||
|
||||
@@ -42,12 +44,12 @@ __maintainer__ = "Ali Parlakci"
|
||||
__email__ = "parlakciali@gmail.com"
|
||||
|
||||
|
||||
def postFromLog(fileName):
|
||||
def postFromLog(filename):
|
||||
"""Analyze a log file and return a list of dictionaries containing
|
||||
submissions
|
||||
"""
|
||||
if Path.is_file(Path(fileName)):
|
||||
content = JsonFile(fileName).read()
|
||||
if Path.is_file(Path(filename)):
|
||||
content = JsonFile(filename).read()
|
||||
else:
|
||||
print("File not found")
|
||||
sys.exit()
|
||||
@@ -60,23 +62,22 @@ def postFromLog(fileName):
|
||||
posts = []
|
||||
|
||||
for post in content:
|
||||
if content[post][-1]['TYPE'] is not None:
|
||||
if not content[post][-1]['TYPE'] is None:
|
||||
posts.append(content[post][-1])
|
||||
|
||||
return posts
|
||||
|
||||
|
||||
def isPostExists(POST, directory):
|
||||
def isPostExists(post, directory):
|
||||
"""Figure out a file's name and checks if the file already exists"""
|
||||
|
||||
filename = GLOBAL.config['filename'].format(**POST)
|
||||
filename = GLOBAL.config['filename'].format(**post)
|
||||
|
||||
possibleExtensions = [".jpg", ".png", ".mp4",
|
||||
".gif", ".webm", ".md", ".mkv", ".flv"]
|
||||
possible_extensions = [".jpg", ".png", ".mp4", ".gif", ".webm", ".md", ".mkv", ".flv"]
|
||||
|
||||
for extension in possibleExtensions:
|
||||
for extension in possible_extensions:
|
||||
|
||||
path = directory / Path(filename+extension)
|
||||
path = directory / Path(filename + extension)
|
||||
|
||||
if path.exists():
|
||||
return True
|
||||
@@ -84,58 +85,57 @@ def isPostExists(POST, directory):
|
||||
return False
|
||||
|
||||
|
||||
def downloadPost(SUBMISSION, directory):
|
||||
|
||||
def downloadPost(submission, directory):
|
||||
|
||||
downloaders = {
|
||||
"imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":SelfPost,
|
||||
"redgifs":Redgifs, "gifdeliverynetwork": GifDeliveryNetwork,
|
||||
"imgur": Imgur, "gfycat": Gfycat, "erome": Erome, "direct": Direct, "self": SelfPost,
|
||||
"redgifs": Redgifs, "gifdeliverynetwork": GifDeliveryNetwork,
|
||||
"v.redd.it": VReddit, "youtube": Youtube, "gallery": Gallery
|
||||
}
|
||||
|
||||
print()
|
||||
if SUBMISSION['TYPE'] in downloaders:
|
||||
downloaders[SUBMISSION['TYPE']](directory, SUBMISSION)
|
||||
if submission['TYPE'] in downloaders:
|
||||
downloaders[submission['TYPE']](directory, submission)
|
||||
else:
|
||||
raise NoSuitablePost
|
||||
|
||||
|
||||
|
||||
def download(submissions):
|
||||
"""Analyze list of submissions and call the right function
|
||||
to download each one, catch errors, update the log files
|
||||
"""
|
||||
|
||||
downloadedCount = 0
|
||||
downloaded_count = 0
|
||||
duplicates = 0
|
||||
|
||||
FAILED_FILE = createLogFile("FAILED")
|
||||
failed_file = createLogFile("FAILED")
|
||||
|
||||
if GLOBAL.arguments.unsave:
|
||||
reddit = Reddit(GLOBAL.config['credentials']['reddit']).begin()
|
||||
|
||||
subsLenght = len(submissions)
|
||||
subs_length = len(submissions)
|
||||
|
||||
for i in range(len(submissions)):
|
||||
print(f"\n({i+1}/{subsLenght})", end=" — ")
|
||||
print(f"\n({i+1}/{subs_length})", end=" — ")
|
||||
print(submissions[i]['POSTID'],
|
||||
f"r/{submissions[i]['SUBREDDIT']}",
|
||||
f"u/{submissions[i]['REDDITOR']}",
|
||||
submissions[i]['FLAIR'] if submissions[i]['FLAIR'] else "",
|
||||
sep=" — ",
|
||||
end="")
|
||||
print(f" – {submissions[i]['TYPE'].upper()}", end="", noPrint=True)
|
||||
print(f" – {submissions[i]['TYPE'].upper()}", end="", no_print=True)
|
||||
|
||||
directory = GLOBAL.directory / \
|
||||
GLOBAL.config["folderpath"].format(**submissions[i])
|
||||
details = {
|
||||
**submissions[i],
|
||||
**{
|
||||
"TITLE": nameCorrector(
|
||||
submissions[i]['TITLE'],
|
||||
reference=str(directory)
|
||||
+ GLOBAL.config['filename'].format(**submissions[i])
|
||||
+ ".ext"
|
||||
)
|
||||
}
|
||||
**{"TITLE": nameCorrector(
|
||||
submissions[i]['TITLE'],
|
||||
reference=str(directory)
|
||||
+ GLOBAL.config['filename'].format(**submissions[i])
|
||||
+ ".ext")}
|
||||
}
|
||||
filename = GLOBAL.config['filename'].format(**details)
|
||||
|
||||
@@ -164,11 +164,7 @@ def download(submissions):
|
||||
reddit = Reddit().begin()
|
||||
reddit.submission(id=details['POSTID']).unsave()
|
||||
|
||||
if GLOBAL.arguments.download_delay:
|
||||
print(f"Delaying next download for {GLOBAL.arguments.download_delay} seconds...")
|
||||
sleep(GLOBAL.arguments.download_delay)
|
||||
|
||||
downloadedCount += 1
|
||||
downloaded_count += 1
|
||||
|
||||
except FileAlreadyExistsError:
|
||||
print("It already exists")
|
||||
@@ -176,33 +172,18 @@ def download(submissions):
|
||||
duplicates += 1
|
||||
|
||||
except ImgurLoginError:
|
||||
print(
|
||||
"Imgur login failed. \nQuitting the program "
|
||||
"as unexpected errors might occur."
|
||||
)
|
||||
print("Imgur login failed. \nQuitting the program as unexpected errors might occur.")
|
||||
sys.exit()
|
||||
|
||||
except ImgurLimitError as exception:
|
||||
FAILED_FILE.add({int(i+1): [
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__, info=str(
|
||||
exception)
|
||||
),
|
||||
details
|
||||
failed_file.add({int(i + 1): [
|
||||
"{class_name}: {info}".format(class_name=exception.__class__.__name__, info=str(exception)), details
|
||||
]})
|
||||
|
||||
except NotADownloadableLinkError as exception:
|
||||
print(
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__, info=str(
|
||||
exception)
|
||||
)
|
||||
)
|
||||
FAILED_FILE.add({int(i+1): [
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__, info=str(
|
||||
exception)
|
||||
),
|
||||
print("{class_name}: {info}".format(class_name=exception.__class__.__name__, info=str(exception)))
|
||||
failed_file.add({int(i + 1): [
|
||||
"{class_name}: {info}".format(class_name=exception.__class__.__name__, info=str(exception)),
|
||||
submissions[i]
|
||||
]})
|
||||
|
||||
@@ -223,60 +204,48 @@ def download(submissions):
|
||||
print("Failed to download the posts, skipping...")
|
||||
except AlbumNotDownloadedCompletely:
|
||||
print("Album did not downloaded completely.")
|
||||
FAILED_FILE.add({int(i+1): [
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exc.__class__.__name__, info=str(exc)
|
||||
),
|
||||
failed_file.add({int(i + 1): [
|
||||
"{class_name}: {info}".format(class_name=exc.__class__.__name__, info=str(exc)),
|
||||
submissions[i]
|
||||
]})
|
||||
|
||||
except Exception as exc:
|
||||
print(
|
||||
"{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
|
||||
class_name=exc.__class__.__name__, info=str(exc)
|
||||
)
|
||||
print("{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
|
||||
class_name=exc.__class__.__name__, info=str(exc))
|
||||
)
|
||||
|
||||
logging.error(sys.exc_info()[0].__name__,
|
||||
exc_info=full_exc_info(sys.exc_info()))
|
||||
print(GLOBAL.log_stream.getvalue(), noPrint=True)
|
||||
logging.error(sys.exc_info()[0].__name__, exc_info=full_exc_info(sys.exc_info()))
|
||||
print(GLOBAL.log_stream.getvalue(), no_print=True)
|
||||
|
||||
FAILED_FILE.add({int(i+1): [
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exc.__class__.__name__, info=str(exc)
|
||||
),
|
||||
failed_file.add({int(i + 1): [
|
||||
"{class_name}: {info}".format(class_name=exc.__class__.__name__, info=str(exc)),
|
||||
submissions[i]
|
||||
]})
|
||||
|
||||
if duplicates:
|
||||
print(f"\nThere {'were' if duplicates > 1 else 'was'} "
|
||||
f"{duplicates} duplicate{'s' if duplicates > 1 else ''}")
|
||||
print(f"\nThere {'were' if duplicates > 1 else 'was'} {duplicates} duplicate{'s' if duplicates > 1 else ''}")
|
||||
|
||||
if downloadedCount:
|
||||
print(f"Total of {downloadedCount} "
|
||||
f"link{'s' if downloadedCount > 1 else ''} downloaded!")
|
||||
if downloaded_count == 0:
|
||||
print("Nothing is downloaded :(")
|
||||
|
||||
else:
|
||||
print("Nothing is downloaded :(")
|
||||
print(f"Total of {downloaded_count} link{'s' if downloaded_count > 1 else ''} downloaded!")
|
||||
|
||||
|
||||
def printLogo():
|
||||
VanillaPrint(f"\nBulk Downloader for Reddit v{__version__}\n"
|
||||
f"Written by Ali PARLAKCI – parlakciali@gmail.com\n\n"
|
||||
f"https://github.com/aliparlakci/bulk-downloader-for-reddit/\n"
|
||||
)
|
||||
|
||||
VanillaPrint(
|
||||
f"\nBulk Downloader for Reddit v{__version__}\n"
|
||||
f"Written by Ali PARLAKCI – parlakciali@gmail.com\n\n"
|
||||
f"https://github.com/aliparlakci/bulk-downloader-for-reddit/\n"
|
||||
)
|
||||
|
||||
def main():
|
||||
|
||||
if not Path(GLOBAL.defaultConfigDirectory).is_dir():
|
||||
os.makedirs(GLOBAL.defaultConfigDirectory)
|
||||
|
||||
if Path("config.json").exists():
|
||||
GLOBAL.configDirectory = Path("config.json")
|
||||
else:
|
||||
GLOBAL.configDirectory = GLOBAL.defaultConfigDirectory / "config.json"
|
||||
if not Path(GLOBAL.defaultConfigDirectory).is_dir():
|
||||
os.makedirs(GLOBAL.defaultConfigDirectory)
|
||||
GLOBAL.configDirectory = GLOBAL.defaultConfigDirectory / "config.json"
|
||||
try:
|
||||
GLOBAL.config = Config(GLOBAL.configDirectory).generate()
|
||||
except InvalidJSONFile as exception:
|
||||
@@ -307,6 +276,7 @@ def main():
|
||||
|
||||
if arguments.use_local_config:
|
||||
JsonFile("config.json").add(GLOBAL.config)
|
||||
sys.exit()
|
||||
|
||||
if arguments.directory:
|
||||
GLOBAL.directory = Path(arguments.directory.strip())
|
||||
@@ -322,21 +292,20 @@ def main():
|
||||
GLOBAL.downloadedPosts = Store()
|
||||
|
||||
printLogo()
|
||||
print("\n", " ".join(sys.argv), "\n", noPrint=True)
|
||||
print("\n", " ".join(sys.argv), "\n", no_print=True)
|
||||
|
||||
if arguments.log is not None:
|
||||
logDir = Path(arguments.log)
|
||||
download(postFromLog(logDir))
|
||||
log_dir = Path(arguments.log)
|
||||
download(postFromLog(log_dir))
|
||||
sys.exit()
|
||||
|
||||
programMode = ProgramMode(arguments).generate()
|
||||
program_mode = ProgramMode(arguments).generate()
|
||||
|
||||
try:
|
||||
posts = getPosts(programMode)
|
||||
posts = getPosts(program_mode)
|
||||
except Exception as exc:
|
||||
logging.error(sys.exc_info()[0].__name__,
|
||||
exc_info=full_exc_info(sys.exc_info()))
|
||||
print(GLOBAL.log_stream.getvalue(), noPrint=True)
|
||||
logging.error(sys.exc_info()[0].__name__, exc_info=full_exc_info(sys.exc_info()))
|
||||
print(GLOBAL.log_stream.getvalue(), no_print=True)
|
||||
print(exc)
|
||||
sys.exit()
|
||||
|
||||
@@ -358,10 +327,7 @@ if __name__ == "__main__":
|
||||
try:
|
||||
VanillaPrint = print
|
||||
print = printToFile
|
||||
GLOBAL.RUN_TIME = str(time.strftime(
|
||||
"%d-%m-%Y_%H-%M-%S",
|
||||
time.localtime(time.time())
|
||||
))
|
||||
GLOBAL.RUN_TIME = str(time.strftime("%d-%m-%Y_%H-%M-%S", time.localtime(time.time())))
|
||||
main()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
@@ -371,9 +337,8 @@ if __name__ == "__main__":
|
||||
except Exception as exception:
|
||||
if GLOBAL.directory is None:
|
||||
GLOBAL.directory = Path("..\\")
|
||||
logging.error(sys.exc_info()[0].__name__,
|
||||
exc_info=full_exc_info(sys.exc_info()))
|
||||
logging.error(sys.exc_info()[0].__name__, exc_info=full_exc_info(sys.exc_info()))
|
||||
print(GLOBAL.log_stream.getvalue())
|
||||
|
||||
if not GLOBAL.arguments.quit: input("\nPress enter to quit\n")
|
||||
|
||||
if not GLOBAL.arguments.quit:
|
||||
input("\nPress enter to quit\n")
|
||||
|
||||
Reference in New Issue
Block a user