ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(maint) code clean up (#187)

Browse Source 
## bdfr

- Add the bound instance as method parameter
- Change methods not using its bound instance to staticmethods
- Fix dangerous default argument
- Refactor the comparison involving `not`
- Refactor unnecessary `else` / `elif` when `if` block has a `raise` statement
- Refactor unnecessary `else` / `elif` when `if` block has a `return` statement
- Refactor useless `else` block in the loop
- Remove implicit `object` from the base class
- Remove reimported module
- Remove unnecessary generator
- Remove unnecessary return statement
- Remove unnecessary use of comprehension
- Remove unused imports
- Use `is` to compare type of objects
- Using not x can cause unwanted results

## Dockerfile

- use a pinned Python version tag instead of latest
- leverage cached requirements

Signed-off-by: Vladislav Doster <mvdoster@gmail.com>

Co-authored-by: Ali Parlakçı <parlakciali@gmail.com>
This commit is contained in:
vlad doster
2021-02-25 03:32:06 -06:00
committed by GitHub
parent e0a2d2eda0
commit fc42afbabe
24 changed files with 781 additions and 663 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
script.py
View File

@@ -4,15 +4,12 @@
This program downloads imgur, gfycat and direct image and video links of
saved posts from a reddit account. It is written in Python 3.
"""
import argparse
import logging
import os
import sys
import time
import webbrowser
from io import StringIO
from pathlib import Path, PurePath
from pathlib import Path
from prawcore.exceptions import InsufficientScope
from src.downloaders.Direct import Direct
@@ -26,8 +23,7 @@ from src.downloaders.vreddit import VReddit
from src.downloaders.youtube import Youtube
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
from src.downloaders.gallery import gallery
from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info
from src.parser import LinkDesigner
from src.errors import ImgurLimitError, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info
from src.searcher import getPosts
from src.utils import (GLOBAL, createLogFile, nameCorrector,
printToFile)
@@ -44,6 +40,7 @@ __version__ = "1.9.4"
__maintainer__ = "Ali Parlakci"
__email__ = "parlakciali@gmail.com"
def postFromLog(fileName):
"""Analyze a log file and return a list of dictionaries containing
submissions
@@ -62,17 +59,19 @@ def postFromLog(fileName):
posts = []
for post in content:
if not content[post][-1]['TYPE'] == None:
if content[post][-1]['TYPE'] is not None:
posts.append(content[post][-1])
return posts
def isPostExists(POST,directory):
def isPostExists(POST, directory):
"""Figure out a file's name and checks if the file already exists"""
filename = GLOBAL.config['filename'].format(**POST)
possibleExtensions = [".jpg",".png",".mp4",".gif",".webm",".md",".mkv",".flv"]
possibleExtensions = [".jpg", ".png", ".mp4",
".gif", ".webm", ".md", ".mkv", ".flv"]
for extension in possibleExtensions:
@@ -81,10 +80,10 @@ def isPostExists(POST,directory):
if path.exists():
return True
else:
return False
return False
def downloadPost(SUBMISSION,directory):
def downloadPost(SUBMISSION, directory):
downloaders = {
"imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":SelfPost,
@@ -94,11 +93,10 @@ def downloadPost(SUBMISSION,directory):
print()
if SUBMISSION['TYPE'] in downloaders:
downloaders[SUBMISSION['TYPE']] (directory,SUBMISSION)
downloaders[SUBMISSION['TYPE']](directory, SUBMISSION)
else:
raise NoSuitablePost
return None
def download(submissions):
"""Analyze list of submissions and call the right function
@@ -114,32 +112,33 @@ def download(submissions):
reddit = Reddit(GLOBAL.config['credentials']['reddit']).begin()
subsLenght = len(submissions)
for i in range(len(submissions)):
print(f"\n({i+1}/{subsLenght})",end=" — ")
print(f"\n({i+1}/{subsLenght})", end=" — ")
print(submissions[i]['POSTID'],
f"r/{submissions[i]['SUBREDDIT']}",
f"u/{submissions[i]['REDDITOR']}",
submissions[i]['FLAIR'] if submissions[i]['FLAIR'] else "",
sep=" — ",
end="")
print(f" – {submissions[i]['TYPE'].upper()}",end="",noPrint=True)
print(f" – {submissions[i]['TYPE'].upper()}", end="", noPrint=True)
directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**submissions[i])
directory = GLOBAL.directory / \
GLOBAL.config["folderpath"].format(**submissions[i])
details = {
**submissions[i],
**submissions[i],
**{
"TITLE": nameCorrector(
submissions[i]['TITLE'],
reference = str(directory)
+ GLOBAL.config['filename'].format(**submissions[i])
+ ".ext"
reference=str(directory)
+ GLOBAL.config['filename'].format(**submissions[i])
+ ".ext"
)
}
}
filename = GLOBAL.config['filename'].format(**details)
if isPostExists(details,directory):
if isPostExists(details, directory):
print()
print(directory)
print(filename)
@@ -154,7 +153,7 @@ def download(submissions):
continue
try:
downloadPost(details,directory)
downloadPost(details, directory)
GLOBAL.downloadedPosts.add(details['POSTID'])
try:
if GLOBAL.arguments.unsave:
@@ -162,9 +161,9 @@ def download(submissions):
except InsufficientScope:
reddit = Reddit().begin()
reddit.submission(id=details['POSTID']).unsave()
downloadedCount += 1
except FileAlreadyExistsError:
print("It already exists")
GLOBAL.downloadedPosts.add(details['POSTID'])
@@ -172,15 +171,16 @@ def download(submissions):
except ImgurLoginError:
print(
"Imgur login failed. \nQuitting the program "\
"Imgur login failed. \nQuitting the program "
"as unexpected errors might occur."
)
sys.exit()
except ImgurLimitError as exception:
FAILED_FILE.add({int(i+1):[
FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
class_name=exception.__class__.__name__, info=str(
exception)
),
details
]})
@@ -188,12 +188,14 @@ def download(submissions):
except NotADownloadableLinkError as exception:
print(
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
class_name=exception.__class__.__name__, info=str(
exception)
)
)
FAILED_FILE.add({int(i+1):[
FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
class_name=exception.__class__.__name__, info=str(
exception)
),
submissions[i]
]})
@@ -215,61 +217,64 @@ def download(submissions):
print("Failed to download the posts, skipping...")
except AlbumNotDownloadedCompletely:
print("Album did not downloaded completely.")
FAILED_FILE.add({int(i+1):[
FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format(
class_name=exc.__class__.__name__,info=str(exc)
class_name=exc.__class__.__name__, info=str(exc)
),
submissions[i]
]})
except Exception as exc:
print(
"{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
class_name=exc.__class__.__name__,info=str(exc)
class_name=exc.__class__.__name__, info=str(exc)
)
)
logging.error(sys.exc_info()[0].__name__,
exc_info=full_exc_info(sys.exc_info()))
print(GLOBAL.log_stream.getvalue(),noPrint=True)
print(GLOBAL.log_stream.getvalue(), noPrint=True)
FAILED_FILE.add({int(i+1):[
FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format(
class_name=exc.__class__.__name__,info=str(exc)
class_name=exc.__class__.__name__, info=str(exc)
),
submissions[i]
]})
if duplicates:
print(f"\nThere {'were' if duplicates > 1 else 'was'} " \
print(f"\nThere {'were' if duplicates > 1 else 'was'} "
f"{duplicates} duplicate{'s' if duplicates > 1 else ''}")
if downloadedCount == 0:
print("Nothing is downloaded :(")
else:
print(f"Total of {downloadedCount} " \
if downloadedCount:
print(f"Total of {downloadedCount} "
f"link{'s' if downloadedCount > 1 else ''} downloaded!")
else:
print("Nothing is downloaded :(")
def printLogo():
VanillaPrint(
f"\nBulk Downloader for Reddit v{__version__}\n" \
f"Written by Ali PARLAKCI – parlakciali@gmail.com\n\n" \
f"\nBulk Downloader for Reddit v{__version__}\n"
f"Written by Ali PARLAKCI – parlakciali@gmail.com\n\n"
f"https://github.com/aliparlakci/bulk-downloader-for-reddit/\n"
)
def main():
if Path("config.json").exists():
GLOBAL.configDirectory = Path("config.json")
else:
if not Path(GLOBAL.defaultConfigDirectory).is_dir():
os.makedirs(GLOBAL.defaultConfigDirectory)
GLOBAL.configDirectory = GLOBAL.defaultConfigDirectory / "config.json"
GLOBAL.configDirectory = GLOBAL.defaultConfigDirectory / "config.json"
try:
GLOBAL.config = Config(GLOBAL.configDirectory).generate()
except InvalidJSONFile as exception:
VanillaPrint(str(exception.__class__.__name__),">>",str(exception))
VanillaPrint(str(exception.__class__.__name__), ">>", str(exception))
VanillaPrint("Resolve it or remove it to proceed")
input("\nPress enter to quit")
sys.exit()
@@ -298,11 +303,12 @@ def main():
if arguments.use_local_config:
JsonFile("config.json").add(GLOBAL.config)
sys.exit()
if arguments.directory:
GLOBAL.directory = Path(arguments.directory.strip())
elif "default_directory" in GLOBAL.config and GLOBAL.config["default_directory"] != "":
GLOBAL.directory = Path(GLOBAL.config["default_directory"].format(time=GLOBAL.RUN_TIME))
GLOBAL.directory = Path(
GLOBAL.config["default_directory"].format(time=GLOBAL.RUN_TIME))
else:
GLOBAL.directory = Path(input("\ndownload directory: ").strip())
@@ -312,14 +318,13 @@ def main():
GLOBAL.downloadedPosts = Store()
printLogo()
print("\n"," ".join(sys.argv),"\n",noPrint=True)
print("\n", " ".join(sys.argv), "\n", noPrint=True)
if arguments.log is not None:
logDir = Path(arguments.log)
download(postFromLog(logDir))
sys.exit()
programMode = ProgramMode(arguments).generate()
try:
@@ -327,7 +332,7 @@ def main():
except Exception as exc:
logging.error(sys.exc_info()[0].__name__,
exc_info=full_exc_info(sys.exc_info()))
print(GLOBAL.log_stream.getvalue(),noPrint=True)
print(GLOBAL.log_stream.getvalue(), noPrint=True)
print(exc)
sys.exit()
@@ -335,27 +340,30 @@ def main():
print("I could not find any posts in that URL")
sys.exit()
if GLOBAL.arguments.no_download: pass
else: download(posts)
if GLOBAL.arguments.no_download:
pass
else:
download(posts)
if __name__ == "__main__":
GLOBAL.log_stream = StringIO()
GLOBAL.log_stream = StringIO()
logging.basicConfig(stream=GLOBAL.log_stream, level=logging.INFO)
try:
VanillaPrint = print
print = printToFile
GLOBAL.RUN_TIME = str(time.strftime(
"%d-%m-%Y_%H-%M-%S",
time.localtime(time.time())
))
"%d-%m-%Y_%H-%M-%S",
time.localtime(time.time())
))
main()
except KeyboardInterrupt:
if GLOBAL.directory is None:
GLOBAL.directory = Path("..\\")
except Exception as exception:
if GLOBAL.directory is None:
GLOBAL.directory = Path("..\\")
@@ -363,5 +371,5 @@ if __name__ == "__main__":
exc_info=full_exc_info(sys.exc_info()))
print(GLOBAL.log_stream.getvalue())
if not GLOBAL.arguments.quit: input("\nPress enter to quit\n")
if not GLOBAL.arguments.quit:
input("\nPress enter to quit\n")