ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(maint) code clean up (#187)

Browse Source 
## bdfr

- Add the bound instance as method parameter
- Change methods not using its bound instance to staticmethods
- Fix dangerous default argument
- Refactor the comparison involving `not`
- Refactor unnecessary `else` / `elif` when `if` block has a `raise` statement
- Refactor unnecessary `else` / `elif` when `if` block has a `return` statement
- Refactor useless `else` block in the loop
- Remove implicit `object` from the base class
- Remove reimported module
- Remove unnecessary generator
- Remove unnecessary return statement
- Remove unnecessary use of comprehension
- Remove unused imports
- Use `is` to compare type of objects
- Using not x can cause unwanted results

## Dockerfile

- use a pinned Python version tag instead of latest
- leverage cached requirements

Signed-off-by: Vladislav Doster <mvdoster@gmail.com>

Co-authored-by: Ali Parlakçı <parlakciali@gmail.com>
This commit is contained in:
vlad doster
2021-02-25 03:32:06 -06:00
committed by GitHub
parent e0a2d2eda0
commit fc42afbabe
24 changed files with 781 additions and 663 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
Dockerfile
View File

@@ -1,9 +1,31 @@
FROM python:latest # Bulk Downloader for Reddit
#
# VERSION 0.0.1
FROM python:3.8-slim-buster
LABEL Description="This image enables running Buld Downloader for Reddit with in a container environment" Version="0.0.1"
ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1
WORKDIR "/root/Bulk Downloader for Reddit"
COPY ./requirements.txt ./
RUN ["pip", "install", "-r", "requirements.txt"]
EXPOSE 8080 EXPOSE 8080
EXPOSE 7634 EXPOSE 7634
CMD ["python", "script.py", "-d", "downloads"] # Install dependencies for building Python packages
RUN apt-get update \
&& apt-get install -y build-essential \
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
&& rm -rf /var/lib/apt/lists/*
# Requirements are installed here to ensure they will be cached.
COPY requirements.txt /requirements.txt
RUN pip install --no-cache-dir -r /requirements.txt \
&& rm -rf /requirements.txt
# Copy project files into container
COPY . /bdfr
WORKDIR /bdfr
# This is useful because the image name can double as a reference to the binary
ENTRYPOINT ["python", "script.py"]
CMD ["--help"]

9
docker-compose.yml
View File

@@ -1,12 +1,17 @@
version: "3" version: "3"
services: services:
bdfr: bdfr:
build:
context: .
dockerfile: ./Dockerfile
image: bdfr image: bdfr
build: . container_name: bdfr
ports: ports:
- "8080:8080" - "8080:8080"
- "7634:7634" - "7634:7634"
volumes: volumes:
- "./:/root/Bulk Downloader for Reddit" - .:/bdfr:z
container_name: bdfr_container container_name: bdfr_container
network_mode: bridge network_mode: bridge

66
script.py
View File

@@ -4,15 +4,12 @@
This program downloads imgur, gfycat and direct image and video links of This program downloads imgur, gfycat and direct image and video links of
saved posts from a reddit account. It is written in Python 3. saved posts from a reddit account. It is written in Python 3.
""" """
import argparse
import logging import logging
import os import os
import sys import sys
import time import time
import webbrowser
from io import StringIO from io import StringIO
from pathlib import Path, PurePath from pathlib import Path
from prawcore.exceptions import InsufficientScope from prawcore.exceptions import InsufficientScope
from src.downloaders.Direct import Direct from src.downloaders.Direct import Direct
@@ -26,8 +23,7 @@ from src.downloaders.vreddit import VReddit
from src.downloaders.youtube import Youtube from src.downloaders.youtube import Youtube
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
from src.downloaders.gallery import gallery from src.downloaders.gallery import gallery
from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info from src.errors import ImgurLimitError, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info
from src.parser import LinkDesigner
from src.searcher import getPosts from src.searcher import getPosts
from src.utils import (GLOBAL, createLogFile, nameCorrector, from src.utils import (GLOBAL, createLogFile, nameCorrector,
printToFile) printToFile)
@@ -44,6 +40,7 @@ __version__ = "1.9.4"
__maintainer__ = "Ali Parlakci" __maintainer__ = "Ali Parlakci"
__email__ = "parlakciali@gmail.com" __email__ = "parlakciali@gmail.com"
def postFromLog(fileName): def postFromLog(fileName):
"""Analyze a log file and return a list of dictionaries containing """Analyze a log file and return a list of dictionaries containing
submissions submissions
@@ -62,17 +59,19 @@ def postFromLog(fileName):
posts = [] posts = []
for post in content: for post in content:
if not content[post][-1]['TYPE'] == None: if content[post][-1]['TYPE'] is not None:
posts.append(content[post][-1]) posts.append(content[post][-1])
return posts return posts
def isPostExists(POST, directory): def isPostExists(POST, directory):
"""Figure out a file's name and checks if the file already exists""" """Figure out a file's name and checks if the file already exists"""
filename = GLOBAL.config['filename'].format(**POST) filename = GLOBAL.config['filename'].format(**POST)
possibleExtensions = [".jpg",".png",".mp4",".gif",".webm",".md",".mkv",".flv"] possibleExtensions = [".jpg", ".png", ".mp4",
".gif", ".webm", ".md", ".mkv", ".flv"]
for extension in possibleExtensions: for extension in possibleExtensions:
@@ -81,9 +80,9 @@ def isPostExists(POST,directory):
if path.exists(): if path.exists():
return True return True
else:
return False return False
def downloadPost(SUBMISSION, directory): def downloadPost(SUBMISSION, directory):
downloaders = { downloaders = {
@@ -98,7 +97,6 @@ def downloadPost(SUBMISSION,directory):
else: else:
raise NoSuitablePost raise NoSuitablePost
return None
def download(submissions): def download(submissions):
"""Analyze list of submissions and call the right function """Analyze list of submissions and call the right function
@@ -125,7 +123,8 @@ def download(submissions):
end="") end="")
print(f" – {submissions[i]['TYPE'].upper()}", end="", noPrint=True) print(f" – {submissions[i]['TYPE'].upper()}", end="", noPrint=True)
directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**submissions[i]) directory = GLOBAL.directory / \
GLOBAL.config["folderpath"].format(**submissions[i])
details = { details = {
**submissions[i], **submissions[i],
**{ **{
@@ -172,7 +171,7 @@ def download(submissions):
except ImgurLoginError: except ImgurLoginError:
print( print(
"Imgur login failed. \nQuitting the program "\ "Imgur login failed. \nQuitting the program "
"as unexpected errors might occur." "as unexpected errors might occur."
) )
sys.exit() sys.exit()
@@ -180,7 +179,8 @@ def download(submissions):
except ImgurLimitError as exception: except ImgurLimitError as exception:
FAILED_FILE.add({int(i+1): [ FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format( "{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception) class_name=exception.__class__.__name__, info=str(
exception)
), ),
details details
]}) ]})
@@ -188,12 +188,14 @@ def download(submissions):
except NotADownloadableLinkError as exception: except NotADownloadableLinkError as exception:
print( print(
"{class_name}: {info}".format( "{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception) class_name=exception.__class__.__name__, info=str(
exception)
) )
) )
FAILED_FILE.add({int(i+1): [ FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format( "{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception) class_name=exception.__class__.__name__, info=str(
exception)
), ),
submissions[i] submissions[i]
]}) ]})
@@ -241,23 +243,26 @@ def download(submissions):
]}) ]})
if duplicates: if duplicates:
print(f"\nThere {'were' if duplicates > 1 else 'was'} " \ print(f"\nThere {'were' if duplicates > 1 else 'was'} "
f"{duplicates} duplicate{'s' if duplicates > 1 else ''}") f"{duplicates} duplicate{'s' if duplicates > 1 else ''}")
if downloadedCount == 0: if downloadedCount:
print("Nothing is downloaded :(") print(f"Total of {downloadedCount} "
else:
print(f"Total of {downloadedCount} " \
f"link{'s' if downloadedCount > 1 else ''} downloaded!") f"link{'s' if downloadedCount > 1 else ''} downloaded!")
else:
print("Nothing is downloaded :(")
def printLogo(): def printLogo():
VanillaPrint( VanillaPrint(
f"\nBulk Downloader for Reddit v{__version__}\n" \ f"\nBulk Downloader for Reddit v{__version__}\n"
f"Written by Ali PARLAKCI – parlakciali@gmail.com\n\n" \ f"Written by Ali PARLAKCI – parlakciali@gmail.com\n\n"
f"https://github.com/aliparlakci/bulk-downloader-for-reddit/\n" f"https://github.com/aliparlakci/bulk-downloader-for-reddit/\n"
) )
def main(): def main():
if Path("config.json").exists(): if Path("config.json").exists():
@@ -302,7 +307,8 @@ def main():
if arguments.directory: if arguments.directory:
GLOBAL.directory = Path(arguments.directory.strip()) GLOBAL.directory = Path(arguments.directory.strip())
elif "default_directory" in GLOBAL.config and GLOBAL.config["default_directory"] != "": elif "default_directory" in GLOBAL.config and GLOBAL.config["default_directory"] != "":
GLOBAL.directory = Path(GLOBAL.config["default_directory"].format(time=GLOBAL.RUN_TIME)) GLOBAL.directory = Path(
GLOBAL.config["default_directory"].format(time=GLOBAL.RUN_TIME))
else: else:
GLOBAL.directory = Path(input("\ndownload directory: ").strip()) GLOBAL.directory = Path(input("\ndownload directory: ").strip())
@@ -319,7 +325,6 @@ def main():
download(postFromLog(logDir)) download(postFromLog(logDir))
sys.exit() sys.exit()
programMode = ProgramMode(arguments).generate() programMode = ProgramMode(arguments).generate()
try: try:
@@ -335,8 +340,11 @@ def main():
print("I could not find any posts in that URL") print("I could not find any posts in that URL")
sys.exit() sys.exit()
if GLOBAL.arguments.no_download: pass if GLOBAL.arguments.no_download:
else: download(posts) pass
else:
download(posts)
if __name__ == "__main__": if __name__ == "__main__":
@@ -363,5 +371,5 @@ if __name__ == "__main__":
exc_info=full_exc_info(sys.exc_info())) exc_info=full_exc_info(sys.exc_info()))
print(GLOBAL.log_stream.getvalue()) print(GLOBAL.log_stream.getvalue())
if not GLOBAL.arguments.quit: input("\nPress enter to quit\n") if not GLOBAL.arguments.quit:
input("\nPress enter to quit\n")

62
src/arguments.py
View File

@@ -1,17 +1,20 @@
import argparse import argparse
import sys import sys
class Arguments: class Arguments:
@staticmethod @staticmethod
def parse(arguments=[]): def parse(arguments=None):
"""Initialize argparse and add arguments""" """Initialize argparse and add arguments"""
if arguments is None:
arguments = []
parser = argparse.ArgumentParser(allow_abbrev=False, parser = argparse.ArgumentParser(allow_abbrev=False,
description="This program downloads " \ description="This program downloads "
"media from reddit " \ "media from reddit "
"posts") "posts")
parser.add_argument("--directory", "-d", parser.add_argument("--directory", "-d",
help="Specifies the directory where posts will be " \ help="Specifies the directory where posts will be "
"downloaded to", "downloaded to",
metavar="DIRECTORY") metavar="DIRECTORY")
@@ -47,40 +50,42 @@ class Arguments:
help="Gets upvoted posts of --user") help="Gets upvoted posts of --user")
parser.add_argument("--log", parser.add_argument("--log",
help="Takes a log file which created by itself " \ help="Takes a log file which created by itself "
"(json files), reads posts and tries downloadin" \ "(json files), reads posts and tries downloadin"
"g them again.", "g them again.",
# type=argparse.FileType('r'), # type=argparse.FileType('r'),
metavar="LOG FILE") metavar="LOG FILE")
parser.add_argument("--subreddit", parser.add_argument(
"--subreddit",
nargs="+", nargs="+",
help="Triggers subreddit mode and takes subreddit's " \ help="Triggers subreddit mode and takes subreddit's "
"name without r/. use \"frontpage\" for frontpage", "name without r/. use \"frontpage\" for frontpage",
metavar="SUBREDDIT", metavar="SUBREDDIT",
type=str) type=str)
parser.add_argument("--multireddit", parser.add_argument("--multireddit",
help="Triggers multireddit mode and takes "\ help="Triggers multireddit mode and takes "
"multireddit's name without m/", "multireddit's name without m/",
metavar="MULTIREDDIT", metavar="MULTIREDDIT",
type=str) type=str)
parser.add_argument("--user", parser.add_argument("--user",
help="reddit username if needed. use \"me\" for " \ help="reddit username if needed. use \"me\" for "
"current user", "current user",
required="--multireddit" in sys.argv or \ required="--multireddit" in sys.argv or
"--submitted" in sys.argv, "--submitted" in sys.argv,
metavar="redditor", metavar="redditor",
type=str) type=str)
parser.add_argument("--search", parser.add_argument(
"--search",
help="Searches for given query in given subreddits", help="Searches for given query in given subreddits",
metavar="query", metavar="query",
type=str) type=str)
parser.add_argument("--sort", parser.add_argument("--sort",
help="Either hot, top, new, controversial, rising " \ help="Either hot, top, new, controversial, rising "
"or relevance default: hot", "or relevance default: hot",
choices=[ choices=[
"hot", "top", "new", "controversial", "rising", "hot", "top", "new", "controversial", "rising",
@@ -95,9 +100,10 @@ class Arguments:
type=int) type=int)
parser.add_argument("--time", parser.add_argument("--time",
help="Either hour, day, week, month, year or all." \ help="Either hour, day, week, month, year or all."
" default: all", " default: all",
choices=["all","hour","day","week","month","year"], choices=["all", "hour", "day",
"week", "month", "year"],
metavar="TIME_LIMIT", metavar="TIME_LIMIT",
type=str) type=str)
@@ -124,38 +130,40 @@ class Arguments:
help="Set custom filename", help="Set custom filename",
) )
parser.add_argument("--set-default-directory", parser.add_argument(
"--set-default-directory",
action="store_true", action="store_true",
help="Set a default directory to be used in case no directory is given", help="Set a default directory to be used in case no directory is given",
) )
parser.add_argument("--set-default-options", parser.add_argument(
"--set-default-options",
action="store_true", action="store_true",
help="Set default options to use everytime program runs", help="Set default options to use everytime program runs",
) )
parser.add_argument("--use-local-config", parser.add_argument(
"--use-local-config",
action="store_true", action="store_true",
help="Creates a config file in the program's directory and uses it. Useful for having multiple configs", help="Creates a config file in the program's directory and uses it. Useful for having multiple configs",
) )
parser.add_argument("--no-dupes", parser.add_argument(
"--no-dupes",
action="store_true", action="store_true",
help="Do not download duplicate posts on different subreddits", help="Do not download duplicate posts on different subreddits",
) )
parser.add_argument("--downloaded-posts", parser.add_argument(
"--downloaded-posts",
help="Use a hash file to keep track of downloaded files", help="Use a hash file to keep track of downloaded files",
type=str type=str)
)
parser.add_argument("--no-download", parser.add_argument(
"--no-download",
action="store_true", action="store_true",
help="Just saved posts into a the POSTS.json file without downloading" help="Just saved posts into a the POSTS.json file without downloading")
)
if arguments == []: if arguments == []:
return parser.parse_args() return parser.parse_args()
else:
return parser.parse_args(arguments) return parser.parse_args(arguments)

16
src/config.py
View File

@@ -1,12 +1,9 @@
import os
import socket
import webbrowser
import random
from src.reddit import Reddit from src.reddit import Reddit
from src.jsonHelper import JsonFile from src.jsonHelper import JsonFile
from src.utils import nameCorrector from src.utils import nameCorrector
class Config(): class Config():
def __init__(self, filename): def __init__(self, filename):
@@ -45,7 +42,7 @@ Existing filename template:""", None if "filename" not in self.file.read() else
def _readCustomFileName(self): def _readCustomFileName(self):
content = self.file.read() content = self.file.read()
if not "filename" in content: if "filename" not in content:
self.file.add({ self.file.add({
"filename": "{REDDITOR}_{TITLE}_{POSTID}" "filename": "{REDDITOR}_{TITLE}_{POSTID}"
}) })
@@ -77,7 +74,7 @@ Existing folder structure""", None if "folderpath" not in self.file.read() else
def _readCustomFolderPath(self, path=None): def _readCustomFolderPath(self, path=None):
content = self.file.read() content = self.file.read()
if not "folderpath" in content: if "folderpath" not in content:
self.file.add({ self.file.add({
"folderpath": "{SUBREDDIT}" "folderpath": "{SUBREDDIT}"
}) })
@@ -98,7 +95,7 @@ Existing default options:""", None if "options" not in self.file.read() else sel
def _readDefaultOptions(self, path=None): def _readDefaultOptions(self, path=None):
content = self.file.read() content = self.file.read()
if not "options" in content: if "options" not in content:
self.file.add({ self.file.add({
"options": "" "options": ""
}) })
@@ -108,7 +105,7 @@ Existing default options:""", None if "options" not in self.file.read() else sel
try: try:
content = self.file.read()["credentials"] content = self.file.read()["credentials"]
except: except BaseException:
self.file.add({ self.file.add({
"credentials": {} "credentials": {}
}) })
@@ -126,7 +123,8 @@ Existing default options:""", None if "options" not in self.file.read() else sel
Leave blank to reset it. You can use {time} in foler names to use to timestamp it Leave blank to reset it. You can use {time} in foler names to use to timestamp it
For example: D:/archive/BDFR_{time} For example: D:/archive/BDFR_{time}
""") """)
print("Current default directory:", self.file.read()["default_directory"] if "default_directory" in self.file.read() else "") print("Current default directory:", self.file.read()[
"default_directory"] if "default_directory" in self.file.read() else "")
self.file.add({ self.file.add({
"default_directory": input(">> ") "default_directory": input(">> ")
}) })

8
src/downloaders/Direct.py
View File

@@ -1,18 +1,16 @@
import os import os
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile, getExtension
from src.errors import FileNameTooLong
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print
class Direct: class Direct:
def __init__(self, directory, POST): def __init__(self, directory, POST):
POST['EXTENSION'] = getExtension(POST['CONTENTURL']) POST['EXTENSION'] = getExtension(POST['CONTENTURL'])
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"] filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"]
shortFilename = POST['POSTID'] + POST['EXTENSION'] shortFilename = POST['POSTID'] + POST['EXTENSION']
getFile(filename, shortFilename, directory, POST['CONTENTURL']) getFile(filename, shortFilename, directory, POST['CONTENTURL'])

15
src/downloaders/Erome.py
View File

@@ -1,17 +1,16 @@
import os import os
import logging
import sys
import urllib.request import urllib.request
from html.parser import HTMLParser from html.parser import HTMLParser
from src.downloaders.downloaderUtils import getFile from src.downloaders.downloaderUtils import getFile
from src.downloaders.downloaderUtils import getExtension from src.downloaders.downloaderUtils import getExtension
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely, from src.errors import (AlbumNotDownloadedCompletely,
NotADownloadableLinkError, FileAlreadyExistsError, full_exc_info) NotADownloadableLinkError, FileAlreadyExistsError)
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
class Erome: class Erome:
def __init__(self, directory, post): def __init__(self, directory, post):
try: try:
@@ -29,7 +28,8 @@ class Erome:
"""Filenames are declared here""" """Filenames are declared here"""
filename = GLOBAL.config['filename'].format(**post)+post["EXTENSION"] filename = GLOBAL.config['filename'].format(
**post) + post["EXTENSION"]
shortFilename = post['POSTID'] + extension shortFilename = post['POSTID'] + extension
imageURL = IMAGES[0] imageURL = IMAGES[0]
@@ -87,7 +87,7 @@ class Erome:
if duplicates == imagesLenght: if duplicates == imagesLenght:
raise FileAlreadyExistsError raise FileAlreadyExistsError
elif howManyDownloaded + duplicates < imagesLenght: if howManyDownloaded + duplicates < imagesLenght:
raise AlbumNotDownloadedCompletely( raise AlbumNotDownloadedCompletely(
"Album Not Downloaded Completely" "Album Not Downloaded Completely"
) )
@@ -99,6 +99,7 @@ class Erome:
class EromeParser(HTMLParser): class EromeParser(HTMLParser):
tag = None tag = None
def handle_starttag(self, tag, attrs): def handle_starttag(self, tag, attrs):
self.tag = {tag: {attr[0]: attr[1] for attr in attrs}} self.tag = {tag: {attr[0]: attr[1] for attr in attrs}}
@@ -130,6 +131,6 @@ class Erome:
content.append(tag["source"]["src"]) content.append(tag["source"]["src"])
return [ return [
link for link in content \ link for link in content
if link.endswith("_480p.mp4") or not link.endswith(".mp4") if link.endswith("_480p.mp4") or not link.endswith(".mp4")
] ]

11
src/downloaders/Gfycat.py
View File

@@ -4,12 +4,11 @@ import urllib.request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile, getExtension
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely, from src.errors import (NotADownloadableLinkError)
NotADownloadableLinkError, FileAlreadyExistsError)
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
class Gfycat: class Gfycat:
def __init__(self, directory, POST): def __init__(self, directory, POST):
try: try:
@@ -19,7 +18,8 @@ class Gfycat:
POST['EXTENSION'] = getExtension(POST['MEDIAURL']) POST['EXTENSION'] = getExtension(POST['MEDIAURL'])
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"] filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"]
shortFilename = POST['POSTID'] + POST['EXTENSION'] shortFilename = POST['POSTID'] + POST['EXTENSION']
@@ -43,7 +43,8 @@ class Gfycat:
pageSource = (urllib.request.urlopen(url).read().decode()) pageSource = (urllib.request.urlopen(url).read().decode())
soup = BeautifulSoup(pageSource, "html.parser") soup = BeautifulSoup(pageSource, "html.parser")
attributes = {"data-react-helmet":"true","type":"application/ld+json"} attributes = {"data-react-helmet": "true",
"type": "application/ld+json"}
content = soup.find("script", attrs=attributes) content = soup.find("script", attrs=attributes)
if content is None: if content is None:

43
src/downloaders/Imgur.py
View File

@@ -1,7 +1,5 @@
import urllib
import json import json
import os import os
import time
import requests import requests
from src.utils import GLOBAL, nameCorrector from src.utils import GLOBAL, nameCorrector
@@ -10,6 +8,7 @@ from src.downloaders.Direct import Direct
from src.downloaders.downloaderUtils import getFile from src.downloaders.downloaderUtils import getFile
from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, ExtensionError, NotADownloadableLinkError, TypeInSkip from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, ExtensionError, NotADownloadableLinkError, TypeInSkip
class Imgur: class Imgur:
IMGUR_IMAGE_DOMAIN = "https://i.imgur.com/" IMGUR_IMAGE_DOMAIN = "https://i.imgur.com/"
@@ -57,11 +56,12 @@ class Imgur:
extension = self.validateExtension(images["images"][i]["ext"]) extension = self.validateExtension(images["images"][i]["ext"])
imageURL = self.IMGUR_IMAGE_DOMAIN + images["images"][i]["hash"] + extension imageURL = self.IMGUR_IMAGE_DOMAIN + \
images["images"][i]["hash"] + extension
filename = "_".join([ filename = "_".join([str(i + 1),
str(i+1), nameCorrector(images["images"][i]['title']), images["images"][i]['hash'] nameCorrector(images["images"][i]['title']),
]) + extension images["images"][i]['hash']]) + extension
shortFilename = str(i + 1) + "_" + images["images"][i]['hash'] shortFilename = str(i + 1) + "_" + images["images"][i]['hash']
print("\n ({}/{})".format(i + 1, imagesLenght)) print("\n ({}/{})".format(i + 1, imagesLenght))
@@ -82,18 +82,16 @@ class Imgur:
except Exception as exception: except Exception as exception:
print("\n Could not get the file") print("\n Could not get the file")
print( print(
" " " " +
+ "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format( "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
class_name=exception.__class__.__name__, class_name=exception.__class__.__name__,
info=str(exception) info=str(exception)) +
) "\n")
+ "\n"
)
print(GLOBAL.log_stream.getvalue(), noPrint=True) print(GLOBAL.log_stream.getvalue(), noPrint=True)
if duplicates == imagesLenght: if duplicates == imagesLenght:
raise FileAlreadyExistsError raise FileAlreadyExistsError
elif howManyDownloaded + duplicates < imagesLenght: if howManyDownloaded + duplicates < imagesLenght:
raise AlbumNotDownloadedCompletely( raise AlbumNotDownloadedCompletely(
"Album Not Downloaded Completely" "Album Not Downloaded Completely"
) )
@@ -116,7 +114,9 @@ class Imgur:
cookies = {"over18": "1", "postpagebeta": "0"} cookies = {"over18": "1", "postpagebeta": "0"}
res = requests.get(link, cookies=cookies) res = requests.get(link, cookies=cookies)
if res.status_code != 200: raise ImageNotFound(f"Server responded with {res.status_code} to {link}") if res.status_code != 200:
raise ImageNotFound(
f"Server responded with {res.status_code} to {link}")
pageSource = requests.get(link, cookies=cookies).text pageSource = requests.get(link, cookies=cookies).text
STARTING_STRING = "image : " STARTING_STRING = "image : "
@@ -124,16 +124,18 @@ class Imgur:
STARTING_STRING_LENGHT = len(STARTING_STRING) STARTING_STRING_LENGHT = len(STARTING_STRING)
try: try:
startIndex = pageSource.index(STARTING_STRING) + STARTING_STRING_LENGHT startIndex = pageSource.index(
STARTING_STRING) + STARTING_STRING_LENGHT
endIndex = pageSource.index(ENDING_STRING, startIndex) endIndex = pageSource.index(ENDING_STRING, startIndex)
except ValueError: except ValueError:
raise NotADownloadableLinkError(f"Could not read the page source on {link}") raise NotADownloadableLinkError(
f"Could not read the page source on {link}")
while pageSource[endIndex] != "}": while pageSource[endIndex] != "}":
endIndex = endIndex - 1 endIndex = endIndex - 1
try: try:
data = pageSource[startIndex:endIndex + 2].strip()[:-1] data = pageSource[startIndex:endIndex + 2].strip()[:-1]
except: except BaseException:
pageSource[endIndex + 1] = '}' pageSource[endIndex + 1] = '}'
data = pageSource[startIndex:endIndex + 3].strip()[:-1] data = pageSource[startIndex:endIndex + 3].strip()[:-1]
@@ -144,5 +146,8 @@ class Imgur:
POSSIBLE_EXTENSIONS = [".jpg", ".png", ".mp4", ".gif"] POSSIBLE_EXTENSIONS = [".jpg", ".png", ".mp4", ".gif"]
for extension in POSSIBLE_EXTENSIONS: for extension in POSSIBLE_EXTENSIONS:
if extension in string: return extension if extension in string:
else: raise ExtensionError(f"\"{string}\" is not recognized as a valid extension.") return extension
raise ExtensionError(
f"\"{string}\" is not recognized as a valid extension.")

41
src/downloaders/downloaderUtils.py
View File

@@ -1,14 +1,13 @@
import sys import sys
import os import os
import time
from urllib.error import HTTPError
import urllib.request import urllib.request
from pathlib import Path from pathlib import Path
import hashlib import hashlib
from src.utils import nameCorrector, GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
from src.errors import FileAlreadyExistsError, FileNameTooLong, FailedToDownload, TypeInSkip, DomainInSkip from src.errors import FileAlreadyExistsError, FailedToDownload, TypeInSkip, DomainInSkip
def dlProgress(count, blockSize, totalSize): def dlProgress(count, blockSize, totalSize):
"""Function for writing download progress to console """Function for writing download progress to console
@@ -19,6 +18,7 @@ def dlProgress(count, blockSize, totalSize):
sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs, fileSize)) sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs, fileSize))
sys.stdout.flush() sys.stdout.flush()
def getExtension(link): def getExtension(link):
"""Extract file extension from image link. """Extract file extension from image link.
If didn't find any, return '.jpg' If didn't find any, return '.jpg'
@@ -29,13 +29,19 @@ def getExtension(link):
for fileType in imageTypes: for fileType in imageTypes:
if fileType in parsed: if fileType in parsed:
return "." + parsed[-1] return "." + parsed[-1]
else:
if not "v.redd.it" in link: if "v.redd.it" not in link:
return '.jpg' return '.jpg'
else:
return '.mp4' return '.mp4'
def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
def getFile(
filename,
shortFilename,
folderDir,
imageURL,
indent=0,
silent=False):
FORMATS = { FORMATS = {
"videos": [".mp4", ".webm"], "videos": [".mp4", ".webm"],
@@ -53,10 +59,10 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
raise DomainInSkip raise DomainInSkip
headers = [ headers = [
("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \ ("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 "\ "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 "
"Safari/537.36 OPR/54.0.2952.64"), "Safari/537.36 OPR/54.0.2952.64"),
("Accept", "text/html,application/xhtml+xml,application/xml;" \ ("Accept", "text/html,application/xhtml+xml,application/xml;"
"q=0.9,image/webp,image/apng,*/*;q=0.8"), "q=0.9,image/webp,image/apng,*/*;q=0.8"),
("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.3"), ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.3"),
("Accept-Encoding", "none"), ("Accept-Encoding", "none"),
@@ -64,18 +70,19 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
("Connection", "keep-alive") ("Connection", "keep-alive")
] ]
if not os.path.exists(folderDir): os.makedirs(folderDir) if not os.path.exists(folderDir):
os.makedirs(folderDir)
opener = urllib.request.build_opener() opener = urllib.request.build_opener()
if not "imgur" in imageURL: if "imgur" not in imageURL:
opener.addheaders = headers opener.addheaders = headers
urllib.request.install_opener(opener) urllib.request.install_opener(opener)
if not silent: print(" "*indent + str(folderDir), if not silent:
print(" " * indent + str(folderDir),
" " * indent + str(filename), " " * indent + str(filename),
sep="\n") sep="\n")
for i in range(3): for i in range(3):
fileDir = Path(folderDir) / filename fileDir = Path(folderDir) / filename
tempDir = Path(folderDir) / (filename + ".tmp") tempDir = Path(folderDir) / (filename + ".tmp")
@@ -94,7 +101,8 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
GLOBAL.downloadedPosts.add(fileHash) GLOBAL.downloadedPosts.add(fileHash)
os.rename(tempDir, fileDir) os.rename(tempDir, fileDir)
if not silent: print(" "*indent+"Downloaded"+" "*10) if not silent:
print(" " * indent + "Downloaded" + " " * 10)
return None return None
except ConnectionResetError: except ConnectionResetError:
raise FailedToDownload raise FailedToDownload
@@ -104,6 +112,7 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
raise FileAlreadyExistsError raise FileAlreadyExistsError
raise FailedToDownload raise FailedToDownload
def createHash(filename): def createHash(filename):
hash_md5 = hashlib.md5() hash_md5 = hashlib.md5()
with open(filename, "rb") as f: with open(filename, "rb") as f:

41
src/downloaders/gallery.py
View File

@@ -1,15 +1,13 @@
import io
import os import os
import json import json
import urllib import urllib
import requests import requests
from pathlib import Path
from src.utils import GLOBAL, nameCorrector from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
from src.downloaders.Direct import Direct
from src.downloaders.downloaderUtils import getFile from src.downloaders.downloaderUtils import getFile
from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, ExtensionError, NotADownloadableLinkError, TypeInSkip from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, NotADownloadableLinkError, TypeInSkip
class gallery: class gallery:
def __init__(self, directory, post): def __init__(self, directory, post):
@@ -26,11 +24,12 @@ class gallery:
try: try:
for item in self.rawData['posts']['models'][model]['media']['gallery']['items']: for item in self.rawData['posts']['models'][model]['media']['gallery']['items']:
try: try:
images[count]={'id':item['mediaId'], 'url':self.rawData['posts']['models'][model]['media']['mediaMetadata'][item['mediaId']]['s']['u']} images[count] = {'id': item['mediaId'], 'url': self.rawData['posts'][
'models'][model]['media']['mediaMetadata'][item['mediaId']]['s']['u']}
count = count + 1 count = count + 1
except: except BaseException:
continue continue
except: except BaseException:
continue continue
self.downloadAlbum(images, count) self.downloadAlbum(images, count)
@@ -43,7 +42,9 @@ class gallery:
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
} }
res = requests.get(link, headers=headers) res = requests.get(link, headers=headers)
if res.status_code != 200: raise ImageNotFound(f"Server responded with {res.status_code} to {link}") if res.status_code != 200:
raise ImageNotFound(
f"Server responded with {res.status_code} to {link}")
pageSource = res.text pageSource = res.text
STARTING_STRING = "_r = {" STARTING_STRING = "_r = {"
@@ -51,10 +52,12 @@ class gallery:
STARTING_STRING_LENGHT = len(STARTING_STRING) STARTING_STRING_LENGHT = len(STARTING_STRING)
try: try:
startIndex = pageSource.index(STARTING_STRING) + STARTING_STRING_LENGHT startIndex = pageSource.index(
STARTING_STRING) + STARTING_STRING_LENGHT
endIndex = pageSource.index(ENDING_STRING, startIndex) endIndex = pageSource.index(ENDING_STRING, startIndex)
except ValueError: except ValueError:
raise NotADownloadableLinkError(f"Could not read the page source on {link}") raise NotADownloadableLinkError(
f"Could not read the page source on {link}")
data = json.loads(pageSource[startIndex - 1:endIndex + 1].strip()[:-1]) data = json.loads(pageSource[startIndex - 1:endIndex + 1].strip()[:-1])
return data return data
@@ -87,7 +90,8 @@ class gallery:
print("\n ({}/{})".format(i + 1, count)) print("\n ({}/{})".format(i + 1, count))
try: try:
getFile(filename,shortFilename,folderDir,images[i]['url'],indent=2) getFile(filename, shortFilename, folderDir,
images[i]['url'], indent=2)
howManyDownloaded += 1 howManyDownloaded += 1
print() print()
@@ -102,19 +106,16 @@ class gallery:
except Exception as exception: except Exception as exception:
print("\n Could not get the file") print("\n Could not get the file")
print( print(
" " " " +
+ "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format( "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
class_name=exception.__class__.__name__, class_name=exception.__class__.__name__,
info=str(exception) info=str(exception)) +
) "\n")
+ "\n"
)
print(GLOBAL.log_stream.getvalue(), noPrint=True) print(GLOBAL.log_stream.getvalue(), noPrint=True)
if duplicates == count: if duplicates == count:
raise FileAlreadyExistsError raise FileAlreadyExistsError
elif howManyDownloaded + duplicates < count: if howManyDownloaded + duplicates < count:
raise AlbumNotDownloadedCompletely( raise AlbumNotDownloadedCompletely(
"Album Not Downloaded Completely" "Album Not Downloaded Completely"
) )

12
src/downloaders/gifDeliveryNetwork.py
View File

@@ -1,13 +1,11 @@
import json
import os import os
import urllib.request import urllib.request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile, getExtension
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely, from src.errors import (NotADownloadableLinkError)
NotADownloadableLinkError, FileAlreadyExistsError)
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print
class GifDeliveryNetwork: class GifDeliveryNetwork:
def __init__(self, directory, POST): def __init__(self, directory, POST):
@@ -18,7 +16,8 @@ class GifDeliveryNetwork:
POST['EXTENSION'] = getExtension(POST['MEDIAURL']) POST['EXTENSION'] = getExtension(POST['MEDIAURL'])
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"] filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"]
shortFilename = POST['POSTID'] + POST['EXTENSION'] shortFilename = POST['POSTID'] + POST['EXTENSION']
@@ -31,7 +30,8 @@ class GifDeliveryNetwork:
and return it and return it
""" """
if '.webm' in url.split('/')[-1] or '.mp4' in url.split('/')[-1] or '.gif' in url.split('/')[-1]: if '.webm' in url.split(
'/')[-1] or '.mp4' in url.split('/')[-1] or '.gif' in url.split('/')[-1]:
return url return url
if url[-1:] == '/': if url[-1:] == '/':

21
src/downloaders/redgifs.py
View File

@@ -4,10 +4,9 @@ import urllib.request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile, getExtension
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely, from src.errors import (NotADownloadableLinkError)
NotADownloadableLinkError, FileAlreadyExistsError)
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print
class Redgifs: class Redgifs:
def __init__(self, directory, POST): def __init__(self, directory, POST):
@@ -18,14 +17,16 @@ class Redgifs:
POST['EXTENSION'] = getExtension(POST['MEDIAURL']) POST['EXTENSION'] = getExtension(POST['MEDIAURL'])
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"] filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"]
shortFilename = POST['POSTID'] + POST['EXTENSION'] shortFilename = POST['POSTID'] + POST['EXTENSION']
getFile(filename, shortFilename, directory, POST['MEDIAURL']) getFile(filename, shortFilename, directory, POST['MEDIAURL'])
def getLink(self, url): @staticmethod
def getLink(url):
"""Extract direct link to the video from page's source """Extract direct link to the video from page's source
and return it and return it
""" """
@@ -36,14 +37,18 @@ class Redgifs:
if url[-1:] == '/': if url[-1:] == '/':
url = url[:-1] url = url[:-1]
url = urllib.request.Request("https://redgifs.com/watch/" + url.split('/')[-1]) url = urllib.request.Request(
"https://redgifs.com/watch/" + url.split('/')[-1])
url.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36 OPR/54.0.2952.64') url.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36 OPR/54.0.2952.64')
pageSource = (urllib.request.urlopen(url).read().decode()) pageSource = (urllib.request.urlopen(url).read().decode())
soup = BeautifulSoup(pageSource, "html.parser") soup = BeautifulSoup(pageSource, "html.parser")
attributes = {"data-react-helmet":"true","type":"application/ld+json"} attributes = {"data-react-helmet": "true",
"type": "application/ld+json"}
content = soup.find("script", attrs=attributes) content = soup.find("script", attrs=attributes)
if content is None: if content is None:

11
src/downloaders/selfPost.py
View File

@@ -1,3 +1,4 @@
from src.utils import printToFile as print
import io import io
import os import os
from pathlib import Path from pathlib import Path
@@ -6,14 +7,16 @@ from src.errors import FileAlreadyExistsError, TypeInSkip
from src.utils import GLOBAL from src.utils import GLOBAL
VanillaPrint = print VanillaPrint = print
from src.utils import printToFile as print
class SelfPost: class SelfPost:
def __init__(self, directory, post): def __init__(self, directory, post):
if "self" in GLOBAL.arguments.skip: raise TypeInSkip if "self" in GLOBAL.arguments.skip:
raise TypeInSkip
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**post) filename = GLOBAL.config['filename'].format(**post)
@@ -21,7 +24,6 @@ class SelfPost:
print(fileDir) print(fileDir)
print(filename + ".md") print(filename + ".md")
if Path.is_file(fileDir): if Path.is_file(fileDir):
raise FileAlreadyExistsError raise FileAlreadyExistsError
@@ -35,7 +37,6 @@ class SelfPost:
@staticmethod @staticmethod
def writeToFile(directory, post): def writeToFile(directory, post):
"""Self posts are formatted here""" """Self posts are formatted here"""
content = ("## [" content = ("## ["
+ post["TITLE"] + post["TITLE"]

10
src/downloaders/vreddit.py
View File

@@ -1,16 +1,16 @@
import os import os
import subprocess import subprocess
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile
from src.errors import FileNameTooLong
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
class VReddit: class VReddit:
def __init__(self, directory, post): def __init__(self, directory, post):
extension = ".mp4" extension = ".mp4"
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**post) + extension filename = GLOBAL.config['filename'].format(**post) + extension
shortFilename = post['POSTID'] + extension shortFilename = post['POSTID'] + extension
@@ -18,7 +18,7 @@ class VReddit:
try: try:
FNULL = open(os.devnull, 'w') FNULL = open(os.devnull, 'w')
subprocess.call("ffmpeg", stdout=FNULL, stderr=subprocess.STDOUT) subprocess.call("ffmpeg", stdout=FNULL, stderr=subprocess.STDOUT)
except: except BaseException:
getFile(filename, shortFilename, directory, post['CONTENTURL']) getFile(filename, shortFilename, directory, post['CONTENTURL'])
print("FFMPEG library not found, skipping merging video and audio") print("FFMPEG library not found, skipping merging video and audio")
else: else:

10
src/downloaders/youtube.py
View File

@@ -2,15 +2,17 @@ import os
import youtube_dl import youtube_dl
import sys import sys
from src.downloaders.downloaderUtils import getExtension, dlProgress, createHash from src.downloaders.downloaderUtils import createHash
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
from src.errors import FileAlreadyExistsError from src.errors import FileAlreadyExistsError
class Youtube: class Youtube:
def __init__(self, directory, post): def __init__(self, directory, post):
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**post) filename = GLOBAL.config['filename'].format(**post)
print(filename) print(filename)
@@ -43,9 +45,9 @@ class Youtube:
@staticmethod @staticmethod
def _hook(d): def _hook(d):
if d['status'] == 'finished': return print("Downloaded") if d['status'] == 'finished':
return print("Downloaded")
downloadedMbs = int(d['downloaded_bytes'] * (10**(-6))) downloadedMbs = int(d['downloaded_bytes'] * (10**(-6)))
fileSize = int(d['total_bytes'] * (10**(-6))) fileSize = int(d['total_bytes'] * (10**(-6)))
sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs, fileSize)) sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs, fileSize))
sys.stdout.flush() sys.stdout.flush()

32
src/errors.py
View File

@@ -1,9 +1,11 @@
import sys import sys
def full_exc_info(exc_info): def full_exc_info(exc_info):
def current_stack(skip=0): def current_stack(skip=0):
try: 1/0 try:
1 / 0
except ZeroDivisionError: except ZeroDivisionError:
f = sys.exc_info()[2].tb_frame f = sys.exc_info()[2].tb_frame
for i in range(skip + 2): for i in range(skip + 2):
@@ -16,7 +18,7 @@ def full_exc_info(exc_info):
def extend_traceback(tb, stack): def extend_traceback(tb, stack):
class FauxTb(object): class FauxTb():
def __init__(self, tb_frame, tb_lineno, tb_next): def __init__(self, tb_frame, tb_lineno, tb_next):
self.tb_frame = tb_frame self.tb_frame = tb_frame
self.tb_lineno = tb_lineno self.tb_lineno = tb_lineno
@@ -33,80 +35,106 @@ def full_exc_info(exc_info):
full_tb = extend_traceback(tb, current_stack(1)) full_tb = extend_traceback(tb, current_stack(1))
return t, v, full_tb return t, v, full_tb
class RedditLoginFailed(Exception): class RedditLoginFailed(Exception):
pass pass
class ImgurLoginError(Exception): class ImgurLoginError(Exception):
pass pass
class FileAlreadyExistsError(Exception): class FileAlreadyExistsError(Exception):
pass pass
class NotADownloadableLinkError(Exception): class NotADownloadableLinkError(Exception):
pass pass
class AlbumNotDownloadedCompletely(Exception): class AlbumNotDownloadedCompletely(Exception):
pass pass
class FileNameTooLong(Exception): class FileNameTooLong(Exception):
pass pass
class InvalidRedditLink(Exception): class InvalidRedditLink(Exception):
pass pass
class ProgramModeError(Exception): class ProgramModeError(Exception):
pass pass
class SearchModeError(Exception): class SearchModeError(Exception):
pass pass
class RedditorNameError(Exception): class RedditorNameError(Exception):
pass pass
class NoMatchingSubmissionFound(Exception): class NoMatchingSubmissionFound(Exception):
pass pass
class NoPrawSupport(Exception): class NoPrawSupport(Exception):
pass pass
class NoRedditSupport(Exception): class NoRedditSupport(Exception):
pass pass
class MultiredditNotFound(Exception): class MultiredditNotFound(Exception):
pass pass
class InsufficientPermission(Exception): class InsufficientPermission(Exception):
pass pass
class InvalidSortingType(Exception): class InvalidSortingType(Exception):
pass pass
class FileNotFoundError(Exception): class FileNotFoundError(Exception):
pass pass
class NoSuitablePost(Exception): class NoSuitablePost(Exception):
pass pass
class ImgurLimitError(Exception): class ImgurLimitError(Exception):
pass pass
class DirectLinkNotFound(Exception): class DirectLinkNotFound(Exception):
pass pass
class InvalidJSONFile(Exception): class InvalidJSONFile(Exception):
pass pass
class FailedToDownload(Exception): class FailedToDownload(Exception):
pass pass
class TypeInSkip(Exception): class TypeInSkip(Exception):
pass pass
class DomainInSkip(Exception): class DomainInSkip(Exception):
pass pass
class ImageNotFound(Exception): class ImageNotFound(Exception):
pass pass
class ExtensionError(Exception): class ExtensionError(Exception):
pass pass

7
src/jsonHelper.py
View File

@@ -3,6 +3,7 @@ from os import path, remove
from src.errors import InvalidJSONFile from src.errors import InvalidJSONFile
class JsonFile: class JsonFile:
""" Write and read JSON files """ Write and read JSON files
@@ -32,8 +33,10 @@ class JsonFile:
""" """
data = self.read() data = self.read()
if sub: data[sub] = {**data[sub], **toBeAdded} if sub:
else: data = {**data, **toBeAdded} data[sub] = {**data[sub], **toBeAdded}
else:
data = {**data, **toBeAdded}
self.__writeToFile(data) self.__writeToFile(data)
return self.read() return self.read()

29
src/parser.py
View File

@@ -5,6 +5,7 @@ try:
except ModuleNotFoundError: except ModuleNotFoundError:
from errors import InvalidRedditLink from errors import InvalidRedditLink
def QueryParser(PassedQueries, index): def QueryParser(PassedQueries, index):
ExtractedQueries = {} ExtractedQueries = {}
@@ -24,11 +25,12 @@ def QueryParser(PassedQueries,index):
return ExtractedQueries return ExtractedQueries
def LinkParser(LINK): def LinkParser(LINK):
RESULT = {} RESULT = {}
ShortLink = False ShortLink = False
if not "reddit.com" in LINK: if "reddit.com" not in LINK:
raise InvalidRedditLink("Invalid reddit link") raise InvalidRedditLink("Invalid reddit link")
SplittedLink = LINK.split("/") SplittedLink = LINK.split("/")
@@ -37,7 +39,7 @@ def LinkParser(LINK):
SplittedLink = SplittedLink[2:] SplittedLink = SplittedLink[2:]
try: try:
if (SplittedLink[-2].endswith("reddit.com") and \ if (SplittedLink[-2].endswith("reddit.com") and
SplittedLink[-1] == "") or \ SplittedLink[-1] == "") or \
SplittedLink[-1].endswith("reddit.com"): SplittedLink[-1].endswith("reddit.com"):
@@ -58,7 +60,7 @@ def LinkParser(LINK):
RESULT = {"post": LINK} RESULT = {"post": LINK}
return RESULT return RESULT
elif "me" in SplittedLink or \ if "me" in SplittedLink or \
"u" in SplittedLink or \ "u" in SplittedLink or \
"user" in SplittedLink or \ "user" in SplittedLink or \
"r" in SplittedLink or \ "r" in SplittedLink or \
@@ -81,7 +83,6 @@ def LinkParser(LINK):
elif SplittedLink[index] == "me": elif SplittedLink[index] == "me":
RESULT["user"] = "me" RESULT["user"] = "me"
for index in range(len(SplittedLink)): for index in range(len(SplittedLink)):
if SplittedLink[index] in [ if SplittedLink[index] in [
"hot", "top", "new", "controversial", "rising" "hot", "top", "new", "controversial", "rising"
@@ -118,15 +119,16 @@ def LinkParser(LINK):
del ParsedQuery["HEADER"] del ParsedQuery["HEADER"]
RESULT["queries"] = ParsedQuery RESULT["queries"] = ParsedQuery
if not ("upvoted" in RESULT or \ if not ("upvoted" in RESULT or
"saved" in RESULT or \ "saved" in RESULT or
"submitted" in RESULT or \ "submitted" in RESULT or
"multireddit" in RESULT) and \ "multireddit" in RESULT) and \
"user" in RESULT: "user" in RESULT:
RESULT["submitted"] = {} RESULT["submitted"] = {}
return RESULT return RESULT
def LinkDesigner(LINK): def LinkDesigner(LINK):
attributes = LinkParser(LINK) attributes = LinkParser(LINK)
@@ -138,13 +140,13 @@ def LinkDesigner(LINK):
MODE["time"] = "" MODE["time"] = ""
return MODE return MODE
elif "search" in attributes: if "search" in attributes:
MODE["search"] = attributes["search"]["q"] MODE["search"] = attributes["search"]["q"]
if "restrict_sr" in attributes["search"]: if "restrict_sr" in attributes["search"]:
if not (attributes["search"]["restrict_sr"] == 0 or \ if not (attributes["search"]["restrict_sr"] == 0 or
attributes["search"]["restrict_sr"] == "off" or \ attributes["search"]["restrict_sr"] == "off" or
attributes["search"]["restrict_sr"] == ""): attributes["search"]["restrict_sr"] == ""):
if "subreddit" in attributes: if "subreddit" in attributes:
@@ -176,7 +178,7 @@ def LinkDesigner(LINK):
else: else:
if "queries" in attributes: if "queries" in attributes:
if not ("submitted" in attributes or \ if not ("submitted" in attributes or
"posts" in attributes): "posts" in attributes):
if "t" in attributes["queries"]: if "t" in attributes["queries"]:
@@ -196,10 +198,10 @@ def LinkDesigner(LINK):
else: else:
MODE["time"] = "day" MODE["time"] = "day"
if "subreddit" in attributes and not "search" in attributes: if "subreddit" in attributes and "search" not in attributes:
MODE["subreddit"] = attributes["subreddit"] MODE["subreddit"] = attributes["subreddit"]
elif "user" in attributes and not "search" in attributes: elif "user" in attributes and "search" not in attributes:
MODE["user"] = attributes["user"] MODE["user"] = attributes["user"]
if "submitted" in attributes: if "submitted" in attributes:
@@ -234,6 +236,7 @@ def LinkDesigner(LINK):
return MODE return MODE
if __name__ == "__main__": if __name__ == "__main__":
while True: while True:
link = input("> ") link = input("> ")

31
src/programMode.py
View File

@@ -1,9 +1,9 @@
from src.errors import SearchModeError, RedditorNameError, ProgramModeError, InvalidSortingType from src.errors import SearchModeError, RedditorNameError, ProgramModeError, InvalidSortingType
from src.utils import GLOBAL
from src.parser import LinkDesigner from src.parser import LinkDesigner
from pathlib import Path from pathlib import Path
import sys import sys
class ProgramMode: class ProgramMode:
def __init__(self, arguments): def __init__(self, arguments):
@@ -57,7 +57,7 @@ class ProgramMode:
programMode["time"] = self.arguments.time programMode["time"] = self.arguments.time
elif self.arguments.subreddit is not None: elif self.arguments.subreddit is not None:
if type(self.arguments.subreddit) == list: if isinstance(self.arguments.subreddit, list):
self.arguments.subreddit = "+".join(self.arguments.subreddit) self.arguments.subreddit = "+".join(self.arguments.subreddit)
programMode["subreddit"] = self.arguments.subreddit programMode["subreddit"] = self.arguments.subreddit
@@ -84,7 +84,7 @@ class ProgramMode:
@staticmethod @staticmethod
def _chooseFrom(choices): def _chooseFrom(choices):
print() print()
choicesByIndex = list(str(x) for x in range(len(choices)+1)) choicesByIndex = [str(x) for x in range(len(choices) + 1)]
for i in range(len(choices)): for i in range(len(choices)):
print("{indent}[{order}] {mode}".format( print("{indent}[{order}] {mode}".format(
indent=" " * 4, order=i + 1, mode=choices[i] indent=" " * 4, order=i + 1, mode=choices[i]
@@ -130,9 +130,10 @@ class ProgramMode:
if programMode == "subreddit": if programMode == "subreddit":
subredditInput = input("(type frontpage for all subscribed subreddits,\n" \ subredditInput = input(
" use plus to seperate multi subreddits:" \ "(type frontpage for all subscribed subreddits,\n"
" pics+funny+me_irl etc.)\n\n" \ " use plus to seperate multi subreddits:"
" pics+funny+me_irl etc.)\n\n"
"subreddit: ") "subreddit: ")
self.arguments.subreddit = subredditInput self.arguments.subreddit = subredditInput
@@ -141,7 +142,8 @@ class ProgramMode:
# self.arguments.subreddit += "+" + subredditInput # self.arguments.subreddit += "+" + subredditInput
if " " in self.arguments.subreddit: if " " in self.arguments.subreddit:
self.arguments.subreddit = "+".join(self.arguments.subreddit.split()) self.arguments.subreddit = "+".join(
self.arguments.subreddit.split())
# DELETE THE PLUS (+) AT THE END # DELETE THE PLUS (+) AT THE END
if not subredditInput.lower() == "frontpage" \ if not subredditInput.lower() == "frontpage" \
@@ -241,13 +243,18 @@ class ProgramMode:
search = 1 if self.arguments.search else 0 search = 1 if self.arguments.search else 0
modes = [ modes = [
"saved","subreddit","submitted","log","link","upvoted","multireddit" "saved",
] "subreddit",
"submitted",
"log",
"link",
"upvoted",
"multireddit"]
values = { values = {
x: 0 if getattr(self.arguments,x) is None or \ x: 0 if getattr(self.arguments, x) is None or
getattr(self.arguments,x) is False \ getattr(self.arguments, x) is False
else 1 \ else 1
for x in modes for x in modes
} }

30
src/reddit.py
View File

@@ -2,12 +2,13 @@ import praw
import random import random
import socket import socket
import webbrowser import webbrowser
from prawcore.exceptions import NotFound, ResponseException, Forbidden from prawcore.exceptions import ResponseException
from src.utils import GLOBAL from src.utils import GLOBAL
from src.jsonHelper import JsonFile from src.jsonHelper import JsonFile
from src. errors import RedditLoginFailed from src. errors import RedditLoginFailed
class Reddit: class Reddit:
def __init__(self, refresh_token=None): def __init__(self, refresh_token=None):
@@ -30,11 +31,13 @@ class Reddit:
self.redditInstance.auth.scopes() self.redditInstance.auth.scopes()
return self.redditInstance return self.redditInstance
except ResponseException: except ResponseException:
self.arguments["redirect_uri"] = "http://localhost:" + str(self.PORT) self.arguments["redirect_uri"] = "http://localhost:" + \
str(self.PORT)
self.redditInstance = praw.Reddit(**self.arguments) self.redditInstance = praw.Reddit(**self.arguments)
reddit, refresh_token = self.getRefreshToken(*self.SCOPES) reddit, refresh_token = self.getRefreshToken(*self.SCOPES)
else: else:
self.arguments["redirect_uri"] = "http://localhost:" + str(self.PORT) self.arguments["redirect_uri"] = "http://localhost:" + \
str(self.PORT)
self.redditInstance = praw.Reddit(**self.arguments) self.redditInstance = praw.Reddit(**self.arguments)
reddit, refresh_token = self.getRefreshToken(*self.SCOPES) reddit, refresh_token = self.getRefreshToken(*self.SCOPES)
@@ -57,7 +60,8 @@ class Reddit:
server.close() server.close()
return client return client
def send_message(self, client, message): @staticmethod
def send_message(client, message):
"""Send message to client and close the connection.""" """Send message to client and close the connection."""
client.send( client.send(
'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8') 'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8')
@@ -68,31 +72,33 @@ class Reddit:
state = str(random.randint(0, 65000)) state = str(random.randint(0, 65000))
url = self.redditInstance.auth.url(scopes, state, 'permanent') url = self.redditInstance.auth.url(scopes, state, 'permanent')
print("---Setting up the Reddit API---\n") print("---Setting up the Reddit API---\n")
print("Go to this URL and login to reddit:\n",url,sep="\n",end="\n\n") print(
"Go to this URL and login to reddit:\n",
url,
sep="\n",
end="\n\n")
webbrowser.open(url, new=2) webbrowser.open(url, new=2)
client = self.recieve_connection() client = self.recieve_connection()
data = client.recv(1024).decode('utf-8') data = client.recv(1024).decode('utf-8')
str(data) str(data)
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&') param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
params = { params = dict([token.split('=')
key: value for (key, value) in [token.split('=') \ for token in param_tokens])
for token in param_tokens]
}
if state != params['state']: if state != params['state']:
self.send_message( self.send_message(
client, 'State mismatch. Expected: {} Received: {}' client, 'State mismatch. Expected: {} Received: {}'
.format(state, params['state']) .format(state, params['state'])
) )
raise RedditLoginFailed raise RedditLoginFailed
elif 'error' in params: if 'error' in params:
self.send_message(client, params['error']) self.send_message(client, params['error'])
raise RedditLoginFailed raise RedditLoginFailed
refresh_token = self.redditInstance.auth.authorize(params['code']) refresh_token = self.redditInstance.auth.authorize(params['code'])
self.send_message(client, self.send_message(client,
"<script>" \ "<script>"
"alert(\"You can go back to terminal window now.\");" \ "alert(\"You can go back to terminal window now.\");"
"</script>" "</script>"
) )
return (self.redditInstance, refresh_token) return (self.redditInstance, refresh_token)

79
src/searcher.py
View File

@@ -1,25 +1,17 @@
import os
import sys import sys
import random
import socket
import time import time
import webbrowser
import urllib.request import urllib.request
from urllib.error import HTTPError from prawcore.exceptions import NotFound, Forbidden
import praw
from prawcore.exceptions import NotFound, ResponseException, Forbidden
from src.reddit import Reddit from src.reddit import Reddit
from src.utils import GLOBAL, createLogFile, printToFile from src.utils import GLOBAL, createLogFile, printToFile
from src.jsonHelper import JsonFile
from src.errors import (NoMatchingSubmissionFound, NoPrawSupport, from src.errors import (NoMatchingSubmissionFound, NoPrawSupport,
NoRedditSupport, MultiredditNotFound, MultiredditNotFound,
InvalidSortingType, RedditLoginFailed, InvalidSortingType, InsufficientPermission)
InsufficientPermission, DirectLinkNotFound)
print = printToFile print = printToFile
def getPosts(programMode): def getPosts(programMode):
"""Call PRAW regarding to arguments and pass it to extractDetails. """Call PRAW regarding to arguments and pass it to extractDetails.
Return what extractDetails has returned. Return what extractDetails has returned.
@@ -39,7 +31,7 @@ def getPosts(programMode):
if programMode["user"] == "me": if programMode["user"] == "me":
programMode["user"] = str(reddit.user.me()) programMode["user"] = str(reddit.user.me())
if not "search" in programMode: if "search" not in programMode:
if programMode["sort"] == "top" or programMode["sort"] == "controversial": if programMode["sort"] == "top" or programMode["sort"] == "controversial":
keyword_params = { keyword_params = {
"time_filter": programMode["time"], "time_filter": programMode["time"],
@@ -62,8 +54,8 @@ def getPosts(programMode):
if "subreddit" in programMode: if "subreddit" in programMode:
print( print(
"search for \"{search}\" in\n" \ "search for \"{search}\" in\n"
"subreddit: {subreddit}\nsort: {sort}\n" \ "subreddit: {subreddit}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
search=programMode["search"], search=programMode["search"],
limit=programMode["limit"], limit=programMode["limit"],
@@ -81,13 +73,13 @@ def getPosts(programMode):
) )
) )
elif "multireddit" in programMode: if "multireddit" in programMode:
raise NoPrawSupport("PRAW does not support that") raise NoPrawSupport("PRAW does not support that")
elif "user" in programMode: if "user" in programMode:
raise NoPrawSupport("PRAW does not support that") raise NoPrawSupport("PRAW does not support that")
elif "saved" in programMode: if "saved" in programMode:
raise ("Reddit does not support that") raise ("Reddit does not support that")
if programMode["sort"] == "relevance": if programMode["sort"] == "relevance":
@@ -100,14 +92,16 @@ def getPosts(programMode):
limit=programMode["limit"] limit=programMode["limit"]
).upper(), noPrint=True ).upper(), noPrint=True
) )
return extractDetails(reddit.user.me().saved(limit=programMode["limit"])) return extractDetails(
reddit.user.me().saved(
limit=programMode["limit"]))
if "subreddit" in programMode: if "subreddit" in programMode:
if programMode["subreddit"] == "frontpage": if programMode["subreddit"] == "frontpage":
print( print(
"subreddit: {subreddit}\nsort: {sort}\n" \ "subreddit: {subreddit}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
limit=programMode["limit"], limit=programMode["limit"],
sort=programMode["sort"], sort=programMode["sort"],
@@ -118,10 +112,8 @@ def getPosts(programMode):
return extractDetails( return extractDetails(
getattr(reddit.front, programMode["sort"])(**keyword_params) getattr(reddit.front, programMode["sort"])(**keyword_params)
) )
else:
print( print(
"subreddit: {subreddit}\nsort: {sort}\n" \ "subreddit: {subreddit}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
limit=programMode["limit"], limit=programMode["limit"],
sort=programMode["sort"], sort=programMode["sort"],
@@ -135,10 +127,10 @@ def getPosts(programMode):
)(**keyword_params) )(**keyword_params)
) )
elif "multireddit" in programMode: if "multireddit" in programMode:
print( print(
"user: {user}\n" \ "user: {user}\n"
"multireddit: {multireddit}\nsort: {sort}\n" \ "multireddit: {multireddit}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
user=programMode["user"], user=programMode["user"],
limit=programMode["limit"], limit=programMode["limit"],
@@ -160,7 +152,7 @@ def getPosts(programMode):
elif "submitted" in programMode: elif "submitted" in programMode:
print( print(
"submitted posts of {user}\nsort: {sort}\n" \ "submitted posts of {user}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
limit=programMode["limit"], limit=programMode["limit"],
sort=programMode["sort"], sort=programMode["sort"],
@@ -170,7 +162,8 @@ def getPosts(programMode):
) )
return extractDetails( return extractDetails(
getattr( getattr(
reddit.redditor(programMode["user"]).submissions,programMode["sort"] reddit.redditor(programMode["user"]
).submissions, programMode["sort"]
)(**keyword_params) )(**keyword_params)
) )
@@ -183,17 +176,21 @@ def getPosts(programMode):
) )
try: try:
return extractDetails( return extractDetails(
reddit.redditor(programMode["user"]).upvoted(limit=programMode["limit"]) reddit.redditor(programMode["user"]).upvoted(
limit=programMode["limit"])
) )
except Forbidden: except Forbidden:
raise InsufficientPermission("You do not have permission to do that") raise InsufficientPermission(
"You do not have permission to do that")
elif "post" in programMode: elif "post" in programMode:
print("post: {post}\n".format(post=programMode["post"]).upper(),noPrint=True) print("post: {post}\n".format(
post=programMode["post"]).upper(), noPrint=True)
return extractDetails( return extractDetails(
reddit.submission(url=programMode["post"]), SINGLE_POST=True reddit.submission(url=programMode["post"]), SINGLE_POST=True
) )
def extractDetails(posts, SINGLE_POST=False): def extractDetails(posts, SINGLE_POST=False):
"""Check posts and decide if it can be downloaded. """Check posts and decide if it can be downloaded.
If so, create a dictionary with post details and append them to a list. If so, create a dictionary with post details and append them to a list.
@@ -229,7 +226,8 @@ def extractDetails(posts,SINGLE_POST=False):
except AttributeError: except AttributeError:
pass pass
if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): if not any(
domain in submission.domain for domain in GLOBAL.arguments.skip_domain):
result = matchWithDownloader(submission) result = matchWithDownloader(submission)
if result is not None: if result is not None:
@@ -267,9 +265,11 @@ def extractDetails(posts,SINGLE_POST=False):
except AttributeError: except AttributeError:
continue continue
if details['POSTID'] in GLOBAL.downloadedPosts(): continue if details['POSTID'] in GLOBAL.downloadedPosts():
continue
if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): if not any(
domain in submission.domain for domain in GLOBAL.arguments.skip_domain):
result = matchWithDownloader(submission) result = matchWithDownloader(submission)
if result is not None: if result is not None:
@@ -284,12 +284,12 @@ def extractDetails(posts,SINGLE_POST=False):
postsFile.add(allPosts) postsFile.add(allPosts)
if not len(postList) == 0: if len(postList) != 0:
print() print()
return postList return postList
else:
raise NoMatchingSubmissionFound("No matching submission was found") raise NoMatchingSubmissionFound("No matching submission was found")
def matchWithDownloader(submission): def matchWithDownloader(submission):
if 'gallery' in submission.url: if 'gallery' in submission.url:
@@ -301,7 +301,7 @@ def matchWithDownloader(submission):
'CONTENTURL': directLink} 'CONTENTURL': directLink}
if 'v.redd.it' in submission.domain: if 'v.redd.it' in submission.domain:
bitrates = ["DASH_1080","DASH_720","DASH_600", \ bitrates = ["DASH_1080", "DASH_720", "DASH_600",
"DASH_480", "DASH_360", "DASH_240"] "DASH_480", "DASH_360", "DASH_240"]
for bitrate in bitrates: for bitrate in bitrates:
@@ -342,10 +342,11 @@ def matchWithDownloader(submission):
if 'reddit.com/gallery' in submission.url: if 'reddit.com/gallery' in submission.url:
return {'TYPE': 'gallery'} return {'TYPE': 'gallery'}
elif submission.is_self and 'self' not in GLOBAL.arguments.skip: if submission.is_self and 'self' not in GLOBAL.arguments.skip:
return {'TYPE': 'self', return {'TYPE': 'self',
'CONTENT': submission.selftext} 'CONTENT': submission.selftext}
def extractDirectLink(URL): def extractDirectLink(URL):
"""Check if link is a direct image link. """Check if link is a direct image link.
If so, return URL, If so, return URL,
@@ -362,7 +363,7 @@ def extractDirectLink(URL):
for extension in imageTypes: for extension in imageTypes:
if extension == URL.split(".")[-1]: if extension == URL.split(".")[-1]:
return URL return URL
else:
return None return None
def genLinksifGallery(metadata): def genLinksifGallery(metadata):

1
src/store.py
View File

@@ -1,5 +1,6 @@
from os import path from os import path
class Store: class Store:
def __init__(self, directory=None): def __init__(self, directory=None):
self.directory = directory self.directory = directory

19
src/utils.py
View File

@@ -1,11 +1,10 @@
import io import io
import json
import sys import sys
from os import makedirs, path, remove from os import makedirs, path
from pathlib import Path from pathlib import Path
from src.jsonHelper import JsonFile from src.jsonHelper import JsonFile
from src.errors import FileNotFoundError
class GLOBAL: class GLOBAL:
"""Declare global variables""" """Declare global variables"""
@@ -18,11 +17,13 @@ class GLOBAL:
configDirectory = "" configDirectory = ""
reddit_client_id = "U-6gk4ZCh3IeNQ" reddit_client_id = "U-6gk4ZCh3IeNQ"
reddit_client_secret = "7CZHY6AmKweZME5s50SfDGylaPg" reddit_client_secret = "7CZHY6AmKweZME5s50SfDGylaPg"
downloadedPosts = lambda: [] @staticmethod
def downloadedPosts(): return []
printVanilla = print printVanilla = print
log_stream = None log_stream = None
def createLogFile(TITLE): def createLogFile(TITLE):
"""Create a log file with given name """Create a log file with given name
inside a folder time stampt in its name and inside a folder time stampt in its name and
@@ -42,12 +43,14 @@ def createLogFile(TITLE):
return FILE return FILE
def printToFile(*args, noPrint=False, **kwargs): def printToFile(*args, noPrint=False, **kwargs):
"""Print to both CONSOLE and """Print to both CONSOLE and
CONSOLE LOG file in a folder time stampt in the name CONSOLE LOG file in a folder time stampt in the name
""" """
folderDirectory = GLOBAL.directory / Path("LOG_FILES") / Path(GLOBAL.RUN_TIME) folderDirectory = GLOBAL.directory / \
Path("LOG_FILES") / Path(GLOBAL.RUN_TIME)
if not noPrint or \ if not noPrint or \
GLOBAL.arguments.verbose or \ GLOBAL.arguments.verbose or \
@@ -58,12 +61,13 @@ def printToFile(*args, noPrint=False,**kwargs):
if not path.exists(folderDirectory): if not path.exists(folderDirectory):
makedirs(folderDirectory) makedirs(folderDirectory)
if not "file" in kwargs: if "file" not in kwargs:
with io.open( with io.open(
folderDirectory / "CONSOLE_LOG.txt", "a", encoding="utf-8" folderDirectory / "CONSOLE_LOG.txt", "a", encoding="utf-8"
) as FILE: ) as FILE:
print(*args, file=FILE, **kwargs) print(*args, file=FILE, **kwargs)
def nameCorrector(string, reference=None): def nameCorrector(string, reference=None):
"""Swap strange characters from given string """Swap strange characters from given string
with underscore (_) and shorten it. with underscore (_) and shorten it.
@@ -89,7 +93,8 @@ def nameCorrector(string,reference=None):
if len(string.split('\n')) > 1: if len(string.split('\n')) > 1:
string = "".join(string.split('\n')) string = "".join(string.split('\n'))
BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','#', '.', '@' ,'“', '’', '\'', '!'] BAD_CHARS = ['\\', '/', ':', '*', '?', '"', '<',
'>', '|', '#', '.', '@', '“', '’', '\'', '!']
string = "".join([i if i not in BAD_CHARS else "_" for i in string]) string = "".join([i if i not in BAD_CHARS else "_" for i in string])
return string return string