ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(maint) code clean up (#187)

Browse Source 
## bdfr

- Add the bound instance as method parameter
- Change methods not using its bound instance to staticmethods
- Fix dangerous default argument
- Refactor the comparison involving `not`
- Refactor unnecessary `else` / `elif` when `if` block has a `raise` statement
- Refactor unnecessary `else` / `elif` when `if` block has a `return` statement
- Refactor useless `else` block in the loop
- Remove implicit `object` from the base class
- Remove reimported module
- Remove unnecessary generator
- Remove unnecessary return statement
- Remove unnecessary use of comprehension
- Remove unused imports
- Use `is` to compare type of objects
- Using not x can cause unwanted results

## Dockerfile

- use a pinned Python version tag instead of latest
- leverage cached requirements

Signed-off-by: Vladislav Doster <mvdoster@gmail.com>

Co-authored-by: Ali Parlakçı <parlakciali@gmail.com>
This commit is contained in:
vlad doster
2021-02-25 03:32:06 -06:00
committed by GitHub
parent e0a2d2eda0
commit fc42afbabe
24 changed files with 781 additions and 663 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
Dockerfile
View File

@@ -1,9 +1,31 @@
FROM python:latest # Bulk Downloader for Reddit
#
# VERSION 0.0.1
FROM python:3.8-slim-buster
LABEL Description="This image enables running Buld Downloader for Reddit with in a container environment" Version="0.0.1"
ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1
WORKDIR "/root/Bulk Downloader for Reddit"
COPY ./requirements.txt ./
RUN ["pip", "install", "-r", "requirements.txt"]
EXPOSE 8080 EXPOSE 8080
EXPOSE 7634 EXPOSE 7634
CMD ["python", "script.py", "-d", "downloads"] # Install dependencies for building Python packages
RUN apt-get update \
&& apt-get install -y build-essential \
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
&& rm -rf /var/lib/apt/lists/*
# Requirements are installed here to ensure they will be cached.
COPY requirements.txt /requirements.txt
RUN pip install --no-cache-dir -r /requirements.txt \
&& rm -rf /requirements.txt
# Copy project files into container
COPY . /bdfr
WORKDIR /bdfr
# This is useful because the image name can double as a reference to the binary
ENTRYPOINT ["python", "script.py"]
CMD ["--help"]

9
docker-compose.yml
View File

@@ -1,12 +1,17 @@
version: "3" version: "3"
services: services:
bdfr: bdfr:
build:
context: .
dockerfile: ./Dockerfile
image: bdfr image: bdfr
build: . container_name: bdfr
ports: ports:
- "8080:8080" - "8080:8080"
- "7634:7634" - "7634:7634"
volumes: volumes:
- "./:/root/Bulk Downloader for Reddit" - .:/bdfr:z
container_name: bdfr_container container_name: bdfr_container
network_mode: bridge network_mode: bridge

104
script.py
View File

@@ -4,15 +4,12 @@
This program downloads imgur, gfycat and direct image and video links of This program downloads imgur, gfycat and direct image and video links of
saved posts from a reddit account. It is written in Python 3. saved posts from a reddit account. It is written in Python 3.
""" """
import argparse
import logging import logging
import os import os
import sys import sys
import time import time
import webbrowser
from io import StringIO from io import StringIO
from pathlib import Path, PurePath from pathlib import Path
from prawcore.exceptions import InsufficientScope from prawcore.exceptions import InsufficientScope
from src.downloaders.Direct import Direct from src.downloaders.Direct import Direct
@@ -26,8 +23,7 @@ from src.downloaders.vreddit import VReddit
from src.downloaders.youtube import Youtube from src.downloaders.youtube import Youtube
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
from src.downloaders.gallery import gallery from src.downloaders.gallery import gallery
from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info from src.errors import ImgurLimitError, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info
from src.parser import LinkDesigner
from src.searcher import getPosts from src.searcher import getPosts
from src.utils import (GLOBAL, createLogFile, nameCorrector, from src.utils import (GLOBAL, createLogFile, nameCorrector,
printToFile) printToFile)
@@ -44,6 +40,7 @@ __version__ = "1.9.4"
__maintainer__ = "Ali Parlakci" __maintainer__ = "Ali Parlakci"
__email__ = "parlakciali@gmail.com" __email__ = "parlakciali@gmail.com"
def postFromLog(fileName): def postFromLog(fileName):
"""Analyze a log file and return a list of dictionaries containing """Analyze a log file and return a list of dictionaries containing
submissions submissions
@@ -62,17 +59,19 @@ def postFromLog(fileName):
posts = [] posts = []
for post in content: for post in content:
if not content[post][-1]['TYPE'] == None: if content[post][-1]['TYPE'] is not None:
posts.append(content[post][-1]) posts.append(content[post][-1])
return posts return posts
def isPostExists(POST,directory):
def isPostExists(POST, directory):
"""Figure out a file's name and checks if the file already exists""" """Figure out a file's name and checks if the file already exists"""
filename = GLOBAL.config['filename'].format(**POST) filename = GLOBAL.config['filename'].format(**POST)
possibleExtensions = [".jpg",".png",".mp4",".gif",".webm",".md",".mkv",".flv"] possibleExtensions = [".jpg", ".png", ".mp4",
".gif", ".webm", ".md", ".mkv", ".flv"]
for extension in possibleExtensions: for extension in possibleExtensions:
@@ -81,10 +80,10 @@ def isPostExists(POST,directory):
if path.exists(): if path.exists():
return True return True
else:
return False return False
def downloadPost(SUBMISSION,directory):
def downloadPost(SUBMISSION, directory):
downloaders = { downloaders = {
"imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":SelfPost, "imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":SelfPost,
@@ -94,11 +93,10 @@ def downloadPost(SUBMISSION,directory):
print() print()
if SUBMISSION['TYPE'] in downloaders: if SUBMISSION['TYPE'] in downloaders:
downloaders[SUBMISSION['TYPE']] (directory,SUBMISSION) downloaders[SUBMISSION['TYPE']](directory, SUBMISSION)
else: else:
raise NoSuitablePost raise NoSuitablePost
return None
def download(submissions): def download(submissions):
"""Analyze list of submissions and call the right function """Analyze list of submissions and call the right function
@@ -116,22 +114,23 @@ def download(submissions):
subsLenght = len(submissions) subsLenght = len(submissions)
for i in range(len(submissions)): for i in range(len(submissions)):
print(f"\n({i+1}/{subsLenght})",end=" — ") print(f"\n({i+1}/{subsLenght})", end=" — ")
print(submissions[i]['POSTID'], print(submissions[i]['POSTID'],
f"r/{submissions[i]['SUBREDDIT']}", f"r/{submissions[i]['SUBREDDIT']}",
f"u/{submissions[i]['REDDITOR']}", f"u/{submissions[i]['REDDITOR']}",
submissions[i]['FLAIR'] if submissions[i]['FLAIR'] else "", submissions[i]['FLAIR'] if submissions[i]['FLAIR'] else "",
sep=" — ", sep=" — ",
end="") end="")
print(f" – {submissions[i]['TYPE'].upper()}",end="",noPrint=True) print(f" – {submissions[i]['TYPE'].upper()}", end="", noPrint=True)
directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**submissions[i]) directory = GLOBAL.directory / \
GLOBAL.config["folderpath"].format(**submissions[i])
details = { details = {
**submissions[i], **submissions[i],
**{ **{
"TITLE": nameCorrector( "TITLE": nameCorrector(
submissions[i]['TITLE'], submissions[i]['TITLE'],
reference = str(directory) reference=str(directory)
+ GLOBAL.config['filename'].format(**submissions[i]) + GLOBAL.config['filename'].format(**submissions[i])
+ ".ext" + ".ext"
) )
@@ -139,7 +138,7 @@ def download(submissions):
} }
filename = GLOBAL.config['filename'].format(**details) filename = GLOBAL.config['filename'].format(**details)
if isPostExists(details,directory): if isPostExists(details, directory):
print() print()
print(directory) print(directory)
print(filename) print(filename)
@@ -154,7 +153,7 @@ def download(submissions):
continue continue
try: try:
downloadPost(details,directory) downloadPost(details, directory)
GLOBAL.downloadedPosts.add(details['POSTID']) GLOBAL.downloadedPosts.add(details['POSTID'])
try: try:
if GLOBAL.arguments.unsave: if GLOBAL.arguments.unsave:
@@ -172,15 +171,16 @@ def download(submissions):
except ImgurLoginError: except ImgurLoginError:
print( print(
"Imgur login failed. \nQuitting the program "\ "Imgur login failed. \nQuitting the program "
"as unexpected errors might occur." "as unexpected errors might occur."
) )
sys.exit() sys.exit()
except ImgurLimitError as exception: except ImgurLimitError as exception:
FAILED_FILE.add({int(i+1):[ FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format( "{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception) class_name=exception.__class__.__name__, info=str(
exception)
), ),
details details
]}) ]})
@@ -188,12 +188,14 @@ def download(submissions):
except NotADownloadableLinkError as exception: except NotADownloadableLinkError as exception:
print( print(
"{class_name}: {info}".format( "{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception) class_name=exception.__class__.__name__, info=str(
exception)
) )
) )
FAILED_FILE.add({int(i+1):[ FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format( "{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception) class_name=exception.__class__.__name__, info=str(
exception)
), ),
submissions[i] submissions[i]
]}) ]})
@@ -215,9 +217,9 @@ def download(submissions):
print("Failed to download the posts, skipping...") print("Failed to download the posts, skipping...")
except AlbumNotDownloadedCompletely: except AlbumNotDownloadedCompletely:
print("Album did not downloaded completely.") print("Album did not downloaded completely.")
FAILED_FILE.add({int(i+1):[ FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format( "{class_name}: {info}".format(
class_name=exc.__class__.__name__,info=str(exc) class_name=exc.__class__.__name__, info=str(exc)
), ),
submissions[i] submissions[i]
]}) ]})
@@ -225,39 +227,42 @@ def download(submissions):
except Exception as exc: except Exception as exc:
print( print(
"{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format( "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
class_name=exc.__class__.__name__,info=str(exc) class_name=exc.__class__.__name__, info=str(exc)
) )
) )
logging.error(sys.exc_info()[0].__name__, logging.error(sys.exc_info()[0].__name__,
exc_info=full_exc_info(sys.exc_info())) exc_info=full_exc_info(sys.exc_info()))
print(GLOBAL.log_stream.getvalue(),noPrint=True) print(GLOBAL.log_stream.getvalue(), noPrint=True)
FAILED_FILE.add({int(i+1):[ FAILED_FILE.add({int(i+1): [
"{class_name}: {info}".format( "{class_name}: {info}".format(
class_name=exc.__class__.__name__,info=str(exc) class_name=exc.__class__.__name__, info=str(exc)
), ),
submissions[i] submissions[i]
]}) ]})
if duplicates: if duplicates:
print(f"\nThere {'were' if duplicates > 1 else 'was'} " \ print(f"\nThere {'were' if duplicates > 1 else 'was'} "
f"{duplicates} duplicate{'s' if duplicates > 1 else ''}") f"{duplicates} duplicate{'s' if duplicates > 1 else ''}")
if downloadedCount == 0: if downloadedCount:
print("Nothing is downloaded :(") print(f"Total of {downloadedCount} "
else:
print(f"Total of {downloadedCount} " \
f"link{'s' if downloadedCount > 1 else ''} downloaded!") f"link{'s' if downloadedCount > 1 else ''} downloaded!")
else:
print("Nothing is downloaded :(")
def printLogo(): def printLogo():
VanillaPrint( VanillaPrint(
f"\nBulk Downloader for Reddit v{__version__}\n" \ f"\nBulk Downloader for Reddit v{__version__}\n"
f"Written by Ali PARLAKCI – parlakciali@gmail.com\n\n" \ f"Written by Ali PARLAKCI – parlakciali@gmail.com\n\n"
f"https://github.com/aliparlakci/bulk-downloader-for-reddit/\n" f"https://github.com/aliparlakci/bulk-downloader-for-reddit/\n"
) )
def main(): def main():
if Path("config.json").exists(): if Path("config.json").exists():
@@ -269,7 +274,7 @@ def main():
try: try:
GLOBAL.config = Config(GLOBAL.configDirectory).generate() GLOBAL.config = Config(GLOBAL.configDirectory).generate()
except InvalidJSONFile as exception: except InvalidJSONFile as exception:
VanillaPrint(str(exception.__class__.__name__),">>",str(exception)) VanillaPrint(str(exception.__class__.__name__), ">>", str(exception))
VanillaPrint("Resolve it or remove it to proceed") VanillaPrint("Resolve it or remove it to proceed")
input("\nPress enter to quit") input("\nPress enter to quit")
sys.exit() sys.exit()
@@ -302,7 +307,8 @@ def main():
if arguments.directory: if arguments.directory:
GLOBAL.directory = Path(arguments.directory.strip()) GLOBAL.directory = Path(arguments.directory.strip())
elif "default_directory" in GLOBAL.config and GLOBAL.config["default_directory"] != "": elif "default_directory" in GLOBAL.config and GLOBAL.config["default_directory"] != "":
GLOBAL.directory = Path(GLOBAL.config["default_directory"].format(time=GLOBAL.RUN_TIME)) GLOBAL.directory = Path(
GLOBAL.config["default_directory"].format(time=GLOBAL.RUN_TIME))
else: else:
GLOBAL.directory = Path(input("\ndownload directory: ").strip()) GLOBAL.directory = Path(input("\ndownload directory: ").strip())
@@ -312,14 +318,13 @@ def main():
GLOBAL.downloadedPosts = Store() GLOBAL.downloadedPosts = Store()
printLogo() printLogo()
print("\n"," ".join(sys.argv),"\n",noPrint=True) print("\n", " ".join(sys.argv), "\n", noPrint=True)
if arguments.log is not None: if arguments.log is not None:
logDir = Path(arguments.log) logDir = Path(arguments.log)
download(postFromLog(logDir)) download(postFromLog(logDir))
sys.exit() sys.exit()
programMode = ProgramMode(arguments).generate() programMode = ProgramMode(arguments).generate()
try: try:
@@ -327,7 +332,7 @@ def main():
except Exception as exc: except Exception as exc:
logging.error(sys.exc_info()[0].__name__, logging.error(sys.exc_info()[0].__name__,
exc_info=full_exc_info(sys.exc_info())) exc_info=full_exc_info(sys.exc_info()))
print(GLOBAL.log_stream.getvalue(),noPrint=True) print(GLOBAL.log_stream.getvalue(), noPrint=True)
print(exc) print(exc)
sys.exit() sys.exit()
@@ -335,8 +340,11 @@ def main():
print("I could not find any posts in that URL") print("I could not find any posts in that URL")
sys.exit() sys.exit()
if GLOBAL.arguments.no_download: pass if GLOBAL.arguments.no_download:
else: download(posts) pass
else:
download(posts)
if __name__ == "__main__": if __name__ == "__main__":
@@ -363,5 +371,5 @@ if __name__ == "__main__":
exc_info=full_exc_info(sys.exc_info())) exc_info=full_exc_info(sys.exc_info()))
print(GLOBAL.log_stream.getvalue()) print(GLOBAL.log_stream.getvalue())
if not GLOBAL.arguments.quit: input("\nPress enter to quit\n") if not GLOBAL.arguments.quit:
input("\nPress enter to quit\n")

74
src/arguments.py
View File

@@ -1,31 +1,34 @@
import argparse import argparse
import sys import sys
class Arguments: class Arguments:
@staticmethod @staticmethod
def parse(arguments=[]): def parse(arguments=None):
"""Initialize argparse and add arguments""" """Initialize argparse and add arguments"""
if arguments is None:
arguments = []
parser = argparse.ArgumentParser(allow_abbrev=False, parser = argparse.ArgumentParser(allow_abbrev=False,
description="This program downloads " \ description="This program downloads "
"media from reddit " \ "media from reddit "
"posts") "posts")
parser.add_argument("--directory","-d", parser.add_argument("--directory", "-d",
help="Specifies the directory where posts will be " \ help="Specifies the directory where posts will be "
"downloaded to", "downloaded to",
metavar="DIRECTORY") metavar="DIRECTORY")
parser.add_argument("--verbose","-v", parser.add_argument("--verbose", "-v",
help="Verbose Mode", help="Verbose Mode",
action="store_true", action="store_true",
default=False) default=False)
parser.add_argument("--quit","-q", parser.add_argument("--quit", "-q",
help="Auto quit afer the process finishes", help="Auto quit afer the process finishes",
action="store_true", action="store_true",
default=False) default=False)
parser.add_argument("--link","-l", parser.add_argument("--link", "-l",
help="Get posts from link", help="Get posts from link",
metavar="link") metavar="link")
@@ -47,43 +50,45 @@ class Arguments:
help="Gets upvoted posts of --user") help="Gets upvoted posts of --user")
parser.add_argument("--log", parser.add_argument("--log",
help="Takes a log file which created by itself " \ help="Takes a log file which created by itself "
"(json files), reads posts and tries downloadin" \ "(json files), reads posts and tries downloadin"
"g them again.", "g them again.",
# type=argparse.FileType('r'), # type=argparse.FileType('r'),
metavar="LOG FILE") metavar="LOG FILE")
parser.add_argument("--subreddit", parser.add_argument(
"--subreddit",
nargs="+", nargs="+",
help="Triggers subreddit mode and takes subreddit's " \ help="Triggers subreddit mode and takes subreddit's "
"name without r/. use \"frontpage\" for frontpage", "name without r/. use \"frontpage\" for frontpage",
metavar="SUBREDDIT", metavar="SUBREDDIT",
type=str) type=str)
parser.add_argument("--multireddit", parser.add_argument("--multireddit",
help="Triggers multireddit mode and takes "\ help="Triggers multireddit mode and takes "
"multireddit's name without m/", "multireddit's name without m/",
metavar="MULTIREDDIT", metavar="MULTIREDDIT",
type=str) type=str)
parser.add_argument("--user", parser.add_argument("--user",
help="reddit username if needed. use \"me\" for " \ help="reddit username if needed. use \"me\" for "
"current user", "current user",
required="--multireddit" in sys.argv or \ required="--multireddit" in sys.argv or
"--submitted" in sys.argv, "--submitted" in sys.argv,
metavar="redditor", metavar="redditor",
type=str) type=str)
parser.add_argument("--search", parser.add_argument(
"--search",
help="Searches for given query in given subreddits", help="Searches for given query in given subreddits",
metavar="query", metavar="query",
type=str) type=str)
parser.add_argument("--sort", parser.add_argument("--sort",
help="Either hot, top, new, controversial, rising " \ help="Either hot, top, new, controversial, rising "
"or relevance default: hot", "or relevance default: hot",
choices=[ choices=[
"hot","top","new","controversial","rising", "hot", "top", "new", "controversial", "rising",
"relevance" "relevance"
], ],
metavar="SORT TYPE", metavar="SORT TYPE",
@@ -95,9 +100,10 @@ class Arguments:
type=int) type=int)
parser.add_argument("--time", parser.add_argument("--time",
help="Either hour, day, week, month, year or all." \ help="Either hour, day, week, month, year or all."
" default: all", " default: all",
choices=["all","hour","day","week","month","year"], choices=["all", "hour", "day",
"week", "month", "year"],
metavar="TIME_LIMIT", metavar="TIME_LIMIT",
type=str) type=str)
@@ -105,7 +111,7 @@ class Arguments:
nargs="+", nargs="+",
help="Skip posts with given type", help="Skip posts with given type",
type=str, type=str,
choices=["images","videos","gifs","self"], choices=["images", "videos", "gifs", "self"],
default=[]) default=[])
parser.add_argument("--skip-domain", parser.add_argument("--skip-domain",
@@ -124,38 +130,40 @@ class Arguments:
help="Set custom filename", help="Set custom filename",
) )
parser.add_argument("--set-default-directory", parser.add_argument(
"--set-default-directory",
action="store_true", action="store_true",
help="Set a default directory to be used in case no directory is given", help="Set a default directory to be used in case no directory is given",
) )
parser.add_argument("--set-default-options", parser.add_argument(
"--set-default-options",
action="store_true", action="store_true",
help="Set default options to use everytime program runs", help="Set default options to use everytime program runs",
) )
parser.add_argument("--use-local-config", parser.add_argument(
"--use-local-config",
action="store_true", action="store_true",
help="Creates a config file in the program's directory and uses it. Useful for having multiple configs", help="Creates a config file in the program's directory and uses it. Useful for having multiple configs",
) )
parser.add_argument("--no-dupes", parser.add_argument(
"--no-dupes",
action="store_true", action="store_true",
help="Do not download duplicate posts on different subreddits", help="Do not download duplicate posts on different subreddits",
) )
parser.add_argument("--downloaded-posts", parser.add_argument(
"--downloaded-posts",
help="Use a hash file to keep track of downloaded files", help="Use a hash file to keep track of downloaded files",
type=str type=str)
)
parser.add_argument("--no-download", parser.add_argument(
"--no-download",
action="store_true", action="store_true",
help="Just saved posts into a the POSTS.json file without downloading" help="Just saved posts into a the POSTS.json file without downloading")
)
if arguments == []: if arguments == []:
return parser.parse_args() return parser.parse_args()
else:
return parser.parse_args(arguments) return parser.parse_args(arguments)

24
src/config.py
View File

@@ -1,15 +1,12 @@
import os
import socket
import webbrowser
import random
from src.reddit import Reddit from src.reddit import Reddit
from src.jsonHelper import JsonFile from src.jsonHelper import JsonFile
from src.utils import nameCorrector from src.utils import nameCorrector
class Config(): class Config():
def __init__(self,filename): def __init__(self, filename):
self.filename = filename self.filename = filename
self.file = JsonFile(self.filename) self.file = JsonFile(self.filename)
@@ -45,7 +42,7 @@ Existing filename template:""", None if "filename" not in self.file.read() else
def _readCustomFileName(self): def _readCustomFileName(self):
content = self.file.read() content = self.file.read()
if not "filename" in content: if "filename" not in content:
self.file.add({ self.file.add({
"filename": "{REDDITOR}_{TITLE}_{POSTID}" "filename": "{REDDITOR}_{TITLE}_{POSTID}"
}) })
@@ -75,9 +72,9 @@ Existing folder structure""", None if "folderpath" not in self.file.read() else
"folderpath": folderpath "folderpath": folderpath
}) })
def _readCustomFolderPath(self,path=None): def _readCustomFolderPath(self, path=None):
content = self.file.read() content = self.file.read()
if not "folderpath" in content: if "folderpath" not in content:
self.file.add({ self.file.add({
"folderpath": "{SUBREDDIT}" "folderpath": "{SUBREDDIT}"
}) })
@@ -96,9 +93,9 @@ Existing default options:""", None if "options" not in self.file.read() else sel
"options": options "options": options
}) })
def _readDefaultOptions(self,path=None): def _readDefaultOptions(self, path=None):
content = self.file.read() content = self.file.read()
if not "options" in content: if "options" not in content:
self.file.add({ self.file.add({
"options": "" "options": ""
}) })
@@ -108,9 +105,9 @@ Existing default options:""", None if "options" not in self.file.read() else sel
try: try:
content = self.file.read()["credentials"] content = self.file.read()["credentials"]
except: except BaseException:
self.file.add({ self.file.add({
"credentials":{} "credentials": {}
}) })
content = self.file.read()["credentials"] content = self.file.read()["credentials"]
@@ -126,7 +123,8 @@ Existing default options:""", None if "options" not in self.file.read() else sel
Leave blank to reset it. You can use {time} in foler names to use to timestamp it Leave blank to reset it. You can use {time} in foler names to use to timestamp it
For example: D:/archive/BDFR_{time} For example: D:/archive/BDFR_{time}
""") """)
print("Current default directory:", self.file.read()["default_directory"] if "default_directory" in self.file.read() else "") print("Current default directory:", self.file.read()[
"default_directory"] if "default_directory" in self.file.read() else "")
self.file.add({ self.file.add({
"default_directory": input(">> ") "default_directory": input(">> ")
}) })

16
src/downloaders/Direct.py
View File

@@ -1,18 +1,16 @@
import os import os
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile, getExtension
from src.errors import FileNameTooLong
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print
class Direct: class Direct:
def __init__(self,directory,POST): def __init__(self, directory, POST):
POST['EXTENSION'] = getExtension(POST['CONTENTURL']) POST['EXTENSION'] = getExtension(POST['CONTENTURL'])
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**POST)+POST["EXTENSION"] filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"]
shortFilename = POST['POSTID']+POST['EXTENSION'] shortFilename = POST['POSTID'] + POST['EXTENSION']
getFile(filename,shortFilename,directory,POST['CONTENTURL'])
getFile(filename, shortFilename, directory, POST['CONTENTURL'])

33
src/downloaders/Erome.py
View File

@@ -1,19 +1,18 @@
import os import os
import logging
import sys
import urllib.request import urllib.request
from html.parser import HTMLParser from html.parser import HTMLParser
from src.downloaders.downloaderUtils import getFile from src.downloaders.downloaderUtils import getFile
from src.downloaders.downloaderUtils import getExtension from src.downloaders.downloaderUtils import getExtension
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely, from src.errors import (AlbumNotDownloadedCompletely,
NotADownloadableLinkError, FileAlreadyExistsError, full_exc_info) NotADownloadableLinkError, FileAlreadyExistsError)
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
class Erome: class Erome:
def __init__(self,directory,post): def __init__(self, directory, post):
try: try:
IMAGES = self.getLinks(post['CONTENTURL']) IMAGES = self.getLinks(post['CONTENTURL'])
except urllib.error.HTTPError: except urllib.error.HTTPError:
@@ -29,14 +28,15 @@ class Erome:
"""Filenames are declared here""" """Filenames are declared here"""
filename = GLOBAL.config['filename'].format(**post)+post["EXTENSION"] filename = GLOBAL.config['filename'].format(
**post) + post["EXTENSION"]
shortFilename = post['POSTID'] + extension shortFilename = post['POSTID'] + extension
imageURL = IMAGES[0] imageURL = IMAGES[0]
if 'https://' not in imageURL or 'http://' not in imageURL: if 'https://' not in imageURL or 'http://' not in imageURL:
imageURL = "https://" + imageURL imageURL = "https://" + imageURL
getFile(filename,shortFilename,directory,imageURL) getFile(filename, shortFilename, directory, imageURL)
else: else:
filename = GLOBAL.config['filename'].format(**post) filename = GLOBAL.config['filename'].format(**post)
@@ -56,19 +56,19 @@ class Erome:
extension = getExtension(IMAGES[i]) extension = getExtension(IMAGES[i])
filename = str(i+1)+extension filename = str(i + 1) + extension
imageURL = IMAGES[i] imageURL = IMAGES[i]
if 'https://' not in imageURL and 'http://' not in imageURL: if 'https://' not in imageURL and 'http://' not in imageURL:
imageURL = "https://" + imageURL imageURL = "https://" + imageURL
print(" ({}/{})".format(i+1,imagesLenght)) print(" ({}/{})".format(i + 1, imagesLenght))
print(" {}".format(filename)) print(" {}".format(filename))
try: try:
getFile(filename,filename,folderDir,imageURL,indent=2) getFile(filename, filename, folderDir, imageURL, indent=2)
print() print()
except FileAlreadyExistsError: except FileAlreadyExistsError:
print(" The file already exists" + " "*10,end="\n\n") print(" The file already exists" + " " * 10, end="\n\n")
duplicates += 1 duplicates += 1
howManyDownloaded -= 1 howManyDownloaded -= 1
@@ -87,20 +87,21 @@ class Erome:
if duplicates == imagesLenght: if duplicates == imagesLenght:
raise FileAlreadyExistsError raise FileAlreadyExistsError
elif howManyDownloaded + duplicates < imagesLenght: if howManyDownloaded + duplicates < imagesLenght:
raise AlbumNotDownloadedCompletely( raise AlbumNotDownloadedCompletely(
"Album Not Downloaded Completely" "Album Not Downloaded Completely"
) )
def getLinks(self,url,lineNumber=129): def getLinks(self, url, lineNumber=129):
content = [] content = []
lineNumber = None lineNumber = None
class EromeParser(HTMLParser): class EromeParser(HTMLParser):
tag = None tag = None
def handle_starttag(self, tag, attrs): def handle_starttag(self, tag, attrs):
self.tag = {tag:{attr[0]: attr[1] for attr in attrs}} self.tag = {tag: {attr[0]: attr[1] for attr in attrs}}
pageSource = (urllib.request.urlopen(url).read().decode().split('\n')) pageSource = (urllib.request.urlopen(url).read().decode().split('\n'))
@@ -124,12 +125,12 @@ class Erome:
if tag is not None: if tag is not None:
if "img" in tag: if "img" in tag:
if "class" in tag["img"]: if "class" in tag["img"]:
if tag["img"]["class"]=="img-front": if tag["img"]["class"] == "img-front":
content.append(tag["img"]["src"]) content.append(tag["img"]["src"])
elif "source" in tag: elif "source" in tag:
content.append(tag["source"]["src"]) content.append(tag["source"]["src"])
return [ return [
link for link in content \ link for link in content
if link.endswith("_480p.mp4") or not link.endswith(".mp4") if link.endswith("_480p.mp4") or not link.endswith(".mp4")
] ]

21
src/downloaders/Gfycat.py
View File

@@ -4,14 +4,13 @@ import urllib.request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile, getExtension
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely, from src.errors import (NotADownloadableLinkError)
NotADownloadableLinkError, FileAlreadyExistsError)
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
class Gfycat: class Gfycat:
def __init__(self,directory,POST): def __init__(self, directory, POST):
try: try:
POST['MEDIAURL'] = self.getLink(POST['CONTENTURL']) POST['MEDIAURL'] = self.getLink(POST['CONTENTURL'])
except IndexError: except IndexError:
@@ -19,12 +18,13 @@ class Gfycat:
POST['EXTENSION'] = getExtension(POST['MEDIAURL']) POST['EXTENSION'] = getExtension(POST['MEDIAURL'])
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**POST)+POST["EXTENSION"] filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"]
shortFilename = POST['POSTID']+POST['EXTENSION'] shortFilename = POST['POSTID'] + POST['EXTENSION']
getFile(filename,shortFilename,directory,POST['MEDIAURL']) getFile(filename, shortFilename, directory, POST['MEDIAURL'])
@staticmethod @staticmethod
def getLink(url): def getLink(url):
@@ -43,8 +43,9 @@ class Gfycat:
pageSource = (urllib.request.urlopen(url).read().decode()) pageSource = (urllib.request.urlopen(url).read().decode())
soup = BeautifulSoup(pageSource, "html.parser") soup = BeautifulSoup(pageSource, "html.parser")
attributes = {"data-react-helmet":"true","type":"application/ld+json"} attributes = {"data-react-helmet": "true",
content = soup.find("script",attrs=attributes) "type": "application/ld+json"}
content = soup.find("script", attrs=attributes)
if content is None: if content is None:
return GifDeliveryNetwork.getLink(url) return GifDeliveryNetwork.getLink(url)

69
src/downloaders/Imgur.py
View File

@@ -1,7 +1,5 @@
import urllib
import json import json
import os import os
import time
import requests import requests
from src.utils import GLOBAL, nameCorrector from src.utils import GLOBAL, nameCorrector
@@ -10,16 +8,17 @@ from src.downloaders.Direct import Direct
from src.downloaders.downloaderUtils import getFile from src.downloaders.downloaderUtils import getFile
from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, ExtensionError, NotADownloadableLinkError, TypeInSkip from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, ExtensionError, NotADownloadableLinkError, TypeInSkip
class Imgur: class Imgur:
IMGUR_IMAGE_DOMAIN = "https://i.imgur.com/" IMGUR_IMAGE_DOMAIN = "https://i.imgur.com/"
def __init__(self,directory, post): def __init__(self, directory, post):
link = post['CONTENTURL'] link = post['CONTENTURL']
if link.endswith(".gifv"): if link.endswith(".gifv"):
link = link.replace(".gifv",".mp4") link = link.replace(".gifv", ".mp4")
Direct(directory, {**post, 'CONTENTURL': link}) Direct(directory, {**post, 'CONTENTURL': link})
return None return None
@@ -57,22 +56,23 @@ class Imgur:
extension = self.validateExtension(images["images"][i]["ext"]) extension = self.validateExtension(images["images"][i]["ext"])
imageURL = self.IMGUR_IMAGE_DOMAIN + images["images"][i]["hash"] + extension imageURL = self.IMGUR_IMAGE_DOMAIN + \
images["images"][i]["hash"] + extension
filename = "_".join([ filename = "_".join([str(i + 1),
str(i+1), nameCorrector(images["images"][i]['title']), images["images"][i]['hash'] nameCorrector(images["images"][i]['title']),
]) + extension images["images"][i]['hash']]) + extension
shortFilename = str(i+1) + "_" + images["images"][i]['hash'] shortFilename = str(i + 1) + "_" + images["images"][i]['hash']
print("\n ({}/{})".format(i+1,imagesLenght)) print("\n ({}/{})".format(i + 1, imagesLenght))
try: try:
getFile(filename,shortFilename,folderDir,imageURL,indent=2) getFile(filename, shortFilename, folderDir, imageURL, indent=2)
howManyDownloaded += 1 howManyDownloaded += 1
print() print()
except FileAlreadyExistsError: except FileAlreadyExistsError:
print(" The file already exists" + " "*10,end="\n\n") print(" The file already exists" + " " * 10, end="\n\n")
duplicates += 1 duplicates += 1
except TypeInSkip: except TypeInSkip:
@@ -82,18 +82,16 @@ class Imgur:
except Exception as exception: except Exception as exception:
print("\n Could not get the file") print("\n Could not get the file")
print( print(
" " " " +
+ "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format( "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
class_name=exception.__class__.__name__, class_name=exception.__class__.__name__,
info=str(exception) info=str(exception)) +
) "\n")
+ "\n" print(GLOBAL.log_stream.getvalue(), noPrint=True)
)
print(GLOBAL.log_stream.getvalue(),noPrint=True)
if duplicates == imagesLenght: if duplicates == imagesLenght:
raise FileAlreadyExistsError raise FileAlreadyExistsError
elif howManyDownloaded + duplicates < imagesLenght: if howManyDownloaded + duplicates < imagesLenght:
raise AlbumNotDownloadedCompletely( raise AlbumNotDownloadedCompletely(
"Album Not Downloaded Completely" "Album Not Downloaded Completely"
) )
@@ -103,9 +101,9 @@ class Imgur:
imageURL = self.IMGUR_IMAGE_DOMAIN + image["hash"] + extension imageURL = self.IMGUR_IMAGE_DOMAIN + image["hash"] + extension
filename = GLOBAL.config['filename'].format(**self.post) + extension filename = GLOBAL.config['filename'].format(**self.post) + extension
shortFilename = self.post['POSTID']+extension shortFilename = self.post['POSTID'] + extension
getFile(filename,shortFilename,self.directory,imageURL) getFile(filename, shortFilename, self.directory, imageURL)
@property @property
def isAlbum(self): def isAlbum(self):
@@ -116,7 +114,9 @@ class Imgur:
cookies = {"over18": "1", "postpagebeta": "0"} cookies = {"over18": "1", "postpagebeta": "0"}
res = requests.get(link, cookies=cookies) res = requests.get(link, cookies=cookies)
if res.status_code != 200: raise ImageNotFound(f"Server responded with {res.status_code} to {link}") if res.status_code != 200:
raise ImageNotFound(
f"Server responded with {res.status_code} to {link}")
pageSource = requests.get(link, cookies=cookies).text pageSource = requests.get(link, cookies=cookies).text
STARTING_STRING = "image : " STARTING_STRING = "image : "
@@ -124,18 +124,20 @@ class Imgur:
STARTING_STRING_LENGHT = len(STARTING_STRING) STARTING_STRING_LENGHT = len(STARTING_STRING)
try: try:
startIndex = pageSource.index(STARTING_STRING) + STARTING_STRING_LENGHT startIndex = pageSource.index(
STARTING_STRING) + STARTING_STRING_LENGHT
endIndex = pageSource.index(ENDING_STRING, startIndex) endIndex = pageSource.index(ENDING_STRING, startIndex)
except ValueError: except ValueError:
raise NotADownloadableLinkError(f"Could not read the page source on {link}") raise NotADownloadableLinkError(
f"Could not read the page source on {link}")
while pageSource[endIndex] != "}": while pageSource[endIndex] != "}":
endIndex=endIndex-1 endIndex = endIndex - 1
try: try:
data = pageSource[startIndex:endIndex+2].strip()[:-1] data = pageSource[startIndex:endIndex + 2].strip()[:-1]
except: except BaseException:
pageSource[endIndex+1]='}' pageSource[endIndex + 1] = '}'
data = pageSource[startIndex:endIndex+3].strip()[:-1] data = pageSource[startIndex:endIndex + 3].strip()[:-1]
return json.loads(data) return json.loads(data)
@@ -144,5 +146,8 @@ class Imgur:
POSSIBLE_EXTENSIONS = [".jpg", ".png", ".mp4", ".gif"] POSSIBLE_EXTENSIONS = [".jpg", ".png", ".mp4", ".gif"]
for extension in POSSIBLE_EXTENSIONS: for extension in POSSIBLE_EXTENSIONS:
if extension in string: return extension if extension in string:
else: raise ExtensionError(f"\"{string}\" is not recognized as a valid extension.") return extension
raise ExtensionError(
f"\"{string}\" is not recognized as a valid extension.")

59
src/downloaders/downloaderUtils.py
View File

@@ -1,45 +1,51 @@
import sys import sys
import os import os
import time
from urllib.error import HTTPError
import urllib.request import urllib.request
from pathlib import Path from pathlib import Path
import hashlib import hashlib
from src.utils import nameCorrector, GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
from src.errors import FileAlreadyExistsError, FileNameTooLong, FailedToDownload, TypeInSkip, DomainInSkip from src.errors import FileAlreadyExistsError, FailedToDownload, TypeInSkip, DomainInSkip
def dlProgress(count, blockSize, totalSize): def dlProgress(count, blockSize, totalSize):
"""Function for writing download progress to console """Function for writing download progress to console
""" """
downloadedMbs = int(count*blockSize*(10**(-6))) downloadedMbs = int(count * blockSize * (10**(-6)))
fileSize = int(totalSize*(10**(-6))) fileSize = int(totalSize * (10**(-6)))
sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs,fileSize)) sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs, fileSize))
sys.stdout.flush() sys.stdout.flush()
def getExtension(link): def getExtension(link):
"""Extract file extension from image link. """Extract file extension from image link.
If didn't find any, return '.jpg' If didn't find any, return '.jpg'
""" """
imageTypes = ['jpg','png','mp4','webm','gif'] imageTypes = ['jpg', 'png', 'mp4', 'webm', 'gif']
parsed = link.split('.') parsed = link.split('.')
for fileType in imageTypes: for fileType in imageTypes:
if fileType in parsed: if fileType in parsed:
return "."+parsed[-1] return "." + parsed[-1]
else:
if not "v.redd.it" in link: if "v.redd.it" not in link:
return '.jpg' return '.jpg'
else:
return '.mp4' return '.mp4'
def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
def getFile(
filename,
shortFilename,
folderDir,
imageURL,
indent=0,
silent=False):
FORMATS = { FORMATS = {
"videos": [".mp4", ".webm"], "videos": [".mp4", ".webm"],
"images": [".jpg",".jpeg",".png",".bmp"], "images": [".jpg", ".jpeg", ".png", ".bmp"],
"gifs": [".gif"], "gifs": [".gif"],
"self": [] "self": []
} }
@@ -53,10 +59,10 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
raise DomainInSkip raise DomainInSkip
headers = [ headers = [
("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \ ("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 "\ "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 "
"Safari/537.36 OPR/54.0.2952.64"), "Safari/537.36 OPR/54.0.2952.64"),
("Accept", "text/html,application/xhtml+xml,application/xml;" \ ("Accept", "text/html,application/xhtml+xml,application/xml;"
"q=0.9,image/webp,image/apng,*/*;q=0.8"), "q=0.9,image/webp,image/apng,*/*;q=0.8"),
("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.3"), ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.3"),
("Accept-Encoding", "none"), ("Accept-Encoding", "none"),
@@ -64,21 +70,22 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
("Connection", "keep-alive") ("Connection", "keep-alive")
] ]
if not os.path.exists(folderDir): os.makedirs(folderDir) if not os.path.exists(folderDir):
os.makedirs(folderDir)
opener = urllib.request.build_opener() opener = urllib.request.build_opener()
if not "imgur" in imageURL: if "imgur" not in imageURL:
opener.addheaders = headers opener.addheaders = headers
urllib.request.install_opener(opener) urllib.request.install_opener(opener)
if not silent: print(" "*indent + str(folderDir), if not silent:
" "*indent + str(filename), print(" " * indent + str(folderDir),
" " * indent + str(filename),
sep="\n") sep="\n")
for i in range(3): for i in range(3):
fileDir = Path(folderDir) / filename fileDir = Path(folderDir) / filename
tempDir = Path(folderDir) / (filename+".tmp") tempDir = Path(folderDir) / (filename + ".tmp")
if not (os.path.isfile(fileDir)): if not (os.path.isfile(fileDir)):
try: try:
@@ -93,8 +100,9 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
raise FileAlreadyExistsError raise FileAlreadyExistsError
GLOBAL.downloadedPosts.add(fileHash) GLOBAL.downloadedPosts.add(fileHash)
os.rename(tempDir,fileDir) os.rename(tempDir, fileDir)
if not silent: print(" "*indent+"Downloaded"+" "*10) if not silent:
print(" " * indent + "Downloaded" + " " * 10)
return None return None
except ConnectionResetError: except ConnectionResetError:
raise FailedToDownload raise FailedToDownload
@@ -104,6 +112,7 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False):
raise FileAlreadyExistsError raise FileAlreadyExistsError
raise FailedToDownload raise FailedToDownload
def createHash(filename): def createHash(filename):
hash_md5 = hashlib.md5() hash_md5 = hashlib.md5()
with open(filename, "rb") as f: with open(filename, "rb") as f:

67
src/downloaders/gallery.py
View File

@@ -1,18 +1,16 @@
import io
import os import os
import json import json
import urllib import urllib
import requests import requests
from pathlib import Path
from src.utils import GLOBAL, nameCorrector from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
from src.downloaders.Direct import Direct
from src.downloaders.downloaderUtils import getFile from src.downloaders.downloaderUtils import getFile
from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, ExtensionError, NotADownloadableLinkError, TypeInSkip from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, NotADownloadableLinkError, TypeInSkip
class gallery: class gallery:
def __init__(self,directory,post): def __init__(self, directory, post):
link = post['CONTENTURL'] link = post['CONTENTURL']
self.rawData = self.getData(link) self.rawData = self.getData(link)
@@ -20,30 +18,33 @@ class gallery:
self.directory = directory self.directory = directory
self.post = post self.post = post
images={} images = {}
count=0 count = 0
for model in self.rawData['posts']['models']: for model in self.rawData['posts']['models']:
try: try:
for item in self.rawData['posts']['models'][model]['media']['gallery']['items']: for item in self.rawData['posts']['models'][model]['media']['gallery']['items']:
try: try:
images[count]={'id':item['mediaId'], 'url':self.rawData['posts']['models'][model]['media']['mediaMetadata'][item['mediaId']]['s']['u']} images[count] = {'id': item['mediaId'], 'url': self.rawData['posts'][
count=count+1 'models'][model]['media']['mediaMetadata'][item['mediaId']]['s']['u']}
except: count = count + 1
except BaseException:
continue continue
except: except BaseException:
continue continue
self.downloadAlbum(images,count) self.downloadAlbum(images, count)
@staticmethod @staticmethod
def getData(link): def getData(link):
headers = { headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36 OPR/54.0.2952.64", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36 OPR/54.0.2952.64",
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
} }
res = requests.get(link, headers=headers) res = requests.get(link, headers=headers)
if res.status_code != 200: raise ImageNotFound(f"Server responded with {res.status_code} to {link}") if res.status_code != 200:
raise ImageNotFound(
f"Server responded with {res.status_code} to {link}")
pageSource = res.text pageSource = res.text
STARTING_STRING = "_r = {" STARTING_STRING = "_r = {"
@@ -51,12 +52,14 @@ class gallery:
STARTING_STRING_LENGHT = len(STARTING_STRING) STARTING_STRING_LENGHT = len(STARTING_STRING)
try: try:
startIndex = pageSource.index(STARTING_STRING) + STARTING_STRING_LENGHT startIndex = pageSource.index(
STARTING_STRING) + STARTING_STRING_LENGHT
endIndex = pageSource.index(ENDING_STRING, startIndex) endIndex = pageSource.index(ENDING_STRING, startIndex)
except ValueError: except ValueError:
raise NotADownloadableLinkError(f"Could not read the page source on {link}") raise NotADownloadableLinkError(
f"Could not read the page source on {link}")
data = json.loads(pageSource[startIndex-1:endIndex+1].strip()[:-1]) data = json.loads(pageSource[startIndex - 1:endIndex + 1].strip()[:-1])
return data return data
def downloadAlbum(self, images, count): def downloadAlbum(self, images, count):
@@ -80,19 +83,20 @@ class gallery:
extension = os.path.splitext(path)[1] extension = os.path.splitext(path)[1]
filename = "_".join([ filename = "_".join([
str(i+1), images[i]['id'] str(i + 1), images[i]['id']
]) + extension ]) + extension
shortFilename = str(i+1) + "_" + images[i]['id'] shortFilename = str(i + 1) + "_" + images[i]['id']
print("\n ({}/{})".format(i+1,count)) print("\n ({}/{})".format(i + 1, count))
try: try:
getFile(filename,shortFilename,folderDir,images[i]['url'],indent=2) getFile(filename, shortFilename, folderDir,
images[i]['url'], indent=2)
howManyDownloaded += 1 howManyDownloaded += 1
print() print()
except FileAlreadyExistsError: except FileAlreadyExistsError:
print(" The file already exists" + " "*10,end="\n\n") print(" The file already exists" + " " * 10, end="\n\n")
duplicates += 1 duplicates += 1
except TypeInSkip: except TypeInSkip:
@@ -102,19 +106,16 @@ class gallery:
except Exception as exception: except Exception as exception:
print("\n Could not get the file") print("\n Could not get the file")
print( print(
" " " " +
+ "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format( "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
class_name=exception.__class__.__name__, class_name=exception.__class__.__name__,
info=str(exception) info=str(exception)) +
) "\n")
+ "\n" print(GLOBAL.log_stream.getvalue(), noPrint=True)
)
print(GLOBAL.log_stream.getvalue(),noPrint=True)
if duplicates == count: if duplicates == count:
raise FileAlreadyExistsError raise FileAlreadyExistsError
elif howManyDownloaded + duplicates < count: if howManyDownloaded + duplicates < count:
raise AlbumNotDownloadedCompletely( raise AlbumNotDownloadedCompletely(
"Album Not Downloaded Completely" "Album Not Downloaded Completely"
) )

24
src/downloaders/gifDeliveryNetwork.py
View File

@@ -1,16 +1,14 @@
import json
import os import os
import urllib.request import urllib.request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile, getExtension
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely, from src.errors import (NotADownloadableLinkError)
NotADownloadableLinkError, FileAlreadyExistsError)
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print
class GifDeliveryNetwork: class GifDeliveryNetwork:
def __init__(self,directory,POST): def __init__(self, directory, POST):
try: try:
POST['MEDIAURL'] = self.getLink(POST['CONTENTURL']) POST['MEDIAURL'] = self.getLink(POST['CONTENTURL'])
except IndexError: except IndexError:
@@ -18,12 +16,13 @@ class GifDeliveryNetwork:
POST['EXTENSION'] = getExtension(POST['MEDIAURL']) POST['EXTENSION'] = getExtension(POST['MEDIAURL'])
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**POST)+POST["EXTENSION"] filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"]
shortFilename = POST['POSTID']+POST['EXTENSION'] shortFilename = POST['POSTID'] + POST['EXTENSION']
getFile(filename,shortFilename,directory,POST['MEDIAURL']) getFile(filename, shortFilename, directory, POST['MEDIAURL'])
@staticmethod @staticmethod
def getLink(url): def getLink(url):
@@ -31,7 +30,8 @@ class GifDeliveryNetwork:
and return it and return it
""" """
if '.webm' in url.split('/')[-1] or '.mp4' in url.split('/')[-1] or '.gif' in url.split('/')[-1]: if '.webm' in url.split(
'/')[-1] or '.mp4' in url.split('/')[-1] or '.gif' in url.split('/')[-1]:
return url return url
if url[-1:] == '/': if url[-1:] == '/':
@@ -42,8 +42,8 @@ class GifDeliveryNetwork:
pageSource = (urllib.request.urlopen(url).read().decode()) pageSource = (urllib.request.urlopen(url).read().decode())
soup = BeautifulSoup(pageSource, "html.parser") soup = BeautifulSoup(pageSource, "html.parser")
attributes = {"id":"mp4Source","type":"video/mp4"} attributes = {"id": "mp4Source", "type": "video/mp4"}
content = soup.find("source",attrs=attributes) content = soup.find("source", attrs=attributes)
if content is None: if content is None:

31
src/downloaders/redgifs.py
View File

@@ -4,13 +4,12 @@ import urllib.request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile, getExtension
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely, from src.errors import (NotADownloadableLinkError)
NotADownloadableLinkError, FileAlreadyExistsError)
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print
class Redgifs: class Redgifs:
def __init__(self,directory,POST): def __init__(self, directory, POST):
try: try:
POST['MEDIAURL'] = self.getLink(POST['CONTENTURL']) POST['MEDIAURL'] = self.getLink(POST['CONTENTURL'])
except IndexError: except IndexError:
@@ -18,14 +17,16 @@ class Redgifs:
POST['EXTENSION'] = getExtension(POST['MEDIAURL']) POST['EXTENSION'] = getExtension(POST['MEDIAURL'])
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**POST)+POST["EXTENSION"] filename = GLOBAL.config['filename'].format(**POST) + POST["EXTENSION"]
shortFilename = POST['POSTID']+POST['EXTENSION'] shortFilename = POST['POSTID'] + POST['EXTENSION']
getFile(filename,shortFilename,directory,POST['MEDIAURL']) getFile(filename, shortFilename, directory, POST['MEDIAURL'])
def getLink(self, url): @staticmethod
def getLink(url):
"""Extract direct link to the video from page's source """Extract direct link to the video from page's source
and return it and return it
""" """
@@ -36,15 +37,19 @@ class Redgifs:
if url[-1:] == '/': if url[-1:] == '/':
url = url[:-1] url = url[:-1]
url = urllib.request.Request("https://redgifs.com/watch/" + url.split('/')[-1]) url = urllib.request.Request(
"https://redgifs.com/watch/" + url.split('/')[-1])
url.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36 OPR/54.0.2952.64') url.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36 OPR/54.0.2952.64')
pageSource = (urllib.request.urlopen(url).read().decode()) pageSource = (urllib.request.urlopen(url).read().decode())
soup = BeautifulSoup(pageSource, "html.parser") soup = BeautifulSoup(pageSource, "html.parser")
attributes = {"data-react-helmet":"true","type":"application/ld+json"} attributes = {"data-react-helmet": "true",
content = soup.find("script",attrs=attributes) "type": "application/ld+json"}
content = soup.find("script", attrs=attributes)
if content is None: if content is None:
raise NotADownloadableLinkError("Could not read the page source") raise NotADownloadableLinkError("Could not read the page source")

29
src/downloaders/selfPost.py
View File

@@ -1,3 +1,4 @@
from src.utils import printToFile as print
import io import io
import os import os
from pathlib import Path from pathlib import Path
@@ -6,36 +7,36 @@ from src.errors import FileAlreadyExistsError, TypeInSkip
from src.utils import GLOBAL from src.utils import GLOBAL
VanillaPrint = print VanillaPrint = print
from src.utils import printToFile as print
class SelfPost: class SelfPost:
def __init__(self,directory,post): def __init__(self, directory, post):
if "self" in GLOBAL.arguments.skip: raise TypeInSkip if "self" in GLOBAL.arguments.skip:
raise TypeInSkip
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**post) filename = GLOBAL.config['filename'].format(**post)
fileDir = directory / (filename+".md") fileDir = directory / (filename + ".md")
print(fileDir) print(fileDir)
print(filename+".md") print(filename + ".md")
if Path.is_file(fileDir): if Path.is_file(fileDir):
raise FileAlreadyExistsError raise FileAlreadyExistsError
try: try:
self.writeToFile(fileDir,post) self.writeToFile(fileDir, post)
except FileNotFoundError: except FileNotFoundError:
fileDir = post['POSTID']+".md" fileDir = post['POSTID'] + ".md"
fileDir = directory / fileDir fileDir = directory / fileDir
self.writeToFile(fileDir,post) self.writeToFile(fileDir, post)
@staticmethod @staticmethod
def writeToFile(directory,post): def writeToFile(directory, post):
"""Self posts are formatted here""" """Self posts are formatted here"""
content = ("## [" content = ("## ["
+ post["TITLE"] + post["TITLE"]
@@ -54,7 +55,7 @@ class SelfPost:
+ post["REDDITOR"] + post["REDDITOR"]
+ ")") + ")")
with io.open(directory,"w",encoding="utf-8") as FILE: with io.open(directory, "w", encoding="utf-8") as FILE:
VanillaPrint(content,file=FILE) VanillaPrint(content, file=FILE)
print("Downloaded") print("Downloaded")

30
src/downloaders/vreddit.py
View File

@@ -1,25 +1,25 @@
import os import os
import subprocess import subprocess
from src.downloaders.downloaderUtils import getFile, getExtension from src.downloaders.downloaderUtils import getFile
from src.errors import FileNameTooLong
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
class VReddit:
def __init__(self,directory,post):
extension = ".mp4"
if not os.path.exists(directory): os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**post)+extension class VReddit:
shortFilename = post['POSTID']+extension def __init__(self, directory, post):
extension = ".mp4"
if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**post) + extension
shortFilename = post['POSTID'] + extension
try: try:
FNULL = open(os.devnull, 'w') FNULL = open(os.devnull, 'w')
subprocess.call("ffmpeg", stdout=FNULL, stderr=subprocess.STDOUT) subprocess.call("ffmpeg", stdout=FNULL, stderr=subprocess.STDOUT)
except: except BaseException:
getFile(filename,shortFilename,directory,post['CONTENTURL']) getFile(filename, shortFilename, directory, post['CONTENTURL'])
print("FFMPEG library not found, skipping merging video and audio") print("FFMPEG library not found, skipping merging video and audio")
else: else:
videoName = post['POSTID'] + "_video" videoName = post['POSTID'] + "_video"
@@ -27,10 +27,10 @@ class VReddit:
audioName = post['POSTID'] + "_audio" audioName = post['POSTID'] + "_audio"
audioURL = videoURL[:videoURL.rfind('/')] + '/DASH_audio.mp4' audioURL = videoURL[:videoURL.rfind('/')] + '/DASH_audio.mp4'
print(directory,filename,sep="\n") print(directory, filename, sep="\n")
getFile(videoName,videoName,directory,videoURL,silent=True) getFile(videoName, videoName, directory, videoURL, silent=True)
getFile(audioName,audioName,directory,audioURL,silent=True) getFile(audioName, audioName, directory, audioURL, silent=True)
try: try:
self._mergeAudio(videoName, self._mergeAudio(videoName,
audioName, audioName,
@@ -44,7 +44,7 @@ class VReddit:
os.rename(directory / videoName, directory / filename) os.rename(directory / videoName, directory / filename)
@staticmethod @staticmethod
def _mergeAudio(video,audio,filename,shortFilename,directory): def _mergeAudio(video, audio, filename, shortFilename, directory):
inputVideo = str(directory / video) inputVideo = str(directory / video)
inputAudio = str(directory / audio) inputAudio = str(directory / audio)

22
src/downloaders/youtube.py
View File

@@ -2,22 +2,24 @@ import os
import youtube_dl import youtube_dl
import sys import sys
from src.downloaders.downloaderUtils import getExtension, dlProgress, createHash from src.downloaders.downloaderUtils import createHash
from src.utils import GLOBAL from src.utils import GLOBAL
from src.utils import printToFile as print from src.utils import printToFile as print
from src.errors import FileAlreadyExistsError from src.errors import FileAlreadyExistsError
class Youtube: class Youtube:
def __init__(self,directory,post): def __init__(self, directory, post):
if not os.path.exists(directory): os.makedirs(directory) if not os.path.exists(directory):
os.makedirs(directory)
filename = GLOBAL.config['filename'].format(**post) filename = GLOBAL.config['filename'].format(**post)
print(filename) print(filename)
self.download(filename,directory,post['CONTENTURL']) self.download(filename, directory, post['CONTENTURL'])
def download(self,filename,directory,url): def download(self, filename, directory, url):
ydl_opts = { ydl_opts = {
"format": "best", "format": "best",
"outtmpl": str(directory / (filename + ".%(ext)s")), "outtmpl": str(directory / (filename + ".%(ext)s")),
@@ -29,7 +31,7 @@ class Youtube:
with youtube_dl.YoutubeDL(ydl_opts) as ydl: with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([url]) ydl.download([url])
location = directory/(filename+".mp4") location = directory / (filename + ".mp4")
if GLOBAL.arguments.no_dupes: if GLOBAL.arguments.no_dupes:
try: try:
@@ -43,9 +45,9 @@ class Youtube:
@staticmethod @staticmethod
def _hook(d): def _hook(d):
if d['status'] == 'finished': return print("Downloaded") if d['status'] == 'finished':
return print("Downloaded")
downloadedMbs = int(d['downloaded_bytes'] * (10**(-6))) downloadedMbs = int(d['downloaded_bytes'] * (10**(-6)))
fileSize = int(d['total_bytes']*(10**(-6))) fileSize = int(d['total_bytes'] * (10**(-6)))
sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs,fileSize)) sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs, fileSize))
sys.stdout.flush() sys.stdout.flush()

32
src/errors.py
View File

@@ -1,9 +1,11 @@
import sys import sys
def full_exc_info(exc_info): def full_exc_info(exc_info):
def current_stack(skip=0): def current_stack(skip=0):
try: 1/0 try:
1 / 0
except ZeroDivisionError: except ZeroDivisionError:
f = sys.exc_info()[2].tb_frame f = sys.exc_info()[2].tb_frame
for i in range(skip + 2): for i in range(skip + 2):
@@ -16,7 +18,7 @@ def full_exc_info(exc_info):
def extend_traceback(tb, stack): def extend_traceback(tb, stack):
class FauxTb(object): class FauxTb():
def __init__(self, tb_frame, tb_lineno, tb_next): def __init__(self, tb_frame, tb_lineno, tb_next):
self.tb_frame = tb_frame self.tb_frame = tb_frame
self.tb_lineno = tb_lineno self.tb_lineno = tb_lineno
@@ -33,80 +35,106 @@ def full_exc_info(exc_info):
full_tb = extend_traceback(tb, current_stack(1)) full_tb = extend_traceback(tb, current_stack(1))
return t, v, full_tb return t, v, full_tb
class RedditLoginFailed(Exception): class RedditLoginFailed(Exception):
pass pass
class ImgurLoginError(Exception): class ImgurLoginError(Exception):
pass pass
class FileAlreadyExistsError(Exception): class FileAlreadyExistsError(Exception):
pass pass
class NotADownloadableLinkError(Exception): class NotADownloadableLinkError(Exception):
pass pass
class AlbumNotDownloadedCompletely(Exception): class AlbumNotDownloadedCompletely(Exception):
pass pass
class FileNameTooLong(Exception): class FileNameTooLong(Exception):
pass pass
class InvalidRedditLink(Exception): class InvalidRedditLink(Exception):
pass pass
class ProgramModeError(Exception): class ProgramModeError(Exception):
pass pass
class SearchModeError(Exception): class SearchModeError(Exception):
pass pass
class RedditorNameError(Exception): class RedditorNameError(Exception):
pass pass
class NoMatchingSubmissionFound(Exception): class NoMatchingSubmissionFound(Exception):
pass pass
class NoPrawSupport(Exception): class NoPrawSupport(Exception):
pass pass
class NoRedditSupport(Exception): class NoRedditSupport(Exception):
pass pass
class MultiredditNotFound(Exception): class MultiredditNotFound(Exception):
pass pass
class InsufficientPermission(Exception): class InsufficientPermission(Exception):
pass pass
class InvalidSortingType(Exception): class InvalidSortingType(Exception):
pass pass
class FileNotFoundError(Exception): class FileNotFoundError(Exception):
pass pass
class NoSuitablePost(Exception): class NoSuitablePost(Exception):
pass pass
class ImgurLimitError(Exception): class ImgurLimitError(Exception):
pass pass
class DirectLinkNotFound(Exception): class DirectLinkNotFound(Exception):
pass pass
class InvalidJSONFile(Exception): class InvalidJSONFile(Exception):
pass pass
class FailedToDownload(Exception): class FailedToDownload(Exception):
pass pass
class TypeInSkip(Exception): class TypeInSkip(Exception):
pass pass
class DomainInSkip(Exception): class DomainInSkip(Exception):
pass pass
class ImageNotFound(Exception): class ImageNotFound(Exception):
pass pass
class ExtensionError(Exception): class ExtensionError(Exception):
pass pass

17
src/jsonHelper.py
View File

@@ -3,6 +3,7 @@ from os import path, remove
from src.errors import InvalidJSONFile from src.errors import InvalidJSONFile
class JsonFile: class JsonFile:
""" Write and read JSON files """ Write and read JSON files
@@ -13,10 +14,10 @@ class JsonFile:
FILEDIR = "" FILEDIR = ""
def __init__(self,FILEDIR): def __init__(self, FILEDIR):
self.FILEDIR = FILEDIR self.FILEDIR = FILEDIR
if not path.exists(self.FILEDIR): if not path.exists(self.FILEDIR):
self.__writeToFile({},create=True) self.__writeToFile({}, create=True)
def read(self): def read(self):
try: try:
@@ -25,19 +26,21 @@ class JsonFile:
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
raise InvalidJSONFile(f"{self.FILEDIR} cannot be read") raise InvalidJSONFile(f"{self.FILEDIR} cannot be read")
def add(self,toBeAdded,sub=None): def add(self, toBeAdded, sub=None):
"""Takes a dictionary and merges it with json file. """Takes a dictionary and merges it with json file.
It uses new key's value if a key already exists. It uses new key's value if a key already exists.
Returns the new content as a dictionary. Returns the new content as a dictionary.
""" """
data = self.read() data = self.read()
if sub: data[sub] = {**data[sub], **toBeAdded} if sub:
else: data = {**data, **toBeAdded} data[sub] = {**data[sub], **toBeAdded}
else:
data = {**data, **toBeAdded}
self.__writeToFile(data) self.__writeToFile(data)
return self.read() return self.read()
def delete(self,*deleteKeys): def delete(self, *deleteKeys):
"""Delete given keys from JSON file. """Delete given keys from JSON file.
Returns the new content as a dictionary. Returns the new content as a dictionary.
""" """
@@ -51,7 +54,7 @@ class JsonFile:
return False return False
self.__writeToFile(data) self.__writeToFile(data)
def __writeToFile(self,content,create=False): def __writeToFile(self, content, create=False):
if not create: if not create:
remove(self.FILEDIR) remove(self.FILEDIR)
with open(self.FILEDIR, 'w') as f: with open(self.FILEDIR, 'w') as f:

45
src/parser.py
View File

@@ -5,13 +5,14 @@ try:
except ModuleNotFoundError: except ModuleNotFoundError:
from errors import InvalidRedditLink from errors import InvalidRedditLink
def QueryParser(PassedQueries,index):
def QueryParser(PassedQueries, index):
ExtractedQueries = {} ExtractedQueries = {}
QuestionMarkIndex = PassedQueries.index("?") QuestionMarkIndex = PassedQueries.index("?")
Header = PassedQueries[:QuestionMarkIndex] Header = PassedQueries[:QuestionMarkIndex]
ExtractedQueries["HEADER"] = Header ExtractedQueries["HEADER"] = Header
Queries = PassedQueries[QuestionMarkIndex+1:] Queries = PassedQueries[QuestionMarkIndex + 1:]
ParsedQueries = Queries.split("&") ParsedQueries = Queries.split("&")
@@ -20,15 +21,16 @@ def QueryParser(PassedQueries,index):
ExtractedQueries[Query[0]] = Query[1] ExtractedQueries[Query[0]] = Query[1]
if ExtractedQueries["HEADER"] == "search": if ExtractedQueries["HEADER"] == "search":
ExtractedQueries["q"] = ExtractedQueries["q"].replace("%20"," ") ExtractedQueries["q"] = ExtractedQueries["q"].replace("%20", " ")
return ExtractedQueries return ExtractedQueries
def LinkParser(LINK): def LinkParser(LINK):
RESULT = {} RESULT = {}
ShortLink = False ShortLink = False
if not "reddit.com" in LINK: if "reddit.com" not in LINK:
raise InvalidRedditLink("Invalid reddit link") raise InvalidRedditLink("Invalid reddit link")
SplittedLink = LINK.split("/") SplittedLink = LINK.split("/")
@@ -37,7 +39,7 @@ def LinkParser(LINK):
SplittedLink = SplittedLink[2:] SplittedLink = SplittedLink[2:]
try: try:
if (SplittedLink[-2].endswith("reddit.com") and \ if (SplittedLink[-2].endswith("reddit.com") and
SplittedLink[-1] == "") or \ SplittedLink[-1] == "") or \
SplittedLink[-1].endswith("reddit.com"): SplittedLink[-1].endswith("reddit.com"):
@@ -55,10 +57,10 @@ def LinkParser(LINK):
SplittedLink = SplittedLink[1:] SplittedLink = SplittedLink[1:]
if "comments" in SplittedLink: if "comments" in SplittedLink:
RESULT = {"post":LINK} RESULT = {"post": LINK}
return RESULT return RESULT
elif "me" in SplittedLink or \ if "me" in SplittedLink or \
"u" in SplittedLink or \ "u" in SplittedLink or \
"user" in SplittedLink or \ "user" in SplittedLink or \
"r" in SplittedLink or \ "r" in SplittedLink or \
@@ -76,15 +78,14 @@ def LinkParser(LINK):
if SplittedLink[index] == "u" or \ if SplittedLink[index] == "u" or \
SplittedLink[index] == "user": SplittedLink[index] == "user":
RESULT["user"] = SplittedLink[index+1] RESULT["user"] = SplittedLink[index + 1]
elif SplittedLink[index] == "me": elif SplittedLink[index] == "me":
RESULT["user"] = "me" RESULT["user"] = "me"
for index in range(len(SplittedLink)): for index in range(len(SplittedLink)):
if SplittedLink[index] in [ if SplittedLink[index] in [
"hot","top","new","controversial","rising" "hot", "top", "new", "controversial", "rising"
]: ]:
RESULT["sort"] = SplittedLink[index] RESULT["sort"] = SplittedLink[index]
@@ -92,7 +93,7 @@ def LinkParser(LINK):
if index == 0: if index == 0:
RESULT["subreddit"] = "frontpage" RESULT["subreddit"] = "frontpage"
elif SplittedLink[index] in ["submitted","saved","posts","upvoted"]: elif SplittedLink[index] in ["submitted", "saved", "posts", "upvoted"]:
if SplittedLink[index] == "submitted" or \ if SplittedLink[index] == "submitted" or \
SplittedLink[index] == "posts": SplittedLink[index] == "posts":
RESULT["submitted"] = {} RESULT["submitted"] = {}
@@ -104,7 +105,7 @@ def LinkParser(LINK):
RESULT["upvoted"] = True RESULT["upvoted"] = True
elif "?" in SplittedLink[index]: elif "?" in SplittedLink[index]:
ParsedQuery = QueryParser(SplittedLink[index],index) ParsedQuery = QueryParser(SplittedLink[index], index)
if ParsedQuery["HEADER"] == "search": if ParsedQuery["HEADER"] == "search":
del ParsedQuery["HEADER"] del ParsedQuery["HEADER"]
RESULT["search"] = ParsedQuery RESULT["search"] = ParsedQuery
@@ -118,15 +119,16 @@ def LinkParser(LINK):
del ParsedQuery["HEADER"] del ParsedQuery["HEADER"]
RESULT["queries"] = ParsedQuery RESULT["queries"] = ParsedQuery
if not ("upvoted" in RESULT or \ if not ("upvoted" in RESULT or
"saved" in RESULT or \ "saved" in RESULT or
"submitted" in RESULT or \ "submitted" in RESULT or
"multireddit" in RESULT) and \ "multireddit" in RESULT) and \
"user" in RESULT: "user" in RESULT:
RESULT["submitted"] = {} RESULT["submitted"] = {}
return RESULT return RESULT
def LinkDesigner(LINK): def LinkDesigner(LINK):
attributes = LinkParser(LINK) attributes = LinkParser(LINK)
@@ -138,13 +140,13 @@ def LinkDesigner(LINK):
MODE["time"] = "" MODE["time"] = ""
return MODE return MODE
elif "search" in attributes: if "search" in attributes:
MODE["search"] = attributes["search"]["q"] MODE["search"] = attributes["search"]["q"]
if "restrict_sr" in attributes["search"]: if "restrict_sr" in attributes["search"]:
if not (attributes["search"]["restrict_sr"] == 0 or \ if not (attributes["search"]["restrict_sr"] == 0 or
attributes["search"]["restrict_sr"] == "off" or \ attributes["search"]["restrict_sr"] == "off" or
attributes["search"]["restrict_sr"] == ""): attributes["search"]["restrict_sr"] == ""):
if "subreddit" in attributes: if "subreddit" in attributes:
@@ -176,7 +178,7 @@ def LinkDesigner(LINK):
else: else:
if "queries" in attributes: if "queries" in attributes:
if not ("submitted" in attributes or \ if not ("submitted" in attributes or
"posts" in attributes): "posts" in attributes):
if "t" in attributes["queries"]: if "t" in attributes["queries"]:
@@ -196,10 +198,10 @@ def LinkDesigner(LINK):
else: else:
MODE["time"] = "day" MODE["time"] = "day"
if "subreddit" in attributes and not "search" in attributes: if "subreddit" in attributes and "search" not in attributes:
MODE["subreddit"] = attributes["subreddit"] MODE["subreddit"] = attributes["subreddit"]
elif "user" in attributes and not "search" in attributes: elif "user" in attributes and "search" not in attributes:
MODE["user"] = attributes["user"] MODE["user"] = attributes["user"]
if "submitted" in attributes: if "submitted" in attributes:
@@ -234,6 +236,7 @@ def LinkDesigner(LINK):
return MODE return MODE
if __name__ == "__main__": if __name__ == "__main__":
while True: while True:
link = input("> ") link = input("> ")

75
src/programMode.py
View File

@@ -1,12 +1,12 @@
from src.errors import SearchModeError, RedditorNameError, ProgramModeError, InvalidSortingType from src.errors import SearchModeError, RedditorNameError, ProgramModeError, InvalidSortingType
from src.utils import GLOBAL
from src.parser import LinkDesigner from src.parser import LinkDesigner
from pathlib import Path from pathlib import Path
import sys import sys
class ProgramMode: class ProgramMode:
def __init__(self,arguments): def __init__(self, arguments):
self.arguments = arguments self.arguments = arguments
def generate(self): def generate(self):
@@ -57,7 +57,7 @@ class ProgramMode:
programMode["time"] = self.arguments.time programMode["time"] = self.arguments.time
elif self.arguments.subreddit is not None: elif self.arguments.subreddit is not None:
if type(self.arguments.subreddit) == list: if isinstance(self.arguments.subreddit, list):
self.arguments.subreddit = "+".join(self.arguments.subreddit) self.arguments.subreddit = "+".join(self.arguments.subreddit)
programMode["subreddit"] = self.arguments.subreddit programMode["subreddit"] = self.arguments.subreddit
@@ -84,29 +84,29 @@ class ProgramMode:
@staticmethod @staticmethod
def _chooseFrom(choices): def _chooseFrom(choices):
print() print()
choicesByIndex = list(str(x) for x in range(len(choices)+1)) choicesByIndex = [str(x) for x in range(len(choices) + 1)]
for i in range(len(choices)): for i in range(len(choices)):
print("{indent}[{order}] {mode}".format( print("{indent}[{order}] {mode}".format(
indent=" "*4,order=i+1,mode=choices[i] indent=" " * 4, order=i + 1, mode=choices[i]
)) ))
print(" "*4+"[0] exit\n") print(" " * 4 + "[0] exit\n")
choice = input("> ") choice = input("> ")
while not choice.lower() in choices+choicesByIndex+["exit"]: while not choice.lower() in choices + choicesByIndex + ["exit"]:
print("Invalid input\n") print("Invalid input\n")
input("> ") input("> ")
if choice == "0" or choice == "exit": if choice == "0" or choice == "exit":
sys.exit() sys.exit()
elif choice in choicesByIndex: elif choice in choicesByIndex:
return choices[int(choice)-1] return choices[int(choice) - 1]
else: else:
return choice return choice
def _promptUser(self): def _promptUser(self):
print("select program mode:") print("select program mode:")
programModes = [ programModes = [
"search","subreddit","multireddit", "search", "subreddit", "multireddit",
"submitted","upvoted","saved","log" "submitted", "upvoted", "saved", "log"
] ]
programMode = self._chooseFrom(programModes) programMode = self._chooseFrom(programModes)
@@ -116,23 +116,24 @@ class ProgramMode:
print("\nselect sort type:") print("\nselect sort type:")
sortTypes = [ sortTypes = [
"relevance","top","new" "relevance", "top", "new"
] ]
sortType = self._chooseFrom(sortTypes) sortType = self._chooseFrom(sortTypes)
self.arguments.sort = sortType self.arguments.sort = sortType
print("\nselect time filter:") print("\nselect time filter:")
timeFilters = [ timeFilters = [
"hour","day","week","month","year","all" "hour", "day", "week", "month", "year", "all"
] ]
timeFilter = self._chooseFrom(timeFilters) timeFilter = self._chooseFrom(timeFilters)
self.arguments.time = timeFilter self.arguments.time = timeFilter
if programMode == "subreddit": if programMode == "subreddit":
subredditInput = input("(type frontpage for all subscribed subreddits,\n" \ subredditInput = input(
" use plus to seperate multi subreddits:" \ "(type frontpage for all subscribed subreddits,\n"
" pics+funny+me_irl etc.)\n\n" \ " use plus to seperate multi subreddits:"
" pics+funny+me_irl etc.)\n\n"
"subreddit: ") "subreddit: ")
self.arguments.subreddit = subredditInput self.arguments.subreddit = subredditInput
@@ -141,7 +142,8 @@ class ProgramMode:
# self.arguments.subreddit += "+" + subredditInput # self.arguments.subreddit += "+" + subredditInput
if " " in self.arguments.subreddit: if " " in self.arguments.subreddit:
self.arguments.subreddit = "+".join(self.arguments.subreddit.split()) self.arguments.subreddit = "+".join(
self.arguments.subreddit.split())
# DELETE THE PLUS (+) AT THE END # DELETE THE PLUS (+) AT THE END
if not subredditInput.lower() == "frontpage" \ if not subredditInput.lower() == "frontpage" \
@@ -150,15 +152,15 @@ class ProgramMode:
print("\nselect sort type:") print("\nselect sort type:")
sortTypes = [ sortTypes = [
"hot","top","new","rising","controversial" "hot", "top", "new", "rising", "controversial"
] ]
sortType = self._chooseFrom(sortTypes) sortType = self._chooseFrom(sortTypes)
self.arguments.sort = sortType self.arguments.sort = sortType
if sortType in ["top","controversial"]: if sortType in ["top", "controversial"]:
print("\nselect time filter:") print("\nselect time filter:")
timeFilters = [ timeFilters = [
"hour","day","week","month","year","all" "hour", "day", "week", "month", "year", "all"
] ]
timeFilter = self._chooseFrom(timeFilters) timeFilter = self._chooseFrom(timeFilters)
self.arguments.time = timeFilter self.arguments.time = timeFilter
@@ -171,15 +173,15 @@ class ProgramMode:
print("\nselect sort type:") print("\nselect sort type:")
sortTypes = [ sortTypes = [
"hot","top","new","rising","controversial" "hot", "top", "new", "rising", "controversial"
] ]
sortType = self._chooseFrom(sortTypes) sortType = self._chooseFrom(sortTypes)
self.arguments.sort = sortType self.arguments.sort = sortType
if sortType in ["top","controversial"]: if sortType in ["top", "controversial"]:
print("\nselect time filter:") print("\nselect time filter:")
timeFilters = [ timeFilters = [
"hour","day","week","month","year","all" "hour", "day", "week", "month", "year", "all"
] ]
timeFilter = self._chooseFrom(timeFilters) timeFilter = self._chooseFrom(timeFilters)
self.arguments.time = timeFilter self.arguments.time = timeFilter
@@ -192,7 +194,7 @@ class ProgramMode:
print("\nselect sort type:") print("\nselect sort type:")
sortTypes = [ sortTypes = [
"hot","top","new","controversial" "hot", "top", "new", "controversial"
] ]
sortType = self._chooseFrom(sortTypes) sortType = self._chooseFrom(sortTypes)
self.arguments.sort = sortType self.arguments.sort = sortType
@@ -200,7 +202,7 @@ class ProgramMode:
if sortType == "top": if sortType == "top":
print("\nselect time filter:") print("\nselect time filter:")
timeFilters = [ timeFilters = [
"hour","day","week","month","year","all" "hour", "day", "week", "month", "year", "all"
] ]
timeFilter = self._chooseFrom(timeFilters) timeFilter = self._chooseFrom(timeFilters)
self.arguments.time = timeFilter self.arguments.time = timeFilter
@@ -241,30 +243,35 @@ class ProgramMode:
search = 1 if self.arguments.search else 0 search = 1 if self.arguments.search else 0
modes = [ modes = [
"saved","subreddit","submitted","log","link","upvoted","multireddit" "saved",
] "subreddit",
"submitted",
"log",
"link",
"upvoted",
"multireddit"]
values = { values = {
x: 0 if getattr(self.arguments,x) is None or \ x: 0 if getattr(self.arguments, x) is None or
getattr(self.arguments,x) is False \ getattr(self.arguments, x) is False
else 1 \ else 1
for x in modes for x in modes
} }
if not sum(values[x] for x in values) == 1: if not sum(values[x] for x in values) == 1:
raise ProgramModeError("Invalid program mode") raise ProgramModeError("Invalid program mode")
if search+values["saved"] == 2: if search + values["saved"] == 2:
raise SearchModeError("You cannot search in your saved posts") raise SearchModeError("You cannot search in your saved posts")
if search+values["submitted"] == 2: if search + values["submitted"] == 2:
raise SearchModeError("You cannot search in submitted posts") raise SearchModeError("You cannot search in submitted posts")
if search+values["upvoted"] == 2: if search + values["upvoted"] == 2:
raise SearchModeError("You cannot search in upvoted posts") raise SearchModeError("You cannot search in upvoted posts")
if search+values["log"] == 2: if search + values["log"] == 2:
raise SearchModeError("You cannot search in log files") raise SearchModeError("You cannot search in log files")
if values["upvoted"]+values["submitted"] == 1 and user == 0: if values["upvoted"] + values["submitted"] == 1 and user == 0:
raise RedditorNameError("No redditor name given") raise RedditorNameError("No redditor name given")

48
src/reddit.py
View File

@@ -2,23 +2,24 @@ import praw
import random import random
import socket import socket
import webbrowser import webbrowser
from prawcore.exceptions import NotFound, ResponseException, Forbidden from prawcore.exceptions import ResponseException
from src.utils import GLOBAL from src.utils import GLOBAL
from src.jsonHelper import JsonFile from src.jsonHelper import JsonFile
from src. errors import RedditLoginFailed from src. errors import RedditLoginFailed
class Reddit: class Reddit:
def __init__(self,refresh_token=None): def __init__(self, refresh_token=None):
self.SCOPES = ['identity','history','read','save'] self.SCOPES = ['identity', 'history', 'read', 'save']
self.PORT = 7634 self.PORT = 7634
self.refresh_token = refresh_token self.refresh_token = refresh_token
self.redditInstance = None self.redditInstance = None
self.arguments = { self.arguments = {
"client_id":GLOBAL.reddit_client_id, "client_id": GLOBAL.reddit_client_id,
"client_secret":GLOBAL.reddit_client_secret, "client_secret": GLOBAL.reddit_client_secret,
"user_agent":str(socket.gethostname()) "user_agent": str(socket.gethostname())
} }
def begin(self): def begin(self):
@@ -30,18 +31,20 @@ class Reddit:
self.redditInstance.auth.scopes() self.redditInstance.auth.scopes()
return self.redditInstance return self.redditInstance
except ResponseException: except ResponseException:
self.arguments["redirect_uri"] = "http://localhost:" + str(self.PORT) self.arguments["redirect_uri"] = "http://localhost:" + \
str(self.PORT)
self.redditInstance = praw.Reddit(**self.arguments) self.redditInstance = praw.Reddit(**self.arguments)
reddit, refresh_token = self.getRefreshToken(*self.SCOPES) reddit, refresh_token = self.getRefreshToken(*self.SCOPES)
else: else:
self.arguments["redirect_uri"] = "http://localhost:" + str(self.PORT) self.arguments["redirect_uri"] = "http://localhost:" + \
str(self.PORT)
self.redditInstance = praw.Reddit(**self.arguments) self.redditInstance = praw.Reddit(**self.arguments)
reddit, refresh_token = self.getRefreshToken(*self.SCOPES) reddit, refresh_token = self.getRefreshToken(*self.SCOPES)
JsonFile(GLOBAL.configDirectory).add({ JsonFile(GLOBAL.configDirectory).add({
"reddit_username": str(reddit.user.me()), "reddit_username": str(reddit.user.me()),
"reddit": refresh_token "reddit": refresh_token
},"credentials") }, "credentials")
return self.redditInstance return self.redditInstance
@@ -57,42 +60,45 @@ class Reddit:
server.close() server.close()
return client return client
def send_message(self, client, message): @staticmethod
def send_message(client, message):
"""Send message to client and close the connection.""" """Send message to client and close the connection."""
client.send( client.send(
'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8') 'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8')
) )
client.close() client.close()
def getRefreshToken(self,*scopes): def getRefreshToken(self, *scopes):
state = str(random.randint(0, 65000)) state = str(random.randint(0, 65000))
url = self.redditInstance.auth.url(scopes, state, 'permanent') url = self.redditInstance.auth.url(scopes, state, 'permanent')
print("---Setting up the Reddit API---\n") print("---Setting up the Reddit API---\n")
print("Go to this URL and login to reddit:\n",url,sep="\n",end="\n\n") print(
webbrowser.open(url,new=2) "Go to this URL and login to reddit:\n",
url,
sep="\n",
end="\n\n")
webbrowser.open(url, new=2)
client = self.recieve_connection() client = self.recieve_connection()
data = client.recv(1024).decode('utf-8') data = client.recv(1024).decode('utf-8')
str(data) str(data)
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&') param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
params = { params = dict([token.split('=')
key: value for (key, value) in [token.split('=') \ for token in param_tokens])
for token in param_tokens]
}
if state != params['state']: if state != params['state']:
self.send_message( self.send_message(
client, 'State mismatch. Expected: {} Received: {}' client, 'State mismatch. Expected: {} Received: {}'
.format(state, params['state']) .format(state, params['state'])
) )
raise RedditLoginFailed raise RedditLoginFailed
elif 'error' in params: if 'error' in params:
self.send_message(client, params['error']) self.send_message(client, params['error'])
raise RedditLoginFailed raise RedditLoginFailed
refresh_token = self.redditInstance.auth.authorize(params['code']) refresh_token = self.redditInstance.auth.authorize(params['code'])
self.send_message(client, self.send_message(client,
"<script>" \ "<script>"
"alert(\"You can go back to terminal window now.\");" \ "alert(\"You can go back to terminal window now.\");"
"</script>" "</script>"
) )
return (self.redditInstance,refresh_token) return (self.redditInstance, refresh_token)

177
src/searcher.py
View File

@@ -1,25 +1,17 @@
import os
import sys import sys
import random
import socket
import time import time
import webbrowser
import urllib.request import urllib.request
from urllib.error import HTTPError from prawcore.exceptions import NotFound, Forbidden
import praw
from prawcore.exceptions import NotFound, ResponseException, Forbidden
from src.reddit import Reddit from src.reddit import Reddit
from src.utils import GLOBAL, createLogFile, printToFile from src.utils import GLOBAL, createLogFile, printToFile
from src.jsonHelper import JsonFile
from src.errors import (NoMatchingSubmissionFound, NoPrawSupport, from src.errors import (NoMatchingSubmissionFound, NoPrawSupport,
NoRedditSupport, MultiredditNotFound, MultiredditNotFound,
InvalidSortingType, RedditLoginFailed, InvalidSortingType, InsufficientPermission)
InsufficientPermission, DirectLinkNotFound)
print = printToFile print = printToFile
def getPosts(programMode): def getPosts(programMode):
"""Call PRAW regarding to arguments and pass it to extractDetails. """Call PRAW regarding to arguments and pass it to extractDetails.
Return what extractDetails has returned. Return what extractDetails has returned.
@@ -39,38 +31,38 @@ def getPosts(programMode):
if programMode["user"] == "me": if programMode["user"] == "me":
programMode["user"] = str(reddit.user.me()) programMode["user"] = str(reddit.user.me())
if not "search" in programMode: if "search" not in programMode:
if programMode["sort"] == "top" or programMode["sort"] == "controversial": if programMode["sort"] == "top" or programMode["sort"] == "controversial":
keyword_params = { keyword_params = {
"time_filter":programMode["time"], "time_filter": programMode["time"],
"limit":programMode["limit"] "limit": programMode["limit"]
} }
# OTHER SORT TYPES DON'T TAKE TIME_FILTER # OTHER SORT TYPES DON'T TAKE TIME_FILTER
else: else:
keyword_params = { keyword_params = {
"limit":programMode["limit"] "limit": programMode["limit"]
} }
else: else:
keyword_params = { keyword_params = {
"time_filter":programMode["time"], "time_filter": programMode["time"],
"limit":programMode["limit"] "limit": programMode["limit"]
} }
if "search" in programMode: if "search" in programMode:
if programMode["sort"] in ["hot","rising","controversial"]: if programMode["sort"] in ["hot", "rising", "controversial"]:
raise InvalidSortingType("Invalid sorting type has given") raise InvalidSortingType("Invalid sorting type has given")
if "subreddit" in programMode: if "subreddit" in programMode:
print ( print(
"search for \"{search}\" in\n" \ "search for \"{search}\" in\n"
"subreddit: {subreddit}\nsort: {sort}\n" \ "subreddit: {subreddit}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
search=programMode["search"], search=programMode["search"],
limit=programMode["limit"], limit=programMode["limit"],
sort=programMode["sort"], sort=programMode["sort"],
subreddit=programMode["subreddit"], subreddit=programMode["subreddit"],
time=programMode["time"] time=programMode["time"]
).upper(),noPrint=True ).upper(), noPrint=True
) )
return extractDetails( return extractDetails(
reddit.subreddit(programMode["subreddit"]).search( reddit.subreddit(programMode["subreddit"]).search(
@@ -81,13 +73,13 @@ def getPosts(programMode):
) )
) )
elif "multireddit" in programMode: if "multireddit" in programMode:
raise NoPrawSupport("PRAW does not support that") raise NoPrawSupport("PRAW does not support that")
elif "user" in programMode: if "user" in programMode:
raise NoPrawSupport("PRAW does not support that") raise NoPrawSupport("PRAW does not support that")
elif "saved" in programMode: if "saved" in programMode:
raise ("Reddit does not support that") raise ("Reddit does not support that")
if programMode["sort"] == "relevance": if programMode["sort"] == "relevance":
@@ -98,103 +90,108 @@ def getPosts(programMode):
"saved posts\nuser:{username}\nlimit={limit}\n".format( "saved posts\nuser:{username}\nlimit={limit}\n".format(
username=reddit.user.me(), username=reddit.user.me(),
limit=programMode["limit"] limit=programMode["limit"]
).upper(),noPrint=True ).upper(), noPrint=True
) )
return extractDetails(reddit.user.me().saved(limit=programMode["limit"])) return extractDetails(
reddit.user.me().saved(
limit=programMode["limit"]))
if "subreddit" in programMode: if "subreddit" in programMode:
if programMode["subreddit"] == "frontpage": if programMode["subreddit"] == "frontpage":
print ( print(
"subreddit: {subreddit}\nsort: {sort}\n" \ "subreddit: {subreddit}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
limit=programMode["limit"], limit=programMode["limit"],
sort=programMode["sort"], sort=programMode["sort"],
subreddit=programMode["subreddit"], subreddit=programMode["subreddit"],
time=programMode["time"] time=programMode["time"]
).upper(),noPrint=True ).upper(), noPrint=True
) )
return extractDetails( return extractDetails(
getattr(reddit.front,programMode["sort"]) (**keyword_params) getattr(reddit.front, programMode["sort"])(**keyword_params)
) )
print(
else: "subreddit: {subreddit}\nsort: {sort}\n"
print (
"subreddit: {subreddit}\nsort: {sort}\n" \
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
limit=programMode["limit"], limit=programMode["limit"],
sort=programMode["sort"], sort=programMode["sort"],
subreddit=programMode["subreddit"], subreddit=programMode["subreddit"],
time=programMode["time"] time=programMode["time"]
).upper(),noPrint=True ).upper(), noPrint=True
) )
return extractDetails( return extractDetails(
getattr( getattr(
reddit.subreddit(programMode["subreddit"]),programMode["sort"] reddit.subreddit(programMode["subreddit"]), programMode["sort"]
) (**keyword_params) )(**keyword_params)
) )
elif "multireddit" in programMode: if "multireddit" in programMode:
print ( print(
"user: {user}\n" \ "user: {user}\n"
"multireddit: {multireddit}\nsort: {sort}\n" \ "multireddit: {multireddit}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
user=programMode["user"], user=programMode["user"],
limit=programMode["limit"], limit=programMode["limit"],
sort=programMode["sort"], sort=programMode["sort"],
multireddit=programMode["multireddit"], multireddit=programMode["multireddit"],
time=programMode["time"] time=programMode["time"]
).upper(),noPrint=True ).upper(), noPrint=True
) )
try: try:
return extractDetails( return extractDetails(
getattr( getattr(
reddit.multireddit( reddit.multireddit(
programMode["user"], programMode["multireddit"] programMode["user"], programMode["multireddit"]
),programMode["sort"] ), programMode["sort"]
) (**keyword_params) )(**keyword_params)
) )
except NotFound: except NotFound:
raise MultiredditNotFound("Multireddit not found") raise MultiredditNotFound("Multireddit not found")
elif "submitted" in programMode: elif "submitted" in programMode:
print ( print(
"submitted posts of {user}\nsort: {sort}\n" \ "submitted posts of {user}\nsort: {sort}\n"
"time: {time}\nlimit: {limit}\n".format( "time: {time}\nlimit: {limit}\n".format(
limit=programMode["limit"], limit=programMode["limit"],
sort=programMode["sort"], sort=programMode["sort"],
user=programMode["user"], user=programMode["user"],
time=programMode["time"] time=programMode["time"]
).upper(),noPrint=True ).upper(), noPrint=True
) )
return extractDetails( return extractDetails(
getattr( getattr(
reddit.redditor(programMode["user"]).submissions,programMode["sort"] reddit.redditor(programMode["user"]
) (**keyword_params) ).submissions, programMode["sort"]
)(**keyword_params)
) )
elif "upvoted" in programMode: elif "upvoted" in programMode:
print ( print(
"upvoted posts of {user}\nlimit: {limit}\n".format( "upvoted posts of {user}\nlimit: {limit}\n".format(
user=programMode["user"], user=programMode["user"],
limit=programMode["limit"] limit=programMode["limit"]
).upper(),noPrint=True ).upper(), noPrint=True
) )
try: try:
return extractDetails( return extractDetails(
reddit.redditor(programMode["user"]).upvoted(limit=programMode["limit"]) reddit.redditor(programMode["user"]).upvoted(
limit=programMode["limit"])
) )
except Forbidden: except Forbidden:
raise InsufficientPermission("You do not have permission to do that") raise InsufficientPermission(
"You do not have permission to do that")
elif "post" in programMode: elif "post" in programMode:
print("post: {post}\n".format(post=programMode["post"]).upper(),noPrint=True) print("post: {post}\n".format(
post=programMode["post"]).upper(), noPrint=True)
return extractDetails( return extractDetails(
reddit.submission(url=programMode["post"]),SINGLE_POST=True reddit.submission(url=programMode["post"]), SINGLE_POST=True
) )
def extractDetails(posts,SINGLE_POST=False):
def extractDetails(posts, SINGLE_POST=False):
"""Check posts and decide if it can be downloaded. """Check posts and decide if it can be downloaded.
If so, create a dictionary with post details and append them to a list. If so, create a dictionary with post details and append them to a list.
Write all of posts to file. Return the list Write all of posts to file. Return the list
@@ -212,15 +209,15 @@ def extractDetails(posts,SINGLE_POST=False):
submission = posts submission = posts
postCount += 1 postCount += 1
try: try:
details = {'POSTID':submission.id, details = {'POSTID': submission.id,
'TITLE':submission.title, 'TITLE': submission.title,
'REDDITOR':str(submission.author), 'REDDITOR': str(submission.author),
'TYPE':None, 'TYPE': None,
'CONTENTURL':submission.url, 'CONTENTURL': submission.url,
'SUBREDDIT':submission.subreddit.display_name, 'SUBREDDIT': submission.subreddit.display_name,
'UPVOTES': submission.score, 'UPVOTES': submission.score,
'FLAIR':submission.link_flair_text, 'FLAIR': submission.link_flair_text,
'DATE':str(time.strftime( 'DATE': str(time.strftime(
"%Y-%m-%d_%H-%M", "%Y-%m-%d_%H-%M",
time.localtime(submission.created_utc) time.localtime(submission.created_utc)
))} ))}
@@ -229,13 +226,14 @@ def extractDetails(posts,SINGLE_POST=False):
except AttributeError: except AttributeError:
pass pass
if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): if not any(
domain in submission.domain for domain in GLOBAL.arguments.skip_domain):
result = matchWithDownloader(submission) result = matchWithDownloader(submission)
if result is not None: if result is not None:
details = {**details, **result} details = {**details, **result}
postList.append(details) postList.append(details)
postsFile.add({postCount:details}) postsFile.add({postCount: details})
else: else:
try: try:
@@ -246,19 +244,19 @@ def extractDetails(posts,SINGLE_POST=False):
sys.stdout.flush() sys.stdout.flush()
if postCount % 1000 == 0: if postCount % 1000 == 0:
sys.stdout.write("\n"+" "*14) sys.stdout.write("\n" + " " * 14)
sys.stdout.flush() sys.stdout.flush()
try: try:
details = {'POSTID':submission.id, details = {'POSTID': submission.id,
'TITLE':submission.title, 'TITLE': submission.title,
'REDDITOR':str(submission.author), 'REDDITOR': str(submission.author),
'TYPE':None, 'TYPE': None,
'CONTENTURL':submission.url, 'CONTENTURL': submission.url,
'SUBREDDIT':submission.subreddit.display_name, 'SUBREDDIT': submission.subreddit.display_name,
'UPVOTES': submission.score, 'UPVOTES': submission.score,
'FLAIR':submission.link_flair_text, 'FLAIR': submission.link_flair_text,
'DATE':str(time.strftime( 'DATE': str(time.strftime(
"%Y-%m-%d_%H-%M", "%Y-%m-%d_%H-%M",
time.localtime(submission.created_utc) time.localtime(submission.created_utc)
))} ))}
@@ -267,9 +265,11 @@ def extractDetails(posts,SINGLE_POST=False):
except AttributeError: except AttributeError:
continue continue
if details['POSTID'] in GLOBAL.downloadedPosts(): continue if details['POSTID'] in GLOBAL.downloadedPosts():
continue
if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): if not any(
domain in submission.domain for domain in GLOBAL.arguments.skip_domain):
result = matchWithDownloader(submission) result = matchWithDownloader(submission)
if result is not None: if result is not None:
@@ -280,16 +280,16 @@ def extractDetails(posts,SINGLE_POST=False):
postCount += 1 postCount += 1
except KeyboardInterrupt: except KeyboardInterrupt:
print("\nKeyboardInterrupt",noPrint=True) print("\nKeyboardInterrupt", noPrint=True)
postsFile.add(allPosts) postsFile.add(allPosts)
if not len(postList) == 0: if len(postList) != 0:
print() print()
return postList return postList
else:
raise NoMatchingSubmissionFound("No matching submission was found") raise NoMatchingSubmissionFound("No matching submission was found")
def matchWithDownloader(submission): def matchWithDownloader(submission):
if 'gallery' in submission.url: if 'gallery' in submission.url:
@@ -301,11 +301,11 @@ def matchWithDownloader(submission):
'CONTENTURL': directLink} 'CONTENTURL': directLink}
if 'v.redd.it' in submission.domain: if 'v.redd.it' in submission.domain:
bitrates = ["DASH_1080","DASH_720","DASH_600", \ bitrates = ["DASH_1080", "DASH_720", "DASH_600",
"DASH_480","DASH_360","DASH_240"] "DASH_480", "DASH_360", "DASH_240"]
for bitrate in bitrates: for bitrate in bitrates:
videoURL = submission.url+"/"+bitrate+".mp4" videoURL = submission.url + "/" + bitrate + ".mp4"
try: try:
responseCode = urllib.request.urlopen(videoURL).getcode() responseCode = urllib.request.urlopen(videoURL).getcode()
@@ -342,17 +342,18 @@ def matchWithDownloader(submission):
if 'reddit.com/gallery' in submission.url: if 'reddit.com/gallery' in submission.url:
return {'TYPE': 'gallery'} return {'TYPE': 'gallery'}
elif submission.is_self and 'self' not in GLOBAL.arguments.skip: if submission.is_self and 'self' not in GLOBAL.arguments.skip:
return {'TYPE': 'self', return {'TYPE': 'self',
'CONTENT': submission.selftext} 'CONTENT': submission.selftext}
def extractDirectLink(URL): def extractDirectLink(URL):
"""Check if link is a direct image link. """Check if link is a direct image link.
If so, return URL, If so, return URL,
if not, return False if not, return False
""" """
imageTypes = ['jpg','jpeg','png','mp4','webm','gif'] imageTypes = ['jpg', 'jpeg', 'png', 'mp4', 'webm', 'gif']
if URL[-1] == "/": if URL[-1] == "/":
URL = URL[:-1] URL = URL[:-1]
@@ -362,7 +363,7 @@ def extractDirectLink(URL):
for extension in imageTypes: for extension in imageTypes:
if extension == URL.split(".")[-1]: if extension == URL.split(".")[-1]:
return URL return URL
else:
return None return None
def genLinksifGallery(metadata): def genLinksifGallery(metadata):

3
src/store.py
View File

@@ -1,7 +1,8 @@
from os import path from os import path
class Store: class Store:
def __init__(self,directory=None): def __init__(self, directory=None):
self.directory = directory self.directory = directory
if self.directory: if self.directory:
if path.exists(directory): if path.exists(directory):

37
src/utils.py
View File

@@ -1,27 +1,28 @@
import io import io
import json
import sys import sys
from os import makedirs, path, remove from os import makedirs, path
from pathlib import Path from pathlib import Path
from src.jsonHelper import JsonFile from src.jsonHelper import JsonFile
from src.errors import FileNotFoundError
class GLOBAL: class GLOBAL:
"""Declare global variables""" """Declare global variables"""
RUN_TIME = "" RUN_TIME = ""
config = {'imgur_client_id':None, 'imgur_client_secret': None} config = {'imgur_client_id': None, 'imgur_client_secret': None}
arguments = None arguments = None
directory = None directory = None
defaultConfigDirectory = Path.home() / "Bulk Downloader for Reddit" defaultConfigDirectory = Path.home() / "Bulk Downloader for Reddit"
configDirectory = "" configDirectory = ""
reddit_client_id = "U-6gk4ZCh3IeNQ" reddit_client_id = "U-6gk4ZCh3IeNQ"
reddit_client_secret = "7CZHY6AmKweZME5s50SfDGylaPg" reddit_client_secret = "7CZHY6AmKweZME5s50SfDGylaPg"
downloadedPosts = lambda: [] @staticmethod
def downloadedPosts(): return []
printVanilla = print printVanilla = print
log_stream= None log_stream = None
def createLogFile(TITLE): def createLogFile(TITLE):
"""Create a log file with given name """Create a log file with given name
@@ -31,40 +32,43 @@ def createLogFile(TITLE):
folderDirectory = GLOBAL.directory / "LOG_FILES" / GLOBAL.RUN_TIME folderDirectory = GLOBAL.directory / "LOG_FILES" / GLOBAL.RUN_TIME
logFilename = TITLE.upper()+'.json' logFilename = TITLE.upper() + '.json'
if not path.exists(folderDirectory): if not path.exists(folderDirectory):
makedirs(folderDirectory) makedirs(folderDirectory)
FILE = JsonFile(folderDirectory / Path(logFilename)) FILE = JsonFile(folderDirectory / Path(logFilename))
HEADER = " ".join(sys.argv) HEADER = " ".join(sys.argv)
FILE.add({"HEADER":HEADER}) FILE.add({"HEADER": HEADER})
return FILE return FILE
def printToFile(*args, noPrint=False,**kwargs):
def printToFile(*args, noPrint=False, **kwargs):
"""Print to both CONSOLE and """Print to both CONSOLE and
CONSOLE LOG file in a folder time stampt in the name CONSOLE LOG file in a folder time stampt in the name
""" """
folderDirectory = GLOBAL.directory / Path("LOG_FILES") / Path(GLOBAL.RUN_TIME) folderDirectory = GLOBAL.directory / \
Path("LOG_FILES") / Path(GLOBAL.RUN_TIME)
if not noPrint or \ if not noPrint or \
GLOBAL.arguments.verbose or \ GLOBAL.arguments.verbose or \
"file" in kwargs: "file" in kwargs:
print(*args,**kwargs) print(*args, **kwargs)
if not path.exists(folderDirectory): if not path.exists(folderDirectory):
makedirs(folderDirectory) makedirs(folderDirectory)
if not "file" in kwargs: if "file" not in kwargs:
with io.open( with io.open(
folderDirectory / "CONSOLE_LOG.txt","a",encoding="utf-8" folderDirectory / "CONSOLE_LOG.txt", "a", encoding="utf-8"
) as FILE: ) as FILE:
print(*args, file=FILE, **kwargs) print(*args, file=FILE, **kwargs)
def nameCorrector(string,reference=None):
def nameCorrector(string, reference=None):
"""Swap strange characters from given string """Swap strange characters from given string
with underscore (_) and shorten it. with underscore (_) and shorten it.
Return the string Return the string
@@ -82,14 +86,15 @@ def nameCorrector(string,reference=None):
if totalLenght > LIMIT: if totalLenght > LIMIT:
limit = LIMIT - referenceLenght limit = LIMIT - referenceLenght
string = string[:limit-1] string = string[:limit - 1]
string = string.replace(" ", "_") string = string.replace(" ", "_")
if len(string.split('\n')) > 1: if len(string.split('\n')) > 1:
string = "".join(string.split('\n')) string = "".join(string.split('\n'))
BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','#', '.', '@' ,'“', '’', '\'', '!'] BAD_CHARS = ['\\', '/', ':', '*', '?', '"', '<',
'>', '|', '#', '.', '@', '“', '’', '\'', '!']
string = "".join([i if i not in BAD_CHARS else "_" for i in string]) string = "".join([i if i not in BAD_CHARS else "_" for i in string])
return string return string