diff options
| -rw-r--r-- | Dockerfile | 8 | ||||
| -rw-r--r-- | dev-requirements.txt | 8 | ||||
| -rw-r--r-- | requirements.txt | 13 | ||||
| -rw-r--r-- | src/downloader.py (renamed from download.py) | 0 | ||||
| -rw-r--r-- | src/main.py | 70 | ||||
| -rw-r--r-- | src/uploader.py | 19 | ||||
| -rw-r--r-- | test/test_download.py (renamed from test_download.py) | 4 | ||||
| -rw-r--r-- | test/test_uploader.py | 20 | ||||
| -rw-r--r-- | user.py | 45 | ||||
| -rw-r--r-- | util.py | 40 |
10 files changed, 124 insertions, 103 deletions
@@ -7,9 +7,9 @@ WORKDIR /opt COPY requirements.txt . RUN pip install -r requirements.txt -COPY download.py . -COPY user.py . -COPY util.py . +COPY src/downloader.py . +COPY src/main.py . +COPY src/uploader.py . -ENTRYPOINT python user.py
\ No newline at end of file +ENTRYPOINT python main.py
\ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index dec7c39..acde5d7 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ -r ./requirements.txt -pytest -pytest-mock -pytest-xdist -coverage
\ No newline at end of file +pytest==6.2.4 +pytest-mock==3.6.1 +pytest-xdist==2.3.0 +coverage==5.5
\ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cb804f0..896d94d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ -kafka-python -youtube-dl -requests -praw -pika -jsonpickle -requests -nextcloud-api-wrapper +youtube_dl==2021.6.6 +requests==2.26.0 +praw==7.3.0 +jsonpickle==2.0.0 +nextcloud-api-wrapper==0.2.1.5 diff --git a/download.py b/src/downloader.py index c67834b..c67834b 100644 --- a/download.py +++ b/src/downloader.py diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..266bb53 --- /dev/null +++ b/src/main.py @@ -0,0 +1,70 @@ +import argparse +import os + +import praw +from time import sleep + +from nextcloud import NextCloud +from praw.models.util import stream_generator + +from downloader import Downloader +from uploader import create_folders, upload_file + +parser = argparse.ArgumentParser(description="Monitor saved") +parser.add_argument('-c', '--client-id', + help="Reddit client id", + default=os.environ.get('CLIENT_ID', '')) +parser.add_argument('-s', '--client-secret', + help="Reddit client secret", + default=os.environ.get('CLIENT_SECRET', '')) +parser.add_argument('-u', '--reddit-username', + help="Reddit username", + default=os.environ.get('REDDIT_USERNAME', '')) +parser.add_argument('-p', '--reddit-password', + help="Reddit user password", + default=os.environ.get('REDDIT_PASSWORD', '')) +parser.add_argument('-P', '--nextcloud-password', + help="Nextcloud Password", + default=os.environ.get('NEXTCLOUD_PASSWORD', '')) +parser.add_argument('-U', '--nextcloud-username', + help="Nextcloud Username", + default=os.environ.get('NEXTCLOUD_USERNAME', '')) +parser.add_argument('-o', '--nextcloud-host', + help="Nextcloud Host", + default=os.environ.get('NEXTCLOUD_HOST', 'localhost')) +parser.add_argument('-d', '--nextcloud-path', + help="Nextcloud root folder", + default=os.environ.get('NEXTCLOUD_PATH', 'im')) + +if __name__ == "__main__": + args = parser.parse_args() + reddit = praw.Reddit(client_id=args.client_id, + client_secret=args.client_secret, + password=args.reddit_password, + user_agent="hcrawler", + username=args.reddit_username) + + nxc = NextCloud( + args.nextcloud_host, + user=args.nextcloud_username, + password=args.nextcloud_password, + session_kwargs={'verify': False} + ) + + redditor = reddit.redditor(args.reddit_username) + + + def upload(post): + url = post.url + create_folders(f"{args.nextcloud_path}/{post.subreddit}/", nxc) + with Downloader(url=url, reddit=reddit) as d: + d.download() + for path in d.paths: + if "-mobile" in path: # Remove mobile version + continue + upload_file(path, f"im/{post.subreddit}/{path}", nxc) + + + generator = stream_generator(redditor.saved, attribute_name="name") + for post in generator: + upload(post) diff --git a/src/uploader.py b/src/uploader.py new file mode 100644 index 0000000..7c7641a --- /dev/null +++ b/src/uploader.py @@ -0,0 +1,19 @@ +from functools import reduce + +from nextcloud import NextCloud + + +def _create_folder(folder: str, nxc: NextCloud) -> str: + nxc.create_folder(folder, True) + return folder + + +def create_folders(path: str, nxc: NextCloud): + # remove first "/" if there is one. + path = path if path[0] != '/' else path[1:] + folders = path.split("/") + reduce(lambda x, y: _create_folder(f"{x}/{y}", nxc), folders, "") + + +def upload_file(local_filename: str, remote_filename: str, nxc: NextCloud): + nxc.upload_file(local_filename, remote_filename) diff --git a/test_download.py b/test/test_download.py index 6f86bbc..9bafc1c 100644 --- a/test_download.py +++ b/test/test_download.py @@ -2,7 +2,7 @@ import os import pytest -from download import SourceType, Downloader +from src.downloader import SourceType, Downloader reddit_env = pytest.mark.skipif( os.environ.get('CLIENT_ID', '') == '' or @@ -16,7 +16,7 @@ reddit_env = pytest.mark.skipif( @pytest.fixture def mock_ydl_download(mocker): # this function is responsible for downloading the file - return mocker.patch('download.youtube_dl.YoutubeDL.process_info') + return mocker.patch('downloader.youtube_dl.YoutubeDL.process_info') @pytest.mark.parametrize('url,source_type', [ diff --git a/test/test_uploader.py b/test/test_uploader.py new file mode 100644 index 0000000..bcdf04b --- /dev/null +++ b/test/test_uploader.py @@ -0,0 +1,20 @@ +import pytest + +from uploader import create_folders + + +@pytest.fixture +def nxc(mocker): + return mocker.MagicMock() + + +@pytest.mark.parametrize("path,folders", [ + ("folder1", ["/folder1"]), + ("folder1/folder2", ["/folder1", "/folder1/folder2"]), + ("/folder1/folder2", ["/folder1", "/folder1/folder2"]) +]) +def test_create_folders(nxc, path, folders): + create_folders(path, nxc) + + for call, folder in zip(nxc.method_calls, folders): + assert folder == call.args[0] diff --git a/user.py b/user.py deleted file mode 100644 index a12cb59..0000000 --- a/user.py +++ /dev/null @@ -1,45 +0,0 @@ -import praw -from time import sleep - -from nextcloud import NextCloud -from praw.models.util import stream_generator - -from download import Downloader -from util import jsonfy, try_post, parser - -if __name__ == "__main__": - args = parser.parse_args() - reddit = praw.Reddit(client_id=args.client_id, - client_secret=args.client_secret, - password=args.reddit_password, - user_agent="hcrawler", - username=args.reddit_username) - - nxc = NextCloud( - args.nextcloud_host, - user=args.nextcloud_username, - password=args.nextcloud_password, - session_kwargs={'verify': False} - ) - - nxc.create_folder(f"im", True) - - redditor = reddit.redditor(args.reddit_username) - - - def uplaod(post): - url = post.url - nxc.create_folder(f"im/{post.subreddit}/", True) - with Downloader(url=url, reddit=reddit) as d: - d.download() - for path in d.paths: - if "-mobile" in path: # Remove mobile version - continue - nxc.upload_file(path, f"im/{post.subreddit}/{path}") - - - sleep(60) - - generator = stream_generator(redditor.saved, attribute_name="name") - for post in generator: - uplaod(post) diff --git a/util.py b/util.py deleted file mode 100644 index cd80506..0000000 --- a/util.py +++ /dev/null @@ -1,40 +0,0 @@ -import argparse -import json -import os -from time import sleep -from typing import Dict - -import jsonpickle -import requests - -headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} - -parser = argparse.ArgumentParser(description="Monitor saved") -parser.add_argument('-c', '--client-id', help="Reddit client id", default=os.environ.get('CLIENT_ID', '')) -parser.add_argument('-s', '--client-secret', help="Reddit client secret", default=os.environ.get('CLIENT_SECRET', '')) -parser.add_argument('-u', '--reddit-username', help="Reddit username", default=os.environ.get('REDDIT_USERNAME', '')) -parser.add_argument('-p', '--reddit-password', help="Reddit user password", default=os.environ.get('REDDIT_PASSWORD', '')) -parser.add_argument('-P', '--nextcloud-password', help="Nextcloud Password", default=os.environ.get('NEXTCLOUD_PASSWORD', '')) -parser.add_argument('-U', '--nextcloud-username', help="Nextcloud Username", default=os.environ.get('NEXTCLOUD_USERNAME', '')) -parser.add_argument('-o', '--nextcloud-host', help="Nextcloud Host", default=os.environ.get('NEXTCLOUD_HOST', 'localhost')) - - -def try_post(url, json_string, count=0): - try: - if count > 10: - return - r = requests.post(url, data=json_string, headers=headers) - if r.status_code != 200: - sleep(60 * count) - try_post(url, json_string, count + 1) - except: - sleep(60 * count) - try_post(url, json_string, count + 1) - - -def jsonfy(post): - json_string = jsonpickle.encode(post) - json_dict: Dict = json.loads(json_string) - json_dict.pop('_reddit') - json_dict.pop('py/object') - return json.dumps(json_dict) |