aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Dockerfile8
-rw-r--r--dev-requirements.txt8
-rw-r--r--requirements.txt13
-rw-r--r--src/downloader.py (renamed from download.py)0
-rw-r--r--src/main.py70
-rw-r--r--src/uploader.py19
-rw-r--r--test/test_download.py (renamed from test_download.py)4
-rw-r--r--test/test_uploader.py20
-rw-r--r--user.py45
-rw-r--r--util.py40
10 files changed, 124 insertions, 103 deletions
diff --git a/Dockerfile b/Dockerfile
index 7d5fbaf..307aede 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,9 +7,9 @@ WORKDIR /opt
COPY requirements.txt .
RUN pip install -r requirements.txt
-COPY download.py .
-COPY user.py .
-COPY util.py .
+COPY src/downloader.py .
+COPY src/main.py .
+COPY src/uploader.py .
-ENTRYPOINT python user.py \ No newline at end of file
+ENTRYPOINT python main.py \ No newline at end of file
diff --git a/dev-requirements.txt b/dev-requirements.txt
index dec7c39..acde5d7 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,5 +1,5 @@
-r ./requirements.txt
-pytest
-pytest-mock
-pytest-xdist
-coverage \ No newline at end of file
+pytest==6.2.4
+pytest-mock==3.6.1
+pytest-xdist==2.3.0
+coverage==5.5 \ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index cb804f0..896d94d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,5 @@
-kafka-python
-youtube-dl
-requests
-praw
-pika
-jsonpickle
-requests
-nextcloud-api-wrapper
+youtube_dl==2021.6.6
+requests==2.26.0
+praw==7.3.0
+jsonpickle==2.0.0
+nextcloud-api-wrapper==0.2.1.5
diff --git a/download.py b/src/downloader.py
index c67834b..c67834b 100644
--- a/download.py
+++ b/src/downloader.py
diff --git a/src/main.py b/src/main.py
new file mode 100644
index 0000000..266bb53
--- /dev/null
+++ b/src/main.py
@@ -0,0 +1,70 @@
+import argparse
+import os
+
+import praw
+from time import sleep
+
+from nextcloud import NextCloud
+from praw.models.util import stream_generator
+
+from downloader import Downloader
+from uploader import create_folders, upload_file
+
+parser = argparse.ArgumentParser(description="Monitor saved")
+parser.add_argument('-c', '--client-id',
+ help="Reddit client id",
+ default=os.environ.get('CLIENT_ID', ''))
+parser.add_argument('-s', '--client-secret',
+ help="Reddit client secret",
+ default=os.environ.get('CLIENT_SECRET', ''))
+parser.add_argument('-u', '--reddit-username',
+ help="Reddit username",
+ default=os.environ.get('REDDIT_USERNAME', ''))
+parser.add_argument('-p', '--reddit-password',
+ help="Reddit user password",
+ default=os.environ.get('REDDIT_PASSWORD', ''))
+parser.add_argument('-P', '--nextcloud-password',
+ help="Nextcloud Password",
+ default=os.environ.get('NEXTCLOUD_PASSWORD', ''))
+parser.add_argument('-U', '--nextcloud-username',
+ help="Nextcloud Username",
+ default=os.environ.get('NEXTCLOUD_USERNAME', ''))
+parser.add_argument('-o', '--nextcloud-host',
+ help="Nextcloud Host",
+ default=os.environ.get('NEXTCLOUD_HOST', 'localhost'))
+parser.add_argument('-d', '--nextcloud-path',
+ help="Nextcloud root folder",
+ default=os.environ.get('NEXTCLOUD_PATH', 'im'))
+
+if __name__ == "__main__":
+ args = parser.parse_args()
+ reddit = praw.Reddit(client_id=args.client_id,
+ client_secret=args.client_secret,
+ password=args.reddit_password,
+ user_agent="hcrawler",
+ username=args.reddit_username)
+
+ nxc = NextCloud(
+ args.nextcloud_host,
+ user=args.nextcloud_username,
+ password=args.nextcloud_password,
+ session_kwargs={'verify': False}
+ )
+
+ redditor = reddit.redditor(args.reddit_username)
+
+
+ def upload(post):
+ url = post.url
+ create_folders(f"{args.nextcloud_path}/{post.subreddit}/", nxc)
+ with Downloader(url=url, reddit=reddit) as d:
+ d.download()
+ for path in d.paths:
+ if "-mobile" in path: # Remove mobile version
+ continue
+ upload_file(path, f"im/{post.subreddit}/{path}", nxc)
+
+
+ generator = stream_generator(redditor.saved, attribute_name="name")
+ for post in generator:
+ upload(post)
diff --git a/src/uploader.py b/src/uploader.py
new file mode 100644
index 0000000..7c7641a
--- /dev/null
+++ b/src/uploader.py
@@ -0,0 +1,19 @@
+from functools import reduce
+
+from nextcloud import NextCloud
+
+
+def _create_folder(folder: str, nxc: NextCloud) -> str:
+ nxc.create_folder(folder, True)
+ return folder
+
+
+def create_folders(path: str, nxc: NextCloud):
+ # remove first "/" if there is one.
+ path = path if path[0] != '/' else path[1:]
+ folders = path.split("/")
+ reduce(lambda x, y: _create_folder(f"{x}/{y}", nxc), folders, "")
+
+
+def upload_file(local_filename: str, remote_filename: str, nxc: NextCloud):
+ nxc.upload_file(local_filename, remote_filename)
diff --git a/test_download.py b/test/test_download.py
index 6f86bbc..9bafc1c 100644
--- a/test_download.py
+++ b/test/test_download.py
@@ -2,7 +2,7 @@ import os
import pytest
-from download import SourceType, Downloader
+from src.downloader import SourceType, Downloader
reddit_env = pytest.mark.skipif(
os.environ.get('CLIENT_ID', '') == '' or
@@ -16,7 +16,7 @@ reddit_env = pytest.mark.skipif(
@pytest.fixture
def mock_ydl_download(mocker):
# this function is responsible for downloading the file
- return mocker.patch('download.youtube_dl.YoutubeDL.process_info')
+ return mocker.patch('downloader.youtube_dl.YoutubeDL.process_info')
@pytest.mark.parametrize('url,source_type', [
diff --git a/test/test_uploader.py b/test/test_uploader.py
new file mode 100644
index 0000000..bcdf04b
--- /dev/null
+++ b/test/test_uploader.py
@@ -0,0 +1,20 @@
+import pytest
+
+from uploader import create_folders
+
+
+@pytest.fixture
+def nxc(mocker):
+ return mocker.MagicMock()
+
+
+@pytest.mark.parametrize("path,folders", [
+ ("folder1", ["/folder1"]),
+ ("folder1/folder2", ["/folder1", "/folder1/folder2"]),
+ ("/folder1/folder2", ["/folder1", "/folder1/folder2"])
+])
+def test_create_folders(nxc, path, folders):
+ create_folders(path, nxc)
+
+ for call, folder in zip(nxc.method_calls, folders):
+ assert folder == call.args[0]
diff --git a/user.py b/user.py
deleted file mode 100644
index a12cb59..0000000
--- a/user.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import praw
-from time import sleep
-
-from nextcloud import NextCloud
-from praw.models.util import stream_generator
-
-from download import Downloader
-from util import jsonfy, try_post, parser
-
-if __name__ == "__main__":
- args = parser.parse_args()
- reddit = praw.Reddit(client_id=args.client_id,
- client_secret=args.client_secret,
- password=args.reddit_password,
- user_agent="hcrawler",
- username=args.reddit_username)
-
- nxc = NextCloud(
- args.nextcloud_host,
- user=args.nextcloud_username,
- password=args.nextcloud_password,
- session_kwargs={'verify': False}
- )
-
- nxc.create_folder(f"im", True)
-
- redditor = reddit.redditor(args.reddit_username)
-
-
- def uplaod(post):
- url = post.url
- nxc.create_folder(f"im/{post.subreddit}/", True)
- with Downloader(url=url, reddit=reddit) as d:
- d.download()
- for path in d.paths:
- if "-mobile" in path: # Remove mobile version
- continue
- nxc.upload_file(path, f"im/{post.subreddit}/{path}")
-
-
- sleep(60)
-
- generator = stream_generator(redditor.saved, attribute_name="name")
- for post in generator:
- uplaod(post)
diff --git a/util.py b/util.py
deleted file mode 100644
index cd80506..0000000
--- a/util.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import argparse
-import json
-import os
-from time import sleep
-from typing import Dict
-
-import jsonpickle
-import requests
-
-headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-
-parser = argparse.ArgumentParser(description="Monitor saved")
-parser.add_argument('-c', '--client-id', help="Reddit client id", default=os.environ.get('CLIENT_ID', ''))
-parser.add_argument('-s', '--client-secret', help="Reddit client secret", default=os.environ.get('CLIENT_SECRET', ''))
-parser.add_argument('-u', '--reddit-username', help="Reddit username", default=os.environ.get('REDDIT_USERNAME', ''))
-parser.add_argument('-p', '--reddit-password', help="Reddit user password", default=os.environ.get('REDDIT_PASSWORD', ''))
-parser.add_argument('-P', '--nextcloud-password', help="Nextcloud Password", default=os.environ.get('NEXTCLOUD_PASSWORD', ''))
-parser.add_argument('-U', '--nextcloud-username', help="Nextcloud Username", default=os.environ.get('NEXTCLOUD_USERNAME', ''))
-parser.add_argument('-o', '--nextcloud-host', help="Nextcloud Host", default=os.environ.get('NEXTCLOUD_HOST', 'localhost'))
-
-
-def try_post(url, json_string, count=0):
- try:
- if count > 10:
- return
- r = requests.post(url, data=json_string, headers=headers)
- if r.status_code != 200:
- sleep(60 * count)
- try_post(url, json_string, count + 1)
- except:
- sleep(60 * count)
- try_post(url, json_string, count + 1)
-
-
-def jsonfy(post):
- json_string = jsonpickle.encode(post)
- json_dict: Dict = json.loads(json_string)
- json_dict.pop('_reddit')
- json_dict.pop('py/object')
- return json.dumps(json_dict)