feat: Add nextcloud path param

Add a new param to input the nextcloud root path.

10 files changed, 124 insertions, 103 deletions

diff --git a/Dockerfile b/Dockerfile
index 7d5fbaf..307aede 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,9 +7,9 @@ WORKDIR /opt
 COPY requirements.txt .
 RUN pip install -r requirements.txt
 
-COPY download.py .
-COPY user.py .
-COPY util.py .
+COPY src/downloader.py .
+COPY src/main.py .
+COPY src/uploader.py .
 
 
-ENTRYPOINT python user.py
\ No newline at end of file
+ENTRYPOINT python main.py
\ No newline at end of file
diff --git a/dev-requirements.txt b/dev-requirements.txt
index dec7c39..acde5d7 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,5 +1,5 @@
 -r ./requirements.txt
-pytest
-pytest-mock
-pytest-xdist
-coverage
\ No newline at end of file
+pytest==6.2.4
+pytest-mock==3.6.1
+pytest-xdist==2.3.0
+coverage==5.5
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index cb804f0..896d94d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,5 @@
-kafka-python
-youtube-dl
-requests
-praw
-pika
-jsonpickle
-requests
-nextcloud-api-wrapper
+youtube_dl==2021.6.6
+requests==2.26.0
+praw==7.3.0
+jsonpickle==2.0.0
+nextcloud-api-wrapper==0.2.1.5
diff --git a/download.py b/src/downloader.py
index c67834b..c67834b 100644
--- a/download.py
+++ b/src/downloader.py
diff --git a/src/main.py b/src/main.py
new file mode 100644
index 0000000..266bb53
--- /dev/null
+++ b/src/main.py
@@ -0,0 +1,70 @@
+import argparse
+import os
+
+import praw
+from time import sleep
+
+from nextcloud import NextCloud
+from praw.models.util import stream_generator
+
+from downloader import Downloader
+from uploader import create_folders, upload_file
+
+parser = argparse.ArgumentParser(description="Monitor saved")
+parser.add_argument('-c', '--client-id',
+                    help="Reddit client id",
+                    default=os.environ.get('CLIENT_ID', ''))
+parser.add_argument('-s', '--client-secret',
+                    help="Reddit client secret",
+                    default=os.environ.get('CLIENT_SECRET', ''))
+parser.add_argument('-u', '--reddit-username',
+                    help="Reddit username",
+                    default=os.environ.get('REDDIT_USERNAME', ''))
+parser.add_argument('-p', '--reddit-password',
+                    help="Reddit user password",
+                    default=os.environ.get('REDDIT_PASSWORD', ''))
+parser.add_argument('-P', '--nextcloud-password',
+                    help="Nextcloud Password",
+                    default=os.environ.get('NEXTCLOUD_PASSWORD', ''))
+parser.add_argument('-U', '--nextcloud-username',
+                    help="Nextcloud Username",
+                    default=os.environ.get('NEXTCLOUD_USERNAME', ''))
+parser.add_argument('-o', '--nextcloud-host',
+                    help="Nextcloud Host",
+                    default=os.environ.get('NEXTCLOUD_HOST', 'localhost'))
+parser.add_argument('-d', '--nextcloud-path',
+                    help="Nextcloud root folder",
+                    default=os.environ.get('NEXTCLOUD_PATH', 'im'))
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    reddit = praw.Reddit(client_id=args.client_id,
+                         client_secret=args.client_secret,
+                         password=args.reddit_password,
+                         user_agent="hcrawler",
+                         username=args.reddit_username)
+
+    nxc = NextCloud(
+        args.nextcloud_host,
+        user=args.nextcloud_username,
+        password=args.nextcloud_password,
+        session_kwargs={'verify': False}
+    )
+
+    redditor = reddit.redditor(args.reddit_username)
+
+
+    def upload(post):
+        url = post.url
+        create_folders(f"{args.nextcloud_path}/{post.subreddit}/", nxc)
+        with Downloader(url=url, reddit=reddit) as d:
+            d.download()
+            for path in d.paths:
+                if "-mobile" in path:  # Remove mobile version
+                    continue
+                upload_file(path, f"im/{post.subreddit}/{path}", nxc)
+
+
+    generator = stream_generator(redditor.saved, attribute_name="name")
+    for post in generator:
+        upload(post)
diff --git a/src/uploader.py b/src/uploader.py
new file mode 100644
index 0000000..7c7641a
--- /dev/null
+++ b/src/uploader.py
@@ -0,0 +1,19 @@
+from functools import reduce
+
+from nextcloud import NextCloud
+
+
+def _create_folder(folder: str, nxc: NextCloud) -> str:
+    nxc.create_folder(folder, True)
+    return folder
+
+
+def create_folders(path: str, nxc: NextCloud):
+    # remove first "/" if there is one.
+    path = path if path[0] != '/' else path[1:]
+    folders = path.split("/")
+    reduce(lambda x, y: _create_folder(f"{x}/{y}", nxc), folders, "")
+
+
+def upload_file(local_filename: str, remote_filename: str, nxc: NextCloud):
+    nxc.upload_file(local_filename, remote_filename)
diff --git a/test_download.py b/test/test_download.py
index 6f86bbc..9bafc1c 100644
--- a/test_download.py
+++ b/test/test_download.py
@@ -2,7 +2,7 @@ import os
 
 import pytest
 
-from download import SourceType, Downloader
+from src.downloader import SourceType, Downloader
 
 reddit_env = pytest.mark.skipif(
     os.environ.get('CLIENT_ID', '') == '' or
@@ -16,7 +16,7 @@ reddit_env = pytest.mark.skipif(
 @pytest.fixture
 def mock_ydl_download(mocker):
     # this function is responsible for downloading the file
-    return mocker.patch('download.youtube_dl.YoutubeDL.process_info')
+    return mocker.patch('downloader.youtube_dl.YoutubeDL.process_info')
 
 
 @pytest.mark.parametrize('url,source_type', [
diff --git a/test/test_uploader.py b/test/test_uploader.py
new file mode 100644
index 0000000..bcdf04b
--- /dev/null
+++ b/test/test_uploader.py
@@ -0,0 +1,20 @@
+import pytest
+
+from uploader import create_folders
+
+
+@pytest.fixture
+def nxc(mocker):
+    return mocker.MagicMock()
+
+
+@pytest.mark.parametrize("path,folders", [
+    ("folder1", ["/folder1"]),
+    ("folder1/folder2", ["/folder1", "/folder1/folder2"]),
+    ("/folder1/folder2", ["/folder1", "/folder1/folder2"])
+])
+def test_create_folders(nxc, path, folders):
+    create_folders(path, nxc)
+
+    for call, folder in zip(nxc.method_calls, folders):
+        assert folder == call.args[0]
diff --git a/user.py b/user.py
deleted file mode 100644
index a12cb59..0000000
--- a/user.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import praw
-from time import sleep
-
-from nextcloud import NextCloud
-from praw.models.util import stream_generator
-
-from download import Downloader
-from util import jsonfy, try_post, parser
-
-if __name__ == "__main__":
-    args = parser.parse_args()
-    reddit = praw.Reddit(client_id=args.client_id,
-                         client_secret=args.client_secret,
-                         password=args.reddit_password,
-                         user_agent="hcrawler",
-                         username=args.reddit_username)
-
-    nxc = NextCloud(
-        args.nextcloud_host,
-        user=args.nextcloud_username,
-        password=args.nextcloud_password,
-        session_kwargs={'verify': False}
-    )
-
-    nxc.create_folder(f"im", True)
-
-    redditor = reddit.redditor(args.reddit_username)
-
-
-    def uplaod(post):
-        url = post.url
-        nxc.create_folder(f"im/{post.subreddit}/", True)
-        with Downloader(url=url, reddit=reddit) as d:
-            d.download()
-            for path in d.paths:
-                if "-mobile" in path:  # Remove mobile version
-                    continue
-                nxc.upload_file(path, f"im/{post.subreddit}/{path}")
-
-
-    sleep(60)
-
-    generator = stream_generator(redditor.saved, attribute_name="name")
-    for post in generator:
-        uplaod(post)
diff --git a/util.py b/util.py
deleted file mode 100644
index cd80506..0000000
--- a/util.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import argparse
-import json
-import os
-from time import sleep
-from typing import Dict
-
-import jsonpickle
-import requests
-
-headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-
-parser = argparse.ArgumentParser(description="Monitor saved")
-parser.add_argument('-c', '--client-id', help="Reddit client id", default=os.environ.get('CLIENT_ID', ''))
-parser.add_argument('-s', '--client-secret', help="Reddit client secret", default=os.environ.get('CLIENT_SECRET', ''))
-parser.add_argument('-u', '--reddit-username', help="Reddit username", default=os.environ.get('REDDIT_USERNAME', ''))
-parser.add_argument('-p', '--reddit-password', help="Reddit user password", default=os.environ.get('REDDIT_PASSWORD', ''))
-parser.add_argument('-P', '--nextcloud-password', help="Nextcloud Password", default=os.environ.get('NEXTCLOUD_PASSWORD', ''))
-parser.add_argument('-U', '--nextcloud-username', help="Nextcloud Username", default=os.environ.get('NEXTCLOUD_USERNAME', ''))
-parser.add_argument('-o', '--nextcloud-host', help="Nextcloud Host", default=os.environ.get('NEXTCLOUD_HOST', 'localhost'))
-
-
-def try_post(url, json_string, count=0):
-    try:
-        if count > 10:
-            return
-        r = requests.post(url, data=json_string, headers=headers)
-        if r.status_code != 200:
-            sleep(60 * count)
-            try_post(url, json_string, count + 1)
-    except:
-        sleep(60 * count)
-        try_post(url, json_string, count + 1)
-
-
-def jsonfy(post):
-    json_string = jsonpickle.encode(post)
-    json_dict: Dict = json.loads(json_string)
-    json_dict.pop('_reddit')
-    json_dict.pop('py/object')
-    return json.dumps(json_dict)