aboutsummaryrefslogtreecommitdiff
path: root/reddit_nextcloud_importer/__init__.py
blob: c0d769c4c2ebd1d6e18bfe3ac69f0ee6a9c5dabf (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
import argparse
import logging
import os
from typing import List
from time import sleep

import praw

from nextcloud import NextCloud
from praw.models.util import stream_generator
from gallery_dl.job import DownloadJob

from functools import reduce

from nextcloud import NextCloud

levels = {
    'critical': logging.CRITICAL,
    'error': logging.ERROR,
    'warn': logging.WARNING,
    'warning': logging.WARNING,
    'info': logging.INFO,
    'debug': logging.DEBUG
}

parser = argparse.ArgumentParser(description="Monitor saved")
parser.add_argument('-c', '--client-id',
                    help="Reddit client id",
                    default=os.environ.get('CLIENT_ID', ''))
parser.add_argument('-s', '--client-secret',
                    help="Reddit client secret",
                    default=os.environ.get('CLIENT_SECRET', ''))
parser.add_argument('-u', '--reddit-username',
                    help="Reddit username",
                    default=os.environ.get('REDDIT_USERNAME', ''))
parser.add_argument('-p', '--reddit-password',
                    help="Reddit user password",
                    default=os.environ.get('REDDIT_PASSWORD', ''))
parser.add_argument('-P', '--nextcloud-password',
                    help="Nextcloud Password",
                    default=os.environ.get('NEXTCLOUD_PASSWORD', ''))
parser.add_argument('-U', '--nextcloud-username',
                    help="Nextcloud Username",
                    default=os.environ.get('NEXTCLOUD_USERNAME', ''))
parser.add_argument('-o', '--nextcloud-host',
                    help="Nextcloud Host",
                    default=os.environ.get('NEXTCLOUD_HOST', 'localhost'))
parser.add_argument('-d', '--nextcloud-path',
                    help="Nextcloud root folder",
                    default=os.environ.get('NEXTCLOUD_PATH', 'im'))
parser.add_argument('-f', '--from-beginning',
                    dest='from_beginning',
                    help="it will attempt to download all saved posts from the beginning.",
                    action='store_true',
                    default=os.environ.get("FROM_BEGINNING") is not None)
parser.add_argument('-l', '--log-level',
                    default=os.environ.get('LOG_LEVEL', 'info'),
                    choices=levels.keys(),
                    help=f'it will set log level.')


def _create_folder(folder: str, nxc: NextCloud) -> str:
    nxc.create_folder(folder, True)
    return folder


def create_folders(path: str, nxc: NextCloud):
    # remove first "/" if there is one.
    path = path if path[0] != '/' else path[1:]
    folders = path.split("/")
    reduce(lambda x, y: _create_folder(f"{x}/{y}", nxc), folders, "")


def upload_file(local_filename: str, remote_filename: str, nxc: NextCloud):
    nxc.upload_file(local_filename, remote_filename)


def get_list_of_files(dirName):
    # create a list of file and sub directories
    # names in the given directory
    listOfFile = os.listdir(dirName)
    allFiles = list()
    # Iterate over all the entries
    for entry in listOfFile:
        # Create full path
        fullPath = os.path.join(dirName, entry)
        # If entry is a directory then get the list of files in this directory
        if os.path.isdir(fullPath):
            allFiles = allFiles + get_list_of_files(fullPath)
        else:
            allFiles.append(fullPath)

    return allFiles


def download(url) -> List[str]:
    d = DownloadJob(url=url)
    d.run()
    basedir = d.pathfmt.basedirectory
    files = get_list_of_files(basedir)
    return files


def main():
    args = parser.parse_args()

    level = levels.get(args.log_level.lower())
    logging.basicConfig(level=level)

    reddit = praw.Reddit(client_id=args.client_id,
                         client_secret=args.client_secret,
                         password=args.reddit_password,
                         user_agent="hcrawler",
                         username=args.reddit_username)

    nxc = NextCloud(
        args.nextcloud_host,
        user=args.nextcloud_username,
        password=args.nextcloud_password,
        session_kwargs={'verify': False}
    )

    redditor = reddit.redditor(args.reddit_username)


    def upload(post):
        try:
            url = post.url
            create_folders(f"{args.nextcloud_path}/{post.subreddit}/", nxc)
            logging.info(f"{post.id} from {post.subreddit} downloaded")
            for path in download(url):
                filename = os.path.basename(path)
                upload_file(path, f"{args.nextcloud_path}/{post.subreddit}/{filename}", nxc)
                os.unlink(path)
                logging.info(f"{path} uploaded")
        except Exception as e:
            logging.error(e)


    if args.from_beginning:
        logging.info(f"Downloading from the beginning")
        for post in redditor.saved(limit=None):
            upload(post)
            sleep(60)

    generator = stream_generator(redditor.saved, attribute_name="name")
    for post in generator:
        upload(post)