1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
import argparse
import logging
import os
from time import sleep
import praw
from nextcloud import NextCloud
from praw.models.util import stream_generator
from importer.downloader import Downloader
from importer.uploader import upload_file, create_folders
levels = {
'critical': logging.CRITICAL,
'error': logging.ERROR,
'warn': logging.WARNING,
'warning': logging.WARNING,
'info': logging.INFO,
'debug': logging.DEBUG
}
parser = argparse.ArgumentParser(description="Monitor saved")
parser.add_argument('-c', '--client-id',
help="Reddit client id",
default=os.environ.get('CLIENT_ID', ''))
parser.add_argument('-s', '--client-secret',
help="Reddit client secret",
default=os.environ.get('CLIENT_SECRET', ''))
parser.add_argument('-u', '--reddit-username',
help="Reddit username",
default=os.environ.get('REDDIT_USERNAME', ''))
parser.add_argument('-p', '--reddit-password',
help="Reddit user password",
default=os.environ.get('REDDIT_PASSWORD', ''))
parser.add_argument('-P', '--nextcloud-password',
help="Nextcloud Password",
default=os.environ.get('NEXTCLOUD_PASSWORD', ''))
parser.add_argument('-U', '--nextcloud-username',
help="Nextcloud Username",
default=os.environ.get('NEXTCLOUD_USERNAME', ''))
parser.add_argument('-o', '--nextcloud-host',
help="Nextcloud Host",
default=os.environ.get('NEXTCLOUD_HOST', 'localhost'))
parser.add_argument('-d', '--nextcloud-path',
help="Nextcloud root folder",
default=os.environ.get('NEXTCLOUD_PATH', 'im'))
parser.add_argument('-f', '--from-beginning',
dest='from_beginning',
help="it will attempt to download all saved posts from the beginning.",
action='store_true',
default=os.environ.get("FROM_BEGINNING") is not None)
parser.add_argument('-l', '--log-level',
default=os.environ.get('LOG_LEVEL', 'info'),
choices=levels.keys(),
help=f'it will set log level.')
if __name__ == "__main__":
args = parser.parse_args()
level = levels.get(args.log_level.lower())
logging.basicConfig(level=level)
reddit = praw.Reddit(client_id=args.client_id,
client_secret=args.client_secret,
password=args.reddit_password,
user_agent="hcrawler",
username=args.reddit_username)
nxc = NextCloud(
args.nextcloud_host,
user=args.nextcloud_username,
password=args.nextcloud_password,
session_kwargs={'verify': False}
)
redditor = reddit.redditor(args.reddit_username)
def upload(post):
try:
url = post.url
create_folders(f"{args.nextcloud_path}/{post.subreddit}/", nxc)
d = Downloader(url=url, reddit=reddit)
with d.provider() as provider:
provider.download()
logging.info(f"{post.id} from {post.subreddit} downloaded")
for path in provider.paths:
if "-mobile" in path: # Remove mobile version
continue
upload_file(path, f"{args.nextcloud_path}/{post.subreddit}/{path}", nxc)
logging.info(f"{path} uploaded")
except Exception as e:
logging.error(e)
if args.from_beginning:
logging.info(f"Downloading from the beginning")
for post in redditor.saved(limit=None):
upload(post)
sleep(60)
generator = stream_generator(redditor.saved, attribute_name="name")
for post in generator:
upload(post)
|
