aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cmd/server/main.go12
-rw-r--r--pkg/database/localfs/filesystem.go6
-rw-r--r--pkg/worker/file_scanner.go83
-rw-r--r--pkg/worker/scanner/exif_scanner.go (renamed from pkg/worker/exif_scanner.go)5
-rw-r--r--pkg/worker/scanner/file_scanner.go99
-rw-r--r--pkg/worker/scanner/thumbnail_scanner.go (renamed from pkg/worker/thumbnail_scanner.go)5
6 files changed, 112 insertions, 98 deletions
diff --git a/cmd/server/main.go b/cmd/server/main.go
index c064d56..b81b291 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -24,6 +24,7 @@ import (
"git.sr.ht/~gabrielgio/img/pkg/service"
"git.sr.ht/~gabrielgio/img/pkg/view"
"git.sr.ht/~gabrielgio/img/pkg/worker"
+ "git.sr.ht/~gabrielgio/img/pkg/worker/scanner"
)
func main() {
@@ -34,9 +35,6 @@ func main() {
logLevel = flag.String("log-level", "error", "Log level: Choose either trace, debug, info, warning, error, fatal or panic")
schedulerCount = flag.Uint("scheduler-count", 10, "How many workers are created to process media files")
cachePath = flag.String("cache-path", "", "Folder to store thumbnail image")
-
- // TODO: this will later be replaced by user specific root folder
- root = flag.String("root", "", "root folder for the whole application. All the workers will use it as working directory")
)
flag.Parse()
@@ -76,7 +74,7 @@ func main() {
var (
userRepository = sql.NewUserRepository(db)
settingsRepository = sql.NewSettingsRespository(db)
- fileSystemRepository = localfs.NewFileSystemRepository(*root)
+ fileSystemRepository = localfs.NewFileSystemRepository()
mediaRepository = sql.NewMediaRepository(db)
)
@@ -113,9 +111,9 @@ func main() {
// processors
var (
- fileScanner = worker.NewFileScanner(*root, mediaRepository)
- exifScanner = worker.NewEXIFScanner(mediaRepository)
- thumbnailScanner = worker.NewThumbnailScanner(*cachePath, mediaRepository)
+ fileScanner = scanner.NewFileScanner(mediaRepository, userRepository)
+ exifScanner = scanner.NewEXIFScanner(mediaRepository)
+ thumbnailScanner = scanner.NewThumbnailScanner(*cachePath, mediaRepository)
)
// worker
diff --git a/pkg/database/localfs/filesystem.go b/pkg/database/localfs/filesystem.go
index c7c6458..d516ce9 100644
--- a/pkg/database/localfs/filesystem.go
+++ b/pkg/database/localfs/filesystem.go
@@ -11,10 +11,8 @@ type FileSystemRepository struct {
root string
}
-func NewFileSystemRepository(root string) *FileSystemRepository {
- return &FileSystemRepository{
- root: root,
- }
+func NewFileSystemRepository() *FileSystemRepository {
+ return &FileSystemRepository{}
}
func (self *FileSystemRepository) getFilesFromPath(filepath string) ([]fs.FileInfo, error) {
diff --git a/pkg/worker/file_scanner.go b/pkg/worker/file_scanner.go
deleted file mode 100644
index b4f907a..0000000
--- a/pkg/worker/file_scanner.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package worker
-
-import (
- "context"
- "io/fs"
- "mime"
- "path/filepath"
-
- "git.sr.ht/~gabrielgio/img/pkg/database/repository"
- "git.sr.ht/~gabrielgio/img/pkg/fileop"
-)
-
-type (
- FileScanner struct {
- root string
- repository repository.MediaRepository
- }
-)
-
-var _ ChanProcessor[string] = &FileScanner{}
-
-func NewFileScanner(root string, repository repository.MediaRepository) *FileScanner {
- return &FileScanner{
- root: root,
- repository: repository,
- }
-}
-
-func (f *FileScanner) Query(ctx context.Context) (<-chan string, error) {
- c := make(chan string)
- go func() {
- defer close(c)
- _ = filepath.Walk(f.root, func(path string, info fs.FileInfo, err error) error {
- select {
- case <-ctx.Done():
- return filepath.SkipAll
- default:
- }
-
- if info == nil {
- return nil
- }
-
- if info.IsDir() && filepath.Base(info.Name())[0] == '.' {
- return filepath.SkipDir
- }
-
- if info.IsDir() {
- return nil
- }
-
- c <- path
- return nil
- })
- }()
- return c, nil
-}
-
-func (f *FileScanner) Process(ctx context.Context, path string) error {
- mimetype := mime.TypeByExtension(filepath.Ext(path))
- supported := fileop.IsMimeTypeSupported(mimetype)
- if !supported {
- return nil
- }
-
- hash := fileop.GetHashFromPath(path)
-
- exists, err := f.repository.Exists(ctx, hash)
- if err != nil {
- return err
- }
-
- if exists {
- return nil
- }
-
- return f.repository.Create(ctx, &repository.CreateMedia{
- Name: filepath.Base(path),
- Path: path,
- PathHash: hash,
- MIMEType: mimetype,
- })
-}
diff --git a/pkg/worker/exif_scanner.go b/pkg/worker/scanner/exif_scanner.go
index 5ea1810..47d717f 100644
--- a/pkg/worker/exif_scanner.go
+++ b/pkg/worker/scanner/exif_scanner.go
@@ -1,4 +1,4 @@
-package worker
+package scanner
import (
"context"
@@ -6,6 +6,7 @@ import (
"git.sr.ht/~gabrielgio/img/pkg/coroutine"
"git.sr.ht/~gabrielgio/img/pkg/database/repository"
"git.sr.ht/~gabrielgio/img/pkg/fileop"
+ "git.sr.ht/~gabrielgio/img/pkg/worker"
)
type (
@@ -14,7 +15,7 @@ type (
}
)
-var _ BatchProcessor[*repository.Media] = &EXIFScanner{}
+var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{}
func NewEXIFScanner(repository repository.MediaRepository) *EXIFScanner {
return &EXIFScanner{
diff --git a/pkg/worker/scanner/file_scanner.go b/pkg/worker/scanner/file_scanner.go
new file mode 100644
index 0000000..7c19a3d
--- /dev/null
+++ b/pkg/worker/scanner/file_scanner.go
@@ -0,0 +1,99 @@
+package scanner
+
+import (
+ "context"
+ "io/fs"
+ "mime"
+ "path/filepath"
+
+ "git.sr.ht/~gabrielgio/img/pkg/database/repository"
+ "git.sr.ht/~gabrielgio/img/pkg/fileop"
+ "git.sr.ht/~gabrielgio/img/pkg/list"
+ "git.sr.ht/~gabrielgio/img/pkg/worker"
+)
+
+type (
+ FileScanner struct {
+ mediaRepository repository.MediaRepository
+ userRepository repository.UserRepository
+ }
+)
+
+var _ worker.ChanProcessor[string] = &FileScanner{}
+
+func NewFileScanner(
+ mediaRepository repository.MediaRepository,
+ userRepository repository.UserRepository,
+) *FileScanner {
+ return &FileScanner{
+ mediaRepository: mediaRepository,
+ userRepository: userRepository,
+ }
+}
+
+func (f *FileScanner) Query(ctx context.Context) (<-chan string, error) {
+ c := make(chan string)
+
+ users, err := f.userRepository.List(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ // TODO: de duplicate file paths
+ paths := list.Map(users, func(u *repository.User) string { return u.Path })
+
+ go func(paths []string) {
+ defer close(c)
+ for _, p := range paths {
+ _ = filepath.Walk(p, func(path string, info fs.FileInfo, err error) error {
+ select {
+ case <-ctx.Done():
+ return filepath.SkipAll
+ default:
+ }
+
+ if info == nil {
+ return nil
+ }
+
+ if info.IsDir() && filepath.Base(info.Name())[0] == '.' {
+ return filepath.SkipDir
+ }
+
+ if info.IsDir() {
+ return nil
+ }
+
+ c <- path
+ return nil
+ })
+ }
+ }(paths)
+ return c, nil
+}
+
+func (f *FileScanner) Process(ctx context.Context, path string) error {
+ mimetype := mime.TypeByExtension(filepath.Ext(path))
+ supported := fileop.IsMimeTypeSupported(mimetype)
+ if !supported {
+ return nil
+ }
+
+ hash := fileop.GetHashFromPath(path)
+
+ exists, err := f.mediaRepository.Exists(ctx, hash)
+ if err != nil {
+ return err
+ }
+
+ if exists {
+ return nil
+ }
+
+ return f.mediaRepository.Create(ctx, &repository.CreateMedia{
+ Name: filepath.Base(path),
+ Path: path,
+ PathHash: hash,
+ MIMEType: mimetype,
+ })
+}
diff --git a/pkg/worker/thumbnail_scanner.go b/pkg/worker/scanner/thumbnail_scanner.go
index 168abef..02fd4dd 100644
--- a/pkg/worker/thumbnail_scanner.go
+++ b/pkg/worker/scanner/thumbnail_scanner.go
@@ -1,4 +1,4 @@
-package worker
+package scanner
import (
"context"
@@ -9,6 +9,7 @@ import (
"git.sr.ht/~gabrielgio/img/pkg/database/repository"
"git.sr.ht/~gabrielgio/img/pkg/fileop"
+ "git.sr.ht/~gabrielgio/img/pkg/worker"
)
type (
@@ -18,7 +19,7 @@ type (
}
)
-var _ BatchProcessor[*repository.Media] = &EXIFScanner{}
+var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{}
func NewThumbnailScanner(cachePath string, repository repository.MediaRepository) *ThumbnailScanner {
return &ThumbnailScanner{
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-21 15:10:29 +0000