aboutsummaryrefslogtreecommitdiff
path: root/pkg/worker
diff options
context:
space:
mode:
authorGabriel Arakaki Giovanini <mail@gabrielgio.me>2023-07-31 18:25:13 +0200
committerGabriel Arakaki Giovanini <mail@gabrielgio.me>2023-08-06 18:41:34 +0200
commit5f660b309bc695277c67223520499fcc13f3c59f (patch)
treece30f46d8feebac6eb3f5145e9c772be1c32f4ad /pkg/worker
parent5168a9476f0e83264ecafc85bc9145e8bdcbb8dc (diff)
downloadlens-5f660b309bc695277c67223520499fcc13f3c59f.tar.gz
lens-5f660b309bc695277c67223520499fcc13f3c59f.tar.bz2
lens-5f660b309bc695277c67223520499fcc13f3c59f.zip
feat: Add album scanner
Diffstat (limited to 'pkg/worker')
-rw-r--r--pkg/worker/list_processor.go59
-rw-r--r--pkg/worker/scanner/album_scanner.go98
-rw-r--r--pkg/worker/scanner/exif_scanner.go2
-rw-r--r--pkg/worker/scanner/thumbnail_scanner.go2
4 files changed, 156 insertions, 5 deletions
diff --git a/pkg/worker/list_processor.go b/pkg/worker/list_processor.go
index c060583..02817c9 100644
--- a/pkg/worker/list_processor.go
+++ b/pkg/worker/list_processor.go
@@ -20,7 +20,7 @@ type (
OnFail(context.Context, T, error)
}
- BatchProcessor[T any] interface {
+ ListProcessor[T any] interface {
Query(context.Context) ([]T, error)
Process(context.Context, T) error
}
@@ -32,14 +32,20 @@ type (
}
batchProcessorWorker[T any] struct {
- batchProcessor BatchProcessor[T]
+ batchProcessor ListProcessor[T]
+ logrus *logrus.Entry
+ scheduler *Scheduler
+ }
+
+ serialProcessorWorker[T any] struct {
+ batchProcessor ListProcessor[T]
logrus *logrus.Entry
scheduler *Scheduler
}
)
func NewWorkerFromBatchProcessor[T any](
- batchProcessor BatchProcessor[T],
+ batchProcessor ListProcessor[T],
scheduler *Scheduler,
logrus *logrus.Entry,
) Worker {
@@ -50,6 +56,18 @@ func NewWorkerFromBatchProcessor[T any](
}
}
+func NewWorkerFromSerialProcessor[T any](
+ batchProcessor ListProcessor[T],
+ scheduler *Scheduler,
+ logrus *logrus.Entry,
+) Worker {
+ return &serialProcessorWorker[T]{
+ batchProcessor: batchProcessor,
+ scheduler: scheduler,
+ logrus: logrus,
+ }
+}
+
func NewWorkerFromChanProcessor[T any](
chanProcessor ChanProcessor[T],
scheduler *Scheduler,
@@ -105,6 +123,41 @@ func (l *batchProcessorWorker[T]) Start(ctx context.Context) error {
}
}
+func (l *serialProcessorWorker[T]) Start(ctx context.Context) error {
+ for {
+ values, err := l.batchProcessor.Query(ctx)
+ if err != nil {
+ return err
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ }
+
+ if len(values) == 0 {
+ return nil
+ }
+ for _, v := range values {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ }
+
+ l.scheduler.Take()
+ if err := l.batchProcessor.Process(ctx, v); err != nil && !errors.Is(err, context.Canceled) {
+ l.logrus.WithError(err).Error("Error processing batch")
+ if failure, ok := l.batchProcessor.(OnFail[T]); ok {
+ failure.OnFail(ctx, v, err)
+ }
+ }
+ l.scheduler.Return()
+ }
+ }
+}
+
func (l *chanProcessorWorker[T]) Start(ctx context.Context) error {
c, err := l.chanProcessor.Query(ctx)
if err != nil {
diff --git a/pkg/worker/scanner/album_scanner.go b/pkg/worker/scanner/album_scanner.go
new file mode 100644
index 0000000..618a184
--- /dev/null
+++ b/pkg/worker/scanner/album_scanner.go
@@ -0,0 +1,98 @@
+package scanner
+
+import (
+ "context"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+
+ "git.sr.ht/~gabrielgio/img/pkg/database/repository"
+ "git.sr.ht/~gabrielgio/img/pkg/worker"
+)
+
+type (
+ AlbumScanner struct {
+ repository repository.MediaRepository
+ }
+)
+
+var _ worker.ListProcessor[*repository.Media] = &AlbumScanner{}
+
+func NewAlbumScanner(repository repository.MediaRepository) *AlbumScanner {
+ return &AlbumScanner{
+ repository: repository,
+ }
+}
+
+func (e *AlbumScanner) Query(ctx context.Context) ([]*repository.Media, error) {
+ return e.repository.ListEmptyAlbums(ctx, &repository.Pagination{
+ Page: 0,
+ Size: 100,
+ })
+}
+
+// This process will optmize for file over folder, which means it will assume that there will be
+// more file then folder in the overall library.
+// So will try to make as cheap as possible to look for fetching many files in a folder
+// meaning it will start from checking from left to right in the path since it will assume
+// that the path to that point has been registered already, resulting in a single lookup for the media
+func (e *AlbumScanner) Process(ctx context.Context, m *repository.Media) error {
+ // we don't need the name of the file, only its path
+ filePath, _ := path.Split(m.Path)
+
+ parts := strings.Split(filePath, string(os.PathSeparator))
+
+ subPaths := FanInwards(parts)
+ album, err := e.GetAndCreateNestedAlbuns(ctx, subPaths)
+ if err != nil {
+ return err
+ }
+
+ return e.repository.CreateAlbumFile(ctx, &repository.CreateAlbumFile{
+ MediaID: m.ID,
+ AlbumID: album.ID,
+ })
+}
+
+func (e *AlbumScanner) GetAndCreateNestedAlbuns(ctx context.Context, paths []string) (*repository.Album, error) {
+ if len(paths) == 1 {
+ // end of trail, we should create a album without parent
+ return e.repository.CreateAlbum(ctx, &repository.CreateAlbum{
+ ParentID: nil,
+ Name: filepath.Base(paths[0]),
+ Path: paths[0],
+ })
+ }
+
+ exists, err := e.repository.ExistsAlbumByAbsolutePath(ctx, paths[0])
+ if err != nil {
+ return nil, err
+ }
+
+ if exists {
+ return e.repository.GetAlbumByAbsolutePath(ctx, paths[0])
+ }
+
+ //album does not exist, create it and get its parent id
+ a, err := e.GetAndCreateNestedAlbuns(ctx, paths[1:])
+ if err != nil {
+ return nil, err
+ }
+
+ return e.repository.CreateAlbum(ctx, &repository.CreateAlbum{
+ ParentID: &a.ID,
+ Name: filepath.Base(paths[0]),
+ Path: paths[0],
+ })
+
+}
+
+func FanInwards(paths []string) []string {
+ result := make([]string, 0, len(paths))
+ for i := (len(paths) - 1); i >= 0; i-- {
+ subPaths := paths[0:i]
+ result = append(result, path.Join(subPaths...))
+ }
+ return result
+}
diff --git a/pkg/worker/scanner/exif_scanner.go b/pkg/worker/scanner/exif_scanner.go
index 47d717f..da63c0b 100644
--- a/pkg/worker/scanner/exif_scanner.go
+++ b/pkg/worker/scanner/exif_scanner.go
@@ -15,7 +15,7 @@ type (
}
)
-var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{}
+var _ worker.ListProcessor[*repository.Media] = &EXIFScanner{}
func NewEXIFScanner(repository repository.MediaRepository) *EXIFScanner {
return &EXIFScanner{
diff --git a/pkg/worker/scanner/thumbnail_scanner.go b/pkg/worker/scanner/thumbnail_scanner.go
index 9f75464..6446c53 100644
--- a/pkg/worker/scanner/thumbnail_scanner.go
+++ b/pkg/worker/scanner/thumbnail_scanner.go
@@ -19,7 +19,7 @@ type (
}
)
-var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{}
+var _ worker.ListProcessor[*repository.Media] = &EXIFScanner{}
func NewThumbnailScanner(cachePath string, repository repository.MediaRepository) *ThumbnailScanner {
return &ThumbnailScanner{