diff options
author | Gabriel Arakaki Giovanini <mail@gabrielgio.me> | 2023-07-31 18:25:13 +0200 |
---|---|---|
committer | Gabriel Arakaki Giovanini <mail@gabrielgio.me> | 2023-08-06 18:41:34 +0200 |
commit | 5f660b309bc695277c67223520499fcc13f3c59f (patch) | |
tree | ce30f46d8feebac6eb3f5145e9c772be1c32f4ad /pkg/worker | |
parent | 5168a9476f0e83264ecafc85bc9145e8bdcbb8dc (diff) | |
download | lens-5f660b309bc695277c67223520499fcc13f3c59f.tar.gz lens-5f660b309bc695277c67223520499fcc13f3c59f.tar.bz2 lens-5f660b309bc695277c67223520499fcc13f3c59f.zip |
feat: Add album scanner
Diffstat (limited to 'pkg/worker')
-rw-r--r-- | pkg/worker/list_processor.go | 59 | ||||
-rw-r--r-- | pkg/worker/scanner/album_scanner.go | 98 | ||||
-rw-r--r-- | pkg/worker/scanner/exif_scanner.go | 2 | ||||
-rw-r--r-- | pkg/worker/scanner/thumbnail_scanner.go | 2 |
4 files changed, 156 insertions, 5 deletions
diff --git a/pkg/worker/list_processor.go b/pkg/worker/list_processor.go index c060583..02817c9 100644 --- a/pkg/worker/list_processor.go +++ b/pkg/worker/list_processor.go @@ -20,7 +20,7 @@ type ( OnFail(context.Context, T, error) } - BatchProcessor[T any] interface { + ListProcessor[T any] interface { Query(context.Context) ([]T, error) Process(context.Context, T) error } @@ -32,14 +32,20 @@ type ( } batchProcessorWorker[T any] struct { - batchProcessor BatchProcessor[T] + batchProcessor ListProcessor[T] + logrus *logrus.Entry + scheduler *Scheduler + } + + serialProcessorWorker[T any] struct { + batchProcessor ListProcessor[T] logrus *logrus.Entry scheduler *Scheduler } ) func NewWorkerFromBatchProcessor[T any]( - batchProcessor BatchProcessor[T], + batchProcessor ListProcessor[T], scheduler *Scheduler, logrus *logrus.Entry, ) Worker { @@ -50,6 +56,18 @@ func NewWorkerFromBatchProcessor[T any]( } } +func NewWorkerFromSerialProcessor[T any]( + batchProcessor ListProcessor[T], + scheduler *Scheduler, + logrus *logrus.Entry, +) Worker { + return &serialProcessorWorker[T]{ + batchProcessor: batchProcessor, + scheduler: scheduler, + logrus: logrus, + } +} + func NewWorkerFromChanProcessor[T any]( chanProcessor ChanProcessor[T], scheduler *Scheduler, @@ -105,6 +123,41 @@ func (l *batchProcessorWorker[T]) Start(ctx context.Context) error { } } +func (l *serialProcessorWorker[T]) Start(ctx context.Context) error { + for { + values, err := l.batchProcessor.Query(ctx) + if err != nil { + return err + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if len(values) == 0 { + return nil + } + for _, v := range values { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + l.scheduler.Take() + if err := l.batchProcessor.Process(ctx, v); err != nil && !errors.Is(err, context.Canceled) { + l.logrus.WithError(err).Error("Error processing batch") + if failure, ok := l.batchProcessor.(OnFail[T]); ok { + failure.OnFail(ctx, v, err) + } + } + l.scheduler.Return() + } + } +} + func (l *chanProcessorWorker[T]) Start(ctx context.Context) error { c, err := l.chanProcessor.Query(ctx) if err != nil { diff --git a/pkg/worker/scanner/album_scanner.go b/pkg/worker/scanner/album_scanner.go new file mode 100644 index 0000000..618a184 --- /dev/null +++ b/pkg/worker/scanner/album_scanner.go @@ -0,0 +1,98 @@ +package scanner + +import ( + "context" + "os" + "path" + "path/filepath" + "strings" + + "git.sr.ht/~gabrielgio/img/pkg/database/repository" + "git.sr.ht/~gabrielgio/img/pkg/worker" +) + +type ( + AlbumScanner struct { + repository repository.MediaRepository + } +) + +var _ worker.ListProcessor[*repository.Media] = &AlbumScanner{} + +func NewAlbumScanner(repository repository.MediaRepository) *AlbumScanner { + return &AlbumScanner{ + repository: repository, + } +} + +func (e *AlbumScanner) Query(ctx context.Context) ([]*repository.Media, error) { + return e.repository.ListEmptyAlbums(ctx, &repository.Pagination{ + Page: 0, + Size: 100, + }) +} + +// This process will optmize for file over folder, which means it will assume that there will be +// more file then folder in the overall library. +// So will try to make as cheap as possible to look for fetching many files in a folder +// meaning it will start from checking from left to right in the path since it will assume +// that the path to that point has been registered already, resulting in a single lookup for the media +func (e *AlbumScanner) Process(ctx context.Context, m *repository.Media) error { + // we don't need the name of the file, only its path + filePath, _ := path.Split(m.Path) + + parts := strings.Split(filePath, string(os.PathSeparator)) + + subPaths := FanInwards(parts) + album, err := e.GetAndCreateNestedAlbuns(ctx, subPaths) + if err != nil { + return err + } + + return e.repository.CreateAlbumFile(ctx, &repository.CreateAlbumFile{ + MediaID: m.ID, + AlbumID: album.ID, + }) +} + +func (e *AlbumScanner) GetAndCreateNestedAlbuns(ctx context.Context, paths []string) (*repository.Album, error) { + if len(paths) == 1 { + // end of trail, we should create a album without parent + return e.repository.CreateAlbum(ctx, &repository.CreateAlbum{ + ParentID: nil, + Name: filepath.Base(paths[0]), + Path: paths[0], + }) + } + + exists, err := e.repository.ExistsAlbumByAbsolutePath(ctx, paths[0]) + if err != nil { + return nil, err + } + + if exists { + return e.repository.GetAlbumByAbsolutePath(ctx, paths[0]) + } + + //album does not exist, create it and get its parent id + a, err := e.GetAndCreateNestedAlbuns(ctx, paths[1:]) + if err != nil { + return nil, err + } + + return e.repository.CreateAlbum(ctx, &repository.CreateAlbum{ + ParentID: &a.ID, + Name: filepath.Base(paths[0]), + Path: paths[0], + }) + +} + +func FanInwards(paths []string) []string { + result := make([]string, 0, len(paths)) + for i := (len(paths) - 1); i >= 0; i-- { + subPaths := paths[0:i] + result = append(result, path.Join(subPaths...)) + } + return result +} diff --git a/pkg/worker/scanner/exif_scanner.go b/pkg/worker/scanner/exif_scanner.go index 47d717f..da63c0b 100644 --- a/pkg/worker/scanner/exif_scanner.go +++ b/pkg/worker/scanner/exif_scanner.go @@ -15,7 +15,7 @@ type ( } ) -var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{} +var _ worker.ListProcessor[*repository.Media] = &EXIFScanner{} func NewEXIFScanner(repository repository.MediaRepository) *EXIFScanner { return &EXIFScanner{ diff --git a/pkg/worker/scanner/thumbnail_scanner.go b/pkg/worker/scanner/thumbnail_scanner.go index 9f75464..6446c53 100644 --- a/pkg/worker/scanner/thumbnail_scanner.go +++ b/pkg/worker/scanner/thumbnail_scanner.go @@ -19,7 +19,7 @@ type ( } ) -var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{} +var _ worker.ListProcessor[*repository.Media] = &EXIFScanner{} func NewThumbnailScanner(cachePath string, repository repository.MediaRepository) *ThumbnailScanner { return &ThumbnailScanner{ |