From 05a8dbf46792adfef007a0ffbcb654026db036fa Mon Sep 17 00:00:00 2001 From: Gabriel Arakaki Giovanini Date: Tue, 4 Jul 2023 19:01:17 +0200 Subject: feat: Add use based file scanner --- cmd/server/main.go | 12 ++-- pkg/database/localfs/filesystem.go | 6 +- pkg/worker/exif_scanner.go | 39 ------------- pkg/worker/file_scanner.go | 83 --------------------------- pkg/worker/scanner/exif_scanner.go | 40 +++++++++++++ pkg/worker/scanner/file_scanner.go | 99 +++++++++++++++++++++++++++++++++ pkg/worker/scanner/thumbnail_scanner.go | 64 +++++++++++++++++++++ pkg/worker/thumbnail_scanner.go | 63 --------------------- 8 files changed, 210 insertions(+), 196 deletions(-) delete mode 100644 pkg/worker/exif_scanner.go delete mode 100644 pkg/worker/file_scanner.go create mode 100644 pkg/worker/scanner/exif_scanner.go create mode 100644 pkg/worker/scanner/file_scanner.go create mode 100644 pkg/worker/scanner/thumbnail_scanner.go delete mode 100644 pkg/worker/thumbnail_scanner.go diff --git a/cmd/server/main.go b/cmd/server/main.go index c064d56..b81b291 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -24,6 +24,7 @@ import ( "git.sr.ht/~gabrielgio/img/pkg/service" "git.sr.ht/~gabrielgio/img/pkg/view" "git.sr.ht/~gabrielgio/img/pkg/worker" + "git.sr.ht/~gabrielgio/img/pkg/worker/scanner" ) func main() { @@ -34,9 +35,6 @@ func main() { logLevel = flag.String("log-level", "error", "Log level: Choose either trace, debug, info, warning, error, fatal or panic") schedulerCount = flag.Uint("scheduler-count", 10, "How many workers are created to process media files") cachePath = flag.String("cache-path", "", "Folder to store thumbnail image") - - // TODO: this will later be replaced by user specific root folder - root = flag.String("root", "", "root folder for the whole application. All the workers will use it as working directory") ) flag.Parse() @@ -76,7 +74,7 @@ func main() { var ( userRepository = sql.NewUserRepository(db) settingsRepository = sql.NewSettingsRespository(db) - fileSystemRepository = localfs.NewFileSystemRepository(*root) + fileSystemRepository = localfs.NewFileSystemRepository() mediaRepository = sql.NewMediaRepository(db) ) @@ -113,9 +111,9 @@ func main() { // processors var ( - fileScanner = worker.NewFileScanner(*root, mediaRepository) - exifScanner = worker.NewEXIFScanner(mediaRepository) - thumbnailScanner = worker.NewThumbnailScanner(*cachePath, mediaRepository) + fileScanner = scanner.NewFileScanner(mediaRepository, userRepository) + exifScanner = scanner.NewEXIFScanner(mediaRepository) + thumbnailScanner = scanner.NewThumbnailScanner(*cachePath, mediaRepository) ) // worker diff --git a/pkg/database/localfs/filesystem.go b/pkg/database/localfs/filesystem.go index c7c6458..d516ce9 100644 --- a/pkg/database/localfs/filesystem.go +++ b/pkg/database/localfs/filesystem.go @@ -11,10 +11,8 @@ type FileSystemRepository struct { root string } -func NewFileSystemRepository(root string) *FileSystemRepository { - return &FileSystemRepository{ - root: root, - } +func NewFileSystemRepository() *FileSystemRepository { + return &FileSystemRepository{} } func (self *FileSystemRepository) getFilesFromPath(filepath string) ([]fs.FileInfo, error) { diff --git a/pkg/worker/exif_scanner.go b/pkg/worker/exif_scanner.go deleted file mode 100644 index 5ea1810..0000000 --- a/pkg/worker/exif_scanner.go +++ /dev/null @@ -1,39 +0,0 @@ -package worker - -import ( - "context" - - "git.sr.ht/~gabrielgio/img/pkg/coroutine" - "git.sr.ht/~gabrielgio/img/pkg/database/repository" - "git.sr.ht/~gabrielgio/img/pkg/fileop" -) - -type ( - EXIFScanner struct { - repository repository.MediaRepository - } -) - -var _ BatchProcessor[*repository.Media] = &EXIFScanner{} - -func NewEXIFScanner(repository repository.MediaRepository) *EXIFScanner { - return &EXIFScanner{ - repository: repository, - } -} - -func (e *EXIFScanner) Query(ctx context.Context) ([]*repository.Media, error) { - return e.repository.ListEmptyEXIF(ctx, &repository.Pagination{ - Page: 0, - Size: 100, - }) -} - -func (e *EXIFScanner) Process(ctx context.Context, m *repository.Media) error { - exif, err := coroutine.WrapProcess(ctx, func() (*repository.MediaEXIF, error) { return fileop.ReadExif(m.Path) }) - if err != nil { - return err - } - - return e.repository.CreateEXIF(ctx, m.ID, exif) -} diff --git a/pkg/worker/file_scanner.go b/pkg/worker/file_scanner.go deleted file mode 100644 index b4f907a..0000000 --- a/pkg/worker/file_scanner.go +++ /dev/null @@ -1,83 +0,0 @@ -package worker - -import ( - "context" - "io/fs" - "mime" - "path/filepath" - - "git.sr.ht/~gabrielgio/img/pkg/database/repository" - "git.sr.ht/~gabrielgio/img/pkg/fileop" -) - -type ( - FileScanner struct { - root string - repository repository.MediaRepository - } -) - -var _ ChanProcessor[string] = &FileScanner{} - -func NewFileScanner(root string, repository repository.MediaRepository) *FileScanner { - return &FileScanner{ - root: root, - repository: repository, - } -} - -func (f *FileScanner) Query(ctx context.Context) (<-chan string, error) { - c := make(chan string) - go func() { - defer close(c) - _ = filepath.Walk(f.root, func(path string, info fs.FileInfo, err error) error { - select { - case <-ctx.Done(): - return filepath.SkipAll - default: - } - - if info == nil { - return nil - } - - if info.IsDir() && filepath.Base(info.Name())[0] == '.' { - return filepath.SkipDir - } - - if info.IsDir() { - return nil - } - - c <- path - return nil - }) - }() - return c, nil -} - -func (f *FileScanner) Process(ctx context.Context, path string) error { - mimetype := mime.TypeByExtension(filepath.Ext(path)) - supported := fileop.IsMimeTypeSupported(mimetype) - if !supported { - return nil - } - - hash := fileop.GetHashFromPath(path) - - exists, err := f.repository.Exists(ctx, hash) - if err != nil { - return err - } - - if exists { - return nil - } - - return f.repository.Create(ctx, &repository.CreateMedia{ - Name: filepath.Base(path), - Path: path, - PathHash: hash, - MIMEType: mimetype, - }) -} diff --git a/pkg/worker/scanner/exif_scanner.go b/pkg/worker/scanner/exif_scanner.go new file mode 100644 index 0000000..47d717f --- /dev/null +++ b/pkg/worker/scanner/exif_scanner.go @@ -0,0 +1,40 @@ +package scanner + +import ( + "context" + + "git.sr.ht/~gabrielgio/img/pkg/coroutine" + "git.sr.ht/~gabrielgio/img/pkg/database/repository" + "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/worker" +) + +type ( + EXIFScanner struct { + repository repository.MediaRepository + } +) + +var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{} + +func NewEXIFScanner(repository repository.MediaRepository) *EXIFScanner { + return &EXIFScanner{ + repository: repository, + } +} + +func (e *EXIFScanner) Query(ctx context.Context) ([]*repository.Media, error) { + return e.repository.ListEmptyEXIF(ctx, &repository.Pagination{ + Page: 0, + Size: 100, + }) +} + +func (e *EXIFScanner) Process(ctx context.Context, m *repository.Media) error { + exif, err := coroutine.WrapProcess(ctx, func() (*repository.MediaEXIF, error) { return fileop.ReadExif(m.Path) }) + if err != nil { + return err + } + + return e.repository.CreateEXIF(ctx, m.ID, exif) +} diff --git a/pkg/worker/scanner/file_scanner.go b/pkg/worker/scanner/file_scanner.go new file mode 100644 index 0000000..7c19a3d --- /dev/null +++ b/pkg/worker/scanner/file_scanner.go @@ -0,0 +1,99 @@ +package scanner + +import ( + "context" + "io/fs" + "mime" + "path/filepath" + + "git.sr.ht/~gabrielgio/img/pkg/database/repository" + "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/list" + "git.sr.ht/~gabrielgio/img/pkg/worker" +) + +type ( + FileScanner struct { + mediaRepository repository.MediaRepository + userRepository repository.UserRepository + } +) + +var _ worker.ChanProcessor[string] = &FileScanner{} + +func NewFileScanner( + mediaRepository repository.MediaRepository, + userRepository repository.UserRepository, +) *FileScanner { + return &FileScanner{ + mediaRepository: mediaRepository, + userRepository: userRepository, + } +} + +func (f *FileScanner) Query(ctx context.Context) (<-chan string, error) { + c := make(chan string) + + users, err := f.userRepository.List(ctx) + if err != nil { + return nil, err + } + + // TODO: de duplicate file paths + paths := list.Map(users, func(u *repository.User) string { return u.Path }) + + go func(paths []string) { + defer close(c) + for _, p := range paths { + _ = filepath.Walk(p, func(path string, info fs.FileInfo, err error) error { + select { + case <-ctx.Done(): + return filepath.SkipAll + default: + } + + if info == nil { + return nil + } + + if info.IsDir() && filepath.Base(info.Name())[0] == '.' { + return filepath.SkipDir + } + + if info.IsDir() { + return nil + } + + c <- path + return nil + }) + } + }(paths) + return c, nil +} + +func (f *FileScanner) Process(ctx context.Context, path string) error { + mimetype := mime.TypeByExtension(filepath.Ext(path)) + supported := fileop.IsMimeTypeSupported(mimetype) + if !supported { + return nil + } + + hash := fileop.GetHashFromPath(path) + + exists, err := f.mediaRepository.Exists(ctx, hash) + if err != nil { + return err + } + + if exists { + return nil + } + + return f.mediaRepository.Create(ctx, &repository.CreateMedia{ + Name: filepath.Base(path), + Path: path, + PathHash: hash, + MIMEType: mimetype, + }) +} diff --git a/pkg/worker/scanner/thumbnail_scanner.go b/pkg/worker/scanner/thumbnail_scanner.go new file mode 100644 index 0000000..02fd4dd --- /dev/null +++ b/pkg/worker/scanner/thumbnail_scanner.go @@ -0,0 +1,64 @@ +package scanner + +import ( + "context" + "fmt" + "math" + "os" + "path" + + "git.sr.ht/~gabrielgio/img/pkg/database/repository" + "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/worker" +) + +type ( + ThumbnailScanner struct { + repository repository.MediaRepository + cachePath string + } +) + +var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{} + +func NewThumbnailScanner(cachePath string, repository repository.MediaRepository) *ThumbnailScanner { + return &ThumbnailScanner{ + repository: repository, + cachePath: cachePath, + } +} + +func (t *ThumbnailScanner) Query(ctx context.Context) ([]*repository.Media, error) { + return t.repository.ListEmptyThumbnail(ctx, &repository.Pagination{ + Page: 0, + Size: 100, + }) +} + +func (t *ThumbnailScanner) Process(ctx context.Context, media *repository.Media) error { + split := media.PathHash[:2] + filename := media.PathHash[2:] + folder := path.Join(t.cachePath, split) + output := path.Join(folder, filename+".jpeg") + + err := os.MkdirAll(folder, os.ModePerm) + if err != nil { + return err + } + + if media.IsVideo() { + err := fileop.EncodeVideoThumbnail(media.Path, output, 1080, 1080) + if err != nil { + return fmt.Errorf("Error thumbnail video %d; %w", media.ID, err) + } + } else { + err := fileop.EncodeImageThumbnail(media.Path, output, 1080, math.MinInt32) + if err != nil { + return fmt.Errorf("Error thumbnail image %d; %w", media.ID, err) + } + } + + return t.repository.CreateThumbnail(ctx, media.ID, &repository.MediaThumbnail{ + Path: output, + }) +} diff --git a/pkg/worker/thumbnail_scanner.go b/pkg/worker/thumbnail_scanner.go deleted file mode 100644 index 168abef..0000000 --- a/pkg/worker/thumbnail_scanner.go +++ /dev/null @@ -1,63 +0,0 @@ -package worker - -import ( - "context" - "fmt" - "math" - "os" - "path" - - "git.sr.ht/~gabrielgio/img/pkg/database/repository" - "git.sr.ht/~gabrielgio/img/pkg/fileop" -) - -type ( - ThumbnailScanner struct { - repository repository.MediaRepository - cachePath string - } -) - -var _ BatchProcessor[*repository.Media] = &EXIFScanner{} - -func NewThumbnailScanner(cachePath string, repository repository.MediaRepository) *ThumbnailScanner { - return &ThumbnailScanner{ - repository: repository, - cachePath: cachePath, - } -} - -func (t *ThumbnailScanner) Query(ctx context.Context) ([]*repository.Media, error) { - return t.repository.ListEmptyThumbnail(ctx, &repository.Pagination{ - Page: 0, - Size: 100, - }) -} - -func (t *ThumbnailScanner) Process(ctx context.Context, media *repository.Media) error { - split := media.PathHash[:2] - filename := media.PathHash[2:] - folder := path.Join(t.cachePath, split) - output := path.Join(folder, filename+".jpeg") - - err := os.MkdirAll(folder, os.ModePerm) - if err != nil { - return err - } - - if media.IsVideo() { - err := fileop.EncodeVideoThumbnail(media.Path, output, 1080, 1080) - if err != nil { - return fmt.Errorf("Error thumbnail video %d; %w", media.ID, err) - } - } else { - err := fileop.EncodeImageThumbnail(media.Path, output, 1080, math.MinInt32) - if err != nil { - return fmt.Errorf("Error thumbnail image %d; %w", media.ID, err) - } - } - - return t.repository.CreateThumbnail(ctx, media.ID, &repository.MediaThumbnail{ - Path: output, - }) -} -- cgit v1.2.3