diff options
author | Gabriel Arakaki Giovanini <mail@gabrielgio.me> | 2023-07-04 19:01:17 +0200 |
---|---|---|
committer | Gabriel Arakaki Giovanini <mail@gabrielgio.me> | 2023-07-04 19:01:17 +0200 |
commit | 05a8dbf46792adfef007a0ffbcb654026db036fa (patch) | |
tree | 386658904377c695747c3ffe4a288915b0a89917 /pkg | |
parent | 311ab744fe1bf278b18c25892497271988399e9a (diff) | |
download | lens-05a8dbf46792adfef007a0ffbcb654026db036fa.tar.gz lens-05a8dbf46792adfef007a0ffbcb654026db036fa.tar.bz2 lens-05a8dbf46792adfef007a0ffbcb654026db036fa.zip |
feat: Add use based file scanner
Diffstat (limited to 'pkg')
-rw-r--r-- | pkg/database/localfs/filesystem.go | 6 | ||||
-rw-r--r-- | pkg/worker/file_scanner.go | 83 | ||||
-rw-r--r-- | pkg/worker/scanner/exif_scanner.go (renamed from pkg/worker/exif_scanner.go) | 5 | ||||
-rw-r--r-- | pkg/worker/scanner/file_scanner.go | 99 | ||||
-rw-r--r-- | pkg/worker/scanner/thumbnail_scanner.go (renamed from pkg/worker/thumbnail_scanner.go) | 5 |
5 files changed, 107 insertions, 91 deletions
diff --git a/pkg/database/localfs/filesystem.go b/pkg/database/localfs/filesystem.go index c7c6458..d516ce9 100644 --- a/pkg/database/localfs/filesystem.go +++ b/pkg/database/localfs/filesystem.go @@ -11,10 +11,8 @@ type FileSystemRepository struct { root string } -func NewFileSystemRepository(root string) *FileSystemRepository { - return &FileSystemRepository{ - root: root, - } +func NewFileSystemRepository() *FileSystemRepository { + return &FileSystemRepository{} } func (self *FileSystemRepository) getFilesFromPath(filepath string) ([]fs.FileInfo, error) { diff --git a/pkg/worker/file_scanner.go b/pkg/worker/file_scanner.go deleted file mode 100644 index b4f907a..0000000 --- a/pkg/worker/file_scanner.go +++ /dev/null @@ -1,83 +0,0 @@ -package worker - -import ( - "context" - "io/fs" - "mime" - "path/filepath" - - "git.sr.ht/~gabrielgio/img/pkg/database/repository" - "git.sr.ht/~gabrielgio/img/pkg/fileop" -) - -type ( - FileScanner struct { - root string - repository repository.MediaRepository - } -) - -var _ ChanProcessor[string] = &FileScanner{} - -func NewFileScanner(root string, repository repository.MediaRepository) *FileScanner { - return &FileScanner{ - root: root, - repository: repository, - } -} - -func (f *FileScanner) Query(ctx context.Context) (<-chan string, error) { - c := make(chan string) - go func() { - defer close(c) - _ = filepath.Walk(f.root, func(path string, info fs.FileInfo, err error) error { - select { - case <-ctx.Done(): - return filepath.SkipAll - default: - } - - if info == nil { - return nil - } - - if info.IsDir() && filepath.Base(info.Name())[0] == '.' { - return filepath.SkipDir - } - - if info.IsDir() { - return nil - } - - c <- path - return nil - }) - }() - return c, nil -} - -func (f *FileScanner) Process(ctx context.Context, path string) error { - mimetype := mime.TypeByExtension(filepath.Ext(path)) - supported := fileop.IsMimeTypeSupported(mimetype) - if !supported { - return nil - } - - hash := fileop.GetHashFromPath(path) - - exists, err := f.repository.Exists(ctx, hash) - if err != nil { - return err - } - - if exists { - return nil - } - - return f.repository.Create(ctx, &repository.CreateMedia{ - Name: filepath.Base(path), - Path: path, - PathHash: hash, - MIMEType: mimetype, - }) -} diff --git a/pkg/worker/exif_scanner.go b/pkg/worker/scanner/exif_scanner.go index 5ea1810..47d717f 100644 --- a/pkg/worker/exif_scanner.go +++ b/pkg/worker/scanner/exif_scanner.go @@ -1,4 +1,4 @@ -package worker +package scanner import ( "context" @@ -6,6 +6,7 @@ import ( "git.sr.ht/~gabrielgio/img/pkg/coroutine" "git.sr.ht/~gabrielgio/img/pkg/database/repository" "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/worker" ) type ( @@ -14,7 +15,7 @@ type ( } ) -var _ BatchProcessor[*repository.Media] = &EXIFScanner{} +var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{} func NewEXIFScanner(repository repository.MediaRepository) *EXIFScanner { return &EXIFScanner{ diff --git a/pkg/worker/scanner/file_scanner.go b/pkg/worker/scanner/file_scanner.go new file mode 100644 index 0000000..7c19a3d --- /dev/null +++ b/pkg/worker/scanner/file_scanner.go @@ -0,0 +1,99 @@ +package scanner + +import ( + "context" + "io/fs" + "mime" + "path/filepath" + + "git.sr.ht/~gabrielgio/img/pkg/database/repository" + "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/list" + "git.sr.ht/~gabrielgio/img/pkg/worker" +) + +type ( + FileScanner struct { + mediaRepository repository.MediaRepository + userRepository repository.UserRepository + } +) + +var _ worker.ChanProcessor[string] = &FileScanner{} + +func NewFileScanner( + mediaRepository repository.MediaRepository, + userRepository repository.UserRepository, +) *FileScanner { + return &FileScanner{ + mediaRepository: mediaRepository, + userRepository: userRepository, + } +} + +func (f *FileScanner) Query(ctx context.Context) (<-chan string, error) { + c := make(chan string) + + users, err := f.userRepository.List(ctx) + if err != nil { + return nil, err + } + + // TODO: de duplicate file paths + paths := list.Map(users, func(u *repository.User) string { return u.Path }) + + go func(paths []string) { + defer close(c) + for _, p := range paths { + _ = filepath.Walk(p, func(path string, info fs.FileInfo, err error) error { + select { + case <-ctx.Done(): + return filepath.SkipAll + default: + } + + if info == nil { + return nil + } + + if info.IsDir() && filepath.Base(info.Name())[0] == '.' { + return filepath.SkipDir + } + + if info.IsDir() { + return nil + } + + c <- path + return nil + }) + } + }(paths) + return c, nil +} + +func (f *FileScanner) Process(ctx context.Context, path string) error { + mimetype := mime.TypeByExtension(filepath.Ext(path)) + supported := fileop.IsMimeTypeSupported(mimetype) + if !supported { + return nil + } + + hash := fileop.GetHashFromPath(path) + + exists, err := f.mediaRepository.Exists(ctx, hash) + if err != nil { + return err + } + + if exists { + return nil + } + + return f.mediaRepository.Create(ctx, &repository.CreateMedia{ + Name: filepath.Base(path), + Path: path, + PathHash: hash, + MIMEType: mimetype, + }) +} diff --git a/pkg/worker/thumbnail_scanner.go b/pkg/worker/scanner/thumbnail_scanner.go index 168abef..02fd4dd 100644 --- a/pkg/worker/thumbnail_scanner.go +++ b/pkg/worker/scanner/thumbnail_scanner.go @@ -1,4 +1,4 @@ -package worker +package scanner import ( "context" @@ -9,6 +9,7 @@ import ( "git.sr.ht/~gabrielgio/img/pkg/database/repository" "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/worker" ) type ( @@ -18,7 +19,7 @@ type ( } ) -var _ BatchProcessor[*repository.Media] = &EXIFScanner{} +var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{} func NewThumbnailScanner(cachePath string, repository repository.MediaRepository) *ThumbnailScanner { return &ThumbnailScanner{ |