From 05a8dbf46792adfef007a0ffbcb654026db036fa Mon Sep 17 00:00:00 2001 From: Gabriel Arakaki Giovanini Date: Tue, 4 Jul 2023 19:01:17 +0200 Subject: feat: Add use based file scanner --- pkg/worker/scanner/exif_scanner.go | 40 +++++++++++++ pkg/worker/scanner/file_scanner.go | 99 +++++++++++++++++++++++++++++++++ pkg/worker/scanner/thumbnail_scanner.go | 64 +++++++++++++++++++++ 3 files changed, 203 insertions(+) create mode 100644 pkg/worker/scanner/exif_scanner.go create mode 100644 pkg/worker/scanner/file_scanner.go create mode 100644 pkg/worker/scanner/thumbnail_scanner.go (limited to 'pkg/worker/scanner') diff --git a/pkg/worker/scanner/exif_scanner.go b/pkg/worker/scanner/exif_scanner.go new file mode 100644 index 0000000..47d717f --- /dev/null +++ b/pkg/worker/scanner/exif_scanner.go @@ -0,0 +1,40 @@ +package scanner + +import ( + "context" + + "git.sr.ht/~gabrielgio/img/pkg/coroutine" + "git.sr.ht/~gabrielgio/img/pkg/database/repository" + "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/worker" +) + +type ( + EXIFScanner struct { + repository repository.MediaRepository + } +) + +var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{} + +func NewEXIFScanner(repository repository.MediaRepository) *EXIFScanner { + return &EXIFScanner{ + repository: repository, + } +} + +func (e *EXIFScanner) Query(ctx context.Context) ([]*repository.Media, error) { + return e.repository.ListEmptyEXIF(ctx, &repository.Pagination{ + Page: 0, + Size: 100, + }) +} + +func (e *EXIFScanner) Process(ctx context.Context, m *repository.Media) error { + exif, err := coroutine.WrapProcess(ctx, func() (*repository.MediaEXIF, error) { return fileop.ReadExif(m.Path) }) + if err != nil { + return err + } + + return e.repository.CreateEXIF(ctx, m.ID, exif) +} diff --git a/pkg/worker/scanner/file_scanner.go b/pkg/worker/scanner/file_scanner.go new file mode 100644 index 0000000..7c19a3d --- /dev/null +++ b/pkg/worker/scanner/file_scanner.go @@ -0,0 +1,99 @@ +package scanner + +import ( + "context" + "io/fs" + "mime" + "path/filepath" + + "git.sr.ht/~gabrielgio/img/pkg/database/repository" + "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/list" + "git.sr.ht/~gabrielgio/img/pkg/worker" +) + +type ( + FileScanner struct { + mediaRepository repository.MediaRepository + userRepository repository.UserRepository + } +) + +var _ worker.ChanProcessor[string] = &FileScanner{} + +func NewFileScanner( + mediaRepository repository.MediaRepository, + userRepository repository.UserRepository, +) *FileScanner { + return &FileScanner{ + mediaRepository: mediaRepository, + userRepository: userRepository, + } +} + +func (f *FileScanner) Query(ctx context.Context) (<-chan string, error) { + c := make(chan string) + + users, err := f.userRepository.List(ctx) + if err != nil { + return nil, err + } + + // TODO: de duplicate file paths + paths := list.Map(users, func(u *repository.User) string { return u.Path }) + + go func(paths []string) { + defer close(c) + for _, p := range paths { + _ = filepath.Walk(p, func(path string, info fs.FileInfo, err error) error { + select { + case <-ctx.Done(): + return filepath.SkipAll + default: + } + + if info == nil { + return nil + } + + if info.IsDir() && filepath.Base(info.Name())[0] == '.' { + return filepath.SkipDir + } + + if info.IsDir() { + return nil + } + + c <- path + return nil + }) + } + }(paths) + return c, nil +} + +func (f *FileScanner) Process(ctx context.Context, path string) error { + mimetype := mime.TypeByExtension(filepath.Ext(path)) + supported := fileop.IsMimeTypeSupported(mimetype) + if !supported { + return nil + } + + hash := fileop.GetHashFromPath(path) + + exists, err := f.mediaRepository.Exists(ctx, hash) + if err != nil { + return err + } + + if exists { + return nil + } + + return f.mediaRepository.Create(ctx, &repository.CreateMedia{ + Name: filepath.Base(path), + Path: path, + PathHash: hash, + MIMEType: mimetype, + }) +} diff --git a/pkg/worker/scanner/thumbnail_scanner.go b/pkg/worker/scanner/thumbnail_scanner.go new file mode 100644 index 0000000..02fd4dd --- /dev/null +++ b/pkg/worker/scanner/thumbnail_scanner.go @@ -0,0 +1,64 @@ +package scanner + +import ( + "context" + "fmt" + "math" + "os" + "path" + + "git.sr.ht/~gabrielgio/img/pkg/database/repository" + "git.sr.ht/~gabrielgio/img/pkg/fileop" + "git.sr.ht/~gabrielgio/img/pkg/worker" +) + +type ( + ThumbnailScanner struct { + repository repository.MediaRepository + cachePath string + } +) + +var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{} + +func NewThumbnailScanner(cachePath string, repository repository.MediaRepository) *ThumbnailScanner { + return &ThumbnailScanner{ + repository: repository, + cachePath: cachePath, + } +} + +func (t *ThumbnailScanner) Query(ctx context.Context) ([]*repository.Media, error) { + return t.repository.ListEmptyThumbnail(ctx, &repository.Pagination{ + Page: 0, + Size: 100, + }) +} + +func (t *ThumbnailScanner) Process(ctx context.Context, media *repository.Media) error { + split := media.PathHash[:2] + filename := media.PathHash[2:] + folder := path.Join(t.cachePath, split) + output := path.Join(folder, filename+".jpeg") + + err := os.MkdirAll(folder, os.ModePerm) + if err != nil { + return err + } + + if media.IsVideo() { + err := fileop.EncodeVideoThumbnail(media.Path, output, 1080, 1080) + if err != nil { + return fmt.Errorf("Error thumbnail video %d; %w", media.ID, err) + } + } else { + err := fileop.EncodeImageThumbnail(media.Path, output, 1080, math.MinInt32) + if err != nil { + return fmt.Errorf("Error thumbnail image %d; %w", media.ID, err) + } + } + + return t.repository.CreateThumbnail(ctx, media.ID, &repository.MediaThumbnail{ + Path: output, + }) +} -- cgit v1.2.3