From 3451d56ead6e57f503962b876c89284f1fb73a90 Mon Sep 17 00:00:00 2001 From: Gabriel Arakaki Giovanini Date: Sat, 10 Sep 2022 17:33:30 +0200 Subject: ref: Create a storage interface This `Storage` interface will define all the interactions with the storage system. For now I plan to support native file system through go's standard library and Nextcloud through *webdav*. So this is the first step in that direction. --- Makefile | 3 ++ fileop/fileop.go | 46 +++++++++++++++++++------ fileop/fileop_test.go | 84 +++++++++++----------------------------------- main.go | 69 ++++++++++++++++++++++++++++++------- storage.go | 68 ------------------------------------- storage/storage.go | 30 +++++++++++++++++ storage/storage_fs.go | 40 ++++++++++++++++++++++ storage/storage_fs_test.go | 29 ++++++++++++++++ testutil/testutil.go | 50 +++++++++++++++++++++++++++ 9 files changed, 264 insertions(+), 155 deletions(-) delete mode 100644 storage.go create mode 100644 storage/storage.go create mode 100644 storage/storage_fs.go create mode 100644 storage/storage_fs_test.go create mode 100644 testutil/testutil.go diff --git a/Makefile b/Makefile index 3a461dc..1c33c1c 100644 --- a/Makefile +++ b/Makefile @@ -8,3 +8,6 @@ build: test: ${GO} test -cover ./... + +install: build + cp porg ${HOME}/.local/bin diff --git a/fileop/fileop.go b/fileop/fileop.go index d08cb82..1be3f2e 100644 --- a/fileop/fileop.go +++ b/fileop/fileop.go @@ -8,7 +8,15 @@ import ( "path/filepath" ) -func WalkFolder(folder string) <-chan string { +type WalkMode int + +const ( + Folder WalkMode = iota + File + FileFolder +) + +func WalkFolder(folder string, walkMode WalkMode) <-chan string { c := make(chan string) go func(folder string, c chan string) { @@ -16,7 +24,17 @@ func WalkFolder(folder string) <-chan string { file, _ := os.Open(path) defer file.Close() fileInfo, _ := file.Stat() - if !fileInfo.IsDir() { + + switch walkMode { + case Folder: + if fileInfo.IsDir() { + c <- path + } + case File: + if !fileInfo.IsDir() { + c <- path + } + case FileFolder: c <- path } return nil @@ -28,15 +46,9 @@ func WalkFolder(folder string) <-chan string { return c } -func CalculateSHA256(file string) (string, error) { - f, err := os.Open(file) - if err != nil { - return "", err - } - defer f.Close() - +func CalculateSHA256(r io.Reader) (string, error) { h := sha256.New() - if _, err := io.Copy(h, f); err != nil { + if _, err := io.Copy(h, r); err != nil { return "", err } @@ -64,3 +76,17 @@ func Move() chan<- *MoveCommand { return c } + +func IsEmpty(name string) (bool, error) { + f, err := os.Open(name) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, err +} diff --git a/fileop/fileop_test.go b/fileop/fileop_test.go index f2ab864..134090d 100644 --- a/fileop/fileop_test.go +++ b/fileop/fileop_test.go @@ -2,57 +2,14 @@ package fileop import ( "fmt" - "math/rand" "os" + "porg/testutil" "testing" - "time" ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - -func RandomString(n int) string { - var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") - s := make([]rune, n) - for i := range s { - s[i] = letters[rand.Intn(len(letters))] - } - return string(s) -} - -func createFolder() (tmp string) { - tmp = fmt.Sprintf("/tmp/%s", RandomString(10)) - - err := os.Mkdir(tmp, 0755) - - if err != nil { - fmt.Println(err.Error()) - } - - return -} - -func appendEmptyFile(path string) (fullPath string) { - fullPath = fmt.Sprintf("%s/%s", path, RandomString(10)) - os.OpenFile(fullPath, os.O_RDONLY|os.O_CREATE, 0666) - - return -} - -func createEmptyFile(path string) { - os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0666) -} - -func createTmpFile() (fullPath string) { - path := createFolder() - fullPath = appendEmptyFile(path) - return -} - func TestMove(t *testing.T) { - src := createTmpFile() - dest := fmt.Sprintf("/tmp/%s", RandomString(10)) + src := testutil.CreateTmpFile() + dest := fmt.Sprintf("/tmp/%s", testutil.RandomString(10)) c := Move() @@ -65,33 +22,30 @@ func TestMove(t *testing.T) { } -func TestWalk(t *testing.T) { - fileCount := 1000 - folder := createFolder() - files := map[string]struct{}{} - walkedFiles := map[string]struct{}{} +func TestIsEmpty(t *testing.T) { + folder := testutil.CreateFolder() - for i := 0; i < fileCount; i++ { - files[appendEmptyFile(folder)] = struct{}{} - } + empty, err := IsEmpty(folder) - c := WalkFolder(folder) - for file := range c { - walkedFiles[file] = struct{}{} + if err != nil { + t.Fatalf("Error reading the folder %s", err.Error()) } - for k := range files { - _, ok := walkedFiles[k] - if !ok { - t.Errorf("File %s was not walked", k) - } + if !empty { + t.Errorf("Folder is not empty %s", folder) } + } func TestCalculateSHA256(t *testing.T) { - sh256, _ := CalculateSHA256("test_file.txt") - if sh256 != "027cc886f9b8c866f932ef8b8da9a32f0857ef8e16ec98dd2797021b34623b88" { - t.Errorf("Wrong sh256 hash, given %s", sh256) + if f, err := os.Open("test_file.txt"); err != nil { + t.Error(err) + } else { + sh256, _ := CalculateSHA256(f) + if sh256 != "027cc886f9b8c866f932ef8b8da9a32f0857ef8e16ec98dd2797021b34623b88" { + t.Errorf("Wrong sh256 hash, given %s", sh256) + } } + } diff --git a/main.go b/main.go index f09a545..1a3c554 100644 --- a/main.go +++ b/main.go @@ -13,13 +13,26 @@ type SHAInfo struct { sha256 string } +type File struct { + Path string + SHA256 string + Time string +} + type MoveCommand struct { src string dest string } func Calculate(file string) *SHAInfo { - sha, err := fileop.CalculateSHA256(file) + f, err := os.Open(file) + if err != nil { + fmt.Println(err) + return nil + } + defer f.Close() + + sha, err := fileop.CalculateSHA256(f) if err != nil { fmt.Println(err) return nil @@ -28,15 +41,32 @@ func Calculate(file string) *SHAInfo { } } -func Move(info *SHAInfo) *MoveCommand { +func Move(base string) func(*SHAInfo) *MoveCommand { + return func(info *SHAInfo) *MoveCommand { + + ext := filepath.Ext(info.path) + head := info.sha256[0:2] + tail := info.sha256[2:] + newPath := fmt.Sprintf("%s/%s/%s%s", base, head, tail, ext) + return &MoveCommand{src: info.path, dest: newPath} + } +} - base := filepath.Dir(info.path) - ext := filepath.Ext(info.path) - head := info.sha256[0:2] - tail := info.sha256[2:] - newPath := fmt.Sprintf("%s/%s/%s%s", base, head, tail, ext) - return &MoveCommand{src: info.path, dest: newPath} +func Delete(path string) { + empty, err := fileop.IsEmpty(path) + if err != nil { + fmt.Printf("Error reading %s: %s\n", path, err.Error()) + } + + if empty { + err := os.Remove(path) + if err != nil { + fmt.Printf("Error deleting %s: %s\n", path, err.Error()) + } else { + fmt.Printf("Folder deleted %s\n", path) + } + } } func Apply(move *MoveCommand) { @@ -45,11 +75,26 @@ func Apply(move *MoveCommand) { if err := os.Rename(move.src, move.dest); err != nil { fmt.Println(err) } + } func main() { - files := fileop.WalkFolder("") - shas := pipe.Proc(files, 8, Calculate) - cmds := pipe.Proc(shas, 1, Move) - pipe.TailProc(cmds, 4, Apply) + for _, path := range os.Args[1:] { + info, err := os.Stat(path) + + if err != nil || !info.IsDir() { + continue + } + + fmt.Println("Processing folder") + fmt.Println(path) + files := fileop.WalkFolder(path, fileop.File) + shas := pipe.Proc(files, 4, Calculate) + cmds := pipe.Proc(shas, 1, Move(path)) + pipe.TailProc(cmds, 4, Apply) + + fmt.Println("Deleting empty folders") + files = fileop.WalkFolder(path, fileop.Folder) + pipe.TailProc(files, 2, Delete) + } } diff --git a/storage.go b/storage.go deleted file mode 100644 index 28637da..0000000 --- a/storage.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "crypto/sha256" - "fmt" - "io" - "os" - - "github.com/barasher/go-exiftool" -) - -type File struct { - Path string - SHA256 string - Time string -} - -type Storage interface { - Upsert(file *File) error - LoadByPath(path string) *File -} - -func NewFile(path string) *File { - return &File{Path: path} -} - -func (file *File) CalculateSHA256() error { - f, err := os.Open(file.Path) - if err != nil { - return err - } - defer f.Close() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return err - } - - file.SHA256 = fmt.Sprintf("%x", h.Sum(nil)) - - return nil -} - -func (file *File) SetTime() error { - - et, err := exiftool.NewExiftool() - if err != nil { - return err - } - defer et.Close() - - fileInfos := et.ExtractMetadata(file.Path) - - for _, fileInfo := range fileInfos { - if fileInfo.Err != nil { - fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err) - continue - } - - v, ok := fileInfo.Fields["DateTimeOriginal"] - if ok { - t, _ := v.(string) - file.Time = t - } - } - - return nil -} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 0000000..b788efb --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,30 @@ +package storage + +import ( + "crypto/sha256" + "fmt" + "io" +) + +type WalkMode int + +const ( + Folder WalkMode = iota + File + FileFolder +) + +type Storage interface { + Walk(path string, walkMode WalkMode) <-chan string + Get(path string) (io.Reader, error) +} + +func CalculateSHA256(r io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil + +} diff --git a/storage/storage_fs.go b/storage/storage_fs.go new file mode 100644 index 0000000..35ce58b --- /dev/null +++ b/storage/storage_fs.go @@ -0,0 +1,40 @@ +package storage + +import ( + "os" + "path/filepath" +) + +type FileSystem struct { + root string +} + +func WalkFolder(folder string, walkMode WalkMode) <-chan string { + c := make(chan string) + + go func(folder string, c chan string) { + filepath.Walk(folder, func(path string, info os.FileInfo, err error) error { + file, _ := os.Open(path) + defer file.Close() + fileInfo, _ := file.Stat() + + switch walkMode { + case Folder: + if fileInfo.IsDir() { + c <- path + } + case File: + if !fileInfo.IsDir() { + c <- path + } + case FileFolder: + c <- path + } + return nil + }) + close(c) + + }(folder, c) + + return c +} diff --git a/storage/storage_fs_test.go b/storage/storage_fs_test.go new file mode 100644 index 0000000..b746c7e --- /dev/null +++ b/storage/storage_fs_test.go @@ -0,0 +1,29 @@ +package storage + +import ( + "porg/testutil" + "testing" +) + +func TestWalk(t *testing.T) { + fileCount := 1000 + folder := testutil.CreateFolder() + files := map[string]struct{}{} + walkedFiles := map[string]struct{}{} + + for i := 0; i < fileCount; i++ { + files[testutil.AppendEmptyFile(folder)] = struct{}{} + } + + c := WalkFolder(folder, File) + for file := range c { + walkedFiles[file] = struct{}{} + } + + for k := range files { + _, ok := walkedFiles[k] + if !ok { + t.Errorf("File %s was not walked", k) + } + } +} diff --git a/testutil/testutil.go b/testutil/testutil.go new file mode 100644 index 0000000..3b5f7b5 --- /dev/null +++ b/testutil/testutil.go @@ -0,0 +1,50 @@ +package testutil + +import ( + "fmt" + "math/rand" + "os" + "time" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func RandomString(n int) string { + var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + s := make([]rune, n) + for i := range s { + s[i] = letters[rand.Intn(len(letters))] + } + return string(s) +} + +func CreateFolder() (tmp string) { + tmp = fmt.Sprintf("/tmp/%s", RandomString(10)) + + err := os.Mkdir(tmp, 0755) + + if err != nil { + fmt.Println(err.Error()) + } + + return +} + +func AppendEmptyFile(path string) (fullPath string) { + fullPath = fmt.Sprintf("%s/%s", path, RandomString(10)) + os.OpenFile(fullPath, os.O_RDONLY|os.O_CREATE, 0666) + + return +} + +func CreateEmptyFile(path string) { + os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0666) +} + +func CreateTmpFile() (fullPath string) { + path := CreateFolder() + fullPath = AppendEmptyFile(path) + return +} -- cgit v1.2.3