From 544bbeeaf836436305cbed87ae1019511de62535 Mon Sep 17 00:00:00 2001 From: Gabriel Arakaki Giovanini Date: Sun, 28 Aug 2022 17:03:38 +0200 Subject: feat: Add init source code For now only `pipe` is feature complete. The order module needs to be changed a lot. --- Makefile | 10 +++++ README.md | 4 ++ fileop/fileop.go | 66 ++++++++++++++++++++++++++++ fileop/fileop_test.go | 97 +++++++++++++++++++++++++++++++++++++++++ fileop/test_file.txt | 8 ++++ go.mod | 5 +++ go.sum | 6 +++ main.go | 55 ++++++++++++++++++++++++ pipe/pipe.go | 64 +++++++++++++++++++++++++++ pipe/pipe_test.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++ storage.go | 68 +++++++++++++++++++++++++++++ 11 files changed, 500 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 fileop/fileop.go create mode 100644 fileop/fileop_test.go create mode 100644 fileop/test_file.txt create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 pipe/pipe.go create mode 100644 pipe/pipe_test.go create mode 100644 storage.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3a461dc --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +GO=go + + +all: build + +build: + ${GO} build . + +test: + ${GO} test -cover ./... diff --git a/README.md b/README.md new file mode 100644 index 0000000..50e7412 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# Photo organizer + +This project will aim to be a photo organizer where users will be able to pick +from a some file properties to choose how to lay out their file. diff --git a/fileop/fileop.go b/fileop/fileop.go new file mode 100644 index 0000000..d08cb82 --- /dev/null +++ b/fileop/fileop.go @@ -0,0 +1,66 @@ +package fileop + +import ( + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" +) + +func WalkFolder(folder string) <-chan string { + c := make(chan string) + + go func(folder string, c chan string) { + filepath.Walk(folder, func(path string, info os.FileInfo, err error) error { + file, _ := os.Open(path) + defer file.Close() + fileInfo, _ := file.Stat() + if !fileInfo.IsDir() { + c <- path + } + return nil + }) + close(c) + + }(folder, c) + + return c +} + +func CalculateSHA256(file string) (string, error) { + f, err := os.Open(file) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + + shaString := fmt.Sprintf("%x", h.Sum(nil)) + + return shaString, nil +} + +type MoveCommand struct { + Source string + Destination string +} + +func Move() chan<- *MoveCommand { + c := make(chan *MoveCommand) + go func(chan *MoveCommand) { + for cmd := range c { + // TODO: add error handling + err := os.Rename(cmd.Source, cmd.Destination) + if err != nil { + fmt.Println(err.Error()) + } + } + }(c) + + return c +} diff --git a/fileop/fileop_test.go b/fileop/fileop_test.go new file mode 100644 index 0000000..f2ab864 --- /dev/null +++ b/fileop/fileop_test.go @@ -0,0 +1,97 @@ +package fileop + +import ( + "fmt" + "math/rand" + "os" + "testing" + "time" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func RandomString(n int) string { + var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + s := make([]rune, n) + for i := range s { + s[i] = letters[rand.Intn(len(letters))] + } + return string(s) +} + +func createFolder() (tmp string) { + tmp = fmt.Sprintf("/tmp/%s", RandomString(10)) + + err := os.Mkdir(tmp, 0755) + + if err != nil { + fmt.Println(err.Error()) + } + + return +} + +func appendEmptyFile(path string) (fullPath string) { + fullPath = fmt.Sprintf("%s/%s", path, RandomString(10)) + os.OpenFile(fullPath, os.O_RDONLY|os.O_CREATE, 0666) + + return +} + +func createEmptyFile(path string) { + os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0666) +} + +func createTmpFile() (fullPath string) { + path := createFolder() + fullPath = appendEmptyFile(path) + return +} + +func TestMove(t *testing.T) { + src := createTmpFile() + dest := fmt.Sprintf("/tmp/%s", RandomString(10)) + + c := Move() + + c <- &MoveCommand{Source: src, Destination: dest} + close(c) + + if _, err := os.Stat(dest); err != nil { + t.Errorf("File %s not moved to %s", src, dest) + } + +} + +func TestWalk(t *testing.T) { + fileCount := 1000 + folder := createFolder() + files := map[string]struct{}{} + walkedFiles := map[string]struct{}{} + + for i := 0; i < fileCount; i++ { + files[appendEmptyFile(folder)] = struct{}{} + } + + c := WalkFolder(folder) + for file := range c { + walkedFiles[file] = struct{}{} + } + + for k := range files { + _, ok := walkedFiles[k] + if !ok { + t.Errorf("File %s was not walked", k) + } + } +} + +func TestCalculateSHA256(t *testing.T) { + sh256, _ := CalculateSHA256("test_file.txt") + + if sh256 != "027cc886f9b8c866f932ef8b8da9a32f0857ef8e16ec98dd2797021b34623b88" { + t.Errorf("Wrong sh256 hash, given %s", sh256) + } +} diff --git a/fileop/test_file.txt b/fileop/test_file.txt new file mode 100644 index 0000000..db88e33 --- /dev/null +++ b/fileop/test_file.txt @@ -0,0 +1,8 @@ +Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod +tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At +vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd +gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum +dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor +invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero +eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no +sea takimata sanctus est Lorem ipsum dolor sit amet. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5bdbcb7 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module porg + +go 1.18 + +require github.com/barasher/go-exiftool v1.8.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..316fde4 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/barasher/go-exiftool v1.8.0 h1:u8bEi1mhLtpVC5aG/ZJlRS/r+SkK+rcgbZQwcKUb424= +github.com/barasher/go-exiftool v1.8.0/go.mod h1:F9s/a3uHSM8YniVfwF+sbQUtP8Gmh9nyzigNF+8vsWo= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..f09a545 --- /dev/null +++ b/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "porg/fileop" + "porg/pipe" +) + +type SHAInfo struct { + path string + sha256 string +} + +type MoveCommand struct { + src string + dest string +} + +func Calculate(file string) *SHAInfo { + sha, err := fileop.CalculateSHA256(file) + if err != nil { + fmt.Println(err) + return nil + } else { + return &SHAInfo{path: file, sha256: sha} + } +} + +func Move(info *SHAInfo) *MoveCommand { + + base := filepath.Dir(info.path) + ext := filepath.Ext(info.path) + head := info.sha256[0:2] + tail := info.sha256[2:] + newPath := fmt.Sprintf("%s/%s/%s%s", base, head, tail, ext) + return &MoveCommand{src: info.path, dest: newPath} + +} + +func Apply(move *MoveCommand) { + dir := filepath.Dir(move.dest) + os.Mkdir(dir, 0755) + if err := os.Rename(move.src, move.dest); err != nil { + fmt.Println(err) + } +} + +func main() { + files := fileop.WalkFolder("") + shas := pipe.Proc(files, 8, Calculate) + cmds := pipe.Proc(shas, 1, Move) + pipe.TailProc(cmds, 4, Apply) +} diff --git a/pipe/pipe.go b/pipe/pipe.go new file mode 100644 index 0000000..2ca1f8b --- /dev/null +++ b/pipe/pipe.go @@ -0,0 +1,64 @@ +package pipe + +import ( + "sync" +) + +func Proc[T any, V any](cin <-chan T, count int, proc func(T) V) <-chan V { + cout := make(chan V) + var wg sync.WaitGroup + + for i := 0; i < count; i++ { + wg.Add(1) + go func(<-chan T, chan<- V) { + for i := range cin { + cout <- proc(i) + } + wg.Done() + }(cin, cout) + } + + go func() { + wg.Wait() + close(cout) + }() + + return cout +} + +func noop[T any](_ T) {} + +func Wait[T any](cin <-chan T) { + for v := range cin { + noop(v) + } +} + +func TailProc[T any](cin <-chan T, count int, proc func(T)) { + var wg sync.WaitGroup + + for i := 0; i < 4; i++ { + wg.Add(1) + go func(<-chan T) { + for i := range cin { + proc(i) + } + wg.Done() + + }(cin) + } + wg.Wait() +} + +func Yield[T any](in []T) <-chan T { + cout := make(chan T) + + go func(chan<- T) { + for _, a := range in { + cout <- a + } + close(cout) + }(cout) + + return cout +} diff --git a/pipe/pipe_test.go b/pipe/pipe_test.go new file mode 100644 index 0000000..e71b05b --- /dev/null +++ b/pipe/pipe_test.go @@ -0,0 +1,117 @@ +package pipe + +import ( + "testing" +) + +const Values = 999 +const Multiplier = 1000 + +func createCountPipe() <-chan int { + cout := make(chan int) + + go func(<-chan int) { + for i := 1; i < Values; i++ { + cout <- i + } + close(cout) + }(cout) + + return cout +} + +func multiply(in int) int { + return in * Multiplier +} + +func devide(in int) int { + return in / Multiplier +} + +func TestProc(t *testing.T) { + c := createCountPipe() + am := Proc(c, 8, multiply) + ad := Proc(am, 8, devide) + a := Proc(ad, 8, multiply) + + out := make([]int, Values-1) + + count := 0 + for i := range a { + out[count] = i + count++ + } + + for _, v := range out { + if v < Multiplier { + t.Errorf("Invalid output %d", v) + } + } +} + +func TestYield(t *testing.T) { + lin := make([]int, Values) + + for i := 0; i < Values; i++ { + lin[i] = i + } + + cout := Yield(lin) + + count := 0 + for out := range cout { + if out != lin[count] { + t.Errorf("Invalid output %d", out) + } + count++ + } +} + +type buffer struct { + value int +} + +func createBuffers() []*buffer { + buffers := make([]*buffer, Values) + + for i := 0; i < Values; i++ { + buffers[i] = &buffer{value: i} + } + + return buffers +} + +func multiplyBuffer(in *buffer) *buffer { + in.value = in.value * Multiplier + return in +} + +func devideBuffer(in *buffer) { + in.value = in.value / Multiplier +} + +func TestTailProc(t *testing.T) { + buffers := createBuffers() + c := Yield(buffers) + am := Proc(c, 8, multiplyBuffer) + TailProc(am, 8, devideBuffer) + + for i, v := range buffers { + if v.value != i { + t.Errorf("Invalid output %d != %d", v.value, i) + } + } +} + +func TestWait(t *testing.T) { + buffers := createBuffers() + c := Yield(buffers) + am := Proc(c, 8, multiplyBuffer) + Wait(am) + + for i, v := range buffers { + if v.value != (i * Multiplier) { + t.Errorf("Invalid output %d != %d", v.value, i) + } + } +} diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..28637da --- /dev/null +++ b/storage.go @@ -0,0 +1,68 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "io" + "os" + + "github.com/barasher/go-exiftool" +) + +type File struct { + Path string + SHA256 string + Time string +} + +type Storage interface { + Upsert(file *File) error + LoadByPath(path string) *File +} + +func NewFile(path string) *File { + return &File{Path: path} +} + +func (file *File) CalculateSHA256() error { + f, err := os.Open(file.Path) + if err != nil { + return err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return err + } + + file.SHA256 = fmt.Sprintf("%x", h.Sum(nil)) + + return nil +} + +func (file *File) SetTime() error { + + et, err := exiftool.NewExiftool() + if err != nil { + return err + } + defer et.Close() + + fileInfos := et.ExtractMetadata(file.Path) + + for _, fileInfo := range fileInfos { + if fileInfo.Err != nil { + fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err) + continue + } + + v, ok := fileInfo.Fields["DateTimeOriginal"] + if ok { + t, _ := v.(string) + file.Time = t + } + } + + return nil +} -- cgit v1.2.3