aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Makefile3
-rw-r--r--fileop/fileop.go46
-rw-r--r--fileop/fileop_test.go84
-rw-r--r--main.go69
-rw-r--r--storage.go68
-rw-r--r--storage/storage.go30
-rw-r--r--storage/storage_fs.go40
-rw-r--r--storage/storage_fs_test.go29
-rw-r--r--testutil/testutil.go50
9 files changed, 264 insertions, 155 deletions
diff --git a/Makefile b/Makefile
index 3a461dc..1c33c1c 100644
--- a/Makefile
+++ b/Makefile
@@ -8,3 +8,6 @@ build:
test:
${GO} test -cover ./...
+
+install: build
+ cp porg ${HOME}/.local/bin
diff --git a/fileop/fileop.go b/fileop/fileop.go
index d08cb82..1be3f2e 100644
--- a/fileop/fileop.go
+++ b/fileop/fileop.go
@@ -8,7 +8,15 @@ import (
"path/filepath"
)
-func WalkFolder(folder string) <-chan string {
+type WalkMode int
+
+const (
+ Folder WalkMode = iota
+ File
+ FileFolder
+)
+
+func WalkFolder(folder string, walkMode WalkMode) <-chan string {
c := make(chan string)
go func(folder string, c chan string) {
@@ -16,7 +24,17 @@ func WalkFolder(folder string) <-chan string {
file, _ := os.Open(path)
defer file.Close()
fileInfo, _ := file.Stat()
- if !fileInfo.IsDir() {
+
+ switch walkMode {
+ case Folder:
+ if fileInfo.IsDir() {
+ c <- path
+ }
+ case File:
+ if !fileInfo.IsDir() {
+ c <- path
+ }
+ case FileFolder:
c <- path
}
return nil
@@ -28,15 +46,9 @@ func WalkFolder(folder string) <-chan string {
return c
}
-func CalculateSHA256(file string) (string, error) {
- f, err := os.Open(file)
- if err != nil {
- return "", err
- }
- defer f.Close()
-
+func CalculateSHA256(r io.Reader) (string, error) {
h := sha256.New()
- if _, err := io.Copy(h, f); err != nil {
+ if _, err := io.Copy(h, r); err != nil {
return "", err
}
@@ -64,3 +76,17 @@ func Move() chan<- *MoveCommand {
return c
}
+
+func IsEmpty(name string) (bool, error) {
+ f, err := os.Open(name)
+ if err != nil {
+ return false, err
+ }
+ defer f.Close()
+
+ _, err = f.Readdirnames(1)
+ if err == io.EOF {
+ return true, nil
+ }
+ return false, err
+}
diff --git a/fileop/fileop_test.go b/fileop/fileop_test.go
index f2ab864..134090d 100644
--- a/fileop/fileop_test.go
+++ b/fileop/fileop_test.go
@@ -2,57 +2,14 @@ package fileop
import (
"fmt"
- "math/rand"
"os"
+ "porg/testutil"
"testing"
- "time"
)
-func init() {
- rand.Seed(time.Now().UnixNano())
-}
-
-func RandomString(n int) string {
- var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
- s := make([]rune, n)
- for i := range s {
- s[i] = letters[rand.Intn(len(letters))]
- }
- return string(s)
-}
-
-func createFolder() (tmp string) {
- tmp = fmt.Sprintf("/tmp/%s", RandomString(10))
-
- err := os.Mkdir(tmp, 0755)
-
- if err != nil {
- fmt.Println(err.Error())
- }
-
- return
-}
-
-func appendEmptyFile(path string) (fullPath string) {
- fullPath = fmt.Sprintf("%s/%s", path, RandomString(10))
- os.OpenFile(fullPath, os.O_RDONLY|os.O_CREATE, 0666)
-
- return
-}
-
-func createEmptyFile(path string) {
- os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0666)
-}
-
-func createTmpFile() (fullPath string) {
- path := createFolder()
- fullPath = appendEmptyFile(path)
- return
-}
-
func TestMove(t *testing.T) {
- src := createTmpFile()
- dest := fmt.Sprintf("/tmp/%s", RandomString(10))
+ src := testutil.CreateTmpFile()
+ dest := fmt.Sprintf("/tmp/%s", testutil.RandomString(10))
c := Move()
@@ -65,33 +22,30 @@ func TestMove(t *testing.T) {
}
-func TestWalk(t *testing.T) {
- fileCount := 1000
- folder := createFolder()
- files := map[string]struct{}{}
- walkedFiles := map[string]struct{}{}
+func TestIsEmpty(t *testing.T) {
+ folder := testutil.CreateFolder()
- for i := 0; i < fileCount; i++ {
- files[appendEmptyFile(folder)] = struct{}{}
- }
+ empty, err := IsEmpty(folder)
- c := WalkFolder(folder)
- for file := range c {
- walkedFiles[file] = struct{}{}
+ if err != nil {
+ t.Fatalf("Error reading the folder %s", err.Error())
}
- for k := range files {
- _, ok := walkedFiles[k]
- if !ok {
- t.Errorf("File %s was not walked", k)
- }
+ if !empty {
+ t.Errorf("Folder is not empty %s", folder)
}
+
}
func TestCalculateSHA256(t *testing.T) {
- sh256, _ := CalculateSHA256("test_file.txt")
- if sh256 != "027cc886f9b8c866f932ef8b8da9a32f0857ef8e16ec98dd2797021b34623b88" {
- t.Errorf("Wrong sh256 hash, given %s", sh256)
+ if f, err := os.Open("test_file.txt"); err != nil {
+ t.Error(err)
+ } else {
+ sh256, _ := CalculateSHA256(f)
+ if sh256 != "027cc886f9b8c866f932ef8b8da9a32f0857ef8e16ec98dd2797021b34623b88" {
+ t.Errorf("Wrong sh256 hash, given %s", sh256)
+ }
}
+
}
diff --git a/main.go b/main.go
index f09a545..1a3c554 100644
--- a/main.go
+++ b/main.go
@@ -13,13 +13,26 @@ type SHAInfo struct {
sha256 string
}
+type File struct {
+ Path string
+ SHA256 string
+ Time string
+}
+
type MoveCommand struct {
src string
dest string
}
func Calculate(file string) *SHAInfo {
- sha, err := fileop.CalculateSHA256(file)
+ f, err := os.Open(file)
+ if err != nil {
+ fmt.Println(err)
+ return nil
+ }
+ defer f.Close()
+
+ sha, err := fileop.CalculateSHA256(f)
if err != nil {
fmt.Println(err)
return nil
@@ -28,15 +41,32 @@ func Calculate(file string) *SHAInfo {
}
}
-func Move(info *SHAInfo) *MoveCommand {
+func Move(base string) func(*SHAInfo) *MoveCommand {
+ return func(info *SHAInfo) *MoveCommand {
+
+ ext := filepath.Ext(info.path)
+ head := info.sha256[0:2]
+ tail := info.sha256[2:]
+ newPath := fmt.Sprintf("%s/%s/%s%s", base, head, tail, ext)
+ return &MoveCommand{src: info.path, dest: newPath}
+ }
+}
- base := filepath.Dir(info.path)
- ext := filepath.Ext(info.path)
- head := info.sha256[0:2]
- tail := info.sha256[2:]
- newPath := fmt.Sprintf("%s/%s/%s%s", base, head, tail, ext)
- return &MoveCommand{src: info.path, dest: newPath}
+func Delete(path string) {
+ empty, err := fileop.IsEmpty(path)
+ if err != nil {
+ fmt.Printf("Error reading %s: %s\n", path, err.Error())
+ }
+
+ if empty {
+ err := os.Remove(path)
+ if err != nil {
+ fmt.Printf("Error deleting %s: %s\n", path, err.Error())
+ } else {
+ fmt.Printf("Folder deleted %s\n", path)
+ }
+ }
}
func Apply(move *MoveCommand) {
@@ -45,11 +75,26 @@ func Apply(move *MoveCommand) {
if err := os.Rename(move.src, move.dest); err != nil {
fmt.Println(err)
}
+
}
func main() {
- files := fileop.WalkFolder("")
- shas := pipe.Proc(files, 8, Calculate)
- cmds := pipe.Proc(shas, 1, Move)
- pipe.TailProc(cmds, 4, Apply)
+ for _, path := range os.Args[1:] {
+ info, err := os.Stat(path)
+
+ if err != nil || !info.IsDir() {
+ continue
+ }
+
+ fmt.Println("Processing folder")
+ fmt.Println(path)
+ files := fileop.WalkFolder(path, fileop.File)
+ shas := pipe.Proc(files, 4, Calculate)
+ cmds := pipe.Proc(shas, 1, Move(path))
+ pipe.TailProc(cmds, 4, Apply)
+
+ fmt.Println("Deleting empty folders")
+ files = fileop.WalkFolder(path, fileop.Folder)
+ pipe.TailProc(files, 2, Delete)
+ }
}
diff --git a/storage.go b/storage.go
deleted file mode 100644
index 28637da..0000000
--- a/storage.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package main
-
-import (
- "crypto/sha256"
- "fmt"
- "io"
- "os"
-
- "github.com/barasher/go-exiftool"
-)
-
-type File struct {
- Path string
- SHA256 string
- Time string
-}
-
-type Storage interface {
- Upsert(file *File) error
- LoadByPath(path string) *File
-}
-
-func NewFile(path string) *File {
- return &File{Path: path}
-}
-
-func (file *File) CalculateSHA256() error {
- f, err := os.Open(file.Path)
- if err != nil {
- return err
- }
- defer f.Close()
-
- h := sha256.New()
- if _, err := io.Copy(h, f); err != nil {
- return err
- }
-
- file.SHA256 = fmt.Sprintf("%x", h.Sum(nil))
-
- return nil
-}
-
-func (file *File) SetTime() error {
-
- et, err := exiftool.NewExiftool()
- if err != nil {
- return err
- }
- defer et.Close()
-
- fileInfos := et.ExtractMetadata(file.Path)
-
- for _, fileInfo := range fileInfos {
- if fileInfo.Err != nil {
- fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err)
- continue
- }
-
- v, ok := fileInfo.Fields["DateTimeOriginal"]
- if ok {
- t, _ := v.(string)
- file.Time = t
- }
- }
-
- return nil
-}
diff --git a/storage/storage.go b/storage/storage.go
new file mode 100644
index 0000000..b788efb
--- /dev/null
+++ b/storage/storage.go
@@ -0,0 +1,30 @@
+package storage
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "io"
+)
+
+type WalkMode int
+
+const (
+ Folder WalkMode = iota
+ File
+ FileFolder
+)
+
+type Storage interface {
+ Walk(path string, walkMode WalkMode) <-chan string
+ Get(path string) (io.Reader, error)
+}
+
+func CalculateSHA256(r io.Reader) (string, error) {
+ h := sha256.New()
+ if _, err := io.Copy(h, r); err != nil {
+ return "", err
+ }
+
+ return fmt.Sprintf("%x", h.Sum(nil)), nil
+
+}
diff --git a/storage/storage_fs.go b/storage/storage_fs.go
new file mode 100644
index 0000000..35ce58b
--- /dev/null
+++ b/storage/storage_fs.go
@@ -0,0 +1,40 @@
+package storage
+
+import (
+ "os"
+ "path/filepath"
+)
+
+type FileSystem struct {
+ root string
+}
+
+func WalkFolder(folder string, walkMode WalkMode) <-chan string {
+ c := make(chan string)
+
+ go func(folder string, c chan string) {
+ filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+ file, _ := os.Open(path)
+ defer file.Close()
+ fileInfo, _ := file.Stat()
+
+ switch walkMode {
+ case Folder:
+ if fileInfo.IsDir() {
+ c <- path
+ }
+ case File:
+ if !fileInfo.IsDir() {
+ c <- path
+ }
+ case FileFolder:
+ c <- path
+ }
+ return nil
+ })
+ close(c)
+
+ }(folder, c)
+
+ return c
+}
diff --git a/storage/storage_fs_test.go b/storage/storage_fs_test.go
new file mode 100644
index 0000000..b746c7e
--- /dev/null
+++ b/storage/storage_fs_test.go
@@ -0,0 +1,29 @@
+package storage
+
+import (
+ "porg/testutil"
+ "testing"
+)
+
+func TestWalk(t *testing.T) {
+ fileCount := 1000
+ folder := testutil.CreateFolder()
+ files := map[string]struct{}{}
+ walkedFiles := map[string]struct{}{}
+
+ for i := 0; i < fileCount; i++ {
+ files[testutil.AppendEmptyFile(folder)] = struct{}{}
+ }
+
+ c := WalkFolder(folder, File)
+ for file := range c {
+ walkedFiles[file] = struct{}{}
+ }
+
+ for k := range files {
+ _, ok := walkedFiles[k]
+ if !ok {
+ t.Errorf("File %s was not walked", k)
+ }
+ }
+}
diff --git a/testutil/testutil.go b/testutil/testutil.go
new file mode 100644
index 0000000..3b5f7b5
--- /dev/null
+++ b/testutil/testutil.go
@@ -0,0 +1,50 @@
+package testutil
+
+import (
+ "fmt"
+ "math/rand"
+ "os"
+ "time"
+)
+
+func init() {
+ rand.Seed(time.Now().UnixNano())
+}
+
+func RandomString(n int) string {
+ var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+ s := make([]rune, n)
+ for i := range s {
+ s[i] = letters[rand.Intn(len(letters))]
+ }
+ return string(s)
+}
+
+func CreateFolder() (tmp string) {
+ tmp = fmt.Sprintf("/tmp/%s", RandomString(10))
+
+ err := os.Mkdir(tmp, 0755)
+
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+
+ return
+}
+
+func AppendEmptyFile(path string) (fullPath string) {
+ fullPath = fmt.Sprintf("%s/%s", path, RandomString(10))
+ os.OpenFile(fullPath, os.O_RDONLY|os.O_CREATE, 0666)
+
+ return
+}
+
+func CreateEmptyFile(path string) {
+ os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0666)
+}
+
+func CreateTmpFile() (fullPath string) {
+ path := CreateFolder()
+ fullPath = AppendEmptyFile(path)
+ return
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-21 11:55:59 +0000