ref: Create a storage interface

This `Storage` interface will define all the interactions with the
storage system. For now I plan to support native file system through
go's standard library and Nextcloud through *webdav*. So this is the
first step in that direction.

9 files changed, 264 insertions, 155 deletions

diff --git a/Makefile b/Makefile
index 3a461dc..1c33c1c 100644
--- a/Makefile
+++ b/Makefile
@@ -8,3 +8,6 @@ build:
 
 test:
 	${GO} test -cover ./...
+
+install: build
+	cp porg ${HOME}/.local/bin
diff --git a/fileop/fileop.go b/fileop/fileop.go
index d08cb82..1be3f2e 100644
--- a/fileop/fileop.go
+++ b/fileop/fileop.go
@@ -8,7 +8,15 @@ import (
 	"path/filepath"
 )
 
-func WalkFolder(folder string) <-chan string {
+type WalkMode int
+
+const (
+	Folder WalkMode = iota
+	File
+	FileFolder
+)
+
+func WalkFolder(folder string, walkMode WalkMode) <-chan string {
 	c := make(chan string)
 
 	go func(folder string, c chan string) {
@@ -16,7 +24,17 @@ func WalkFolder(folder string) <-chan string {
 			file, _ := os.Open(path)
 			defer file.Close()
 			fileInfo, _ := file.Stat()
-			if !fileInfo.IsDir() {
+
+			switch walkMode {
+			case Folder:
+				if fileInfo.IsDir() {
+					c <- path
+				}
+			case File:
+				if !fileInfo.IsDir() {
+					c <- path
+				}
+			case FileFolder:
 				c <- path
 			}
 			return nil
@@ -28,15 +46,9 @@ func WalkFolder(folder string) <-chan string {
 	return c
 }
 
-func CalculateSHA256(file string) (string, error) {
-	f, err := os.Open(file)
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-
+func CalculateSHA256(r io.Reader) (string, error) {
 	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
+	if _, err := io.Copy(h, r); err != nil {
 		return "", err
 	}
 
@@ -64,3 +76,17 @@ func Move() chan<- *MoveCommand {
 
 	return c
 }
+
+func IsEmpty(name string) (bool, error) {
+	f, err := os.Open(name)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	_, err = f.Readdirnames(1)
+	if err == io.EOF {
+		return true, nil
+	}
+	return false, err
+}
diff --git a/fileop/fileop_test.go b/fileop/fileop_test.go
index f2ab864..134090d 100644
--- a/fileop/fileop_test.go
+++ b/fileop/fileop_test.go
@@ -2,57 +2,14 @@ package fileop
 
 import (
 	"fmt"
-	"math/rand"
 	"os"
+	"porg/testutil"
 	"testing"
-	"time"
 )
 
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
-func RandomString(n int) string {
-	var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
-	s := make([]rune, n)
-	for i := range s {
-		s[i] = letters[rand.Intn(len(letters))]
-	}
-	return string(s)
-}
-
-func createFolder() (tmp string) {
-	tmp = fmt.Sprintf("/tmp/%s", RandomString(10))
-
-	err := os.Mkdir(tmp, 0755)
-
-	if err != nil {
-		fmt.Println(err.Error())
-	}
-
-	return
-}
-
-func appendEmptyFile(path string) (fullPath string) {
-	fullPath = fmt.Sprintf("%s/%s", path, RandomString(10))
-	os.OpenFile(fullPath, os.O_RDONLY|os.O_CREATE, 0666)
-
-	return
-}
-
-func createEmptyFile(path string) {
-	os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0666)
-}
-
-func createTmpFile() (fullPath string) {
-	path := createFolder()
-	fullPath = appendEmptyFile(path)
-	return
-}
-
 func TestMove(t *testing.T) {
-	src := createTmpFile()
-	dest := fmt.Sprintf("/tmp/%s", RandomString(10))
+	src := testutil.CreateTmpFile()
+	dest := fmt.Sprintf("/tmp/%s", testutil.RandomString(10))
 
 	c := Move()
 
@@ -65,33 +22,30 @@ func TestMove(t *testing.T) {
 
 }
 
-func TestWalk(t *testing.T) {
-	fileCount := 1000
-	folder := createFolder()
-	files := map[string]struct{}{}
-	walkedFiles := map[string]struct{}{}
+func TestIsEmpty(t *testing.T) {
+	folder := testutil.CreateFolder()
 
-	for i := 0; i < fileCount; i++ {
-		files[appendEmptyFile(folder)] = struct{}{}
-	}
+	empty, err := IsEmpty(folder)
 
-	c := WalkFolder(folder)
-	for file := range c {
-		walkedFiles[file] = struct{}{}
+	if err != nil {
+		t.Fatalf("Error reading the folder %s", err.Error())
 	}
 
-	for k := range files {
-		_, ok := walkedFiles[k]
-		if !ok {
-			t.Errorf("File %s was not walked", k)
-		}
+	if !empty {
+		t.Errorf("Folder is not empty %s", folder)
 	}
+
 }
 
 func TestCalculateSHA256(t *testing.T) {
-	sh256, _ := CalculateSHA256("test_file.txt")
 
-	if sh256 != "027cc886f9b8c866f932ef8b8da9a32f0857ef8e16ec98dd2797021b34623b88" {
-		t.Errorf("Wrong sh256 hash, given %s", sh256)
+	if f, err := os.Open("test_file.txt"); err != nil {
+		t.Error(err)
+	} else {
+		sh256, _ := CalculateSHA256(f)
+		if sh256 != "027cc886f9b8c866f932ef8b8da9a32f0857ef8e16ec98dd2797021b34623b88" {
+			t.Errorf("Wrong sh256 hash, given %s", sh256)
+		}
 	}
+
 }
diff --git a/main.go b/main.go
index f09a545..1a3c554 100644
--- a/main.go
+++ b/main.go
@@ -13,13 +13,26 @@ type SHAInfo struct {
 	sha256 string
 }
 
+type File struct {
+	Path   string
+	SHA256 string
+	Time   string
+}
+
 type MoveCommand struct {
 	src  string
 	dest string
 }
 
 func Calculate(file string) *SHAInfo {
-	sha, err := fileop.CalculateSHA256(file)
+	f, err := os.Open(file)
+	if err != nil {
+		fmt.Println(err)
+		return nil
+	}
+	defer f.Close()
+
+	sha, err := fileop.CalculateSHA256(f)
 	if err != nil {
 		fmt.Println(err)
 		return nil
@@ -28,15 +41,32 @@ func Calculate(file string) *SHAInfo {
 	}
 }
 
-func Move(info *SHAInfo) *MoveCommand {
+func Move(base string) func(*SHAInfo) *MoveCommand {
+	return func(info *SHAInfo) *MoveCommand {
+
+		ext := filepath.Ext(info.path)
+		head := info.sha256[0:2]
+		tail := info.sha256[2:]
+		newPath := fmt.Sprintf("%s/%s/%s%s", base, head, tail, ext)
+		return &MoveCommand{src: info.path, dest: newPath}
+	}
+}
 
-	base := filepath.Dir(info.path)
-	ext := filepath.Ext(info.path)
-	head := info.sha256[0:2]
-	tail := info.sha256[2:]
-	newPath := fmt.Sprintf("%s/%s/%s%s", base, head, tail, ext)
-	return &MoveCommand{src: info.path, dest: newPath}
+func Delete(path string) {
+	empty, err := fileop.IsEmpty(path)
 
+	if err != nil {
+		fmt.Printf("Error reading %s: %s\n", path, err.Error())
+	}
+
+	if empty {
+		err := os.Remove(path)
+		if err != nil {
+			fmt.Printf("Error deleting %s: %s\n", path, err.Error())
+		} else {
+			fmt.Printf("Folder deleted %s\n", path)
+		}
+	}
 }
 
 func Apply(move *MoveCommand) {
@@ -45,11 +75,26 @@ func Apply(move *MoveCommand) {
 	if err := os.Rename(move.src, move.dest); err != nil {
 		fmt.Println(err)
 	}
+
 }
 
 func main() {
-	files := fileop.WalkFolder("")
-	shas := pipe.Proc(files, 8, Calculate)
-	cmds := pipe.Proc(shas, 1, Move)
-	pipe.TailProc(cmds, 4, Apply)
+	for _, path := range os.Args[1:] {
+		info, err := os.Stat(path)
+
+		if err != nil || !info.IsDir() {
+			continue
+		}
+
+		fmt.Println("Processing folder")
+		fmt.Println(path)
+		files := fileop.WalkFolder(path, fileop.File)
+		shas := pipe.Proc(files, 4, Calculate)
+		cmds := pipe.Proc(shas, 1, Move(path))
+		pipe.TailProc(cmds, 4, Apply)
+
+		fmt.Println("Deleting empty folders")
+		files = fileop.WalkFolder(path, fileop.Folder)
+		pipe.TailProc(files, 2, Delete)
+	}
 }
diff --git a/storage.go b/storage.go
deleted file mode 100644
index 28637da..0000000
--- a/storage.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package main
-
-import (
-	"crypto/sha256"
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/barasher/go-exiftool"
-)
-
-type File struct {
-	Path   string
-	SHA256 string
-	Time   string
-}
-
-type Storage interface {
-	Upsert(file *File) error
-	LoadByPath(path string) *File
-}
-
-func NewFile(path string) *File {
-	return &File{Path: path}
-}
-
-func (file *File) CalculateSHA256() error {
-	f, err := os.Open(file.Path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return err
-	}
-
-	file.SHA256 = fmt.Sprintf("%x", h.Sum(nil))
-
-	return nil
-}
-
-func (file *File) SetTime() error {
-
-	et, err := exiftool.NewExiftool()
-	if err != nil {
-		return err
-	}
-	defer et.Close()
-
-	fileInfos := et.ExtractMetadata(file.Path)
-
-	for _, fileInfo := range fileInfos {
-		if fileInfo.Err != nil {
-			fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err)
-			continue
-		}
-
-		v, ok := fileInfo.Fields["DateTimeOriginal"]
-		if ok {
-			t, _ := v.(string)
-			file.Time = t
-		}
-	}
-
-	return nil
-}
diff --git a/storage/storage.go b/storage/storage.go
new file mode 100644
index 0000000..b788efb
--- /dev/null
+++ b/storage/storage.go
@@ -0,0 +1,30 @@
+package storage
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+)
+
+type WalkMode int
+
+const (
+	Folder WalkMode = iota
+	File
+	FileFolder
+)
+
+type Storage interface {
+	Walk(path string, walkMode WalkMode) <-chan string
+	Get(path string) (io.Reader, error)
+}
+
+func CalculateSHA256(r io.Reader) (string, error) {
+	h := sha256.New()
+	if _, err := io.Copy(h, r); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+
+}
diff --git a/storage/storage_fs.go b/storage/storage_fs.go
new file mode 100644
index 0000000..35ce58b
--- /dev/null
+++ b/storage/storage_fs.go
@@ -0,0 +1,40 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+)
+
+type FileSystem struct {
+	root string
+}
+
+func WalkFolder(folder string, walkMode WalkMode) <-chan string {
+	c := make(chan string)
+
+	go func(folder string, c chan string) {
+		filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+			file, _ := os.Open(path)
+			defer file.Close()
+			fileInfo, _ := file.Stat()
+
+			switch walkMode {
+			case Folder:
+				if fileInfo.IsDir() {
+					c <- path
+				}
+			case File:
+				if !fileInfo.IsDir() {
+					c <- path
+				}
+			case FileFolder:
+				c <- path
+			}
+			return nil
+		})
+		close(c)
+
+	}(folder, c)
+
+	return c
+}
diff --git a/storage/storage_fs_test.go b/storage/storage_fs_test.go
new file mode 100644
index 0000000..b746c7e
--- /dev/null
+++ b/storage/storage_fs_test.go
@@ -0,0 +1,29 @@
+package storage
+
+import (
+	"porg/testutil"
+	"testing"
+)
+
+func TestWalk(t *testing.T) {
+	fileCount := 1000
+	folder := testutil.CreateFolder()
+	files := map[string]struct{}{}
+	walkedFiles := map[string]struct{}{}
+
+	for i := 0; i < fileCount; i++ {
+		files[testutil.AppendEmptyFile(folder)] = struct{}{}
+	}
+
+	c := WalkFolder(folder, File)
+	for file := range c {
+		walkedFiles[file] = struct{}{}
+	}
+
+	for k := range files {
+		_, ok := walkedFiles[k]
+		if !ok {
+			t.Errorf("File %s was not walked", k)
+		}
+	}
+}
diff --git a/testutil/testutil.go b/testutil/testutil.go
new file mode 100644
index 0000000..3b5f7b5
--- /dev/null
+++ b/testutil/testutil.go
@@ -0,0 +1,50 @@
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"time"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func RandomString(n int) string {
+	var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+	s := make([]rune, n)
+	for i := range s {
+		s[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(s)
+}
+
+func CreateFolder() (tmp string) {
+	tmp = fmt.Sprintf("/tmp/%s", RandomString(10))
+
+	err := os.Mkdir(tmp, 0755)
+
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	return
+}
+
+func AppendEmptyFile(path string) (fullPath string) {
+	fullPath = fmt.Sprintf("%s/%s", path, RandomString(10))
+	os.OpenFile(fullPath, os.O_RDONLY|os.O_CREATE, 0666)
+
+	return
+}
+
+func CreateEmptyFile(path string) {
+	os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0666)
+}
+
+func CreateTmpFile() (fullPath string) {
+	path := CreateFolder()
+	fullPath = AppendEmptyFile(path)
+	return
+}