aboutsummaryrefslogtreecommitdiff
path: root/cmd/importer
diff options
context:
space:
mode:
Diffstat (limited to 'cmd/importer')
-rw-r--r--cmd/importer/importer.go131
1 files changed, 131 insertions, 0 deletions
diff --git a/cmd/importer/importer.go b/cmd/importer/importer.go
new file mode 100644
index 0000000..18a7a7b
--- /dev/null
+++ b/cmd/importer/importer.go
@@ -0,0 +1,131 @@
+package importer
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "math"
+ "os"
+ "strings"
+
+ "github.com/urfave/cli/v2"
+
+ "git.gabrielgio.me/dict/db"
+)
+
+var ImportCommand = &cli.Command{
+ Name: "import",
+ Usage: "convert dict.cc dictionary into a queryable sqlite format.",
+ Flags: []cli.Flag{
+ &cli.StringFlag{
+ Name: "output",
+ Value: "main.dict",
+ Usage: "Dictionary database location",
+ },
+ &cli.StringFlag{
+ Name: "input",
+ Value: "dict.txt",
+ Usage: "Dict.cc txt dictionary file",
+ },
+ },
+ Action: func(cCtx *cli.Context) error {
+ input := cCtx.String("input")
+ output := cCtx.String("output")
+ return Import(context.Background(), input, output)
+ },
+}
+
+func Import(ctx context.Context, txtInput, sqliteOutput string) error {
+ db, err := db.Open(":memory:")
+ if err != nil {
+ return err
+ }
+ err = db.Migrate(ctx)
+ if err != nil {
+ return err
+ }
+
+ file, err := os.Open(txtInput)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ count := 0
+ total, err := lineCounter(file)
+ if err != nil {
+ return err
+ }
+
+ _, err = file.Seek(0, 0)
+ if err != nil {
+ return err
+ }
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ if strings.HasPrefix(scanner.Text(), "#") || scanner.Text() == "" {
+ continue
+ }
+
+ var (
+ p = strings.SplitN(scanner.Text(), "\t", 2)
+ word = p[0]
+ line = strings.ReplaceAll(p[1], "\t", " ")
+ )
+
+ if err := db.InsertLine(ctx, word, line); err != nil {
+ return err
+ }
+ count++
+
+ if (count % 1234) == 0 {
+ fmt.Print("\033[G\033[K") // move the cursor left and clear the line
+ per := math.Ceil((float64(count) / float64(total)) * 100.0)
+ fmt.Printf("%d/%d (%.0f%%)", count, total, per)
+ }
+ }
+
+ fmt.Printf("Consolidating")
+ err = db.Consolidade(ctx)
+ if err != nil {
+ return err
+ }
+
+ err = db.Backup(ctx, sqliteOutput)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func lineCounter(r io.Reader) (int, error) {
+ var count int
+ const lineBreak = '\n'
+
+ buf := make([]byte, bufio.MaxScanTokenSize)
+
+ for {
+ bufferSize, err := r.Read(buf)
+ if err != nil && err != io.EOF {
+ return 0, err
+ }
+
+ var buffPosition int
+ for {
+ i := bytes.IndexByte(buf[buffPosition:], lineBreak)
+ if i == -1 || bufferSize == buffPosition {
+ break
+ }
+ buffPosition += i + 1
+ count++
+ }
+ if err == io.EOF {
+ break
+ }
+ }
+
+ return count, nil
+}