package importer import ( "bufio" "bytes" "context" "fmt" "io" "math" "os" "strings" "github.com/urfave/cli/v2" "git.gabrielgio.me/dict/db" ) var ImportCommand = &cli.Command{ Name: "import", Usage: "convert dict.cc dictionary into a queryable sqlite format.", Flags: []cli.Flag{ &cli.StringFlag{ Name: "output", Value: "main.dict", Usage: "Dictionary database location", }, &cli.StringFlag{ Name: "input", Value: "dict.txt", Usage: "Dict.cc txt dictionary file", }, }, Action: func(cCtx *cli.Context) error { input := cCtx.String("input") output := cCtx.String("output") return Import(context.Background(), input, output) }, } func Import(ctx context.Context, txtInput, sqliteOutput string) error { db, err := db.Open(":memory:") if err != nil { return err } err = db.Migrate(ctx) if err != nil { return err } file, err := os.Open(txtInput) if err != nil { return err } defer file.Close() count := 0 total, err := lineCounter(file) if err != nil { return err } _, err = file.Seek(0, 0) if err != nil { return err } scanner := bufio.NewScanner(file) for scanner.Scan() { if strings.HasPrefix(scanner.Text(), "#") || scanner.Text() == "" { continue } var ( p = strings.SplitN(scanner.Text(), "\t", 2) word = p[0] line = strings.ReplaceAll(p[1], "\t", " ") ) if err := db.InsertLine(ctx, word, line); err != nil { return err } count++ if (count % 1234) == 0 { fmt.Print("\033[G\033[K") // move the cursor left and clear the line per := math.Ceil((float64(count) / float64(total)) * 100.0) fmt.Printf("%d/%d (%.0f%%)", count, total, per) } } fmt.Printf("Consolidating") err = db.Consolidade(ctx) if err != nil { return err } err = db.Backup(ctx, sqliteOutput) if err != nil { return err } return nil } func lineCounter(r io.Reader) (int, error) { var count int const lineBreak = '\n' buf := make([]byte, bufio.MaxScanTokenSize) for { bufferSize, err := r.Read(buf) if err != nil && err != io.EOF { return 0, err } var buffPosition int for { i := bytes.IndexByte(buf[buffPosition:], lineBreak) if i == -1 || bufferSize == buffPosition { break } buffPosition += i + 1 count++ } if err == io.EOF { break } } return count, nil }