oh no indexer :(

index/main.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"git.tdpain.net/codemicro/hn84/index/internal/config"
 	"git.tdpain.net/codemicro/hn84/index/internal/database"
 	"git.tdpain.net/codemicro/hn84/util"
@@ -14,6 +15,7 @@ import (
 	"os"
 	"path"
 	"strings"
+	"sync"
 )
 
 func main() {
@@ -42,10 +44,42 @@ func walkDir(db *bun.DB, dir string) error {
 		return util.Wrap("read data dir", err)
 	}
 
-	for _, entry := range de {
+	fmt.Printf("%d entries in %s\n", len(de)/2, dir)
+	defer fmt.Println()
+
+	jobs := make(chan os.DirEntry)
+	stop := new(sync.WaitGroup)
+
+	go worker(jobs, stop, db, dir)
+	go worker(jobs, stop, db, dir)
+	go worker(jobs, stop, db, dir)
+	go worker(jobs, stop, db, dir)
+
+	for i, entry := range de {
+		jobs <- entry
+
+		if i%100 == 0 {
+			fmt.Printf("%d %.0f%%              \r", i, (float32(i)/float32(len(de)))*100)
+		}
+	}
+
+	return nil
+}
+
+func worker(jobs chan os.DirEntry, done *sync.WaitGroup, db *bun.DB, dir string) {
+	done.Add(1)
+	for entry := range jobs {
+		if err := processEntry(db, entry, dir); err != nil {
+			slog.Error("entry process error", err)
+		}
+	}
+	done.Done()
+}
+
+func processEntry(db *bun.DB, entry os.DirEntry, dir string) error {
 	name := entry.Name()
 	if !strings.HasSuffix(name, "html") {
-			continue
+		return nil
 	}
 
 	id := name[:len(name)-5]
@@ -99,9 +133,6 @@ func walkDir(db *bun.DB, dir string) error {
 		return util.Wrap("insert document to database", err)
 	}
 
-		break
-	}
-
 	return nil
 }