From 5bd0720e5bcc787f7e5df82bfc8f8704f0d9e77f Mon Sep 17 00:00:00 2001 From: Julius Lisauskas Date: Tue, 30 Apr 2024 20:39:20 +0300 Subject: [PATCH] fix: improve logging, add extra template --- Dockerfile | 1 + cmd/rok-scanner/main.go | 4 +- docs/install/golang_mac.md | 4 +- internal/pkg/ocrschema/result-schema.go | 3 + internal/pkg/rokocr/tesseractutils/parser.go | 8 +- .../pkg/rokocr/tesseractutils/recognition.go | 18 +- internal/pkg/www/jobs_controller.go | 11 +- templates/cod_more_info_iphone13_pro.json | 244 ++++++++++++++++++ 8 files changed, 267 insertions(+), 26 deletions(-) create mode 100644 templates/cod_more_info_iphone13_pro.json diff --git a/Dockerfile b/Dockerfile index e63db02..4403b26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,4 +8,5 @@ FROM debian:bookworm RUN apt update && apt install -y libtesseract5 COPY --from=build /usr/bin/rok-server /usr/bin/rok-server EXPOSE 8080 +ENV GOMEMLIMIT=100MiB ENTRYPOINT [ "/usr/bin/rok-server" ] diff --git a/cmd/rok-scanner/main.go b/cmd/rok-scanner/main.go index 7cb7776..e1a1dcf 100644 --- a/cmd/rok-scanner/main.go +++ b/cmd/rok-scanner/main.go @@ -2,11 +2,12 @@ package main import ( "fmt" - "github.com/rokmonster/ocr/internal/pkg/rokocr/tesseractutils" "os" "strings" "time" + "github.com/rokmonster/ocr/internal/pkg/rokocr/tesseractutils" + "github.com/olekukonko/tablewriter" config "github.com/rokmonster/ocr/internal/pkg/config/scannerconfig" schema "github.com/rokmonster/ocr/internal/pkg/ocrschema" @@ -50,6 +51,7 @@ func writeCSV(data []schema.OCRResult, template schema.OCRTemplate) { } func main() { + rokocr.Prepare(flags.CommonConfiguration) rokocr.DownloadTesseractData(flags.CommonConfiguration) rokocr.PreloadTemplates(flags.CommonConfiguration) diff --git a/docs/install/golang_mac.md b/docs/install/golang_mac.md index cdf880f..cd7360a 100644 --- a/docs/install/golang_mac.md +++ b/docs/install/golang_mac.md @@ -23,8 +23,8 @@ curl -LO https://go.dev/dl/go1.19.2.darwin-amd64.pkg && open ./go1.19.2.darwin-a # install tesseract & opencv brew install tesseract opencv # You might need these -# export CPATH="/opt/homebrew/include" -# export LIBRARY_PATH="/opt/homebrew/lib" +export CPATH="/opt/homebrew/include" +export LIBRARY_PATH="/opt/homebrew/lib" ``` ### PLEASE CHECK FOR ERRORS!!! diff --git a/internal/pkg/ocrschema/result-schema.go b/internal/pkg/ocrschema/result-schema.go index c838a3d..5bfeea0 100644 --- a/internal/pkg/ocrschema/result-schema.go +++ b/internal/pkg/ocrschema/result-schema.go @@ -1,6 +1,9 @@ package ocrschema +import "time" + type OCRResult struct { Filename string `json:"filename"` Data map[string]interface{} `json:"data"` + Took time.Duration `json:"duration"` } diff --git a/internal/pkg/rokocr/tesseractutils/parser.go b/internal/pkg/rokocr/tesseractutils/parser.go index d41fb12..a73ccaf 100644 --- a/internal/pkg/rokocr/tesseractutils/parser.go +++ b/internal/pkg/rokocr/tesseractutils/parser.go @@ -1,11 +1,13 @@ package tesseractutils import ( - imgutils2 "github.com/rokmonster/ocr/internal/pkg/utils/imgutils" - "github.com/rokmonster/ocr/internal/pkg/utils/stringutils" "image" "os" "path/filepath" + "time" + + imgutils2 "github.com/rokmonster/ocr/internal/pkg/utils/imgutils" + "github.com/rokmonster/ocr/internal/pkg/utils/stringutils" log "github.com/sirupsen/logrus" @@ -14,6 +16,7 @@ import ( func ParseImage(name string, img image.Image, template schema.OCRTemplate, tmpdir, tessdata string) schema.OCRResult { log.Debugf("[%s] Processing with template: %s", filepath.Base(name), template.Title) + start := time.Now() results := make(map[string]interface{}) @@ -35,5 +38,6 @@ func ParseImage(name string, img image.Image, template schema.OCRTemplate, tmpdi return schema.OCRResult{ Filename: filepath.Base(name), Data: results, + Took: time.Since(start), } } diff --git a/internal/pkg/rokocr/tesseractutils/recognition.go b/internal/pkg/rokocr/tesseractutils/recognition.go index 97472b6..ef84f29 100644 --- a/internal/pkg/rokocr/tesseractutils/recognition.go +++ b/internal/pkg/rokocr/tesseractutils/recognition.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/rokmonster/ocr/internal/pkg/utils/fileutils" "github.com/rokmonster/ocr/internal/pkg/utils/imgutils" @@ -13,17 +12,6 @@ import ( schema "github.com/rokmonster/ocr/internal/pkg/ocrschema" ) -// func processSingleFile(index, total int, f string) *schema.OCRResult { -// start := time.Now() -// result, err := ParseSingleFile(f, tessData, template, force) -// if err != nil { -// logrus.Printf("[%04d/%04d] %v - %v", index, total, filepath.Base(f), err) -// return -// } -// logrus.Printf("[%04d/%04d] %v Took: %v ms", index, total, filepath.Base(f), time.Since(start).Milliseconds()) -// return result -// } - func RunRecognitionChan(mediaDir, tessData string, template schema.OCRTemplate, force bool) <-chan schema.OCRResult { out := make(chan schema.OCRResult) @@ -33,13 +21,11 @@ func RunRecognitionChan(mediaDir, tessData string, template schema.OCRTemplate, total := len(files) for index, f := range files { - start := time.Now() result, err := ParseSingleFile(f, tessData, template, force) if err != nil { - logrus.Printf("[%04d/%04d] %v - %v", index, total, filepath.Base(f), err) + logrus.Errorf("[%04d/%04d] %v - %v", index, total, filepath.Base(f), err) continue } - logrus.Printf("[%04d/%04d] %v Took: %v ms", index, total, filepath.Base(f), time.Since(start).Milliseconds()) out <- *result } close(out) @@ -69,5 +55,5 @@ func ParseSingleFile(f, tessData string, template schema.OCRTemplate, force bool return &result, nil } - return nil, fmt.Errorf("image doesn't match the template") + return nil, fmt.Errorf("image doesn't match the template: Template: %s @ %s", template.Title, template.Version) } diff --git a/internal/pkg/www/jobs_controller.go b/internal/pkg/www/jobs_controller.go index cc4d7da..ccaffdd 100644 --- a/internal/pkg/www/jobs_controller.go +++ b/internal/pkg/www/jobs_controller.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "encoding/json" "fmt" - "github.com/rokmonster/ocr/internal/pkg/www/middlewares" "net/http" "os" "path/filepath" @@ -13,12 +12,12 @@ import ( "strings" "time" - "github.com/rokmonster/ocr/internal/pkg/rokocr/tesseractutils" - "github.com/rokmonster/ocr/internal/pkg/utils/fileutils" - "github.com/gin-gonic/gin" "github.com/rokmonster/ocr/internal/pkg/ocrschema" "github.com/rokmonster/ocr/internal/pkg/rokocr" + "github.com/rokmonster/ocr/internal/pkg/rokocr/tesseractutils" + "github.com/rokmonster/ocr/internal/pkg/utils/fileutils" + "github.com/rokmonster/ocr/internal/pkg/www/middlewares" log "github.com/sirupsen/logrus" bolt "go.etcd.io/bbolt" ) @@ -218,11 +217,13 @@ func (controller *JobsController) StartJobByID(c *gin.Context) { if len(templates) > 0 { log.Debugf("Loaded %v templates", len(templates)) template := ocrschema.FindTemplate(mediaDir, templates) + log.Infof("[Job: %04d] Picked template: %s by %s", job.ID, template.Title, template.Author) _ = controller.updateJobTemplate(job.ID, template) var data []ocrschema.OCRResult - for elem := range tesseractutils.RunRecognitionChan(mediaDir, controller.tessdataDir, template, false) { + for elem := range tesseractutils.RunRecognitionChan(mediaDir, controller.tessdataDir, template, true) { data = append(data, elem) + log.Printf("[Job: %04d][%04d/%04d] %v Took: %v ms", job.ID, index, fileCount, elem.Filename, elem.Took.Milliseconds()) index = index + 1 _ = controller.updateJobStatus(job.ID, fmt.Sprintf("Processing: %v/%v", index, fileCount)) _ = controller.updateJobResults(job.ID, data) diff --git a/templates/cod_more_info_iphone13_pro.json b/templates/cod_more_info_iphone13_pro.json new file mode 100644 index 0000000..9d24736 --- /dev/null +++ b/templates/cod_more_info_iphone13_pro.json @@ -0,0 +1,244 @@ +{ + "title": "Call of Dragons (More Info) / iPhone 13 Pro", + "version": "1", + "author": "Julius Lisauskas", + "width": 2532, + "height": 1170, + "ocr_schema": { + "high_power": { + "callback": [], + "lang": [ + "eng" + ], + "oem": 1, + "psm": 7, + "crop": [ + 1843, + 384, + 272, + 61 + ], + "allowlist": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ] + }, + "merits": { + "callback": [], + "lang": [ + "eng" + ], + "oem": 1, + "psm": 7, + "crop": [ + 441, + 1013, + 272, + 61 + ], + "allowlist": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ] + }, + "name": { + "callback": [], + "lang": [ + "eng", + "chi_sim", + "chi_tra", + "fra", + "ita", + "jpn", + "kor", + "rus", + "spa" + ], + "oem": 1, + "psm": 7, + "crop": [ + 445, + 738, + 223, + 61 + ] + }, + "power": { + "callback": [], + "lang": [ + "eng" + ], + "oem": 1, + "psm": 7, + "crop": [ + 441, + 869, + 272, + 61 + ], + "allowlist": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ] + }, + "units_dead": { + "callback": [], + "lang": [ + "eng" + ], + "oem": 1, + "psm": 7, + "crop": [ + 1837, + 806, + 272, + 61 + ], + "allowlist": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ] + }, + "units_killed": { + "callback": [], + "lang": [ + "eng" + ], + "oem": 1, + "psm": 7, + "crop": [ + 1840, + 742, + 272, + 61 + ], + "allowlist": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ] + }, + "victories": { + "callback": [], + "lang": [ + "eng" + ], + "oem": 1, + "psm": 7, + "crop": [ + 1842, + 505, + 272, + 61 + ], + "allowlist": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ] + } + }, + "fingerprint": "c2f0d8f8f8d0d8d8", + "threshold": 6, + "table": [ + [ + "units_killed", + "units_killed", + false, + "" + ], + [ + "units_dead", + "units_dead", + false, + "" + ], + [ + "name", + "name", + false, + "" + ], + [ + "power", + "power", + false, + "" + ], + [ + "merits", + "merits", + false, + "" + ], + [ + "high_power", + "high_power", + false, + "" + ], + [ + "victories", + "victories", + false, + "" + ] + ], + "checkpoints": [ + { + "crop": [ + 167, + 19, + 370, + 70 + ], + "fingerprint": "9be4e4ecccf0f0f1" + } + ] +} \ No newline at end of file