Perform first OCR extraction

moneymgr_mobile/android/app/build.gradle.kts

@@ -59,7 +59,7 @@ android {
         create("development") {
             dimension = "default"
             applicationIdSuffix = ".debug"
-            signingConfig = signingConfigs.getByName("debug")
+            // signingConfig = signingConfigs.getByName("debug")
         }
         create("publish") {
             dimension = "default"

moneymgr_mobile/lib/routes/scan/scan_screen.dart

@@ -1,45 +1,23 @@
-import 'dart:io';
 import 'dart:typed_data';
 
 import 'package:flutter/material.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:logging/logging.dart';
 import 'package:moneymgr_mobile/services/storage/expenses.dart';
+import 'package:moneymgr_mobile/utils/ocr_utils.dart';
+import 'package:moneymgr_mobile/utils/pdf_utils.dart';
 import 'package:moneymgr_mobile/widgets/expense_editor.dart';
 import 'package:riverpod_annotation/riverpod_annotation.dart';
-import 'package:scanbot_sdk/scanbot_sdk.dart';
-import 'package:scanbot_sdk/scanbot_sdk_ui_v2.dart' hide IconButton, EdgeInsets;
 
 part 'scan_screen.g.dart';
 
 /// Scan a document & return generated PDF as byte file
 @riverpod
-Future<Uint8List?> _scanDocument(Ref ref) async {
+Future<(Uint8List?, double?)> _scanDocument(Ref ref) async {
-  var configuration = DocumentScanningFlow(
+  final pdf = await scanDocAsPDF();
-    appearance: DocumentFlowAppearanceConfiguration(
+  final img = await renderPdf(pdfBytes: pdf);
-      statusBarMode: StatusBarMode.DARK,
+  final amount = await extractTotalFromBill(img);
-    ),
+  return (pdf, amount);
-    cleanScanningSession: true,
-    outputSettings: DocumentScannerOutputSettings(pagesScanLimit: 1),
-    screens: DocumentScannerScreens(
-      review: ReviewScreenConfiguration(enabled: false),
-    ),
-  );
-  var documentResult = await ScanbotSdkUiV2.startDocumentScanner(configuration);
-
-  if (documentResult.status != OperationStatus.OK) {
-    throw Exception("Scanner failed with status ${documentResult.status}");
-  }
-
-  // Convert result to PDF
-  var result = await ScanbotSdk.document.createPDFForDocument(
-    PDFFromDocumentParams(
-      documentID: documentResult.data!.uuid,
-      pdfConfiguration: PdfConfiguration(),
-    ),
-  );
-  final pdfPath = result.pdfFileUri.replaceFirst("file://", "");
-  return File(pdfPath).readAsBytes();
 }
 
 class ScanScreen extends HookConsumerWidget {
@@ -62,12 +40,17 @@ class ScanScreen extends HookConsumerWidget {
     return Padding(
       padding: const EdgeInsets.all(8.0),
       child: switch (scanDocProvider) {
-        AsyncData(:final value) when value != null => ExpenseEditor(
+        AsyncData(:final value) when value.$1 != null => ExpenseEditor(
-          file: value,
+          file: value.$1!,
+          initialData: BaseExpenseInfo(
+            label: null,
+            cost: value.$2 ?? 0.0,
+            time: DateTime.now(),
+          ),
           onFinished: (expense) async {
             await expenses.add(
               info: expense,
-              fileContent: value,
+              fileContent: value.$1!,
               fileMimeType: "application/pdf",
             );
             restartScan();
@@ -76,7 +59,7 @@ class ScanScreen extends HookConsumerWidget {
         ),
 
         // No data
-        AsyncData(:final value) when value == null => ScanErrorScreen(
+        AsyncData(:final value) when value.$1 == null => ScanErrorScreen(
           message: "No document scanned!",
           onTryAgain: restartScan,
         ),

moneymgr_mobile/lib/utils/ocr_utils.dart

@@ -0,0 +1,44 @@
+import 'dart:math';
+import 'dart:typed_data';
+import 'dart:ui' as ui;
+
+import 'package:flutter/material.dart';
+import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart';
+import 'package:logging/logging.dart';
+
+/// Attempt to extract total amount from invoice image
+Future<double?> extractTotalFromBill(Uint8List imgBuff) async {
+  final decodedImage = await decodeImageFromList(imgBuff);
+
+  final byteData = await decodedImage.toByteData(
+    format: ui.ImageByteFormat.rawRgba,
+  );
+
+  final image = InputImage.fromBitmap(
+    bitmap: byteData!.buffer.asUint8List(),
+    width: decodedImage.width,
+    height: decodedImage.height,
+  );
+
+  final textRecognizer = TextRecognizer(script: TextRecognitionScript.latin);
+  final extractionResult = await textRecognizer.processImage(image);
+
+  Logger.root.fine("Expense text: ${extractionResult.text}");
+
+  // Check for highest amount on invoice
+  final regexp = RegExp(
+    r'([0-9]+([ ]*(\\.|,)[ ]*[0-9]{1,2}){0,1})[ \\t\\n]*(EUR|eur|€)',
+    multiLine: true,
+    caseSensitive: false,
+  );
+  var highest = 0.0;
+  for (final match in regexp.allMatches(extractionResult.text)) {
+    if (match.groupCount == 0) continue;
+
+    // Process only numeric value
+    final value = (match.group(1) ?? "").replaceAll(",", ".");
+    highest = max(highest, double.tryParse(value) ?? 0.0);
+  }
+
+  return highest == 0.0 ? null : highest;
+}

moneymgr_mobile/lib/utils/pdf_utils.dart

@@ -6,14 +6,40 @@ import 'package:flutter/material.dart';
 import 'package:path/path.dart' as p;
 import 'package:path_provider/path_provider.dart';
 import 'package:pdf_image_renderer/pdf_image_renderer.dart';
+import 'package:scanbot_sdk/scanbot_sdk.dart';
+import 'package:scanbot_sdk/scanbot_sdk_ui_v2.dart' hide IconButton, EdgeInsets;
 
+/// Scan document as PDF
+Future<Uint8List> scanDocAsPDF() async {
+  var configuration = DocumentScanningFlow(
+    appearance: DocumentFlowAppearanceConfiguration(
+      statusBarMode: StatusBarMode.DARK,
+    ),
+    cleanScanningSession: true,
+    outputSettings: DocumentScannerOutputSettings(pagesScanLimit: 1),
+    screens: DocumentScannerScreens(
+      review: ReviewScreenConfiguration(enabled: false),
+    ),
+  );
+  var documentResult = await ScanbotSdkUiV2.startDocumentScanner(configuration);
+
+  if (documentResult.status != OperationStatus.OK) {
+    throw Exception("Scanner failed with status ${documentResult.status}");
+  }
+
+  // Convert result to PDF
+  var result = await ScanbotSdk.document.createPDFForDocument(
+    PDFFromDocumentParams(
+      documentID: documentResult.data!.uuid,
+      pdfConfiguration: PdfConfiguration(),
+    ),
+  );
+  final pdfPath = result.pdfFileUri.replaceFirst("file://", "");
+  return File(pdfPath).readAsBytes();
+}
 
 /// Render PDF to image bits
-Future<Uint8List> renderPdf(
+Future<Uint8List> renderPdf({String? path, Uint8List? pdfBytes}) async {
-     {
-      String? path,
-      Uint8List? pdfBytes,
-    }) async {
   assert(path != null || pdfBytes != null);
 
   // Create temporary file if required

moneymgr_mobile/pubspec.lock

@@ -488,6 +488,22 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "16.0.0"
+  google_mlkit_commons:
+    dependency: transitive
+    description:
+      name: google_mlkit_commons
+      sha256: "8f40fbac10685cad4715d11e6a0d86837d9ad7168684dfcad29610282a88e67a"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.11.0"
+  google_mlkit_text_recognition:
+    dependency: "direct main"
+    description:
+      name: google_mlkit_text_recognition
+      sha256: "96173ad4dd7fd06c660e22ac3f9e9f1798a517fe7e48bee68eeec83853224224"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.15.0"
   graphs:
     dependency: transitive
     description:

moneymgr_mobile/pubspec.yaml

@@ -19,7 +19,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev
 version: 1.0.0+1
 
 environment:
-  sdk: ^3.8.2
+  sdk: ^3.8.1
 
 # Dependencies specify other packages that your package needs in order to work.
 # To automatically upgrade your package dependencies to the latest versions
@@ -93,6 +93,9 @@ dependencies:
   # PDF renderer
   pdf_image_renderer: ^1.0.1
 
+  # Text extraction
+  google_mlkit_text_recognition: ^0.15.0
+
 dev_dependencies:
   flutter_test:
     sdk: flutter