Can extract date of expenses

moneymgr_backend/Cargo.lock

@@ -3232,9 +3232,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.140"
+version = "1.0.141"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
+checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3"
 dependencies = [
  "itoa",
  "memchr",

moneymgr_backend/Cargo.toml

@@ -22,7 +22,7 @@ rust-s3 = "0.36.0-beta.2"
 thiserror = "2.0.12"
 tokio = "1.45.1"
 futures-util = "0.3.31"
-serde_json = "1.0.140"
+serde_json = "1.0.141"
 light-openid = "1.0.4"
 rand = "0.9.1"
 ipnet = { version = "2.11.0", features = ["serde"] }

moneymgr_mobile/lib/routes/scan/scan_screen.dart

@@ -1,45 +1,23 @@
-import 'dart:io';
 import 'dart:typed_data';
 
 import 'package:flutter/material.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:logging/logging.dart';
 import 'package:moneymgr_mobile/services/storage/expenses.dart';
+import 'package:moneymgr_mobile/utils/ocr_utils.dart';
+import 'package:moneymgr_mobile/utils/pdf_utils.dart';
 import 'package:moneymgr_mobile/widgets/expense_editor.dart';
 import 'package:riverpod_annotation/riverpod_annotation.dart';
-import 'package:scanbot_sdk/scanbot_sdk.dart';
-import 'package:scanbot_sdk/scanbot_sdk_ui_v2.dart' hide IconButton, EdgeInsets;
 
 part 'scan_screen.g.dart';
 
 /// Scan a document & return generated PDF as byte file
 @riverpod
-Future<Uint8List?> _scanDocument(Ref ref) async {
-  var configuration = DocumentScanningFlow(
-    appearance: DocumentFlowAppearanceConfiguration(
-      statusBarMode: StatusBarMode.DARK,
-    ),
-    cleanScanningSession: true,
-    outputSettings: DocumentScannerOutputSettings(pagesScanLimit: 1),
-    screens: DocumentScannerScreens(
-      review: ReviewScreenConfiguration(enabled: false),
-    ),
-  );
-  var documentResult = await ScanbotSdkUiV2.startDocumentScanner(configuration);
-
-  if (documentResult.status != OperationStatus.OK) {
-    throw Exception("Scanner failed with status ${documentResult.status}");
-  }
-
-  // Convert result to PDF
-  var result = await ScanbotSdk.document.createPDFForDocument(
-    PDFFromDocumentParams(
-      documentID: documentResult.data!.uuid,
-      pdfConfiguration: PdfConfiguration(),
-    ),
-  );
-  final pdfPath = result.pdfFileUri.replaceFirst("file://", "");
-  return File(pdfPath).readAsBytes();
+Future<(Uint8List?, BaseExpenseInfo?)> _scanDocument(Ref ref) async {
+  final pdf = await scanDocAsPDF();
+  final img = await renderPdf(pdfBytes: pdf);
+  final amount = await extractInfoFromBill(img);
+  return (pdf, amount);
 }
 
 class ScanScreen extends HookConsumerWidget {
@@ -62,12 +40,13 @@ class ScanScreen extends HookConsumerWidget {
     return Padding(
       padding: const EdgeInsets.all(8.0),
       child: switch (scanDocProvider) {
-        AsyncData(:final value) when value != null => ExpenseEditor(
-          file: value,
+        AsyncData(:final value) when value.$1 != null => ExpenseEditor(
+          file: value.$1!,
+          initialData: value.$2,
           onFinished: (expense) async {
             await expenses.add(
               info: expense,
-              fileContent: value,
+              fileContent: value.$1!,
               fileMimeType: "application/pdf",
             );
             restartScan();
@@ -76,7 +55,7 @@ class ScanScreen extends HookConsumerWidget {
         ),
 
         // No data
-        AsyncData(:final value) when value == null => ScanErrorScreen(
+        AsyncData(:final value) when value.$1 == null => ScanErrorScreen(
           message: "No document scanned!",
           onTryAgain: restartScan,
         ),

moneymgr_mobile/lib/utils/ocr_utils.dart

@@ -0,0 +1,79 @@
+import 'dart:math';
+import 'dart:typed_data';
+import 'dart:ui' as ui;
+
+import 'package:flutter/material.dart';
+import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart';
+import 'package:logging/logging.dart';
+import 'package:moneymgr_mobile/services/storage/expenses.dart';
+
+/// Attempt to extract information from invoice image
+Future<BaseExpenseInfo?> extractInfoFromBill(Uint8List imgBuff) async {
+  final decodedImage = await decodeImageFromList(imgBuff);
+
+  final byteData = await decodedImage.toByteData(
+    format: ui.ImageByteFormat.rawRgba,
+  );
+
+  final image = InputImage.fromBitmap(
+    bitmap: byteData!.buffer.asUint8List(),
+    width: decodedImage.width,
+    height: decodedImage.height,
+  );
+
+  final textRecognizer = TextRecognizer(script: TextRecognitionScript.latin);
+  final extractionResult = await textRecognizer.processImage(image);
+
+  Logger.root.fine("Expense text: ${extractionResult.text}");
+
+  // Check for highestCost amount on invoice
+  final costRegexp = RegExp(
+    r'([0-9]+([ ]*(\\.|,)[ ]*[0-9]{1,2}){0,1})([ \\t\\n]*(EUR|eur|€)|E)',
+    multiLine: true,
+    caseSensitive: false,
+  );
+  var highestCost = 0.0;
+  for (final match in costRegexp.allMatches(extractionResult.text)) {
+    if (match.groupCount == 0) continue;
+
+    // Process only numeric value
+    final value = (match.group(1) ?? "").replaceAll(",", ".");
+    highestCost = max(highestCost, double.tryParse(value) ?? 0.0);
+  }
+
+  // Check for highestCost amount on invoice
+  final dateRegexp = RegExp(
+    r'([0-3][0-9])(\/|-)([0-1][0-9])(\/|-)(20[0-9]{2})',
+    multiLine: true,
+    caseSensitive: false,
+  );
+  final currDate = DateTime.now();
+  DateTime? newest;
+  for (final match in dateRegexp.allMatches(extractionResult.text)) {
+    if (match.groupCount < 5) continue;
+
+    try {
+      final date = DateTime(
+        int.tryParse(match.group(5)!) ?? currDate.year,
+        int.tryParse(match.group(3)!) ?? currDate.month,
+        int.tryParse(match.group(1)!) ?? currDate.day,
+      );
+
+      if (newest == null) {
+        newest = date;
+      } else {
+        newest = DateTime.fromMillisecondsSinceEpoch(
+          max(newest.millisecondsSinceEpoch, date.millisecondsSinceEpoch),
+        );
+      }
+    } catch (e, s) {
+      Logger.root.warning("Failed to parse date! $e$s");
+    }
+  }
+
+  return BaseExpenseInfo(
+    label: null,
+    cost: highestCost,
+    time: newest?.isBefore(currDate) ?? false ? newest! : currDate,
+  );
+}

moneymgr_mobile/lib/utils/pdf_utils.dart

@@ -6,14 +6,40 @@ import 'package:flutter/material.dart';
 import 'package:path/path.dart' as p;
 import 'package:path_provider/path_provider.dart';
 import 'package:pdf_image_renderer/pdf_image_renderer.dart';
+import 'package:scanbot_sdk/scanbot_sdk.dart';
+import 'package:scanbot_sdk/scanbot_sdk_ui_v2.dart' hide IconButton, EdgeInsets;
 
+/// Scan document as PDF
+Future<Uint8List> scanDocAsPDF() async {
+  var configuration = DocumentScanningFlow(
+    appearance: DocumentFlowAppearanceConfiguration(
+      statusBarMode: StatusBarMode.DARK,
+    ),
+    cleanScanningSession: true,
+    outputSettings: DocumentScannerOutputSettings(pagesScanLimit: 1),
+    screens: DocumentScannerScreens(
+      review: ReviewScreenConfiguration(enabled: false),
+    ),
+  );
+  var documentResult = await ScanbotSdkUiV2.startDocumentScanner(configuration);
+
+  if (documentResult.status != OperationStatus.OK) {
+    throw Exception("Scanner failed with status ${documentResult.status}");
+  }
+
+  // Convert result to PDF
+  var result = await ScanbotSdk.document.createPDFForDocument(
+    PDFFromDocumentParams(
+      documentID: documentResult.data!.uuid,
+      pdfConfiguration: PdfConfiguration(),
+    ),
+  );
+  final pdfPath = result.pdfFileUri.replaceFirst("file://", "");
+  return File(pdfPath).readAsBytes();
+}
 
 /// Render PDF to image bits
-Future<Uint8List> renderPdf(
-     {
-      String? path,
-      Uint8List? pdfBytes,
-    }) async {
+Future<Uint8List> renderPdf({String? path, Uint8List? pdfBytes}) async {
   assert(path != null || pdfBytes != null);
 
   // Create temporary file if required
@@ -62,4 +88,4 @@ Future<Uint8List> renderPdf(
       await File(path).delete();
     }
   }
-}
+}

moneymgr_mobile/pubspec.lock

@@ -488,6 +488,22 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "16.0.0"
+  google_mlkit_commons:
+    dependency: transitive
+    description:
+      name: google_mlkit_commons
+      sha256: "8f40fbac10685cad4715d11e6a0d86837d9ad7168684dfcad29610282a88e67a"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.11.0"
+  google_mlkit_text_recognition:
+    dependency: "direct main"
+    description:
+      name: google_mlkit_text_recognition
+      sha256: "96173ad4dd7fd06c660e22ac3f9e9f1798a517fe7e48bee68eeec83853224224"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.15.0"
   graphs:
     dependency: transitive
     description:

moneymgr_mobile/pubspec.yaml

@@ -93,6 +93,9 @@ dependencies:
   # PDF renderer
   pdf_image_renderer: ^1.0.1
 
+  # Text extraction
+  google_mlkit_text_recognition: ^0.15.0
+
 dev_dependencies:
   flutter_test:
     sdk: flutter